BiobaseInfernal 0.5.0.0 → 0.5.1.0

raw patch · 7 files changed

+257/−211 lines, 7 filesdep +attoparsec-iterateedep +either-unwrapdep +iterateedep −attoparsec-enumeratordep −enumeratorPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: attoparsec-iteratee, either-unwrap, iteratee

Dependencies removed: attoparsec-enumerator, enumerator

API changes (from Hackage documentation)

Files

• Biobase/Infernal.hs +32/−0
• Biobase/Infernal/TabularHit.hs +19/−0
• Biobase/Infernal/TabularHit/Import.hs +46/−0
• Biobase/Infernal/Taxonomy/Import.hs +46/−35
• Biobase/Infernal/VerboseHit/Export.hs +38/−124
• Biobase/Infernal/VerboseHit/Import.hs +65/−49
• BiobaseInfernal.cabal +11/−3

@@ -0,0 +1,32 @@++-- | Re-export the most import parts.++module Biobase.Infernal+  ( TabularHit(..)+  , thFromFile+  , eneeTabularHit+  , VerboseHit(..)+  , vhFromFile+  , eneeVerboseHit+  , vhEneeByteString+  , Species(..)+  ) where++import Data.ByteString as BS+import Data.Iteratee as I++import Biobase.Infernal.TabularHit+import Biobase.Infernal.TabularHit.Import as TH+import Biobase.Infernal.VerboseHit+import Biobase.Infernal.VerboseHit.Import as VH+import Biobase.Infernal.VerboseHit.Export as VH+import Biobase.Infernal.Taxonomy+import Biobase.Infernal.Taxonomy.Import as T++thFromFile = TH.fromFile++vhFromFile = VH.fromFile+vhEneeByteString :: Monad m => Enumeratee [VerboseHit] ByteString m a+vhEneeByteString = VH.eneeByteString++tFromFile = T.fromFile

@@ -0,0 +1,19 @@++-- | Simple tabular hits as returned by Infernal.++module Biobase.Infernal.TabularHit where++import Data.ByteString.Char8 as BS+++data TabularHit = TabularHit+  { thModel :: ByteString+  , thTarget :: ByteString+  , thScaffoldStart :: Int+  , thScaffoldStop :: Int+  , thQueryStart :: Int+  , thQueryStop :: Int+  , thBitScore :: Double+  , thEvalue :: Double+  , thGCpercent :: Int+  } deriving (Read,Show)

@@ -0,0 +1,46 @@+{-# LANGUAGE OverloadedStrings #-}++-- Importing tabular hits is rather easy, as they are one entry per line.++module Biobase.Infernal.TabularHit.Import where++import Data.ByteString.Char8 as BS+import Data.Iteratee as I+import Data.Iteratee.Iteratee as I+import Data.Iteratee.ListLike as I+import Data.Iteratee.Char as I+import Data.Either.Unwrap+import Data.Attoparsec as A hiding (takeTill)+import Data.Attoparsec.Char8 as A+import Control.Applicative+import Data.Iteratee.IO as I++import Biobase.Infernal.TabularHit++++-- | Transform a stream into tabular hits.++eneeTabularHit :: (Functor m, Monad m) => Enumeratee ByteString [TabularHit] m a+eneeTabularHit = enumLinesBS ><> I.filter (\x -> not $ BS.null x || isPrefixOf "#" x) ><> mapStream f where+  f = fromRight . parseOnly p+  p = TabularHit <$> pString -- model name+                 <*> pString -- target name+                 <*> pDecimal -- target start+                 <*> pDecimal -- target stop+                 <*> pDecimal -- query start+                 <*> pDecimal -- query stop+                 <*> pDouble -- bit score+                 <*> pDouble -- evalue+                 <*> pDecimal -- gc content+  pString = A.skipSpace *> A.takeTill A.isSpace+  pDecimal = A.skipSpace *> A.decimal+  pDouble = A.skipSpace *> A.double++-- | Convenience function to load from file and return a big list of tabular+-- hits.++fromFile :: FilePath -> IO [TabularHit]+fromFile fp = do+  i <- enumFile 8192 fp . joinI $ eneeTabularHit stream2stream+  run i

@@ -1,68 +1,79 @@+{-# LANGUAGE BangPatterns #-} -module Biobase.Infernal.Taxonomy.Import-  ( eeImport-  , iSpeciesMap-  , iTaxIdMap-  ) where+-- | Iteratee-based importer. Provides a simple "fromFile" function that+-- produces both maps in one pass. -import qualified Data.Enumerator as E-import qualified Data.Enumerator.List as EL-import qualified Data.ByteString.Char8 as BS-import qualified Data.Attoparsec as A+module Biobase.Infernal.Taxonomy.Import where+ import Control.Applicative-import qualified Data.Attoparsec.Char8 as A8-import qualified Data.Attoparsec.Enumerator as EAP-import qualified Data.Map as M+import Data.Attoparsec as A+import Data.Attoparsec.Char8 as A8+import Data.Attoparsec.Iteratee+import Data.ByteString.Char8 as BS+import Data.Either.Unwrap as E+import Data.Iteratee as I+import Data.Iteratee.Char as I+import Data.Iteratee.IO as I+import Data.Iteratee.ListLike as I+import Data.List as L+import Data.Map as M  import Biobase.Infernal.Taxonomy -import qualified Data.Enumerator.Binary as EB  - -- | Provide name-based lookup as the most-common usage scenario. -- -- TODO there are 9 duplicates in the names, let's find them and see what is -- going on -iSpeciesMap :: Monad m => E.Iteratee Species m (M.Map BS.ByteString Species)-iSpeciesMap = EL.fold (\m x -> M.insert (name x) x m) M.empty+iSpeciesMap :: Monad m => Iteratee [Species] m (M.Map ByteString Species)+iSpeciesMap = I.foldl' f M.empty where+  f !m x = M.insert (name x) x m  -- | And a map based on taxon id -iTaxIdMap :: Monad m => E.Iteratee Species m (M.Map Int Species)-iTaxIdMap = EL.fold (\m x -> M.insert (taxid x) x m) M.empty+iTaxIdMap :: Monad m => Iteratee [Species] m (M.Map Int Species)+iTaxIdMap = I.foldl' f M.empty where+  f !m x = M.insert (taxid x) x m  -- | Imports taxonomy data.++eneeSpecies :: Monad m => Enumeratee ByteString [Either String Species] m a+eneeSpecies = enumLinesBS ><> mapStream (parseOnly mkSpecies)++-- | Given a 'ByteString', create a species entry. -- -- NOTE The taxonomy format is, for each species, a line consisting of: taxid - -- tab - species name - tab - semicolon separated list of classification names -- - dot - end of line. -eeImport :: Monad m => E.Enumeratee BS.ByteString Species m b-eeImport = E.sequence $ EAP.iterParser mkSpecies---- | Given a 'ByteString', create a species entry.--mkSpecies :: A.Parser Species-mkSpecies = f <$> ptaxid <* tab <*> pname <* tab <*> A8.takeWhile (/='\n') <* A8.endOfLine where+mkSpecies :: Parser Species+mkSpecies = f <$> ptaxid <* tab <*> pname <* tab <*> takeByteString where   f k n xs = let-               cs = map (BS.copy . BS.dropWhile (==' ')) . BS.split ';' . BS.init $ xs-             in Species (BS.copy n) cs k-  ptaxid   = A8.decimal+               cs = L.map (copy . BS.dropWhile (==' ')) . BS.split ';' . BS.init $ xs+             in Species (copy n) cs k+  ptaxid   = decimal   pname    = A8.takeWhile (/='\t')-  tab      = A8.char '\t'+  tab      = char '\t' +-- | Convenience function: given a taxonomy file, produce both maps simultanously. +fromFile :: FilePath -> IO (M.Map ByteString Species, M.Map Int Species)+fromFile fp = do+  i <- enumFile 8192 fp+    . joinI+    . (eneeSpecies ><> I.filter isRight ><> mapStream fromRight)+    $ I.zip iSpeciesMap iTaxIdMap+  run i  -- * Testing +{- test :: IO () test = do-  m1 <- E.run_ $ (EB.enumFile "./Tests/Infernal/taxonomy" E.$$ (eeImport E.=$ iSpeciesMap))-  m2 <- E.run_ $ (EB.enumFile "./Tests/Infernal/taxonomy" E.$$ (eeImport E.=$ iTaxIdMap))-  print $ M.size m1-  print $ m1 M.! BS.pack "Cenarchaeum symbiosum B"-  -- print $ M.size $ M.mapKeys shortenName m1-  -- print $ M.size m2+  (s,t) <- fromFile "/home/choener/tmp/taxonomy"+  print $ M.size s+  print $ M.size t   return ()+-}

@@ -9,15 +9,12 @@ -- use printVerboseHit.  module Biobase.Infernal.VerboseHit.Export where-{--  ( eeFromVerboseHit-  ) where--}  import Control.Monad.Trans.Class (lift)-import qualified Data.ByteString.Char8 as BS-import qualified Data.Enumerator as E-import qualified Data.Enumerator.List as EL+import Data.ByteString.Char8 as BS+import Data.Iteratee as I+import Data.Maybe+import Prelude as P import Text.Printf  import Biobase.Infernal.VerboseHit@@ -25,51 +22,31 @@   --- | Takes a list of 'VerboseHit's and produces a list of bytestrings. Unlining--- those bytestrings produces a file that is \in essence\ an Infernal--- verbose-hit output file and should be parse-able by ours and other--- importers.------ TODO block length (for the alignment of query/sequenc) ?!+-- | Transforms a list of verbose hits into a bytestring. ----- TODO is there a more elegant treatment of the eof condition than asking at--- every verbose hit that is created?+-- TOOD How to append the last line "//" to the finished stream, if at least+-- one element was printed? -{--eeFromVerboseHit :: Monad m => E.Enumeratee VerboseHit BS.ByteString m a-eeFromVerboseHit = goS (AliGo "" "" '?') where-  goS s (E.Continue k) = EL.head >>= go s where-    go s Nothing = return $ E.Continue k-    go s@AliGo{..} (Just l@VerboseHit{..}) = do-      eof <- E.isEOF-      let res =  [ "\n//\n" | vhCM /= aliCM && aliCM /= "" ]-              ++ [ "\nCM: " `BS.append` vhCM `BS.append` "\n" | vhCM /= aliCM]-              ++ [ "\n>" `BS.append` vhSeqName `BS.append` "\n" | vhSeqName /= aliScaffold]-              ++ [ strand vhStrand | vhStrand /= aliStrand]-              ++ [ BS.pack $ printf " Query = %d - %d, Target = %d - %d"-                              (fst vhQuery) (snd vhQuery) (fst vhTarget) (snd vhTarget)-                 , BS.pack $ printf " Score = %.2f, E = %f, P = %.4e, GC = %d"-                              vhScore vhEvalue vhPvalue vhGC-                 , ""-                 , ws11 `BS.append` vhWuss-                 , (BS.pack $ printf "%10d " (fst vhQuery))-                      `BS.append` vhConsensus-                      `BS.append` (BS.pack $ printf " %d" (snd vhQuery))-                 , ws11 `BS.append` vhScoring-                 , (BS.pack $ printf "%10d " (fst vhTarget))-                      `BS.append` vhSequence-                      `BS.append` (BS.pack $ printf " %d" (snd vhTarget))-                 ]-              ++ [ "\n//" | eof]-      newStep <- lift $ E.runIteratee $ k $ E.Chunks res-      goS (AliGo vhCM vhSequence vhStrand) newStep-    strand '+' = "  Plus strand results:\n"-    strand '-' = "  Minus strand results:\n"-    strand _   = "  Unknown strand results:\n"-    ws11 = BS.pack $ replicate 11 ' '-  goS _ step = return step--}+eneeByteString :: Monad m => Enumeratee [VerboseHit] BS.ByteString m a+eneeByteString = unfoldConvStream f (AliGo BS.empty BS.empty '?') where+  f acc = do+    h <- I.head+    let na = newAcc acc h+    return (fst na , BS.unlines $ snd na ++ [showVerboseHit h]) +newAcc a@(AliGo{..}) h@VerboseHit{..}+  | otherwise = ( AliGo vhCM vhScaffold vhStrand, ls )+  where ls = [ "//" | aliCM /= BS.empty && aliCM /= vhCM ] +++             [ "CM: " `BS.append` vhCM | aliCM /= vhCM ] +++             [ ">" `BS.append` vhScaffold `BS.append` "\n" | aliScaffold /= vhScaffold ] +++             [ str `BS.append` " strand results:\n" | aliStrand /= vhStrand ]+        str+          | vhStrand == '+' = "Plus"+          | vhStrand == '-' = "Minus"+          | otherwise       = "Unknown"+++ -- | Convert a 'VerboseHit' to a string, ready for printing as in the input -- file. @@ -89,82 +66,19 @@     `BS.append` vhSequence     `BS.append` (BS.pack $ printf " %d" (snd vhTarget))   ] where-    ws11 = BS.pack $ replicate 11 ' '+    ws11 = BS.pack $ P.replicate 11 ' ' --- | CM information, ready for printing. -showCM :: BS.ByteString -> BS.ByteString-showCM cm = "CM: " `BS.append` cm --- | Scaffold information--showScaffold :: BS.ByteString -> BS.ByteString-showScaffold sc = ">" `BS.append` sc---- | Strand information--showStrand :: Char -> BS.ByteString-showStrand = f where-  f '+' = "  Plus strand results:"-  f '-' = "  Minus strand results:"-  f _   = "  Unknown strand results:"---- | Turning a list of 'VerboseHit's back into lines of characters is, in--- principle, not too hard. But just before we actually stream out, we might--- want to inject arbitrary data into the stream. This is done via--- 'StreamInsertion'. The other constructors merely wrap certain data.------ One way to, say, tag verbose hits is like this (note the output type of--- 'eeHitToStream'):------ > tag [s@(StreamVerboseHit _)] = [StreamInsertion (), s]--- > tag xs = xs--data HitStream a-  = StreamVerboseHit {streamVerboseHit :: VerboseHit}-  | StreamCM {streamCM :: BS.ByteString}-  | StreamScaffold {streamScaffold :: BS.ByteString}-  | StreamStrand {streamStrand :: Char}-  | StreamInsertion {streamInsertion :: a}-  deriving (Show)---- | This enumeratee turns 'VerboseHit's into a 'HitStream'. Each VerboseHit--- can emit one or more elements, depending on if the CM, scaffold, or strand--- changes.------ TODO try to rewrite use Control.Monad.State--eeHitToStream :: Monad m => E.Enumeratee VerboseHit [HitStream z] m a-eeHitToStream = EL.mapAccum go (AliGo "" "" '?') where-  go AliGo{..} vh@VerboseHit{..} = (AliGo vhCM vhScaffold vhStrand,-    [StreamCM vhCM | aliCM /= vhCM] ++-    [StreamScaffold vhScaffold | aliCM /= vhCM || aliScaffold /= vhScaffold] ++-    [StreamStrand vhStrand | aliCM /= vhCM || aliScaffold /= vhScaffold || aliStrand /= vhStrand] ++-    [StreamVerboseHit vh]-    )---- | Flattens a stream from a list of lists to a single list. After this point,--- you probably want to insert elements into the stream, then flatten again.--eeFlattenStream :: Monad m => E.Enumeratee [HitStream z] (HitStream z) m a-eeFlattenStream = EL.concatMap id---- | If the 'HitStream' contains 'StreamInsertion's that are an instance of--- 'Show', this provides a default method to turn the stream into a bytestring.------ TODO add some newline characters for good measure------ TODO on end-of-stream, we should print out "//"--eeStreamToByteString :: (Monad m, Show z) => StreamToByteString m z-eeStreamToByteString = EL.map f where-  f StreamVerboseHit{..} = showVerboseHit streamVerboseHit `BS.snoc` '\n'-  f StreamCM{..} = showCM streamCM `BS.append` "\n\n"-  f StreamScaffold{..} = showScaffold streamScaffold `BS.append` "\n\n"-  f StreamStrand{..} = showStrand streamStrand `BS.append` "\n\n"-  f StreamInsertion{..} = BS.pack . show $ streamInsertion--eeStreamToByteString' :: (Monad m) => StreamToByteString m ()-eeStreamToByteString' = eeStreamToByteString+{-+import Biobase.Infernal.VerboseHit.Import -type StreamToByteString m z = forall a . E.Enumeratee (HitStream z) BS.ByteString m a+test = do+  xs <- fromFile "/home/choener/tmp/infernal-1.0.2/tutorial/tmp.res"+  i <- enumList [xs] $ joinI $ eneeByteString stream2stream+  ys <- run i+  BS.putStrLn ys+  print $ BS.length ys+  print $ P.length $ BS.lines ys+  return ()+-}

@@ -6,59 +6,57 @@ -- | Enumeratee that transforms a stream of 'ByteString's into a stream of -- 'VerboseHit's. -module Biobase.Infernal.VerboseHit.Import -  ( eeToVerboseHit+module Biobase.Infernal.VerboseHit.Import+  ( eneeVerboseHit+  , fromFile   ) where  import Control.Applicative-import Control.Monad.IO.Class (MonadIO)-import Control.Monad.Trans.Class (lift)-import Data.Char (isSpace)+import Data.Attoparsec as A+import Data.Attoparsec.Char8 as A8+import Data.Attoparsec.Iteratee as EAP+import Data.ByteString.Char8 as BS+import Data.Either.Unwrap+import Data.Iteratee as I+import Data.Iteratee.Char as I+import Data.Iteratee.IO as I+import Data.Iteratee.Iteratee as I+import Data.Iteratee.ListLike as I import Data.Tuple.Select-import qualified Data.Attoparsec as A-import qualified Data.Attoparsec.Char8 as A8-import qualified Data.Attoparsec.Enumerator as EAP-import qualified Data.ByteString.Char8 as BS-import qualified Data.Enumerator as E-import qualified Data.Enumerator.List as EL+import Prelude as P  import Biobase.Infernal.VerboseHit import Biobase.Infernal.VerboseHit.Internal   --- | Stateful reading of 'VerboseHit's. This pipe groups certain lines and--- condenses them to one hit. As multiple hits share certain state (cm,--- scaffold, and strand), we thread some state through using "goS" internally.------ TODO is MonadIO really necessary?+-- | Transforms a stream into verbose hits. We need to keep a state in the+-- accumulator to keep track of the current CM, scaffold and strand. -eeToVerboseHit :: MonadIO m => E.Enumeratee BS.ByteString VerboseHit m b-eeToVerboseHit = goS (AliGo "" "" '?') where-  goS s (E.Continue k) = EL.dropWhile BS.null >> E.peek >>= go s where-    go s Nothing = return $ E.Continue k -- ???-    go s (Just (l :: BS.ByteString))-      | "CM: "    `BS.isInfixOf` l = drops >> E.peek >>= go s{aliCM = BS.copy $ BS.drop 4 l}-      | ">"       `BS.isInfixOf` l = drops >> E.peek >>= go s{aliScaffold = BS.copy $ BS.drop 1 l}-      | "  Plus"  `BS.isInfixOf` l = drops >> E.peek >>= go s{aliStrand = '+'}-      | "  Minus" `BS.isInfixOf` l = drops >> E.peek >>= go s{aliStrand = '-'}-      | " Query"  `BS.isInfixOf` l = do-          x <- qs (aliCM s) (aliScaffold s) (aliStrand s)-          newStep <- lift $ E.runIteratee $ k $ E.Chunks [x]-          goS s newStep-    go s l = drops >> E.peek >>= go s-    drops = EL.drop 1 >> EL.dropWhile BS.null-  goS _ step = return step+eneeVerboseHit :: (Functor m, Monad m) => Enumeratee BS.ByteString [VerboseHit] m a+eneeVerboseHit = enumLinesBS ><> I.filter (not . BS.null) ><> unfoldConvStream f (AliGo BS.empty BS.empty '?') where+  f acc = do+    h' <- tryHead+    case h' of+      Nothing -> return (acc, [])+      (Just h)+        | "CM: "    `BS.isInfixOf` h -> return (acc{aliCM = BS.copy $ BS.drop 4 h}, [])+        | ">"       `BS.isInfixOf` h -> return (acc{aliScaffold = BS.copy $ BS.drop 1 h}, [])+        | "  Plus"  `BS.isInfixOf` h -> return (acc{aliStrand = '+'}, [])+        | "  Minus" `BS.isInfixOf` h -> return (acc{aliStrand = '-'}, [])+        | " Query"  `BS.isInfixOf` h -> do+            x <- qs h (aliCM acc) (aliScaffold acc) (aliStrand acc)+            return (acc,x)+        | otherwise -> return (acc,[]) --- | Parses one "Alignment". This is build by Query/Score lines and multiple--- 4-tuples or strings with the actual alignment.+-- | Parses one CM query result. -qs :: Monad m => BS.ByteString -> BS.ByteString -> Char -> E.Iteratee BS.ByteString m VerboseHit-qs cm scaf pm = do-  q <- EAP.iterParser qt-  s <- EAP.iterParser sepg+qs :: Monad m => BS.ByteString -> BS.ByteString -> BS.ByteString -> Char -> Iteratee [BS.ByteString] m [VerboseHit]+qs query cm scaf pm = do+  let q = fromRight . parseOnly qt $ query+  s <- I.head >>= return . fromRight . parseOnly sepg   l <- fourLines $ sel4 q-  return $ VerboseHit+  return . pure $ VerboseHit     { vhScaffold = scaf     , vhCM = cm     , vhStrand = pm@@ -77,20 +75,38 @@     cpy = BS.copy . BS.concat     qt = (,,,) <$ A.string " Query = "   <*> A8.decimal <* A.string " - " <*> A8.decimal                <* A.string ", Target = " <*> A8.decimal <* A.string " - " <*> A8.decimal+               <?> "qt"     sepg = (,,,) <$ A.string " Score = " <*> A8.double                  <* A.string ", E = "    <*> A8.double                  <* A.string ", P = "    <*> A8.double                  <* A.string ", GC = " <* A8.skipSpace <*> A8.decimal+                 <?> "sepg" --- | Parse 4-tuples of cmsearch alignment output until we have hit the last, or--- "to", 4-tuple. Assumes that "to" is correct+-- | Parses multiple four-line elements.  fourLines to = do-  EL.dropWhile BS.null-  ls <- EL.take 4-  let ws = BS.length . BS.takeWhile isSpace . head $ ls-  let cs = BS.length . BS.dropWhile isSpace . head $ ls-  let xs = map (BS.take cs . BS.drop ws) ls-  if (to == (read . BS.unpack . last . BS.words . last $ ls))-  then return . map (:[]) $ xs-  else fourLines to >>= return . (zipWith (:) xs)+  I.dropWhile BS.null+  ls <- joinI $ I.take 4 stream2stream+  let ws = BS.length . BS.takeWhile isSpace . P.head $ ls+  let cs = BS.length . BS.dropWhile isSpace . P.head $ ls+  let xs = P.map (BS.take cs . BS.drop ws) ls+  if (to == (read . BS.unpack . P.last . BS.words . P.last $ ls))+  then return . P.map (:[]) $ xs+  else fourLines to >>= return . (P.zipWith (:) xs)++-- | Convenience function: read all results into a single list.++fromFile :: FilePath -> IO [VerboseHit]+fromFile fp = do+  i <- enumFile 8192 fp . joinI $ eneeVerboseHit stream2list+  run i++{- How to use this enumeratee.+++test = do+  i <- enumFile 8192 "/home/choener/tmp/infernal-1.0.2/tutorial/tmp.res" $ joinI $ eneeVerboseHit stream2list+  xs <- run i+  print xs+  print $ P.length xs+-}

@@ -1,5 +1,5 @@ name:           BiobaseInfernal-version:        0.5.0.0+version:        0.5.1.0 author:         Christian Hoener zu Siederdissen maintainer:     choener@tbi.univie.ac.at homepage:       http://www.tbi.univie.ac.at/~choener/@@ -19,6 +19,10 @@                 With the BioHaskell library split, this package is officially                 back! And the package is not feature-complete yet, take the                 above as a will-include-soon list.+                .+                The taxonomy importer makes use of Iteratee.zip, hence the+                switch from Enumerator. (See the Biohaskell wiki pages for+                discussion on Iteratee/Enumerator).  extra-source-files: @@ -26,14 +30,18 @@   build-depends:     base >3 && <5,     attoparsec,-    attoparsec-enumerator,+    attoparsec-iteratee,     bytestring,     containers,-    enumerator,+    either-unwrap,+    iteratee,     transformers,     tuple    exposed-modules:+    Biobase.Infernal+    Biobase.Infernal.TabularHit+    Biobase.Infernal.TabularHit.Import     Biobase.Infernal.Taxonomy     Biobase.Infernal.Taxonomy.Import     Biobase.Infernal.VerboseHit