iteratee 0.8.3.0 → 0.8.4.0

raw patch · 10 files changed

+456/−104 lines, 10 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• iteratee.cabal +4/−2
• src/Data/Iteratee.hs +41/−2
• src/Data/Iteratee/Base.hs +26/−2
• src/Data/Iteratee/Char.hs +6/−6
• src/Data/Iteratee/Exception.hs +2/−0
• src/Data/Iteratee/IO.hs +42/−14
• src/Data/Iteratee/IO/Fd.hs +27/−0
• src/Data/Iteratee/Iteratee.hs +112/−18
• src/Data/Iteratee/ListLike.hs +189/−60
• tests/testIteratee.hs +7/−0

@@ -1,9 +1,11 @@ name:          iteratee-version:       0.8.3.0+version:       0.8.4.0 synopsis:      Iteratee-based I/O description:   The Iteratee monad provides strict, safe, and functional I/O. In addition   to pure Iteratee processors, file IO and combinator functions are provided.++  See @Data.Iteratee@ for full documentation. category:      System, Data author:        Oleg Kiselyov, John W. Lato maintainer:    John W. Lato <jwlato@gmail.com>@@ -81,7 +83,7 @@   other-modules:     Data.Iteratee.IO.Base -  ghc-options:   -Wall+  ghc-options:   -Wall -O2   if impl(ghc >= 6.8)     ghc-options: -fwarn-tabs 

@@ -1,6 +1,45 @@-{- | Provide iteratee-based IO as described in Oleg Kiselyov's paper http://okmij.org/ftp/Haskell/Iteratee/.+{- | Provide iteratee-based IO as described in Oleg Kiselyov's paper 'http://okmij.org/ftp/Haskell/Iteratee/'. -Oleg's original code uses lists to store buffers of data for reading in the iteratee.  This package allows the use of arbitrary types through use of the StreamChunk type class.  See Data.Iteratee.WrappedByteString for implementation details.+Oleg's original code uses lists to store buffers of data for reading in the iteratee.  This package allows the use of arbitrary types through use of the ListLike type class.++Iteratees can be thought of as stream processor combinators.  Iteratees are combined to run in sequence or in parallel, and then processed by enumerators.  The result of the enumeration is another iteratee, which may then be used again, or have the result obtained via the 'run' function.++> -- count the number of bytes in a file, reading at most 8192 bytes at a time+> import Data.Iteratee as I+> import Data.Iteratee.IO+> import Data.ByteString+> +> byteCounter :: Monad m => Iteratee ByteString m Int+> byteCounter = I.length+> +> countBytes = do+>   i' <- enumFile 8192 "/usr/share/dict/words" byteCounter+>   result <- run i'+>   print result++Iteratees can be combined to perform much more complex tasks.  The iteratee monad allows for sequencing iteratee operations.++> iter2 = do+>   I.drop 4+>   I.head++In addition to enumerations over files and Handles, enumerations can be programmatically generated.++> get5thElement = enumPure1Chunk [1..10] iter2 >>= run >>= print++Iteratees can also work as stream transformers, called 'Enumeratee's.  A very simple example is provided by 'Data.Iteratee.ListLike.filter'.  When working with enumeratees, it's very common to collaps the nested iteratee with 'joinI'.++This function returns the 5th element greater than 5.++> iterfilt = joinI $ I.filter (>5) iter2+> find5thOver5 = enumPure1Chunk [10,1,4,6,7,4,2,8,5,9::Int] iterfilt >>= run >>= print++Another common use of iteratees is 'takeUpTo', which guarantees that an iteratee consumes a bounded number of elements.  This is often useful when parsing data.  You can check how much data an iteratee has consumed with 'enumWith'++> iter3 :: (Num el, Ord el, Monad m) => Iteratee [el] m (el,Int)+> iter3 = joinI (I.takeUpTo 100 (enumWith iterfilt I.length))++Many more functions are provided, and there are many other useful ways to combine iteratees and enumerators.  -} 

@@ -17,6 +17,7 @@   ,run   ,tryRun   ,mapIteratee+  ,ilift   -- ** Creating Iteratees   ,idone   ,icont@@ -165,8 +166,8 @@ instance (MonadCatchIO m, Nullable s, NullPoint s) =>   MonadCatchIO (Iteratee s m) where     m `catch` f = Iteratee $ \od oc -> runIter m od oc `catch` (\e -> runIter (f e) od oc)-    block       = mapIteratee block-    unblock     = mapIteratee unblock+    block       = ilift block+    unblock     = ilift unblock  -- |Send 'EOF' to the @Iteratee@ and disregard the unconsumed part of the -- stream.  If the iteratee is in an exception state, that exception is@@ -202,3 +203,26 @@   -> Iteratee s m a   -> Iteratee s n b mapIteratee f = lift . f . run+{-# DEPRECATED mapIteratee "This function will be removed, compare to 'ilift'" #-}++-- | Lift a computation in the inner monad of an iteratee.+-- +-- A simple use would be to lift a logger iteratee to a monad stack.+-- +-- > logger :: Iteratee String IO ()+-- > logger = mapChunksM_ putStrLn+-- > +-- > loggerG :: MonadIO m => Iteratee String m ()+-- > loggerG = ilift liftIO logger+-- +-- A more complex example would involve lifting an iteratee to work with+-- interleaved streams.  See the example at 'Data.Iteratee.ListLike.merge'.+ilift ::+  (Monad m, Monad n)+  => (forall r. m r -> n r)+  -> Iteratee s m a+  -> Iteratee s n a+ilift f i = Iteratee $  \od oc ->+  let onDone a str  = return $ Left (a,str)+      onCont k mErr = return $ Right (ilift f . k, mErr)+  in f (runIter i onDone onCont) >>= either (uncurry od) (uncurry oc)

@@ -83,12 +83,12 @@     step (Chunk xs)       | LL.null xs = getter       | lChar xs   = idone (LL.lines xs) mempty-      | True       = icont (step' xs) Nothing+      | otherwise  = icont (step' xs) Nothing     step _str      = getter     step' xs (Chunk ys)       | LL.null ys = icont (step' xs) Nothing       | lChar ys   = idone (LL.lines . mappend xs $ ys) mempty-      | True       = let w' = LL.lines $ mappend xs ys+      | otherwise  = let w' = LL.lines $ mappend xs ys                          ws = init w'                          ck = last w'                      in idone ws (Chunk ck)@@ -114,12 +114,12 @@     step (Chunk xs)       | BC.null xs = getter       | lChar xs   = idone (BC.words xs) (Chunk BC.empty)-      | True       = icont (step' xs) Nothing+      | otherwise  = icont (step' xs) Nothing     step str       = idone mempty str     step' xs (Chunk ys)       | BC.null ys = icont (step' xs) Nothing       | lChar ys   = idone (BC.words . BC.append xs $ ys) mempty-      | True       = let w' = BC.words . BC.append xs $ ys+      | otherwise  = let w' = BC.words . BC.append xs $ ys                          ws = init w'                          ck = last w'                      in idone ws (Chunk ck)@@ -138,12 +138,12 @@     step (Chunk xs)       | BC.null xs = getter       | lChar xs   = idone (BC.lines xs) (Chunk BC.empty)-      | True       = icont (step' xs) Nothing+      | otherwise  = icont (step' xs) Nothing     step str       = idone mempty str     step' xs (Chunk ys)       | BC.null ys = icont (step' xs) Nothing       | lChar ys   = idone (BC.lines . BC.append xs $ ys) mempty-      | True       = let w' = BC.lines $ BC.append xs ys+      | otherwise  = let w' = BC.lines $ BC.append xs ys                          ws = init w'                          ck = last w'                      in idone ws (Chunk ck)

@@ -55,6 +55,8 @@   ,enStrExc   ,iterStrExc   ,wrapIterExc+  ,iterExceptionToException+  ,iterExceptionFromException ) where 

@@ -7,14 +7,14 @@   defaultBufSize,   -- * File enumerators   -- ** Handle-based enumerators-  enumHandle,-  enumHandleRandom,+  H.enumHandle,+  H.enumHandleRandom,   enumFile,   enumFileRandom, #if defined(USE_POSIX)   -- ** FileDescriptor based enumerators-  enumFd,-  enumFdRandom,+  FD.enumFd,+  FD.enumFdRandom, #endif   -- * Iteratee drivers   --   These are FileDescriptor-based on POSIX systems, otherwise they are@@ -31,10 +31,10 @@ import Data.Iteratee.Base.ReadableChunk import Data.Iteratee.Iteratee import Data.Iteratee.Binary()-import Data.Iteratee.IO.Handle+import qualified Data.Iteratee.IO.Handle as H  #if defined(USE_POSIX)-import Data.Iteratee.IO.Fd+import qualified Data.Iteratee.IO.Fd as FD #endif  import Control.Monad.CatchIO@@ -46,6 +46,20 @@ -- If Posix is available, use the fileDriverRandomFd as fileDriverRandom.  Otherwise, use a handle-based variant. #if defined(USE_POSIX) +enumFile+  :: (MonadCatchIO m, NullPoint s, ReadableChunk s el) =>+     Int+     -> FilePath+     -> Enumerator s m a+enumFile = FD.enumFile++enumFileRandom+  :: (MonadCatchIO m, NullPoint s, ReadableChunk s el) =>+     Int+     -> FilePath+     -> Enumerator s m a+enumFileRandom = FD.enumFileRandom+ -- |Process a file using the given Iteratee.  This function wraps -- enumFd as a convenience. fileDriver@@ -53,7 +67,7 @@      Iteratee s m a      -> FilePath      -> m a-fileDriver = fileDriverFd defaultBufSize+fileDriver = FD.fileDriverFd defaultBufSize  -- |A version of fileDriver with a user-specified buffer size (in elements). fileDriverVBuf@@ -62,7 +76,7 @@      -> Iteratee s m a      -> FilePath      -> m a-fileDriverVBuf = fileDriverFd+fileDriverVBuf = FD.fileDriverFd  -- |Process a file using the given Iteratee.  This function wraps -- enumFdRandom as a convenience.@@ -71,7 +85,7 @@      Iteratee s m a      -> FilePath      -> m a-fileDriverRandom = fileDriverRandomFd defaultBufSize+fileDriverRandom = FD.fileDriverRandomFd defaultBufSize  fileDriverRandomVBuf   :: (MonadCatchIO m, NullPoint s, ReadableChunk s el) =>@@ -79,7 +93,7 @@      -> Iteratee s m a      -> FilePath      -> m a-fileDriverRandomVBuf = fileDriverRandomFd+fileDriverRandomVBuf = FD.fileDriverRandomFd  #else @@ -93,7 +107,7 @@   Iteratee s m a   -> FilePath   -> m a-fileDriver = fileDriverHandle defaultBufSize+fileDriver = H.fileDriverHandle defaultBufSize  -- |A version of fileDriver with a user-specified buffer size (in elements). fileDriverVBuf ::@@ -102,7 +116,7 @@   -> Iteratee s m a   -> FilePath   -> m a-fileDriverVBuf = fileDriverHandle+fileDriverVBuf = H.fileDriverHandle  -- |Process a file using the given Iteratee.  This function wraps -- @enumRandomHandle@ as a convenience.@@ -111,7 +125,7 @@      Iteratee s m a      -> FilePath      -> m a-fileDriverRandom = fileDriverRandomHandle defaultBufSize+fileDriverRandom = H.fileDriverRandomHandle defaultBufSize  fileDriverRandomVBuf   :: (MonadCatchIO m, NullPoint s, ReadableChunk s el) =>@@ -119,6 +133,20 @@      -> Iteratee s m a      -> FilePath      -> m a-fileDriverRandomVBuf = fileDriverRandomHandle+fileDriverRandomVBuf = H.fileDriverRandomHandle++enumFile+  :: (MonadCatchIO m, NullPoint s, ReadableChunk s el) =>+     Int+     -> FilePath+     -> Enumerator s m a+enumFile = H.enumFile++enumFileRandom+  :: (MonadCatchIO m, NullPoint s, ReadableChunk s el) =>+     Int+     -> FilePath+     -> Enumerator s m a+enumFileRandom = H.enumFileRandom  #endif

@@ -11,6 +11,8 @@   enumFd   ,enumFdCatch   ,enumFdRandom+  ,enumFile+  ,enumFileRandom   -- * Iteratee drivers   ,fileDriverFd   ,fileDriverRandomFd@@ -129,5 +131,30 @@      -> FilePath      -> m a fileDriverRandomFd = fileDriver enumFdRandom++enumFile' :: (NullPoint s, MonadCatchIO m, ReadableChunk s el) =>+  (Int -> Fd -> Enumerator s m a)+  -> Int -- ^Buffer size+  -> FilePath+  -> Enumerator s m a+enumFile' enumf bufsize filepath iter = CIO.bracket+  (liftIO $ openFd filepath ReadOnly Nothing defaultFileFlags)+  (liftIO . closeFd)+  (flip (enumf bufsize) iter)++enumFile ::+  (NullPoint s, MonadCatchIO m, ReadableChunk s el)+  => Int                 -- ^Buffer size+  -> FilePath+  -> Enumerator s m a+enumFile = enumFile' enumFd++enumFileRandom ::+  (NullPoint s, MonadCatchIO m, ReadableChunk s el)+  => Int                 -- ^Buffer size+  -> FilePath+  -> Enumerator s m a+enumFileRandom = enumFile' enumFdRandom+  #endif

@@ -21,7 +21,10 @@   -- ** Chunkwise Iteratees   ,mapChunksM_   ,mapReduce+  ,getChunk+  ,getChunks   -- ** Nested iteratee combinators+  ,mapChunks   ,convStream   ,unfoldConvStream   ,joinI@@ -34,12 +37,14 @@   ,enumEof   ,enumErr   ,enumPure1Chunk+  ,enumList   ,enumCheckIfDone   ,enumFromCallback   ,enumFromCallbackCatch   -- ** Enumerator Combinators   ,(>>>)   ,eneeCheckIfDone+  ,mergeEnums   -- ** Enumeratee Combinators   ,(><>)   ,(<><)@@ -134,7 +139,7 @@ -- | Map a monadic function over the chunks of the stream and ignore the -- result.  Useful for creating efficient monadic iteratee consumers, e.g. -- ---   logger = mapChunksM_ (liftIO . putStrLn)+-- >  logger = mapChunksM_ (liftIO . putStrLn) --  -- these can be efficiently run in parallel with other iteratees via -- @Data.Iteratee.ListLike.zip@.@@ -142,9 +147,9 @@ mapChunksM_ f = liftI step   where     step (Chunk xs)-      | nullC xs = liftI step-      | True     = lift (f xs) >> liftI step-    step s@(EOF _)  = idone () s+      | nullC xs   = liftI step+      | otherwise  = lift (f xs) >> liftI step+    step s@(EOF _) = idone () s {-# INLINE mapChunksM_ #-}  -- | Perform a parallel map/reduce.  The `bufsize` parameter controls@@ -176,6 +181,27 @@   step acc       (EOF (Just err))  =     throwRecoverableErr err (step acc) +-- | Get the current chunk from the stream.+getChunk :: (Monad m, Nullable s, NullPoint s) => Iteratee s m s+getChunk = liftI step+ where+  step (Chunk xs)+    | nullC xs  = liftI step+    | otherwise = idone xs $ Chunk empty+  step (EOF Nothing)  = throwErr $ toException EofException+  step (EOF (Just e)) = throwErr e+{-# INLINE getChunk #-}++-- | Get a list of all chunks from the stream.+getChunks :: (Monad m, Nullable s) => Iteratee s m [s]+getChunks = liftI (step [])+ where+  step acc (Chunk xs)+    | nullC xs    = liftI (step acc)+    | otherwise   = liftI (step (xs:acc))+  step acc stream = idone (reverse acc) stream+{-# INLINE getChunks #-}+ -- --------------------------------------------------- -- The converters show a different way of composing two iteratees: -- `vertical' rather than `horizontal'@@ -187,17 +213,48 @@ -- The following pattern appears often in Enumeratee code {-# INLINE eneeCheckIfDone #-} +-- | Utility function for creating enumeratees.  Typical usage is demonstrated+-- by the @breakE@ definition.+-- +-- > breakE+-- >   :: (Monad m, LL.ListLike s el, NullPoint s)+-- >   => (el -> Bool)+-- >   -> Enumeratee s s m a+-- > breakE cpred = eneeCheckIfDone (liftI . step)+-- >  where+-- >   step k (Chunk s)+-- >       | LL.null s  = liftI (step k)+-- >       | otherwise  = case LL.break cpred s of+-- >         (str', tail')+-- >           | LL.null tail' -> eneeCheckIfDone (liftI . step) . k $ Chunk str'+-- >           | otherwise     -> idone (k $ Chunk str') (Chunk tail')+-- >   step k stream           =  idone (k stream) stream+--  eneeCheckIfDone ::  (Monad m, NullPoint elo) =>   ((Stream eli -> Iteratee eli m a) -> Iteratee elo m (Iteratee eli m a))   -> Enumeratee elo eli m a eneeCheckIfDone f inner = Iteratee $ \od oc -> -  let on_done x s = od (idone x s) (Chunk empty)-      on_cont k Nothing  = runIter (f k) od oc-      on_cont _ (Just e) = runIter (throwErr e) od oc-  in runIter inner on_done on_cont+  let onDone x s = od (idone x s) (Chunk empty)+      onCont k Nothing  = runIter (f k) od oc+      onCont _ (Just e) = runIter (throwErr e) od oc+  in runIter inner onDone onCont +-- | Convert one stream into another with the supplied mapping function.+-- This function operates on whole chunks at a time, contrasting to+-- @mapStream@ which operates on single elements.+-- +-- > unpacker :: Enumeratee B.ByteString [Word8] m a+-- > unpacker = mapChunks B.unpack+-- +mapChunks :: (Monad m, NullPoint s) => (s -> s') -> Enumeratee s s' m a+mapChunks f = eneeCheckIfDone (liftI . step)+ where+  step k (Chunk xs)     = eneeCheckIfDone (liftI . step) . k . Chunk $ f xs+  step k str@(EOF mErr) = idone (k $ EOF mErr) str+{-# INLINE mapChunks #-} + -- |Convert one stream into another, not necessarily in lockstep. -- The transformer mapStream maps one element of the outer stream -- to one element of the nested stream.  The transformer below is more@@ -230,18 +287,27 @@                     eneeCheckIfDone (check acc') . k . Chunk $ s'  +-- | Collapse a nested iteratee.  The inner iteratee is terminated by @EOF@.+--   Errors are propagated through the result.+-- +--  The stream resumes from the point of the outer iteratee; any remaining+--  input in the inner iteratee will be lost.+--  Differs from 'Control.Monad.join' in that the inner iteratee is terminated,+--  and may have a different stream type than the result. joinI ::  (Monad m, Nullable s) =>   Iteratee s m (Iteratee s' m a)   -> Iteratee s m a joinI = (>>=   \inner -> Iteratee $ \od oc ->-  let on_done  x _        = od x (Chunk empty)-      on_cont  k Nothing  = runIter (k (EOF Nothing)) on_done on_cont'-      on_cont  _ (Just e) = runIter (throwErr e) od oc-      on_cont' _ e        = runIter (throwErr (fromMaybe excDivergent e)) od oc-  in runIter inner on_done on_cont)+  let onDone  x _        = od x (Chunk empty)+      onCont  k Nothing  = runIter (k (EOF Nothing)) onDone onCont'+      onCont  _ (Just e) = runIter (throwErr e) od oc+      onCont' _ e        = runIter (throwErr (fromMaybe excDivergent e)) od oc+  in runIter inner onDone onCont)+{-# INLINE joinI #-} +-- | Lift an iteratee inside a monad to an iteratee. joinIM :: (Monad m) => m (Iteratee s m a) -> Iteratee s m a joinIM mIter = Iteratee $ \od oc -> mIter >>= \iter -> runIter iter od oc @@ -320,7 +386,20 @@   -> Enumeratee s1 s3 m a f <>< g = joinI . g . f --- |The pure 1-chunk enumerator+-- | Combine enumeration over two streams.  The merging enumeratee would+-- typically be the result of 'Data.Iteratee.ListLike.merge' or+-- 'Data.Iteratee.ListLike.mergeByChunks' (see @merge@ for example).+mergeEnums :: +  (Nullable s2, Nullable s1, Monad m)+  => Enumerator s1 m a                   -- ^ inner enumerator+  -> Enumerator s2 (Iteratee s1 m) a     -- ^ outer enumerator+  -> Enumeratee s2 s1 (Iteratee s1 m) a  -- ^ merging enumeratee+  -> Enumerator s1 m a+mergeEnums e1 e2 etee i = e1 $ e2 (joinI . etee $ ilift lift i) >>= run+{-# INLINE mergeEnums #-}++-- | The pure 1-chunk enumerator+--  -- It passes a given list of elements to the iteratee in one chunk -- This enumerator does no IO and is useful for testing of base parsing enumPure1Chunk :: (Monad m) => s -> Enumerator s m a@@ -329,7 +408,21 @@     onCont k Nothing = return $ k $ Chunk str     onCont k e       = return $ icont k e --- |Checks if an iteratee has finished.+-- | Enumerate chunks from a list+-- +enumList :: (Monad m) => [s] -> Enumerator s m a+enumList chunks = go chunks+ where+  go [] i = return i+  go xs' i = runIter i idoneM (onCont xs')+   where+    onCont (x:xs) k Nothing = go xs . k $ Chunk x+    onCont _ _ (Just e) = return $ throwErr e+    onCont _ k Nothing  = return $ icont k Nothing+{-# INLINABLE enumList #-}++-- | Checks if an iteratee has finished.+--  -- This enumerator runs the iteratee, performing any monadic actions. -- If the result is True, the returned iteratee is done. enumCheckIfDone :: (Monad m) => Iteratee s m a -> m (Bool, Iteratee s m a)@@ -368,13 +461,14 @@   -> Enumerator s m a enumFromCallbackCatch c handler = loop   where-    loop st iter = runIter iter idoneM (on_cont st)-    on_cont st k Nothing = c st >>=+    loop st iter = runIter iter idoneM (onCont st)+    onCont st k Nothing = c st >>=         either (return . k . EOF . Just) (uncurry check)       where         check (b,st') = if b then loop st' . k . Chunk else return . k . Chunk-    on_cont st k j@(Just e) = case fromException e of+    onCont st k j@(Just e) = case fromException e of       Just e' -> handler e' >>= maybe (loop st . k $ Chunk empty)                                  (return . icont k . Just) . fmap toException       Nothing -> return (icont k j)+{-# INLINE enumFromCallbackCatch #-} 

@@ -1,4 +1,4 @@-{-# LANGUAGE FlexibleContexts, BangPatterns #-}+{-# LANGUAGE FlexibleContexts, BangPatterns, TupleSections #-}  -- |Monadic Iteratees: -- incremental input parsers, processors and transformers@@ -23,6 +23,8 @@   ,peek   ,roll   ,length+  ,chunkLength+  ,takeFromChunk   -- ** Nested iteratee combinators   ,breakE   ,take@@ -32,6 +34,8 @@   ,filter   ,group   ,groupBy+  ,merge+  ,mergeByChunks   -- ** Folds   ,foldl   ,foldl'@@ -54,7 +58,7 @@   -- ** Monadic functions   ,mapM_   ,foldM-  -- * Classes+  -- * Re-exported modules   ,module Data.Iteratee.Iteratee ) where@@ -68,7 +72,7 @@ import Data.Iteratee.Iteratee import Data.Monoid import Data.Maybe (catMaybes)-import Control.Applicative+import Control.Applicative ((<$>), (<*>), (<*)) import Control.Monad (liftM, liftM2, mplus, (<=<)) import Control.Monad.Trans.Class import Data.Word (Word8)@@ -82,8 +86,8 @@ isFinished = liftI check   where   check c@(Chunk xs)-    | nullC xs     = liftI check-    | True        = idone False c+    | nullC xs    = liftI check+    | otherwise   = idone False c   check s@(EOF _) = idone True s {-# INLINE isFinished #-} @@ -97,8 +101,8 @@   where     step acc (Chunk ls)       | nullC ls  = liftI (step acc)-      | True     = liftI (step (acc ++ LL.toList ls))-    step acc str = idone acc str+      | otherwise = liftI (step (acc ++ LL.toList ls))+    step acc str  = idone acc str {-# INLINE stream2list #-}  -- |Read a stream to the end and return all of its elements as a stream.@@ -108,8 +112,8 @@   where     step acc (Chunk ls)       | nullC ls   = icont (step acc) Nothing-      | True      = icont (step (acc `mappend` ls)) Nothing-    step acc str  = idone acc str+      | otherwise  = icont (step (acc `mappend` ls)) Nothing+    step acc str   = idone acc str {-# INLINE stream2stream #-}  @@ -122,7 +126,7 @@ -- If the stream is not terminated, the first character of the remaining stream -- satisfies the predicate. -- --- N.B. @breakE@ should be used in preference to @break@.+-- N.B. 'breakE' should be used in preference to @break@. -- @break@ will retain all data until the predicate is met, which may -- result in a space leak. -- @@ -133,10 +137,10 @@   where     step bfr (Chunk str)       | LL.null str       =  icont (step bfr) Nothing-      | True              =  case LL.break cpred str of+      | otherwise         =  case LL.break cpred str of         (str', tail')           | LL.null tail' -> icont (step (bfr `mappend` str)) Nothing-          | True          -> idone (bfr `mappend` str') (Chunk tail')+          | otherwise     -> idone (bfr `mappend` str') (Chunk tail')     step bfr stream       =  idone bfr stream {-# INLINE break #-} @@ -150,7 +154,7 @@   where   step (Chunk vec)     | LL.null vec  = icont step Nothing-    | True         = idone (LL.head vec) (Chunk $ LL.tail vec)+    | otherwise    = idone (LL.head vec) (Chunk $ LL.tail vec)   step stream      = icont step (Just (setEOF stream)) {-# INLINE head #-} @@ -174,15 +178,15 @@ -- the characters on the stream.  Return the count of how many -- characters matched.  The matched characters are removed from the -- stream.--- For example, if the stream contains "abd", then (heads "abc")--- will remove the characters "ab" and return 2.+-- For example, if the stream contains 'abd', then (heads 'abc')+-- will remove the characters 'ab' and return 2. heads :: (Monad m, Nullable s, LL.ListLike s el, Eq el) => s -> Iteratee s m Int heads st | nullC st = return 0 heads st = loop 0 st   where   loop cnt xs-    | nullC xs = return cnt-    | True     = liftI (step cnt xs)+    | nullC xs  = return cnt+    | otherwise = liftI (step cnt xs)   step cnt str (Chunk xs) | nullC xs  = liftI (step cnt str)   step cnt str stream     | nullC str = idone cnt stream   step cnt str s@(Chunk xs) =@@ -196,22 +200,23 @@ -- |Look ahead at the next element of the stream, without removing -- it from the stream. -- Return @Just c@ if successful, return @Nothing@ if the stream is--- terminated by EOF.+-- terminated by 'EOF'. peek :: (Monad m, LL.ListLike s el) => Iteratee s m (Maybe el) peek = liftI step   where     step s@(Chunk vec)       | LL.null vec = liftI step-      | True        = idone (Just $ LL.head vec) s+      | otherwise   = idone (Just $ LL.head vec) s     step stream     = idone Nothing stream {-# INLINE peek #-} --- | Return a chunk of `t' elements length, while consuming `d' elements---   from the stream.  Useful for creating a "rolling average" with convStream.+-- | Return a chunk of @t@ elements length while consuming @d@ elements+--   from the stream.  Useful for creating a 'rolling average' with+--  'convStream'. roll   :: (Monad m, Functor m, Nullable s, LL.ListLike s el, LL.ListLike s' s)-  => Int-  -> Int+  => Int  -- ^ length of chunk (t)+  -> Int  -- ^ amount to consume (d)   -> Iteratee s m s' roll t d | t > d  = liftI step   where@@ -221,7 +226,7 @@       | LL.length vec >= t =           idone (LL.singleton $ LL.take t vec) mempty <* drop (d-LL.length vec)       | LL.null vec        = liftI step-      | True               = liftI (step' vec)+      | otherwise          = liftI (step' vec)     step stream            = idone LL.empty stream     step' v1 (Chunk vec)   = step . Chunk $ v1 `mappend` vec     step' v1 stream        = idone (LL.singleton v1) stream@@ -238,9 +243,9 @@ drop n' = liftI (step n')   where     step n (Chunk str)-      | LL.length str <= n = liftI (step (n - LL.length str))-      | True               = idone () (Chunk (LL.drop n str))-    step _ stream          = idone () stream+      | LL.length str < n = liftI (step (n - LL.length str))+      | otherwise         = idone () (Chunk (LL.drop n str))+    step _ stream         = idone () stream {-# INLINE drop #-}  -- |Skip all elements while the predicate is true.@@ -251,14 +256,15 @@   where     step (Chunk str)       | LL.null left = liftI step-      | True         = idone () (Chunk left)+      | otherwise    = idone () (Chunk left)       where         left = LL.dropWhile p str     step stream      = idone () stream {-# INLINE dropWhile #-}  --- |Return the total length of the remaining part of the stream.+-- | Return the total length of the remaining part of the stream.+--  -- This forces evaluation of the entire stream. --  -- The analogue of @List.length@@@ -269,7 +275,29 @@     step !i stream     = idone i stream {-# INLINE length #-} +-- | Get the length of the current chunk, or @Nothing@ if 'EOF'.+-- +-- This function consumes no input.+chunkLength :: (Monad m, LL.ListLike s el) => Iteratee s m (Maybe Int)+chunkLength = liftI step+ where+  step s@(Chunk xs) = idone (Just $ LL.length xs) s+  step stream       = idone Nothing stream+{-# INLINE chunkLength #-} +-- | Take @n@ elements from the current chunk, or the whole chunk if+-- @n@ is greater.+takeFromChunk ::+  (Monad m, Nullable s, LL.ListLike s el)+  => Int+  -> Iteratee s m s+takeFromChunk n | n <= 0 = return empty+takeFromChunk n = liftI step+ where+  step (Chunk xs) = let (h,t) = LL.splitAt n xs in idone h $ Chunk t+  step stream     = idone empty stream+{-# INLINE takeFromChunk #-}+ -- --------------------------------------------------- -- The converters show a different way of composing two iteratees: -- `vertical' rather than `horizontal'@@ -301,10 +329,13 @@ -- read exactly n elements, even if the iteratee has accepted fewer. --  -- The analogue of @List.take@-take :: (Monad m, Nullable s, LL.ListLike s el) => Int -> Enumeratee s s m a+take ::+  (Monad m, Nullable s, LL.ListLike s el)+  => Int   -- ^ number of elements to consume+  -> Enumeratee s s m a take n' iter- | n' <= 0 = return iter- | True    = Iteratee $ \od oc -> runIter iter (on_done od oc) (on_cont od oc)+ | n' <= 0   = return iter+ | otherwise = Iteratee $ \od oc -> runIter iter (on_done od oc) (on_cont od oc)   where     on_done od oc x _ = runIter (drop n' >> return (return x)) od oc     on_cont od oc k Nothing = if n' == 0 then od (liftI k) (Chunk mempty)@@ -313,7 +344,7 @@     step n k (Chunk str)       | LL.null str        = liftI (step n k)       | LL.length str <= n = take (n - LL.length str) $ k (Chunk str)-      | True               = idone (k (Chunk s1)) (Chunk s2)+      | otherwise          = idone (k (Chunk s1)) (Chunk s2)       where (s1, s2) = LL.splitAt n str     step _n k stream       = idone (k stream) stream {-# SPECIALIZE take :: Monad m => Int -> Enumeratee [el] [el] m a #-}@@ -323,34 +354,62 @@ -- |Read n elements from a stream and apply the given iteratee to the -- stream of the read elements. If the given iteratee accepted fewer -- elements, we stop.--- This is the variation of `take' with the early termination+-- This is the variation of 'take' with the early termination -- of processing of the outer stream once the processing of the inner stream -- finished early. -- --- N.B. If the inner iteratee finishes early, remaining data within the current--- chunk will be dropped.+-- Iteratees composed with 'takeUpTo' will consume only enough elements to+-- reach a done state.  Any remaining data will be available in the outer+-- stream.+-- +-- > > let iter = do+-- > h <- joinI $ takeUpTo 5 I.head+-- > t <- stream2list+-- > return (h,t)+-- > +-- > > enumPureNChunk [1..10::Int] 3 iter >>= run >>= print+-- > (1,[2,3,4,5,6,7,8,9,10])+-- > +-- > > enumPureNChunk [1..10::Int] 7 iter >>= run >>= print+-- > (1,[2,3,4,5,6,7,8,9,10])+-- +-- in each case, @I.head@ consumes only one element, returning the remaining+-- 4 elements to the outer stream takeUpTo :: (Monad m, Nullable s, LL.ListLike s el) => Int -> Enumeratee s s m a takeUpTo i iter  | i <= 0    = return iter  | otherwise = Iteratee $ \od oc ->     runIter iter (onDone od oc) (onCont od oc)   where-    onDone od oc x _        = runIter (return (return x)) od oc+    onDone od oc x str      = runIter (idone (return x) str) od oc     onCont od oc k Nothing  = if i == 0 then od (liftI k) (Chunk mempty)                                  else runIter (liftI (step i k)) od oc     onCont od oc _ (Just e) = runIter (throwErr e) od oc     step n k (Chunk str)-      | LL.null str         = liftI (step n k)-      | LL.length str <= n  = takeUpTo (n - LL.length str) $ k (Chunk str)-      | True                = idone (k (Chunk s1)) (Chunk s2)-      where (s1, s2) = LL.splitAt n str-    step _ k stream         = idone (k stream) stream+      | LL.null str       = liftI (step n k)+      | LL.length str < n = takeUpTo (n - LL.length str) $ k (Chunk str)+      | otherwise         =+         -- check to see if the inner iteratee has completed, and if so,+         -- grab any remaining stream to put it in the outer iteratee.+         -- the outer iteratee is always complete at this stage, although+         -- the inner may not be.+         let (s1, s2) = LL.splitAt n str+         in Iteratee $ \od' _ -> do+              res <- runIter (k (Chunk s1)) (\a s  -> return $ Left  (a, s))+                                            (\k' e -> return $ Right (k',e))+              case res of+                Left (a,Chunk s1') -> od' (return a)+                                          (Chunk $ s1' `LL.append` s2)+                Left  (a,s')       -> od' (idone a s') (Chunk s2)+                Right (k',e)       -> od' (icont k' e) (Chunk s2)+    step _ k stream       = idone (k stream) stream {-# SPECIALIZE takeUpTo :: Monad m => Int -> Enumeratee [el] [el] m a #-} {-# SPECIALIZE takeUpTo :: Monad m => Int -> Enumeratee B.ByteString B.ByteString m a #-}+{-# INLINABLE takeUpTo #-}   -- |Map the stream: another iteratee transformer--- Given the stream of elements of the type @el@ and the function @el->el'@,+-- Given the stream of elements of the type @el@ and the function @(el->el')@, -- build a nested stream of elements of the type @el'@ and apply the -- given iteratee to it. -- @@ -367,7 +426,7 @@   where     step k (Chunk xs)       | LL.null xs = liftI (step k)-      | True       = mapStream f $ k (Chunk $ lMap f xs)+      | otherwise  = mapStream f $ k (Chunk $ lMap f xs)     step k s       = idone (liftI k) s {-# SPECIALIZE mapStream :: Monad m => (el -> el') -> Enumeratee [el] [el'] m a #-} @@ -384,7 +443,7 @@   where     step k (Chunk xs)       | LL.null xs = liftI (step k)-      | True       = rigidMapStream f $ k (Chunk $ LL.rigidMap f xs)+      | otherwise  = rigidMapStream f $ k (Chunk $ LL.rigidMap f xs)     step k s       = idone (liftI k) s {-# SPECIALIZE rigidMapStream :: Monad m => (el -> el) -> Enumeratee [el] [el] m a #-} {-# SPECIALIZE rigidMapStream :: Monad m => (Word8 -> Word8) -> Enumeratee B.ByteString B.ByteString m a #-}@@ -403,16 +462,16 @@     f' = icont step Nothing     step (Chunk xs)       | LL.null xs = f'-      | True       = idone (LL.filter p xs) mempty+      | otherwise  = idone (LL.filter p xs) mempty     step _ = f' {-# INLINE filter #-} --- |Creates an 'enumeratee' in which elements from the stream are--- grouped into \sz\-sized blocks.  The outer stream is completely+-- |Creates an 'Enumeratee' in which elements from the stream are+-- grouped into @sz@-sized blocks.  The outer stream is completely -- consumed and the final block may be smaller than \sz\. group   :: (LL.ListLike s el, Monad m, Nullable s)-  => Int+  => Int  -- ^ size of group   -> Enumeratee s [s] m a group sz iinit = liftI $ go iinit LL.empty   where go icurr pfx (Chunk s) = case gsplit (pfx `LL.append` s) of @@ -463,6 +522,76 @@                     xs = LL.tail l {-# INLINE groupBy #-} +-- | Merge offers another way to nest iteratees: as a monad stack.+-- This allows for the possibility of interleaving data from multiple+-- streams.+-- +-- > -- print each element from a stream of lines.+-- > logger :: (MonadIO m) => Iteratee [ByteString] m ()+-- > logger = mapM_ (liftIO . putStrLn . B.unpack)+-- >+-- > -- combine alternating lines from two sources+-- > -- To see how this was derived, follow the types from+-- > -- 'ileaveStream logger' and work outwards.+-- > run =<< enumFile 10 "file1" (joinI $ enumLinesBS $+-- >           ( enumFile 10 "file2" . joinI . enumLinesBS $ joinI+-- >                 (ileaveLines logger)) >>= run)+-- > +-- > ileaveLines :: (Functor m, Monad m)+-- >   => Enumeratee [ByteString] [ByteString] (Iteratee [ByteString] m)+-- >        [ByteString]+-- > ileaveLines = merge (\l1 l2 ->+-- >    [B.pack "f1:\n\t" ,l1 ,B.pack "f2:\n\t" ,l2 ]+-- > +-- > +-- +merge ::+  (LL.ListLike s1 el1+   ,LL.ListLike s2 el2+   ,Nullable s1+   ,Nullable s2+   ,Monad m+   ,Functor m)+  => (el1 -> el2 -> b)+  -> Enumeratee s2 b (Iteratee s1 m) a+merge f = convStream $ f <$> lift head <*> head+{-# INLINE merge #-}++-- | A version of merge which operates on chunks instead of elements.+-- +-- mergeByChunks offers more control than 'merge'.  'merge' terminates+-- when the first stream terminates, however mergeByChunks will continue+-- until both streams are exhausted.+-- +-- 'mergeByChunks' guarantees that both chunks passed to the merge function+-- will have the same number of elements, although that number may vary+-- between calls.+mergeByChunks ::+  (Nullable c2, Nullable c1+  ,NullPoint c2, NullPoint c1+  ,LL.ListLike c1 el1, LL.ListLike c2 el2+  ,Functor m, Monad m)+  => (c1 -> c2 -> c3)  -- ^ merge function+  -> (c1 -> c3)+  -> (c2 -> c3)+  -> Enumeratee c2 c3 (Iteratee c1 m) a+mergeByChunks f f1 f2 = unfoldConvStream iter (0 :: Int)+ where+  iter 1 = (1,) . f1 <$> lift getChunk+  iter 2 = (2,) . f2 <$> getChunk+  iter _ = do+    ml1 <- lift chunkLength+    ml2 <- chunkLength+    case (ml1, ml2) of+      (Just l1, Just l2) -> do+        let tval = min l1 l2+        c1 <- lift $ takeFromChunk tval+        c2 <- takeFromChunk tval+        return (0, f c1 c2)+      (Just _, Nothing) -> iter 1+      (Nothing, _)      -> iter 2+{-# INLINE mergeByChunks #-}+ -- ------------------------------------------------------------------------ -- Folds @@ -478,7 +607,7 @@   where     step acc (Chunk xs)       | LL.null xs  = liftI (step acc)-      | True   = liftI (step $ FLL.foldl f acc xs)+      | otherwise   = liftI (step $ FLL.foldl f acc xs)     step acc stream = idone acc stream {-# INLINE foldl #-} @@ -496,7 +625,7 @@   where     step acc (Chunk xs)       | LL.null xs = liftI (step acc)-      | True       = liftI (step $! FLL.foldl' f acc xs)+      | otherwise  = liftI (step $! FLL.foldl' f acc xs)     step acc stream = idone acc stream {-# INLINE foldl' #-} @@ -513,7 +642,7 @@     step (Chunk xs)     -- After the first chunk, just use regular foldl.       | LL.null xs = liftI step-      | True       = foldl f $ FLL.foldl1 f xs+      | otherwise  = foldl f $ FLL.foldl1 f xs     step stream    = icont step (Just (setEOF stream)) {-# INLINE foldl1 #-} @@ -528,7 +657,7 @@     step (Chunk xs)     -- After the first chunk, just use regular foldl'.       | LL.null xs = liftI step-      | True       = foldl' f $ FLL.foldl1 f xs+      | otherwise  = foldl' f $ FLL.foldl1 f xs     step stream    = icont step (Just (setEOF stream)) {-# INLINE foldl1' #-} @@ -539,7 +668,7 @@   where     step acc (Chunk xs)       | LL.null xs = liftI (step acc)-      | True       = liftI (step $! acc + LL.sum xs)+      | otherwise  = liftI (step $! acc + LL.sum xs)     step acc str   = idone acc str {-# INLINE sum #-} @@ -550,7 +679,7 @@   where     step acc (Chunk xs)       | LL.null xs = liftI (step acc)-      | True       = liftI (step $! acc * LL.product xs)+      | otherwise  = liftI (step $! acc * LL.product xs)     step acc str   = idone acc str {-# INLINE product #-} @@ -573,7 +702,7 @@  -- |Enumerate two iteratees over a single stream simultaneously. -- --- Compare to @zip@.+-- Compare to @List.zip@. zip   :: (Monad m, Nullable s, LL.ListLike s el)   => Iteratee s m a@@ -672,7 +801,7 @@ -- and discard the results. This is a different behavior than Prelude's -- sequence_ which runs iteratees in the list one after the other. -- --- Compare to @sequence_@.+-- Compare to @Prelude.sequence_@. sequence_   :: (Monad m, LL.ListLike s el, Nullable s)   => [Iteratee s m a]@@ -691,7 +820,7 @@             then idone () <=< remainingStream $ is'             else self is''         step s@(EOF _) = do-          s' <- remainingStream <=< lift $ mapM (enumChunk s) $ is+          s' <- remainingStream <=< lift $ mapM (enumChunk s) is           case s' of             EOF (Just e) -> throwErr e             _            -> idone () s'@@ -732,11 +861,11 @@ enumPureNChunk str n iter   | LL.null str = return iter   | n > 0       = enum' str iter-  | True        = error $ "enumPureNChunk called with n==" ++ show n+  | otherwise   = error $ "enumPureNChunk called with n==" ++ show n   where     enum' str' iter'       | LL.null str' = return iter'-      | True         = let (s1, s2) = LL.splitAt n str'+      | otherwise    = let (s1, s2) = LL.splitAt n str'                            on_cont k Nothing = enum' s2 . k $ Chunk s1                            on_cont k e = return $ icont k e                        in runIter iter' idoneM on_cont

@@ -291,6 +291,12 @@  runner2 (enumPure1Chunk xs (takeUpTo n identity)) == ()   where types = xs :: [Int] +-- check for final stream state+prop_takeUpTo3 xs n d t = n > 0 ==>+ runner1 (enumPureNChunk xs n (joinI (takeUpTo t (Iter.drop d)) >> stream2list))+ == P.drop (min t d) xs+  where types = xs :: [Int]+ prop_filter xs n f = n > 0 ==>  runner2 (enumSpecial xs n (Iter.filter f stream2list)) == P.filter f xs   where types = xs :: [Int]@@ -391,6 +397,7 @@     ,testProperty "take (finished iteratee)" prop_take2     ,testProperty "takeUpTo" prop_takeUpTo     ,testProperty "takeUpTo (finished iteratee)" prop_takeUpTo2+    ,testProperty "takeUpTo (remaining stream)" prop_takeUpTo3     ,testProperty "filter" prop_filter     ,testProperty "group" prop_group     ,testProperty "groupBy" prop_groupBy