diff --git a/enumerator.cabal b/enumerator.cabal --- a/enumerator.cabal +++ b/enumerator.cabal @@ -1,5 +1,5 @@ name: enumerator -version: 0.4.7 +version: 0.4.8 synopsis: Reliable, high-performance processing with left-fold enumerators license: MIT license-file: license.txt @@ -57,21 +57,28 @@ readme.txt -- src/api-docs.anansi - src/binary.anansi - src/compat.anansi + src/compatibility.anansi src/enumerator.anansi - src/list.anansi + src/io.anansi + src/list-analogues.anansi src/primitives.anansi - src/text.anansi + src/public-interface.anansi + src/summary.anansi + src/text-codecs.anansi src/types.anansi - src/util.anansi + src/utilities.anansi -- examples/cat.hs examples/wc.hs -- - scripts/cabal-dist + scripts/common.bash + scripts/dist + scripts/haddock + scripts/latex + scripts/run-benchmarks scripts/run-tests -- + tests/Benchmarks.hs tests/enumerator-tests.cabal tests/Properties.hs diff --git a/examples/cat.hs b/examples/cat.hs --- a/examples/cat.hs +++ b/examples/cat.hs @@ -8,99 +8,10 @@ -- ----------------------------------------------------------------------------- module Main (main) where -import Prelude as Prelude -import Control.Exception as E -import Control.Monad.IO.Class (liftIO) import Data.Enumerator -import qualified Data.ByteString as B -import qualified Foreign as F -import System.IO -import System.IO.Error (isEOFError) +import Data.Enumerator.Binary (enumFile, enumHandle, iterHandle) +import System.IO (stdin, stdout) import System.Environment (getArgs) - --- The following definitions of 'enumHandle', 'enumFile', and 'iterHandle' are --- copied from "Data.Enumerator.Binary", with additional comments so they're --- easier to understand. - -enumHandle :: Integer -- ^ Buffer size - -> Handle - -> Enumerator B.ByteString IO b -enumHandle bufferSize h = loop where - intSize = fromInteger bufferSize - - -- If more input is required before the enumerator's iteratee can - -- yield a result, feed it from the handle. - loop (Continue k) = do - -- While not strictly necessary to proper operation, catching - -- exceptions here allows more unified exception handling when - -- the enumerator/iteratee is run. - eitherBytes <- liftIO $ E.try $ do - - -- The enumerator must function normally when the - -- handle is something like a slow file, or network - -- socket; if there's not enough data to fill the - -- buffer yet, a partial read is returned. - hasInput <- E.catch - (hWaitForInput h (1)) - (\err -> if isEOFError err - then return False - else E.throwIO err) - - -- An EOF is represented by the empty bytestring - if hasInput - then B.hGetNonBlocking h intSize - else return B.empty - - case eitherBytes of - -- Interacting with the socket threw an IO error of - -- some sort - Left err -> throwError (err :: E.SomeException) - - -- The socket has reached EOF; pass control to the - -- next enumerator - Right bytes | B.null bytes -> continue k - - -- Bytes were read successfully; feed them to the - -- iteratee and continue looping - Right bytes -> k (Chunks [bytes]) >>== loop - - -- If a different step is received ('Error' or 'Yield'), just pass - -- it through. - loop step = returnI step - -enumFile :: FilePath -> Enumerator B.ByteString IO b -enumFile path s = do - -- Opening the file can be performed either inside or outside of the - -- Iteratee. Inside allows exceptions to be caught and propagated - -- through the 'Error' step constructor. - eitherH <- liftIO (E.try (openBinaryFile path ReadMode)) - case eitherH of - Left err -> throwError (err :: E.SomeException) - Right h -> Iteratee $ finally - (runIteratee (enumHandle 4096 h s)) - (hClose h) - --- 'iterHandle' is the opposite of 'enumHandle', in that it *writes to* a --- handle instead of reading from it. An enumerator is a source, an iteratee --- is a sink. -iterHandle :: Handle -> Iteratee B.ByteString IO () - --- Most iteratees start in the 'Continue' state, as they need some --- input before they can produce any value. -iterHandle h = continue step where - - -- This iteratee produces no value; its only purpose is its - -- side-effects. When 'EOF' is received, it simply yields (). - step EOF = yield () EOF - - -- When some chunks are received from the Enumeratee, they're written - -- to the handle. Any exceptions are caught and reported, as in - -- 'enumHandle'. - step (Chunks bytes) = do - eitherErr <- liftIO (E.try (mapM_ (B.hPut h) bytes)) - case eitherErr of - Left err -> throwError (err :: E.SomeException) - _ -> continue step main :: IO () main = do diff --git a/examples/wc.hs b/examples/wc.hs --- a/examples/wc.hs +++ b/examples/wc.hs @@ -12,12 +12,8 @@ import qualified Data.Enumerator.Binary as EB import qualified Data.Enumerator.Text as ET import qualified Data.ByteString as B -import qualified Data.ByteString.Char8 as B8 -import qualified Data.Text as T -- support imports -import Control.Exception as E -import Data.List import Control.Monad (unless, forM_) import System.IO import System.Console.GetOpt @@ -27,23 +23,17 @@ -- support wc modes -c (bytes), -m (characters), and -l (lines) -- iterBytes simply counts how many bytes are in each chunk, accumulates this --- count, and returns it when EOF is received +-- count, and returns it when EOF is received. iterBytes :: Monad m => Iteratee B.ByteString m Integer -iterBytes = continue (step 0) where - step acc EOF = yield acc EOF - step acc (Chunks xs) = continue $ step $! Data.List.foldl' foldStep acc xs - foldStep acc bytes = acc + toInteger (B.length bytes) +iterBytes = EB.fold (\acc _ -> acc + 1) 0 -- iterLines is similar, except it only counts newlines ('\n') --- --- Because it's basically the same as 'iterBytes', we use it to demonstrate --- the 'liftFoldL\'' helper function. iterLines :: Monad m => Iteratee B.ByteString m Integer -iterLines = E.foldl' step 0 where - step acc bytes = acc + countChar '\n' bytes - countChar c = B8.foldl (\acc c' -> if c' == c then acc + 1 else acc) 0 +iterLines = EB.fold step 0 where + step acc 0xA = acc + 1 + step acc _ = acc -- iterChars is a bit more complicated. It has to decode the input (for now, -- assuming UTF-8) before performing any counting. Leftover bytes, not part @@ -55,7 +45,7 @@ iterChars :: Monad m => Iteratee B.ByteString m Integer iterChars = joinI (ET.decode ET.utf8 $$ count) where - count = E.foldl' (\acc t -> acc + toInteger (T.length t)) 0 + count = ET.fold (\acc _ -> acc + 1) 0 main :: IO () main = do @@ -72,7 +62,6 @@ forM_ files $ \filename -> do putStr $ filename ++ ": " - -- see cat.hs for commented implementation of 'Data.Enumerator.IO.enumFile' eitherStat <- run (EB.enumFile filename $$ iter) putStrLn $ case eitherStat of Left err -> "ERROR: " ++ show err diff --git a/hs/Data/Enumerator.hs b/hs/Data/Enumerator.hs --- a/hs/Data/Enumerator.hs +++ b/hs/Data/Enumerator.hs @@ -21,129 +21,104 @@ module Data.Enumerator ( - -- * Core - -- ** Types + -- * Types Stream (..) , Iteratee (..) , Step (..) , Enumerator , Enumeratee - + + -- * Primitives , returnI - , yield , continue - + , yield + -- ** Operators , (>>==) , (==<<) , ($$) , (>==>) , (<==<) - - -- * Primitives - + + -- ** Running iteratees + , run + , run_ + -- ** Error handling , throwError , catchError - - -- ** Iteratees - , Data.Enumerator.foldl - , Data.Enumerator.foldl' - , Data.Enumerator.foldM - - -- ** Enumerators - , Data.Enumerator.iterate - , iterateM - , Data.Enumerator.repeat - , repeatM - , Data.Enumerator.replicate - , replicateM - , generateM - - -- ** Enumeratees - , Data.Enumerator.map - , Data.Enumerator.concatMap - , Data.Enumerator.filter - , Data.Enumerator.mapM - , concatMapM - , Data.Enumerator.filterM - - -- ** Debugging - , printChunks - - -- * Misc. utilities - + + -- * Miscellaneous , concatEnums - , joinI , joinE , Data.Enumerator.sequence - , enumList - , enumEOF - , run - , run_ - - , checkDone , checkDoneEx - + , checkDone , isEOF - - -- * Compatibility - - -- ** Obsolete functions + + -- ** Testing and debugging + , printChunks + , enumList + + -- * Legacy compatibility + + -- ** Obsolete , liftTrans , liftI , peek , Data.Enumerator.last , Data.Enumerator.length - - -- ** Deprecated aliases + + -- ** Aliases , Data.Enumerator.head , Data.Enumerator.drop , Data.Enumerator.dropWhile , Data.Enumerator.span , Data.Enumerator.break - , Data.Enumerator.consume - + , consume + , Data.Enumerator.foldl + , Data.Enumerator.foldl' + , foldM + , Data.Enumerator.iterate + , iterateM + , Data.Enumerator.repeat + , repeatM + , Data.Enumerator.replicate + , replicateM + , generateM + , Data.Enumerator.map + , Data.Enumerator.mapM + , Data.Enumerator.concatMap + , concatMapM + , Data.Enumerator.filter + , filterM , liftFoldL , liftFoldL' , liftFoldM ) where -import qualified Prelude as Prelude -import Prelude hiding ( - - concatMap, - - ) - -import Data.Monoid (Monoid, mempty, mappend, mconcat) - -import qualified Control.Exception as Exc - -import Data.Function (fix) - -import Control.Monad.Trans.Class (MonadTrans, lift) -import Control.Monad.IO.Class (MonadIO, liftIO) - -import qualified Control.Applicative as A -import qualified Control.Monad as CM - import Data.Typeable ( Typeable, typeOf , Typeable1, typeOf1 , mkTyConApp, mkTyCon) -import Data.List (foldl') - import Data.List (genericSplitAt) +import qualified Control.Exception as Exc +import Data.Monoid (Monoid, mempty, mappend, mconcat) +import Control.Monad.Trans.Class (MonadTrans, lift) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Control.Applicative as A +import qualified Control.Monad as CM +import Data.Function (fix) +import {-# SOURCE #-} qualified Data.Enumerator.List as EL import Data.List (genericLength) -import {-# SOURCE #-} qualified Data.Enumerator.List as EL + -- | A 'Stream' is a sequence of chunks generated by an 'Enumerator'. -- -- @('Chunks' [])@ is used to indicate that a stream is still active, but @@ -159,15 +134,6 @@ Chunks xs >>= f = mconcat (fmap f xs) EOF >>= _ = EOF -instance Functor Stream where - fmap f (Chunks xs) = Chunks (fmap f xs) - fmap _ EOF = EOF - --- | Since: 0.4.5 -instance A.Applicative Stream where - pure = return - (<*>) = CM.ap - instance Monoid (Stream a) where mempty = Chunks mempty mappend (Chunks xs) (Chunks ys) = Chunks (xs ++ ys) @@ -209,24 +175,26 @@ { runIteratee :: m (Step a m b) } - --- | @returnI step = 'Iteratee' (return step)@ - -returnI :: Monad m => Step a m b -> Iteratee a m b -returnI step = Iteratee (return step) - - --- | @yield x extra = 'returnI' ('Yield' x extra)@ - -yield :: Monad m => b -> Stream a -> Iteratee a m b -yield x extra = returnI (Yield x extra) +instance Monad m => Monad (Iteratee a m) where + return x = yield x (Chunks []) + m0 >>= f = ($ m0) $ fix $ + \bind m -> Iteratee $ runIteratee m >>= \r1 -> + case r1 of + Continue k -> return (Continue (bind . k)) + Error err -> return (Error err) + Yield x (Chunks []) -> runIteratee (f x) + Yield x extra -> runIteratee (f x) >>= \r2 -> + case r2 of + Continue k -> runIteratee (k extra) + Error err -> return (Error err) + Yield x' _ -> return (Yield x' extra) --- | @continue k = 'returnI' ('Continue' k)@ +instance MonadTrans (Iteratee a) where + lift m = Iteratee (m >>= runIteratee . return) -continue :: Monad m => (Stream a -> Iteratee a m b) - -> Iteratee a m b -continue k = returnI (Continue k) +instance MonadIO m => MonadIO (Iteratee a m) where + liftIO = lift . liftIO -- | While 'Iteratee's consume data, enumerators generate it. Since @@ -246,117 +214,108 @@ -- type is named an 'Enumeratee'. Enumeratees have two input types, -- “outer a” (@aOut@) and “inner a” (@aIn@). -type Enumeratee ao ai m b = Step ai m b - -> Iteratee ao m (Step ai m b) +type Enumeratee ao ai m b = Step ai m b -> Iteratee ao m (Step ai m b) -infixl 1 >>== +-- | Since: 0.4.8 +instance Typeable1 Stream where + typeOf1 _ = mkTyConApp tyCon [] where + tyCon = mkTyCon "Data.Enumerator.Stream" --- | Equivalent to '(>>=)' for @m ('Step' a m b)@; allows 'Iteratee's with --- different input types to be composed. +-- | Since: 0.4.6 +instance (Typeable a, Typeable1 m) => + Typeable1 (Iteratee a m) where + typeOf1 i = let + tyCon = mkTyCon "Data.Enumerator.Iteratee" + (a, m) = peel i + + peel :: Iteratee a m b -> (a, m ()) + peel = undefined + + in mkTyConApp tyCon [typeOf a, typeOf1 m] -(>>==) :: Monad m - => Iteratee a m b - -> (Step a m b -> Iteratee a' m b') - -> Iteratee a' m b' -i >>== f = Iteratee (runIteratee i >>= runIteratee . f) +-- | Since: 0.4.8 +instance (Typeable a, Typeable1 m) => + Typeable1 (Step a m) where + typeOf1 s = let + tyCon = mkTyCon "Data.Enumerator.Step" + (a, m) = peel s + + peel :: Step a m b -> (a, m ()) + peel = undefined + + in mkTyConApp tyCon [typeOf a, typeOf1 m] -infixr 1 ==<< +instance Monad m => Functor (Iteratee a m) where + fmap = CM.liftM +instance Monad m => A.Applicative (Iteratee a m) where + pure = return + (<*>) = CM.ap --- | @(==\<\<) = flip (\>\>==)@ +instance Functor Stream where + fmap f (Chunks xs) = Chunks (fmap f xs) + fmap _ EOF = EOF -(==<<) :: Monad m - => (Step a m b -> Iteratee a' m b') - -> Iteratee a m b - -> Iteratee a' m b' -(==<<) = flip (>>==) +-- | Since: 0.4.5 +instance A.Applicative Stream where + pure = return + (<*>) = CM.ap -infixr 0 $$ --- | @($$) = (==\<\<)@ --- --- This might be easier to read when passing a chain of iteratees to an --- enumerator. --- --- Since: 0.1.1 - -($$) :: Monad m - => (Step a m b -> Iteratee a' m b') - -> Iteratee a m b - -> Iteratee a' m b' -($$) = (==<<) +-- | @'returnI' step = 'Iteratee' (return step)@ -infixr 1 >==> +returnI :: Monad m => Step a m b -> Iteratee a m b +returnI step = Iteratee (return step) --- | @(>==>) e1 e2 s = e1 s >>== e2@ +-- | @'yield' x extra = 'returnI' ('Yield' x extra)@ -- --- Since: 0.1.1 +-- WARNING: due to the current encoding of iteratees in this library, +-- careless use of the 'yield' primitive may violate the monad laws. +-- To prevent this, always make sure that an iteratee never yields +-- extra data unless it has received at least one input element. +-- +-- More strictly, iteratees may not yield data that they did not +-- receive as input. Don't use 'yield' to “inject” elements +-- into the stream. -(>==>) :: Monad m - => Enumerator a m b - -> (Step a m b -> Iteratee a' m b') - -> Step a m b - -> Iteratee a' m b' -(>==>) e1 e2 s = e1 s >>== e2 +yield :: Monad m => b -> Stream a -> Iteratee a m b +yield x extra = returnI (Yield x extra) -infixr 1 <==< +-- | @'continue' k = 'returnI' ('Continue' k)@ --- | @(\<==\<) = flip (>==>)@ --- --- Since: 0.1.1 +continue :: Monad m => (Stream a -> Iteratee a m b) -> Iteratee a m b +continue k = returnI (Continue k) -(<==<) :: Monad m - => (Step a m b -> Iteratee a' m b') - -> Enumerator a m b - -> Step a m b - -> Iteratee a' m b' -(<==<) = flip (>==>) -instance Monad m => Monad (Iteratee a m) where - return x = yield x (Chunks []) - - m0 >>= f = ($ m0) $ fix $ \bind m -> Iteratee $ runIteratee m >>= - \r1 -> case r1 of - Continue k -> return (Continue (bind . k)) - Error err -> return (Error err) - Yield x (Chunks []) -> runIteratee (f x) - Yield x extra -> runIteratee (f x) >>= - \r2 -> case r2 of - Continue k -> runIteratee (k extra) - Error err -> return (Error err) - Yield x' _ -> return (Yield x' extra) +-- | Run an iteratee until it finishes, and return either the final value +-- (if it succeeded) or the error (if it failed). -instance MonadTrans (Iteratee a) where - lift m = Iteratee (m >>= runIteratee . return) +run :: Monad m => Iteratee a m b + -> m (Either Exc.SomeException b) +run i = do + mStep <- runIteratee $ enumEOF ==<< i + case mStep of + Error err -> return $ Left err + Yield x _ -> return $ Right x + Continue _ -> error "run: divergent iteratee" -instance MonadIO m => MonadIO (Iteratee a m) where - liftIO = lift . liftIO -instance Monad m => Functor (Iteratee a m) where - fmap = CM.liftM - -instance Monad m => A.Applicative (Iteratee a m) where - pure = return - (<*>) = CM.ap +-- | Like 'run', except errors are converted to exceptions and thrown. +-- Primarily useful for small scripts or other simple cases. +-- +-- Since: 0.4.1 --- | Since: 0.4.6 -instance (Typeable a, Typeable1 m) => Typeable1 (Iteratee a m) where - typeOf1 i = mkTyConApp tyCon [typeOf a, typeOf1 m] where - tyCon = mkTyCon "Data.Enumerator.Iteratee" - (a, m) = peel i - - peel :: Iteratee a m b -> (a, m ()) - peel = undefined +run_ :: Monad m => Iteratee a m b -> m b +run_ i = run i >>= either Exc.throw return --- | @throwError exc = 'returnI' ('Error' ('Exc.toException' exc))@ +-- | @'throwError' exc = 'returnI' ('Error' ('Exc.toException' exc))@ -throwError :: (Monad m, Exc.Exception e) => e - -> Iteratee a m b +throwError :: (Monad m, Exc.Exception e) => e -> Iteratee a m b throwError exc = returnI (Error (Exc.toException exc)) @@ -367,7 +326,8 @@ -- -- Since: 0.1.1 -catchError :: Monad m => Iteratee a m b +catchError :: Monad m + => Iteratee a m b -> (Exc.SomeException -> Iteratee a m b) -> Iteratee a m b catchError iter h = iter >>== step where @@ -376,202 +336,71 @@ step (Continue k) = continue (\s -> k s >>== step) --- | Run the entire input stream through a pure left fold, yielding when --- there is no more input. --- --- Since: 0.4.5 - -foldl :: Monad m => (b -> a -> b) -> b - -> Iteratee a m b -foldl step = continue . loop where - fold = Prelude.foldl step - loop acc stream = case stream of - Chunks [] -> continue (loop acc) - Chunks xs -> continue (loop (fold acc xs)) - EOF -> yield acc EOF +infixl 1 >>== +infixr 1 ==<< +infixr 0 $$ +infixr 1 >==> +infixr 1 <==< --- | Run the entire input stream through a pure strict left fold, yielding --- when there is no more input. --- --- Since: 0.4.5 +-- | Equivalent to '(>>=)' for @m ('Step' a m b)@; allows 'Iteratee's with +-- different input types to be composed. -foldl' :: Monad m => (b -> a -> b) -> b - -> Iteratee a m b -foldl' step = continue . loop where - fold = Data.List.foldl' step - loop acc stream = case stream of - Chunks [] -> continue (loop acc) - Chunks xs -> continue (loop (fold acc xs)) - EOF -> yield acc EOF +(>>==) :: Monad m + => Iteratee a m b + -> (Step a m b -> Iteratee a' m b') + -> Iteratee a' m b' +i >>== f = Iteratee (runIteratee i >>= runIteratee . f) --- | Run the entire input stream through a monadic left fold, yielding --- when there is no more input. --- --- Since: 0.4.5 +-- | @'(==\<\<)' = flip '(\>\>==)'@ -foldM :: Monad m => (b -> a -> m b) -> b - -> Iteratee a m b -foldM step = continue . loop where - fold acc = lift . CM.foldM step acc - - loop acc stream = case stream of - Chunks [] -> continue (loop acc) - Chunks xs -> fold acc xs >>= continue . loop - EOF -> yield acc EOF +(==<<) :: Monad m + => (Step a m b -> Iteratee a' m b') + -> Iteratee a m b + -> Iteratee a' m b' +(==<<) = flip (>>==) --- | @iterate f x@ enumerates an infinite stream of repeated applications --- of /f/ to /x/. +-- | @'($$)' = '(==\<\<)'@ -- --- Analogous to 'Prelude.iterate'. +-- This might be easier to read when passing a chain of iteratees to an +-- enumerator. -- --- Since: 0.4.5 +-- Since: 0.1.1 -iterate :: Monad m => (a -> a) -> a -> Enumerator a m b -iterate f = loop where - loop a (Continue k) = k (Chunks [a]) >>== loop (f a) - loop _ step = returnI step +($$) :: Monad m + => (Step a m b -> Iteratee a' m b') + -> Iteratee a m b + -> Iteratee a' m b' +($$) = (==<<) --- | Similar to 'iterate', except the iteration function is monadic. +-- | @'(>==>)' e1 e2 s = e1 s '>>==' e2@ -- --- Since: 0.4.5 +-- Since: 0.1.1 -iterateM :: Monad m => (a -> m a) -> a - -> Enumerator a m b -iterateM f base = loop (return base) where - loop m_a (Continue k) = do - a <- lift m_a - k (Chunks [a]) >>== loop (f a) - loop _ step = returnI step +(>==>) :: Monad m + => Enumerator a m b + -> (Step a m b -> Iteratee a' m b') + -> Step a m b + -> Iteratee a' m b' +(>==>) e1 e2 s = e1 s >>== e2 --- | Enumerates an infinite stream of the provided value. --- --- Analogous to 'Prelude.repeat'. +-- | @'(\<==\<)' = flip '(>==>)'@ -- --- Since: 0.4.5 +-- Since: 0.1.1 -repeat :: Monad m => a -> Enumerator a m b -repeat a = Data.Enumerator.iterate (const a) a +(<==<) :: Monad m + => (Step a m b -> Iteratee a' m b') + -> Enumerator a m b + -> Step a m b + -> Iteratee a' m b' +(<==<) = flip (>==>) --- | Enumerates an infinite stream by running the provided computation and --- passing each result to the iteratee. --- --- Since: 0.4.5 -repeatM :: Monad m => m a -> Enumerator a m b -repeatM m_a step = do - a <- lift m_a - iterateM (const m_a) a step - - --- | @replicateM n m_x@ enumerates a stream of /n/ input elements; each --- element is generated by running the input computation /m_x/ once. --- --- Since: 0.4.5 - -replicateM :: Monad m => Integer -> m a - -> Enumerator a m b -replicateM maxCount getNext = loop maxCount where - loop 0 step = returnI step - loop n (Continue k) = do - next <- lift getNext - k (Chunks [next]) >>== loop (n - 1) - loop _ step = returnI step - - --- | @replicate n x = 'replicateM' n (return x)@ --- --- Analogous to 'Prelude.replicate'. --- --- Since: 0.4.5 - -replicate :: Monad m => Integer -> a - -> Enumerator a m b -replicate maxCount a = replicateM maxCount (return a) - - --- | Like 'repeatM', except the computation may terminate the stream by --- returning 'Nothing'. --- --- Since: 0.4.5 - -generateM :: Monad m => m (Maybe a) - -> Enumerator a m b -generateM getNext = loop where - loop (Continue k) = do - next <- lift getNext - case next of - Nothing -> continue k - Just x -> k (Chunks [x]) >>== loop - loop step = returnI step - - --- | @concatMapM f@ applies /f/ to each input element and feeds the --- resulting outputs to the inner iteratee. --- --- Since: 0.4.5 - -concatMapM :: Monad m => (ao -> m [ai]) - -> Enumeratee ao ai m b -concatMapM f = checkDone (continue . step) where - step k EOF = yield (Continue k) EOF - step k (Chunks xs) = loop k xs - - loop k [] = continue (step k) - loop k (x:xs) = do - fx <- lift (f x) - k (Chunks fx) >>== - checkDoneEx (Chunks xs) (\k' -> loop k' xs) - - --- | @concatMap f = 'concatMapM' (return . f)@ --- --- Since: 0.4.3 - -concatMap :: Monad m => (ao -> [ai]) - -> Enumeratee ao ai m b -concatMap f = concatMapM (return . f) - - --- | @map f = 'concatMap' (\x -> 'Prelude.map' f [x])@ - -map :: Monad m => (ao -> ai) - -> Enumeratee ao ai m b -map f = concatMap (\x -> Prelude.map f [x]) - - --- | @filter p = 'concatMap' (\x -> 'Prelude.filter' p [x])@ --- --- Since: 0.4.5 - -filter :: Monad m => (a -> Bool) - -> Enumeratee a a m b -filter p = concatMap (\x -> Prelude.filter p [x]) - - --- | @mapM f = 'concatMapM' (\x -> 'Prelude.mapM' f [x])@ --- --- Since: 0.4.3 - -mapM :: Monad m => (ao -> m ai) - -> Enumeratee ao ai m b -mapM f = concatMapM (\x -> Prelude.mapM f [x]) - - --- | @filterM p = 'concatMapM' (\x -> 'CM.filterM' p [x])@ --- --- Since: 0.4.5 - -filterM :: Monad m => (a -> m Bool) - -> Enumeratee a a m b -filterM p = concatMapM (\x -> CM.filterM p [x]) - - -- | Print chunks as they're received from the enumerator, optionally -- printing empty chunks. @@ -589,6 +418,20 @@ yield () EOF +-- | @'enumList' n xs@ enumerates /xs/ as a stream, passing /n/ inputs per +-- chunk. +-- +-- Primarily useful for testing and debugging. + +enumList :: Monad m => Integer -> [a] -> Enumerator a m b +enumList n = loop where + loop xs (Continue k) | not (null xs) = let + (s1, s2) = genericSplitAt n xs + in k (Chunks s1) >>== loop s2 + loop _ step = returnI step + + + -- | Compose a list of 'Enumerator's using @'(>>==)'@ concatEnums :: Monad m => [Enumerator a m b] @@ -636,33 +479,8 @@ step k = i >>= \v -> k (Chunks [v]) >>== loop --- | @enumList n xs@ enumerates /xs/ as a stream, passing /n/ inputs per --- chunk. --- --- Primarily useful for testing and debugging. - -enumList :: Monad m => Integer -> [a] -> Enumerator a m b -enumList n = loop where - loop xs (Continue k) | not (null xs) = let - (s1, s2) = genericSplitAt n xs - in k (Chunks s1) >>== loop s2 - loop _ step = returnI step - - --- | Run an iteratee until it finishes, and return either the final value --- (if it succeeded) or the error (if it failed). - -run :: Monad m => Iteratee a m b - -> m (Either Exc.SomeException b) -run i = do - mStep <- runIteratee $ enumEOF ==<< i - case mStep of - Error err -> return $ Left err - Yield x _ -> return $ Right x - Continue _ -> error "run: divergent iteratee" - - --- | docs TODO +-- | Sends 'EOF' to its iteratee. Most clients should use 'run' or 'run_' +-- instead. enumEOF :: Monad m => Enumerator a m b enumEOF (Yield x _) = yield x EOF @@ -672,15 +490,6 @@ check s = enumEOF s --- | Like 'run', except errors are converted to exceptions and thrown. --- Primarily useful for small scripts or other simple cases. --- --- Since: 0.4.1 - -run_ :: Monad m => Iteratee a m b -> m b -run_ i = run i >>= either Exc.throw return - - -- | A common pattern in 'Enumeratee' implementations is to check whether -- the inner 'Iteratee' has finished, and if so, to return its output. -- 'checkDone' passes its parameter a continuation if the 'Iteratee' @@ -696,7 +505,7 @@ checkDoneEx extra _ step = yield step extra --- | @checkDone = 'checkDoneEx' ('Chunks' [])@ +-- | @'checkDone' = 'checkDoneEx' ('Chunks' [])@ -- -- Use this for enumeratees which do not have an input buffer. @@ -706,7 +515,8 @@ checkDone = checkDoneEx (Chunks []) --- | docs TODO +-- | Check whether a stream has reached EOF. Most clients should use +-- 'Data.Enumerator.List.head' instead. isEOF :: Monad m => Iteratee a m Bool isEOF = continue $ \s -> case s of @@ -714,6 +524,7 @@ _ -> yield False s + -- | Lift an 'Iteratee' onto a monad transformer, re-wrapping the -- 'Iteratee'’s inner monadic values. -- @@ -728,8 +539,7 @@ Error err -> Error err Continue k -> Continue (liftTrans . k) -{-# DEPRECATED liftI - "Use 'Data.Enumerator.continue' instead" #-} +{-# DEPRECATED liftI "Use 'Data.Enumerator.continue' instead" #-} -- | Deprecated in 0.4.5: use 'Data.Enumerator.continue' instead @@ -771,83 +581,222 @@ loop n (Chunks xs) = continue (loop (n + len xs)) loop n EOF = yield n EOF -{-# DEPRECATED head - "Use 'Data.Enumerator.List.head' instead" #-} +{-# DEPRECATED head "Use 'Data.Enumerator.List.head' instead" #-} + -- | Deprecated in 0.4.5: use 'Data.Enumerator.List.head' instead head :: Monad m => Iteratee a m (Maybe a) head = EL.head -{-# DEPRECATED drop - "Use 'Data.Enumerator.List.drop' instead" #-} +{-# DEPRECATED drop "Use 'Data.Enumerator.List.drop' instead" #-} -- | Deprecated in 0.4.5: use 'Data.Enumerator.List.drop' instead drop :: Monad m => Integer -> Iteratee a m () drop = EL.drop -{-# DEPRECATED dropWhile - "Use 'Data.Enumerator.List.dropWhile' instead" #-} +{-# DEPRECATED dropWhile "Use 'Data.Enumerator.List.dropWhile' instead" #-} -- | Deprecated in 0.4.5: use 'Data.Enumerator.List.dropWhile' instead dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () dropWhile = EL.dropWhile -{-# DEPRECATED span - "Use 'Data.Enumerator.List.takeWhile' instead" #-} +{-# DEPRECATED span "Use 'Data.Enumerator.List.takeWhile' instead" #-} -- | Deprecated in 0.4.5: use 'Data.Enumerator.List.takeWhile' instead span :: Monad m => (a -> Bool) -> Iteratee a m [a] span = EL.takeWhile -{-# DEPRECATED break - "Use 'Data.Enumerator.List.takeWhile' instead" #-} +{-# DEPRECATED break "Use 'Data.Enumerator.List.takeWhile' instead" #-} -- | Deprecated in 0.4.5: use 'Data.Enumerator.List.takeWhile' instead break :: Monad m => (a -> Bool) -> Iteratee a m [a] break p = EL.takeWhile (not . p) -{-# DEPRECATED consume - "Use 'Data.Enumerator.List.consume' instead" #-} +{-# DEPRECATED consume "Use 'Data.Enumerator.List.consume' instead" #-} -- | Deprecated in 0.4.5: use 'Data.Enumerator.List.consume' instead consume :: Monad m => Iteratee a m [a] consume = EL.consume -{-# DEPRECATED liftFoldL - "Use 'Data.Enumerator.foldl' instead" #-} +{-# DEPRECATED foldl "Use Data.Enumerator.List.fold instead" #-} --- | Deprecated in 0.4.5: use 'Data.Enumerator.foldl' instead +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.fold' instead -- +-- Since: 0.4.5 + +foldl :: Monad m => (b -> a -> b) -> b -> Iteratee a m b +foldl step = continue . loop where + fold = Prelude.foldl step + loop acc stream = case stream of + Chunks [] -> continue (loop acc) + Chunks xs -> continue (loop (fold acc xs)) + EOF -> yield acc EOF + +{-# DEPRECATED foldl' "Use Data.Enumerator.List.fold instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.fold' instead +-- +-- Since: 0.4.5 + +foldl' :: Monad m => (b -> a -> b) -> b -> Iteratee a m b +foldl' = EL.fold + +{-# DEPRECATED foldM "Use Data.Enumerator.List.foldM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.foldM' instead +-- +-- Since: 0.4.5 + +foldM :: Monad m => (b -> a -> m b) -> b -> Iteratee a m b +foldM = EL.foldM + +{-# DEPRECATED iterate "Use Data.Enumerator.List.iterate instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.iterate' instead +-- +-- Since: 0.4.5 + +iterate :: Monad m => (a -> a) -> a -> Enumerator a m b +iterate = EL.iterate + +{-# DEPRECATED iterateM "Use Data.Enumerator.List.iterateM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.iterateM' instead +-- +-- Since: 0.4.5 + +iterateM :: Monad m => (a -> m a) -> a -> Enumerator a m b +iterateM = EL.iterateM + +{-# DEPRECATED repeat "Use Data.Enumerator.List.repeat instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.repeat' instead +-- +-- Since: 0.4.5 + +repeat :: Monad m => a -> Enumerator a m b +repeat = EL.repeat + +{-# DEPRECATED repeatM "Use Data.Enumerator.List.repeatM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.repeatM' instead +-- +-- Since: 0.4.5 + +repeatM :: Monad m => m a -> Enumerator a m b +repeatM = EL.repeatM + +{-# DEPRECATED replicate "Use Data.Enumerator.List.replicate instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.replicate' instead +-- +-- Since: 0.4.5 + +replicate :: Monad m => Integer -> a -> Enumerator a m b +replicate = EL.replicate + +{-# DEPRECATED replicateM "Use Data.Enumerator.List.replicateM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.replicateM' instead +-- +-- Since: 0.4.5 + +replicateM :: Monad m => Integer -> m a -> Enumerator a m b +replicateM = EL.replicateM + +{-# DEPRECATED generateM "Use Data.Enumerator.List.generateM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.generateM' instead +-- +-- Since: 0.4.5 + +generateM :: Monad m => m (Maybe a) -> Enumerator a m b +generateM = EL.generateM + +{-# DEPRECATED map "Use Data.Enumerator.List.map instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.map' instead + +map :: Monad m => (ao -> ai) -> Enumeratee ao ai m b +map = EL.map + +{-# DEPRECATED mapM "Use Data.Enumerator.List.mapM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.mapM' instead +-- +-- Since: 0.4.3 + +mapM :: Monad m => (ao -> m ai) -> Enumeratee ao ai m b +mapM = EL.mapM + +{-# DEPRECATED concatMap "Use Data.Enumerator.List.concatMap instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.concatMap' instead +-- +-- Since: 0.4.3 + +concatMap :: Monad m => (ao -> [ai]) -> Enumeratee ao ai m b +concatMap = EL.concatMap + +{-# DEPRECATED concatMapM "Use Data.Enumerator.List.concatMapM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.concatMapM' instead +-- +-- Since: 0.4.5 + +concatMapM :: Monad m => (ao -> m [ai]) -> Enumeratee ao ai m b +concatMapM = EL.concatMapM + +{-# DEPRECATED filter "Use Data.Enumerator.List.filter instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.filter' instead +-- +-- Since: 0.4.5 + +filter :: Monad m => (a -> Bool) -> Enumeratee a a m b +filter = EL.filter + +{-# DEPRECATED filterM "Use Data.Enumerator.List.filterM instead" #-} + +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.filterM' instead +-- +-- Since: 0.4.5 + +filterM :: Monad m => (a -> m Bool) -> Enumeratee a a m b +filterM = EL.filterM + +{-# DEPRECATED liftFoldL "Use Data.Enumerator.List.fold instead" #-} + +-- | Deprecated in 0.4.5: use 'Data.Enumerator.List.fold' instead +-- -- Since: 0.1.1 liftFoldL :: Monad m => (b -> a -> b) -> b -> Iteratee a m b liftFoldL = Data.Enumerator.foldl -{-# DEPRECATED liftFoldL' - "Use 'Data.Enumerator.foldl' ' instead" #-} +{-# DEPRECATED liftFoldL' "Use Data.Enumerator.List.fold instead" #-} --- | Deprecated in 0.4.5: use 'Data.Enumerator.foldl'' instead +-- | Deprecated in 0.4.5: use 'Data.Enumerator.List.fold' instead -- -- Since: 0.1.1 liftFoldL' :: Monad m => (b -> a -> b) -> b -> Iteratee a m b -liftFoldL' = Data.Enumerator.foldl' +liftFoldL' = EL.fold -{-# DEPRECATED liftFoldM - "Use 'Data.Enumerator.foldM' instead" #-} +{-# DEPRECATED liftFoldM "Use Data.Enumerator.List.foldM instead" #-} --- | Deprecated in 0.4.5: use 'Data.Enumerator.foldM' instead +-- | Deprecated in 0.4.5: use 'Data.Enumerator.List.foldM' instead -- -- Since: 0.1.1 liftFoldM :: Monad m => (b -> a -> m b) -> b -> Iteratee a m b -liftFoldM = Data.Enumerator.foldM +liftFoldM = EL.foldM diff --git a/hs/Data/Enumerator.hs-boot b/hs/Data/Enumerator.hs-boot --- a/hs/Data/Enumerator.hs-boot +++ b/hs/Data/Enumerator.hs-boot @@ -9,3 +9,5 @@ newtype Iteratee a m b = Iteratee { runIteratee :: m (Step a m b) } +type Enumerator a m b = Step a m b -> Iteratee a m b +type Enumeratee ao ai m b = Step ai m b -> Iteratee ao m (Step ai m b) diff --git a/hs/Data/Enumerator/Binary.hs b/hs/Data/Enumerator/Binary.hs --- a/hs/Data/Enumerator/Binary.hs +++ b/hs/Data/Enumerator/Binary.hs @@ -21,149 +21,263 @@ module Data.Enumerator.Binary ( - -- * Binary IO + -- * IO enumHandle + , enumHandleRange , enumFile + , enumFileRange , iterHandle - + -- * List analogues + + -- ** Folds + , fold + , foldM + + -- ** Maps + , Data.Enumerator.Binary.map + , Data.Enumerator.Binary.mapM + , Data.Enumerator.Binary.concatMap + , concatMapM + + -- ** Infinite streams + , Data.Enumerator.Binary.iterate + , iterateM + , Data.Enumerator.Binary.repeat + , repeatM + + -- ** Bounded streams + , Data.Enumerator.Binary.replicate + , replicateM + , generateM + , unfold + , unfoldM + + -- ** Filters + , Data.Enumerator.Binary.filter + , filterM + + -- ** Consumers + , Data.Enumerator.Binary.take + , takeWhile + , consume + + -- ** Unsorted , Data.Enumerator.Binary.head , Data.Enumerator.Binary.drop , Data.Enumerator.Binary.dropWhile - , Data.Enumerator.Binary.take - , Data.Enumerator.Binary.takeWhile - , Data.Enumerator.Binary.consume , require , isolate + , splitWhen + ) where -import Prelude hiding (head, drop, takeWhile) -import Data.Enumerator hiding (head, drop) -import qualified Data.ByteString as B -import Data.Enumerator.Util (tryStep) -import qualified Control.Exception as Exc +import Prelude hiding (head, drop, takeWhile) +import Data.Enumerator hiding ( head, drop, iterateM, repeatM, replicateM + , generateM, filterM, consume, foldM + , concatMapM) +import Data.Enumerator.Util (tryIO) import Control.Monad.IO.Class (MonadIO) +import qualified Data.ByteString as B import qualified System.IO as IO +import Data.Function (fix) +import qualified Control.Exception as Exc import System.IO.Error (isEOFError) - import Data.Word (Word8) +import qualified Data.Enumerator.List as EL +import qualified Control.Monad as CM import qualified Data.ByteString.Lazy as BL +import Control.Monad.Trans.Class (lift) +import Control.Monad (liftM) --- | Read bytes (in chunks of the given buffer size) from the handle, and --- stream them to an 'Iteratee'. If an exception occurs during file IO, --- enumeration will stop and 'Error' will be returned. Exceptions from the --- iteratee are not caught. + +-- | Consume the entire input stream with a strict left fold, one byte +-- at a time. -- --- This enumerator blocks until at least one byte is available from the --- handle, and might read less than the maximum buffer size in some --- cases. +-- Since: 0.4.8 + +fold :: Monad m => (b -> Word8 -> b) -> b + -> Iteratee B.ByteString m b +fold step = EL.fold (B.foldl' step) + + +-- | Consume the entire input stream with a strict monadic left fold, one +-- byte at a time. -- --- The handle should be opened with no encoding, and in 'IO.ReadMode' or --- 'IO.ReadWriteMode'. +-- Since: 0.4.8 -enumHandle :: MonadIO m - => Integer -- ^ Buffer size - -> IO.Handle - -> Enumerator B.ByteString m b -enumHandle bufferSize h = loop where - loop (Continue k) = withBytes $ \bytes -> - if B.null bytes - then continue k - else k (Chunks [bytes]) >>== loop - - loop step = returnI step +foldM :: Monad m => (b -> Word8 -> m b) -> b + -> Iteratee B.ByteString m b +foldM step = EL.foldM (\b bytes -> CM.foldM step b (B.unpack bytes)) + + +-- | Enumerates a stream of bytes by repeatedly applying a function to +-- some state. +-- +-- Similar to 'iterate'. +-- +-- Since: 0.4.8 + +unfold :: Monad m => (s -> Maybe (Word8, s)) -> s -> Enumerator B.ByteString m b +unfold f = loop where + loop s (Continue k) = case f s of + Nothing -> continue k + Just (b, s') -> k (Chunks [B.singleton b]) >>== loop s' + loop _ step = returnI step + + +-- | Enumerates a stream of bytes by repeatedly applying a computation to +-- some state. +-- +-- Similar to 'iterateM'. +-- +-- Since: 0.4.8 + +unfoldM :: Monad m => (s -> m (Maybe (Word8, s))) -> s -> Enumerator B.ByteString m b +unfoldM f = loop where + loop s (Continue k) = do + fs <- lift (f s) + case fs of + Nothing -> continue k + Just (b, s') -> k (Chunks [B.singleton b]) >>== loop s' + loop _ step = returnI step + + +-- | @'map' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +map :: Monad m => (Word8 -> Word8) -> Enumeratee B.ByteString B.ByteString m b +map f = Data.Enumerator.Binary.concatMap (\x -> B.singleton (f x)) + + +-- | @'mapM' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +mapM :: Monad m => (Word8 -> m Word8) -> Enumeratee B.ByteString B.ByteString m b +mapM f = Data.Enumerator.Binary.concatMapM (\x -> liftM B.singleton (f x)) + + +-- | @'concatMap' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +concatMap :: Monad m => (Word8 -> B.ByteString) -> Enumeratee B.ByteString B.ByteString m b +concatMap f = Data.Enumerator.Binary.concatMapM (return . f) + + +-- | @'concatMapM' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +concatMapM :: Monad m => (Word8 -> m B.ByteString) -> Enumeratee B.ByteString B.ByteString m b +concatMapM f = checkDone (continue . step) where + step k EOF = yield (Continue k) EOF + step k (Chunks xs) = loop k (BL.unpack (BL.fromChunks xs)) - intSize = fromInteger bufferSize - withBytes = tryStep $ do - hasInput <- Exc.catch - (IO.hWaitForInput h (-1)) - (\err -> if isEOFError err - then return False - else Exc.throwIO err) - if hasInput - then B.hGetNonBlocking h intSize - else return B.empty + loop k [] = continue (step k) + loop k (x:xs) = do + fx <- lift (f x) + k (Chunks [fx]) >>== + checkDoneEx (Chunks [B.pack xs]) (\k' -> loop k' xs) --- | Opens a file path in binary mode, and passes the handle to 'enumHandle'. --- The file will be closed when the 'Iteratee' finishes. +-- | @'iterate' f x@ enumerates an infinite stream of repeated applications +-- of /f/ to /x/. +-- +-- Analogous to 'Prelude.iterate'. +-- +-- Since: 0.4.8 -enumFile :: FilePath -> Enumerator B.ByteString IO b -enumFile path = enum where - withHandle = tryStep (IO.openBinaryFile path IO.ReadMode) - enum step = withHandle $ \h -> do - Iteratee $ Exc.finally - (runIteratee (enumHandle 4096 h step)) - (IO.hClose h) +iterate :: Monad m => (Word8 -> Word8) -> Word8 -> Enumerator B.ByteString m b +iterate f = loop where + loop byte (Continue k) = k (Chunks [B.singleton byte]) >>== loop (f byte) + loop _ step = returnI step --- | Read bytes from a stream and write them to a handle. If an exception --- occurs during file IO, enumeration will stop and 'Error' will be --- returned. +-- | Similar to 'iterate', except the iteration function is monadic. -- --- The handle should be opened with no encoding, and in 'IO.WriteMode' or --- 'IO.ReadWriteMode'. +-- Since: 0.4.8 -iterHandle :: MonadIO m => IO.Handle - -> Iteratee B.ByteString m () -iterHandle h = continue step where - step EOF = yield () EOF - step (Chunks []) = continue step - step (Chunks bytes) = let - put = mapM_ (B.hPut h) bytes - in tryStep put (\_ -> continue step) +iterateM :: Monad m => (Word8 -> m Word8) -> Word8 -> Enumerator B.ByteString m b +iterateM f base = loop (return base) where + loop m_byte (Continue k) = do + byte <- lift m_byte + k (Chunks [B.singleton byte]) >>== loop (f byte) + loop _ step = returnI step -toChunks :: BL.ByteString -> Stream B.ByteString -toChunks = Chunks . BL.toChunks +-- | Enumerates an infinite stream of a single byte. +-- +-- Analogous to 'Prelude.repeat'. +-- +-- Since: 0.4.8 --- | Get the next byte from the stream, or 'Nothing' if the stream has --- ended. +repeat :: Monad m => Word8 -> Enumerator B.ByteString m b +repeat byte = EL.repeat (B.singleton byte) + + +-- | Enumerates an infinite stream of byte. Each byte is computed by the +-- underlying monad. -- --- Since: 0.4.5 +-- Since: 0.4.8 -head :: Monad m => Iteratee B.ByteString m (Maybe Word8) -head = continue loop where - loop (Chunks xs) = case BL.uncons (BL.fromChunks xs) of - Just (char, extra) -> yield (Just char) (toChunks extra) - Nothing -> head - loop EOF = yield Nothing EOF +repeatM :: Monad m => m Word8 -> Enumerator B.ByteString m b +repeatM next = EL.repeatM (liftM B.singleton next) --- | @drop n@ ignores /n/ bytes of input from the stream. +-- | @'replicate' n x@ enumerates a stream containing /n/ copies of /x/. -- --- Since: 0.4.5 +-- Since: 0.4.8 -drop :: Monad m => Integer -> Iteratee B.ByteString m () -drop n | n <= 0 = return () -drop n = continue (loop n) where - loop n' (Chunks xs) = iter where - lazy = BL.fromChunks xs - len = toInteger (BL.length lazy) - iter = if len < n' - then drop (n' - len) - else yield () (toChunks (BL.drop (fromInteger n') lazy)) - loop _ EOF = yield () EOF +replicate :: Monad m => Integer -> Word8 -> Enumerator B.ByteString m b +replicate n byte = EL.replicate n (B.singleton byte) --- | @dropWhile p@ ignores input from the stream until the first byte which --- does not match the predicate. +-- | @'replicateM' n m_x@ enumerates a stream of /n/ bytes, with each byte +-- computed by /m_x/. -- --- Since: 0.4.5 +-- Since: 0.4.8 -dropWhile :: Monad m => (Word8 -> Bool) -> Iteratee B.ByteString m () -dropWhile p = continue loop where - loop (Chunks xs) = iter where - lazy = BL.dropWhile p (BL.fromChunks xs) - iter = if BL.null lazy - then continue loop - else yield () (toChunks lazy) - loop EOF = yield () EOF +replicateM :: Monad m => Integer -> m Word8 -> Enumerator B.ByteString m b +replicateM n next = EL.replicateM n (liftM B.singleton next) --- | @take n@ extracts the next /n/ bytes from the stream, as a lazy +-- | Like 'repeatM', except the computation may terminate the stream by +-- returning 'Nothing'. +-- +-- Since: 0.4.8 + +generateM :: Monad m => m (Maybe Word8) -> Enumerator B.ByteString m b +generateM next = EL.generateM (liftM (liftM B.singleton) next) + + +-- | Applies a predicate to the stream. The inner iteratee only receives +-- characters for which the predicate is @True@. +-- +-- Since: 0.4.8 + +filter :: Monad m => (Word8 -> Bool) -> Enumeratee B.ByteString B.ByteString m b +filter p = Data.Enumerator.Binary.concatMap (\x -> B.pack [x | p x]) + + +-- | Applies a monadic predicate to the stream. The inner iteratee only +-- receives bytes for which the predicate returns @True@. +-- +-- Since: 0.4.8 + +filterM :: Monad m => (Word8 -> m Bool) -> Enumeratee B.ByteString B.ByteString m b +filterM p = Data.Enumerator.Binary.concatMapM (\x -> liftM B.pack (CM.filterM p [x])) + + +-- | @'take' n@ extracts the next /n/ bytes from the stream, as a lazy -- ByteString. -- -- Since: 0.4.5 @@ -183,7 +297,7 @@ loop acc _ EOF = yield (acc BL.empty) EOF --- | @takeWhile p@ extracts input from the stream until the first byte which +-- | @'takeWhile' p@ extracts input from the stream until the first byte which -- does not match the predicate. -- -- Since: 0.4.5 @@ -200,8 +314,7 @@ loop acc EOF = yield (acc BL.empty) EOF --- | Read all remaining input from the stream, and return as a lazy --- ByteString. +-- | @'consume' = 'takeWhile' (const True)@ -- -- Since: 0.4.5 @@ -214,7 +327,51 @@ loop acc EOF = yield (acc BL.empty) EOF --- | @require n@ buffers input until at least /n/ bytes are available, or +-- | Get the next byte from the stream, or 'Nothing' if the stream has +-- ended. +-- +-- Since: 0.4.5 + +head :: Monad m => Iteratee B.ByteString m (Maybe Word8) +head = continue loop where + loop (Chunks xs) = case BL.uncons (BL.fromChunks xs) of + Just (char, extra) -> yield (Just char) (toChunks extra) + Nothing -> head + loop EOF = yield Nothing EOF + + +-- | @'drop' n@ ignores /n/ bytes of input from the stream. +-- +-- Since: 0.4.5 + +drop :: Monad m => Integer -> Iteratee B.ByteString m () +drop n | n <= 0 = return () +drop n = continue (loop n) where + loop n' (Chunks xs) = iter where + lazy = BL.fromChunks xs + len = toInteger (BL.length lazy) + iter = if len < n' + then drop (n' - len) + else yield () (toChunks (BL.drop (fromInteger n') lazy)) + loop _ EOF = yield () EOF + + +-- | @'dropWhile' p@ ignores input from the stream until the first byte +-- which does not match the predicate. +-- +-- Since: 0.4.5 + +dropWhile :: Monad m => (Word8 -> Bool) -> Iteratee B.ByteString m () +dropWhile p = continue loop where + loop (Chunks xs) = iter where + lazy = BL.dropWhile p (BL.fromChunks xs) + iter = if BL.null lazy + then continue loop + else yield () (toChunks lazy) + loop EOF = yield () EOF + + +-- | @'require' n@ buffers input until at least /n/ bytes are available, or -- throws an error if the stream ends early. -- -- Since: 0.4.5 @@ -231,7 +388,7 @@ loop _ _ EOF = throwError (Exc.ErrorCall "require: Unexpected EOF") --- | @isolate n@ reads at most /n/ bytes from the stream, and passes them +-- | @'isolate' n@ reads at most /n/ bytes from the stream, and passes them -- to its iteratee. If the iteratee finishes early, bytes continue to be -- consumed from the outer stream until /n/ have been consumed. -- @@ -252,3 +409,159 @@ in k (toChunks s1) >>== (\step -> yield step (toChunks s2)) loop EOF = k EOF >>== (\step -> yield step EOF) isolate n step = drop n >> return step + + +-- | Split on bytes satisfying a given predicate. +-- +-- Since: 0.4.8 + +splitWhen :: Monad m => (Word8 -> Bool) -> Enumeratee B.ByteString B.ByteString m b +splitWhen p = loop where + loop = checkDone step + step k = isEOF >>= \eof -> if eof + then yield (Continue k) EOF + else do + lazy <- takeWhile (not . p) + let bytes = B.concat (BL.toChunks lazy) + eof <- isEOF + drop 1 + if BL.null lazy && eof + then yield (Continue k) EOF + else k (Chunks [bytes]) >>== loop + + + +-- | Read bytes (in chunks of the given buffer size) from the handle, and +-- stream them to an 'Iteratee'. If an exception occurs during file IO, +-- enumeration will stop and 'Error' will be returned. Exceptions from the +-- iteratee are not caught. +-- +-- This enumerator blocks until at least one byte is available from the +-- handle, and might read less than the maximum buffer size in some +-- cases. +-- +-- The handle should be opened with no encoding, and in 'IO.ReadMode' or +-- 'IO.ReadWriteMode'. +-- +-- Since: 0.4.5 + +enumHandle :: MonadIO m + => Integer -- ^ Buffer size + -> IO.Handle + -> Enumerator B.ByteString m b +enumHandle bufferSize h = do + let intSize = fromInteger bufferSize + + fix $ \loop step -> case step of + Continue k -> do + bytes <- tryIO (getBytes h intSize) + if B.null bytes + then continue k + else k (Chunks [bytes]) >>== loop + _ -> returnI step + + +-- | Read bytes (in chunks of the given buffer size) from the handle, and +-- stream them to an 'Iteratee'. If an exception occurs during file IO, +-- enumeration will stop and 'Error' will be returned. Exceptions from the +-- iteratee are not caught. +-- +-- This enumerator blocks until at least one byte is available from the +-- handle, and might read less than the maximum buffer size in some +-- cases. +-- +-- The handle should be opened with no encoding, and in 'IO.ReadMode' or +-- 'IO.ReadWriteMode'. +-- +-- If an offset is specified, the handle will be seeked to that offset +-- before reading. If the handle cannot be seeked, an error will be +-- thrown. +-- +-- If a maximum count is specified, the number of bytes read will not +-- exceed that count. +-- +-- Since: 0.4.8 + +enumHandleRange :: MonadIO m + => Integer -- ^ Buffer size + -> Maybe Integer -- ^ Offset + -> Maybe Integer -- ^ Maximum count + -> IO.Handle + -> Enumerator B.ByteString m b +enumHandleRange bufferSize offset count h s = seek >> enum where + seek = case offset of + Nothing -> return () + Just off -> tryIO (IO.hSeek h IO.AbsoluteSeek off) + + enum = case count of + Just n -> loop n s + Nothing -> enumHandle bufferSize h s + + loop n (Continue k) = + let rem = fromInteger (min bufferSize n) in + if rem <= 0 + then continue k + else do + bytes <- tryIO (getBytes h rem) + if B.null bytes + then continue k + else k (Chunks [bytes]) >>== loop (n - (toInteger (B.length bytes))) + loop _ step = returnI step + +getBytes :: IO.Handle -> Int -> IO B.ByteString +getBytes h n = do + hasInput <- Exc.catch + (IO.hWaitForInput h (-1)) + (\err -> if isEOFError err + then return False + else Exc.throwIO err) + if hasInput + then B.hGetNonBlocking h n + else return B.empty + + +-- | Opens a file path in binary mode, and passes the handle to +-- 'enumHandle'. The file will be closed when enumeration finishes. +-- +-- Since: 0.4.5 + +enumFile :: FilePath -> Enumerator B.ByteString IO b +enumFile path = enumFileRange path Nothing Nothing + + +-- | Opens a file path in binary mode, and passes the handle to +-- 'enumHandleRange'. The file will be closed when enumeration finishes. +-- +-- Since: 0.4.8 + +enumFileRange :: FilePath + -> Maybe Integer -- ^ Offset + -> Maybe Integer -- ^ Maximum count + -> Enumerator B.ByteString IO b +enumFileRange path offset count step = do + h <- tryIO (IO.openBinaryFile path IO.ReadMode) + let iter = enumHandleRange 4096 offset count h step + Iteratee (Exc.finally (runIteratee iter) (IO.hClose h)) + + +-- | Read bytes from a stream and write them to a handle. If an exception +-- occurs during file IO, enumeration will stop and 'Error' will be +-- returned. +-- +-- The handle should be opened with no encoding, and in 'IO.WriteMode' or +-- 'IO.ReadWriteMode'. +-- +-- Since: 0.4.5 + +iterHandle :: MonadIO m => IO.Handle + -> Iteratee B.ByteString m () +iterHandle h = continue step where + step EOF = yield () EOF + step (Chunks []) = continue step + step (Chunks bytes) = do + tryIO (mapM_ (B.hPut h) bytes) + continue step + + +toChunks :: BL.ByteString -> Stream B.ByteString +toChunks = Chunks . BL.toChunks diff --git a/hs/Data/Enumerator/IO.hs b/hs/Data/Enumerator/IO.hs --- a/hs/Data/Enumerator/IO.hs +++ b/hs/Data/Enumerator/IO.hs @@ -14,8 +14,7 @@ ----------------------------------------------------------------------------- module Data.Enumerator.IO - {-# DEPRECATED - "Use 'Data.Enumerator.Binary' instead" #-} + {-# DEPRECATED "Use 'Data.Enumerator.Binary' instead" #-} ( enumHandle , enumFile , iterHandle @@ -26,10 +25,9 @@ import qualified Data.ByteString as B import qualified System.IO as IO -{-# DEPRECATED enumHandle - "Use 'Data.Enumerator.Binary.enumHandle' instead" #-} +{-# DEPRECATED enumHandle "Use 'Data.Enumerator.Binary.enumHandle' instead" #-} --- | Deprecated in 0.4.5: use 'Data.Enumerator.Binary.enumHandle' instead +-- | Deprecated in 0.4.5: use 'EB.enumHandle' instead enumHandle :: MonadIO m => Integer @@ -37,18 +35,16 @@ -> E.Enumerator B.ByteString m b enumHandle = EB.enumHandle -{-# DEPRECATED enumFile - "Use 'Data.Enumerator.Binary.enumFile' instead" #-} +{-# DEPRECATED enumFile "Use 'Data.Enumerator.Binary.enumFile' instead" #-} --- | Deprecated in 0.4.5: use 'Data.Enumerator.Binary.enumFile' instead +-- | Deprecated in 0.4.5: use 'EB.enumFile' instead enumFile :: FilePath -> E.Enumerator B.ByteString IO b enumFile = EB.enumFile -{-# DEPRECATED iterHandle - "Use 'Data.Enumerator.Binary.iterHandle' instead" #-} +{-# DEPRECATED iterHandle "Use 'Data.Enumerator.Binary.iterHandle' instead" #-} --- | Deprecated in 0.4.5: use 'Data.Enumerator.Binary.iterHandle' instead +-- | Deprecated in 0.4.5: use 'EB.iterHandle' instead iterHandle :: MonadIO m => IO.Handle -> E.Iteratee B.ByteString m () diff --git a/hs/Data/Enumerator/List.hs b/hs/Data/Enumerator/List.hs --- a/hs/Data/Enumerator/List.hs +++ b/hs/Data/Enumerator/List.hs @@ -21,64 +21,282 @@ module Data.Enumerator.List ( - head - , drop - , dropWhile - , take + -- * List analogues + + -- ** Folds + fold + , foldM + + -- ** Maps + , Data.Enumerator.List.map + , Data.Enumerator.List.mapM + , Data.Enumerator.List.concatMap + , concatMapM + + -- ** Infinite streams + , Data.Enumerator.List.iterate + , iterateM + , Data.Enumerator.List.repeat + , repeatM + + -- ** Bounded streams + , Data.Enumerator.List.replicate + , replicateM + , generateM + , unfold + , unfoldM + + -- ** Filters + , Data.Enumerator.List.filter + , filterM + + -- ** Consumers + , Data.Enumerator.List.take , takeWhile , consume + + -- ** Unsorted + , head + , drop + , Data.Enumerator.List.dropWhile , require , isolate + , splitWhen ) where -import Data.Enumerator hiding (consume, head, peek, drop, dropWhile) -import Control.Exception (ErrorCall(..)) -import Prelude hiding (head, drop, dropWhile, take, takeWhile) + +import Prelude hiding (head, drop, sequence, takeWhile) +import Data.Enumerator hiding ( concatMapM, iterateM, replicateM, head, drop + , foldM, repeatM, generateM, filterM, consume) +import Control.Monad.Trans.Class (lift) +import qualified Control.Monad as CM import qualified Data.List as L +import Control.Exception (ErrorCall(..)) --- | Get the next element from the stream, or 'Nothing' if the stream has --- ended. + +-- | Consume the entire input stream with a strict left fold, one element +-- at a time. -- --- Since: 0.4.5 +-- Since: 0.4.8 -head :: Monad m => Iteratee a m (Maybe a) -head = continue loop where - loop (Chunks []) = head - loop (Chunks (x:xs)) = yield (Just x) (Chunks xs) - loop EOF = yield Nothing EOF +fold :: Monad m => (b -> a -> b) -> b + -> Iteratee a m b +fold step = continue . loop where + f = L.foldl' step + loop acc stream = case stream of + Chunks [] -> continue (loop acc) + Chunks xs -> continue (loop (f acc xs)) + EOF -> yield acc EOF --- | @drop n@ ignores /n/ input elements from the stream. +-- | Consume the entire input stream with a strict monadic left fold, one +-- element at a time. -- --- Since: 0.4.5 +-- Since: 0.4.8 -drop :: Monad m => Integer -> Iteratee a m () -drop n | n <= 0 = return () -drop n = continue (loop n) where - loop n' (Chunks xs) = iter where - len = L.genericLength xs - iter = if len < n' - then drop (n' - len) - else yield () (Chunks (L.genericDrop n' xs)) - loop _ EOF = yield () EOF +foldM :: Monad m => (b -> a -> m b) -> b + -> Iteratee a m b +foldM step = continue . loop where + f = CM.foldM step + + loop acc stream = acc `seq` case stream of + Chunks [] -> continue (loop acc) + Chunks xs -> lift (f acc xs) >>= continue . loop + EOF -> yield acc EOF --- | @dropWhile p@ ignores input from the stream until the first element --- which does not match the predicate. +-- | Enumerates a stream of elements by repeatedly applying a function to +-- some state. -- --- Since: 0.4.5 +-- Similar to 'iterate'. +-- +-- Since: 0.4.8 -dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () -dropWhile p = continue loop where - loop (Chunks xs) = case L.dropWhile p xs of - [] -> continue loop - xs' -> yield () (Chunks xs') - loop EOF = yield () EOF +unfold :: Monad m => (s -> Maybe (a, s)) -> s -> Enumerator a m b +unfold f = loop where + loop s (Continue k) = case f s of + Nothing -> continue k + Just (a, s') -> k (Chunks [a]) >>== loop s' + loop _ step = returnI step --- | @take n@ extracts the next /n/ elements from the stream, as a list. +-- | Enumerates a stream of elements by repeatedly applying a computation to +-- some state. -- +-- Similar to 'iterateM'. +-- +-- Since: 0.4.8 + +unfoldM :: Monad m => (s -> m (Maybe (a, s))) -> s -> Enumerator a m b +unfoldM f = loop where + loop s (Continue k) = do + fs <- lift (f s) + case fs of + Nothing -> continue k + Just (a, s') -> k (Chunks [a]) >>== loop s' + loop _ step = returnI step + + +-- | @'concatMapM' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +concatMapM :: Monad m => (ao -> m [ai]) + -> Enumeratee ao ai m b +concatMapM f = checkDone (continue . step) where + step k EOF = yield (Continue k) EOF + step k (Chunks xs) = loop k xs + + loop k [] = continue (step k) + loop k (x:xs) = do + fx <- lift (f x) + k (Chunks fx) >>== + checkDoneEx (Chunks xs) (\k' -> loop k' xs) + + +-- | @'concatMap' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +concatMap :: Monad m => (ao -> [ai]) + -> Enumeratee ao ai m b +concatMap f = concatMapM (return . f) + + +-- | @'map' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +map :: Monad m => (ao -> ai) + -> Enumeratee ao ai m b +map f = Data.Enumerator.List.concatMap (\x -> [f x]) + + +-- | @'mapM' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +mapM :: Monad m => (ao -> m ai) + -> Enumeratee ao ai m b +mapM f = concatMapM (\x -> Prelude.mapM f [x]) + + +-- | @'iterate' f x@ enumerates an infinite stream of repeated applications +-- of /f/ to /x/. +-- +-- Analogous to 'Prelude.iterate'. +-- +-- Since: 0.4.8 + +iterate :: Monad m => (a -> a) -> a -> Enumerator a m b +iterate f = loop where + loop a (Continue k) = k (Chunks [a]) >>== loop (f a) + loop _ step = returnI step + + +-- | Similar to 'iterate', except the iteration function is monadic. +-- +-- Since: 0.4.8 + +iterateM :: Monad m => (a -> m a) -> a + -> Enumerator a m b +iterateM f base = loop (return base) where + loop m_a (Continue k) = do + a <- lift m_a + k (Chunks [a]) >>== loop (f a) + loop _ step = returnI step + + +-- | Enumerates an infinite stream of a single element. +-- +-- Analogous to 'Prelude.repeat'. +-- +-- Since: 0.4.8 + +repeat :: Monad m => a -> Enumerator a m b +repeat a = Data.Enumerator.List.iterate (const a) a + + +-- | Enumerates an infinite stream of element. Each element is computed by +-- the underlying monad. +-- +-- Since: 0.4.8 + +repeatM :: Monad m => m a -> Enumerator a m b +repeatM m_a step = do + a <- lift m_a + iterateM (const m_a) a step + + +-- | @'replicateM' n m_x@ enumerates a stream of /n/ elements, with each +-- element computed by /m_x/. +-- +-- Since: 0.4.8 + +replicateM :: Monad m => Integer -> m a + -> Enumerator a m b +replicateM maxCount getNext = loop maxCount where + loop 0 step = returnI step + loop n (Continue k) = do + next <- lift getNext + k (Chunks [next]) >>== loop (n - 1) + loop _ step = returnI step + + +-- | @'replicate' n x@ enumerates a stream containing /n/ copies of /x/. +-- +-- Analogous to 'Prelude.replicate'. +-- +-- Since: 0.4.8 + +replicate :: Monad m => Integer -> a + -> Enumerator a m b +replicate maxCount a = replicateM maxCount (return a) + + +-- | Like 'repeatM', except the computation may terminate the stream by +-- returning 'Nothing'. +-- +-- Since: 0.4.8 + +generateM :: Monad m => m (Maybe a) + -> Enumerator a m b +generateM getNext = loop where + loop (Continue k) = do + next <- lift getNext + case next of + Nothing -> continue k + Just x -> k (Chunks [x]) >>== loop + loop step = returnI step + + +-- | Applies a predicate to the stream. The inner iteratee only receives +-- elements for which the predicate is @True@. +-- +-- Since: 0.4.8 + +filter :: Monad m => (a -> Bool) + -> Enumeratee a a m b +filter p = Data.Enumerator.List.concatMap (\x -> [x | p x]) + + +-- | Applies a monadic predicate to the stream. The inner iteratee only +-- receives elements for which the predicate returns @True@. +-- +-- Since: 0.4.8 + +filterM :: Monad m => (a -> m Bool) + -> Enumeratee a a m b +filterM p = concatMapM (\x -> CM.filterM p [x]) + + +-- | @'take' n@ extracts the next /n/ elements from the stream, as a list. +-- -- Since: 0.4.5 take :: Monad m => Integer -> Iteratee a m [a] @@ -93,7 +311,7 @@ loop acc _ EOF = yield (acc []) EOF --- | @takeWhile p@ extracts input from the stream until the first element +-- | @'takeWhile' p@ extracts input from the stream until the first element -- which does not match the predicate. -- -- Since: 0.4.5 @@ -107,7 +325,7 @@ loop acc EOF = yield (acc []) EOF --- | Read all remaining input elements from the stream, and return as a list. +-- | @'consume' = 'takeWhile' (const True)@ -- -- Since: 0.4.5 @@ -118,7 +336,47 @@ loop acc EOF = yield (acc []) EOF --- | @require n@ buffers input until at least /n/ elements are available, or +-- | Get the next element from the stream, or 'Nothing' if the stream has +-- ended. +-- +-- Since: 0.4.5 + +head :: Monad m => Iteratee a m (Maybe a) +head = continue loop where + loop (Chunks []) = head + loop (Chunks (x:xs)) = yield (Just x) (Chunks xs) + loop EOF = yield Nothing EOF + + +-- | @'drop' n@ ignores /n/ input elements from the stream. +-- +-- Since: 0.4.5 + +drop :: Monad m => Integer -> Iteratee a m () +drop n | n <= 0 = return () +drop n = continue (loop n) where + loop n' (Chunks xs) = iter where + len = L.genericLength xs + iter = if len < n' + then drop (n' - len) + else yield () (Chunks (L.genericDrop n' xs)) + loop _ EOF = yield () EOF + + +-- | @'dropWhile' p@ ignores input from the stream until the first element +-- which does not match the predicate. +-- +-- Since: 0.4.5 + +dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () +dropWhile p = continue loop where + loop (Chunks xs) = case L.dropWhile p xs of + [] -> continue loop + xs' -> yield () (Chunks xs') + loop EOF = yield () EOF + + +-- | @'require' n@ buffers input until at least /n/ elements are available, or -- throws an error if the stream ends early. -- -- Since: 0.4.5 @@ -133,7 +391,7 @@ loop _ _ EOF = throwError (ErrorCall "require: Unexpected EOF") --- | @isolate n@ reads at most /n/ elements from the stream, and passes them +-- | @'isolate' n@ reads at most /n/ elements from the stream, and passes them -- to its iteratee. If the iteratee finishes early, elements continue to be -- consumed from the outer stream until /n/ have been consumed. -- @@ -152,3 +410,14 @@ in k (Chunks s1) >>== (\step -> yield step (Chunks s2)) loop EOF = k EOF >>== (\step -> yield step EOF) isolate n step = drop n >> return step + + +-- | Split on elements satisfying a given predicate. +-- +-- Since: 0.4.8 + +splitWhen :: Monad m => (a -> Bool) -> Enumeratee a [a] m b +splitWhen p = sequence $ do + as <- takeWhile (not . p) + drop 1 + return as diff --git a/hs/Data/Enumerator/List.hs-boot b/hs/Data/Enumerator/List.hs-boot --- a/hs/Data/Enumerator/List.hs-boot +++ b/hs/Data/Enumerator/List.hs-boot @@ -6,3 +6,18 @@ dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () takeWhile :: Monad m => (a -> Bool) -> Iteratee a m [a] consume :: Monad m => Iteratee a m [a] +fold :: Monad m => (b -> a -> b) -> b -> Iteratee a m b +foldM :: Monad m => (b -> a -> m b) -> b -> Iteratee a m b +iterate :: Monad m => (a -> a) -> a -> Enumerator a m b +iterateM :: Monad m => (a -> m a) -> a -> Enumerator a m b +repeat :: Monad m => a -> Enumerator a m b +repeatM :: Monad m => m a -> Enumerator a m b +replicateM :: Monad m => Integer -> m a -> Enumerator a m b +replicate :: Monad m => Integer -> a -> Enumerator a m b +generateM :: Monad m => m (Maybe a) -> Enumerator a m b +map :: Monad m => (ao -> ai) -> Enumeratee ao ai m b +mapM :: Monad m => (ao -> m ai) -> Enumeratee ao ai m b +concatMap :: Monad m => (ao -> [ai]) -> Enumeratee ao ai m b +concatMapM :: Monad m => (ao -> m [ai]) -> Enumeratee ao ai m b +filter :: Monad m => (a -> Bool) -> Enumeratee a a m b +filterM :: Monad m => (a -> m Bool) -> Enumeratee a a m b diff --git a/hs/Data/Enumerator/Text.hs b/hs/Data/Enumerator/Text.hs --- a/hs/Data/Enumerator/Text.hs +++ b/hs/Data/Enumerator/Text.hs @@ -21,174 +21,282 @@ module Data.Enumerator.Text ( - -- * Text IO + -- * IO enumHandle , enumFile , iterHandle - + -- * List analogues + + -- ** Folds + , fold + , foldM + + -- ** Maps + , Data.Enumerator.Text.map + , Data.Enumerator.Text.mapM + , Data.Enumerator.Text.concatMap + , concatMapM + + -- ** Infinite streams + , Data.Enumerator.Text.iterate + , iterateM + , Data.Enumerator.Text.repeat + , repeatM + + -- ** Bounded streams + , Data.Enumerator.Text.replicate + , replicateM + , generateM + , unfold + , unfoldM + + -- ** Filters + , Data.Enumerator.Text.filter + , filterM + + -- ** Consumers + , Data.Enumerator.Text.take + , takeWhile + , consume + + -- ** Unsorted , Data.Enumerator.Text.head , Data.Enumerator.Text.drop , Data.Enumerator.Text.dropWhile - , Data.Enumerator.Text.take - , Data.Enumerator.Text.takeWhile - , Data.Enumerator.Text.consume , require , isolate - - -- * Codecs + , splitWhen + , lines + + -- * Text codecs , Codec , encode , decode - , utf8 - , utf16_le , utf16_be - , utf32_le , utf32_be - , ascii - , iso8859_1 ) where -import qualified Prelude -import Prelude hiding (head, drop, takeWhile) -import Data.Enumerator hiding (head, drop) -import qualified Data.Text as T -import Data.Enumerator.Util (tryStep) -import qualified Data.Text.IO as TIO - -import qualified Control.Exception as Exc +import Prelude hiding (head, drop, takeWhile, lines) +import qualified Prelude +import Data.Enumerator hiding ( head, drop, generateM, filterM, consume + , concatMapM, iterateM, repeatM, replicateM + , foldM) +import Data.Enumerator.Util (tryIO, tSpanBy, tlSpanBy, reprWord, reprChar, textToStrict) import Control.Monad.IO.Class (MonadIO) +import qualified Control.Exception as Exc +import Control.Arrow (first) +import Data.Maybe (catMaybes) +import qualified Data.Text as T +import qualified Data.ByteString as B +import qualified Data.ByteString.Char8 as B8 +import qualified Data.Text.Encoding as TE +import Data.Word (Word8, Word16) +import Data.Bits ((.&.), (.|.), shiftL) import qualified System.IO as IO import System.IO.Error (isEOFError) - +import qualified Data.Text.IO as TIO +import Data.Char (ord) +import System.IO.Unsafe (unsafePerformIO) import qualified Data.Text.Lazy as TL +import qualified Data.Enumerator.List as EL +import qualified Control.Monad as CM +import Control.Monad.Trans.Class (lift) +import Control.Monad (liftM) -import qualified Data.ByteString as B -import Data.Enumerator.Util (tSpanBy, tlSpanBy, reprWord, reprChar) -import Control.Arrow (first) -import Data.Bits ((.&.), (.|.), shiftL) -import Data.Char (ord) -import Data.Word (Word8, Word16) -import qualified Data.ByteString.Char8 as B8 -import qualified Data.Text.Encoding as TE -import Data.Maybe (catMaybes) +-- | Consume the entire input stream with a strict left fold, one character +-- at a time. +-- +-- Since: 0.4.8 -import System.IO.Unsafe (unsafePerformIO) +fold :: Monad m => (b -> Char -> b) -> b + -> Iteratee T.Text m b +fold step = EL.fold (T.foldl' step) --- | Read lines of text from the handle, and stream them to an 'Iteratee'. --- If an exception occurs during file IO, enumeration will stop and 'Error' --- will be returned. Exceptions from the iteratee are not caught. +-- | Consume the entire input stream with a strict monadic left fold, one +-- character at a time. -- --- The handle should be opened with an appropriate text encoding, and --- in 'IO.ReadMode' or 'IO.ReadWriteMode'. +-- Since: 0.4.8 + +foldM :: Monad m => (b -> Char -> m b) -> b + -> Iteratee T.Text m b +foldM step = EL.foldM (\b txt -> CM.foldM step b (T.unpack txt)) + + +-- | Enumerates a stream of characters by repeatedly applying a function to +-- some state. -- --- Since: 0.2 +-- Similar to 'iterate'. +-- +-- Since: 0.4.8 -enumHandle :: MonadIO m => IO.Handle - -> Enumerator T.Text m b -enumHandle h = loop where - loop (Continue k) = withText $ \maybeText -> - case maybeText of +unfold :: Monad m => (s -> Maybe (Char, s)) -> s -> Enumerator T.Text m b +unfold f = loop where + loop s (Continue k) = case f s of + Nothing -> continue k + Just (c, s') -> k (Chunks [T.singleton c]) >>== loop s' + loop _ step = returnI step + + +-- | Enumerates a stream of characters by repeatedly applying a computation +-- to some state. +-- +-- Similar to 'iterateM'. +-- +-- Since: 0.4.8 + +unfoldM :: Monad m => (s -> m (Maybe (Char, s))) -> s -> Enumerator T.Text m b +unfoldM f = loop where + loop s (Continue k) = do + fs <- lift (f s) + case fs of Nothing -> continue k - Just text -> k (Chunks [text]) >>== loop + Just (c, s') -> k (Chunks [T.singleton c]) >>== loop s' + loop _ step = returnI step + + +-- | @'map' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +map :: Monad m => (Char -> Char) -> Enumeratee T.Text T.Text m b +map f = Data.Enumerator.Text.concatMap (\x -> T.singleton (f x)) + + +-- | @'mapM' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +mapM :: Monad m => (Char -> m Char) -> Enumeratee T.Text T.Text m b +mapM f = Data.Enumerator.Text.concatMapM (\x -> liftM T.singleton (f x)) + + +-- | @'concatMap' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +concatMap :: Monad m => (Char -> T.Text) -> Enumeratee T.Text T.Text m b +concatMap f = Data.Enumerator.Text.concatMapM (return . f) + + +-- | @'concatMapM' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 + +concatMapM :: Monad m => (Char -> m T.Text) -> Enumeratee T.Text T.Text m b +concatMapM f = checkDone (continue . step) where + step k EOF = yield (Continue k) EOF + step k (Chunks xs) = loop k (TL.unpack (TL.fromChunks xs)) - loop step = returnI step - withText = tryStep $ Exc.catch - (Just `fmap` TIO.hGetLine h) - (\err -> if isEOFError err - then return Nothing - else Exc.throwIO err) + loop k [] = continue (step k) + loop k (x:xs) = do + fx <- lift (f x) + k (Chunks [fx]) >>== + checkDoneEx (Chunks [T.pack xs]) (\k' -> loop k' xs) --- | Opens a file path in text mode, and passes the handle to 'enumHandle'. --- The file will be closed when the 'Iteratee' finishes. +-- | @'iterate' f x@ enumerates an infinite stream of repeated applications +-- of /f/ to /x/. -- --- Since: 0.2 +-- Analogous to 'Prelude.iterate'. +-- +-- Since: 0.4.8 -enumFile :: FilePath -> Enumerator T.Text IO b -enumFile path = enum where - withHandle = tryStep (IO.openFile path IO.ReadMode) - enum step = withHandle $ \h -> Iteratee $ Exc.finally - (runIteratee (enumHandle h step)) - (IO.hClose h) +iterate :: Monad m => (Char -> Char) -> Char -> Enumerator T.Text m b +iterate f = loop where + loop char (Continue k) = k (Chunks [T.singleton char]) >>== loop (f char) + loop _ step = returnI step --- | Read text from a stream and write it to a handle. If an exception --- occurs during file IO, enumeration will stop and 'Error' will be --- returned. +-- | Similar to 'iterate', except the iteration function is monadic. -- --- The handle should be opened with an appropriate text encoding, and --- in 'IO.WriteMode' or 'IO.ReadWriteMode'. +-- Since: 0.4.8 + +iterateM :: Monad m => (Char -> m Char) -> Char -> Enumerator T.Text m b +iterateM f base = loop (return base) where + loop m_char (Continue k) = do + char <- lift m_char + k (Chunks [T.singleton char]) >>== loop (f char) + loop _ step = returnI step + + +-- | Enumerates an infinite stream of a single character. -- --- Since: 0.2 +-- Analogous to 'Prelude.repeat'. +-- +-- Since: 0.4.8 -iterHandle :: MonadIO m => IO.Handle - -> Iteratee T.Text m () -iterHandle h = continue step where - step EOF = yield () EOF - step (Chunks []) = continue step - step (Chunks chunks) = let - put = mapM_ (TIO.hPutStr h) chunks - in tryStep put (\_ -> continue step) +repeat :: Monad m => Char -> Enumerator T.Text m b +repeat char = EL.repeat (T.singleton char) -toChunks :: TL.Text -> Stream T.Text -toChunks = Chunks . TL.toChunks +-- | Enumerates an infinite stream of characters. Each character is computed +-- by the underlying monad. +-- +-- Since: 0.4.8 --- | Get the next character from the stream, or 'Nothing' if the stream has --- ended. +repeatM :: Monad m => m Char -> Enumerator T.Text m b +repeatM next = EL.repeatM (liftM T.singleton next) + + +-- | @'replicate' n x@ enumerates a stream containing /n/ copies of /x/. -- --- Since: 0.4.5 +-- Since: 0.4.8 -head :: Monad m => Iteratee T.Text m (Maybe Char) -head = continue loop where - loop (Chunks xs) = case TL.uncons (TL.fromChunks xs) of - Just (char, extra) -> yield (Just char) (toChunks extra) - Nothing -> head - loop EOF = yield Nothing EOF +replicate :: Monad m => Integer -> Char -> Enumerator T.Text m b +replicate n byte = EL.replicate n (T.singleton byte) --- | @drop n@ ignores /n/ characters of input from the stream. +-- | @'replicateM' n m_x@ enumerates a stream of /n/ characters, with each +-- character computed by /m_x/. -- --- Since: 0.4.5 +-- Since: 0.4.8 -drop :: Monad m => Integer -> Iteratee T.Text m () -drop n | n <= 0 = return () -drop n = continue (loop n) where - loop n' (Chunks xs) = iter where - lazy = TL.fromChunks xs - len = toInteger (TL.length lazy) - iter = if len < n' - then drop (n' - len) - else yield () (toChunks (TL.drop (fromInteger n') lazy)) - loop _ EOF = yield () EOF +replicateM :: Monad m => Integer -> m Char -> Enumerator T.Text m b +replicateM n next = EL.replicateM n (liftM T.singleton next) --- | @dropWhile p@ ignores input from the stream until the first character --- which does not match the predicate. +-- | Like 'repeatM', except the computation may terminate the stream by +-- returning 'Nothing'. -- --- Since: 0.4.5 +-- Since: 0.4.8 -dropWhile :: Monad m => (Char -> Bool) -> Iteratee T.Text m () -dropWhile p = continue loop where - loop (Chunks xs) = iter where - lazy = TL.dropWhile p (TL.fromChunks xs) - iter = if TL.null lazy - then continue loop - else yield () (toChunks lazy) - loop EOF = yield () EOF +generateM :: Monad m => m (Maybe Char) -> Enumerator T.Text m b +generateM next = EL.generateM (liftM (liftM T.singleton) next) --- | @take n@ extracts the next /n/ characters from the stream, as a lazy +-- | Applies a predicate to the stream. The inner iteratee only receives +-- characters for which the predicate is @True@. +-- +-- Since: 0.4.8 + +filter :: Monad m => (Char -> Bool) -> Enumeratee T.Text T.Text m b +filter p = Data.Enumerator.Text.concatMap (\x -> T.pack [x | p x]) + + +-- | Applies a monadic predicate to the stream. The inner iteratee only +-- receives characters for which the predicate returns @True@. +-- +-- Since: 0.4.8 + +filterM :: Monad m => (Char -> m Bool) -> Enumeratee T.Text T.Text m b +filterM p = Data.Enumerator.Text.concatMapM (\x -> liftM T.pack (CM.filterM p [x])) + + +-- | @'take' n@ extracts the next /n/ characters from the stream, as a lazy -- Text. -- -- Since: 0.4.5 @@ -208,7 +316,7 @@ loop acc _ EOF = yield (acc TL.empty) EOF --- | @takeWhile p@ extracts input from the stream until the first character +-- | @'takeWhile' p@ extracts input from the stream until the first character -- which does not match the predicate. -- -- Since: 0.4.5 @@ -225,8 +333,7 @@ loop acc EOF = yield (acc TL.empty) EOF --- | Read all remaining input from the stream, and return as a lazy --- Text. +-- | @'consume' = 'takeWhile' (const True)@ -- -- Since: 0.4.5 @@ -239,7 +346,51 @@ loop acc EOF = yield (acc TL.empty) EOF --- | @require n@ buffers input until at least /n/ characters are available, +-- | Get the next character from the stream, or 'Nothing' if the stream has +-- ended. +-- +-- Since: 0.4.5 + +head :: Monad m => Iteratee T.Text m (Maybe Char) +head = continue loop where + loop (Chunks xs) = case TL.uncons (TL.fromChunks xs) of + Just (char, extra) -> yield (Just char) (toChunks extra) + Nothing -> head + loop EOF = yield Nothing EOF + + +-- | @'drop' n@ ignores /n/ characters of input from the stream. +-- +-- Since: 0.4.5 + +drop :: Monad m => Integer -> Iteratee T.Text m () +drop n | n <= 0 = return () +drop n = continue (loop n) where + loop n' (Chunks xs) = iter where + lazy = TL.fromChunks xs + len = toInteger (TL.length lazy) + iter = if len < n' + then drop (n' - len) + else yield () (toChunks (TL.drop (fromInteger n') lazy)) + loop _ EOF = yield () EOF + + +-- | @'dropWhile' p@ ignores input from the stream until the first character +-- which does not match the predicate. +-- +-- Since: 0.4.5 + +dropWhile :: Monad m => (Char -> Bool) -> Iteratee T.Text m () +dropWhile p = continue loop where + loop (Chunks xs) = iter where + lazy = TL.dropWhile p (TL.fromChunks xs) + iter = if TL.null lazy + then continue loop + else yield () (toChunks lazy) + loop EOF = yield () EOF + + +-- | @'require' n@ buffers input until at least /n/ characters are available, -- or throws an error if the stream ends early. -- -- Since: 0.4.5 @@ -256,7 +407,7 @@ loop _ _ EOF = throwError (Exc.ErrorCall "require: Unexpected EOF") --- | @isolate n@ reads at most /n/ characters from the stream, and passes +-- | @'isolate' n@ reads at most /n/ characters from the stream, and passes -- them to its iteratee. If the iteratee finishes early, characters continue -- to be consumed from the outer stream until /n/ have been consumed. -- @@ -278,6 +429,93 @@ loop EOF = k EOF >>== (\step -> yield step EOF) isolate n step = drop n >> return step + +-- | Split on characters satisfying a given predicate. +-- +-- Since: 0.4.8 + +splitWhen :: Monad m => (Char -> Bool) -> Enumeratee T.Text T.Text m b +splitWhen p = loop where + loop = checkDone step + step k = isEOF >>= \eof -> if eof + then yield (Continue k) EOF + else do + lazy <- takeWhile (not . p) + let text = textToStrict lazy + eof <- isEOF + drop 1 + if TL.null lazy && eof + then yield (Continue k) EOF + else k (Chunks [text]) >>== loop + + +-- | @'lines' = 'splitWhen' (== '\n')@ +-- +-- Since: 0.4.8 + +lines :: Monad m => Enumeratee T.Text T.Text m b +lines = splitWhen (== '\n') + + + +-- | Read lines of text from the handle, and stream them to an 'Iteratee'. +-- If an exception occurs during file IO, enumeration will stop and 'Error' +-- will be returned. Exceptions from the iteratee are not caught. +-- +-- The handle should be opened with an appropriate text encoding, and +-- in 'IO.ReadMode' or 'IO.ReadWriteMode'. +-- +-- Since: 0.2 + +enumHandle :: MonadIO m => IO.Handle + -> Enumerator T.Text m b +enumHandle h = loop where + loop (Continue k) = do + maybeText <- tryIO getText + case maybeText of + Nothing -> continue k + Just text -> k (Chunks [text]) >>== loop + + loop step = returnI step + getText = Exc.catch + (Just `fmap` TIO.hGetLine h) + (\err -> if isEOFError err + then return Nothing + else Exc.throwIO err) + + +-- | Opens a file path in text mode, and passes the handle to 'enumHandle'. +-- The file will be closed when the 'Iteratee' finishes. +-- +-- Since: 0.2 + +enumFile :: FilePath -> Enumerator T.Text IO b +enumFile path step = do + h <- tryIO (IO.openFile path IO.ReadMode) + Iteratee $ Exc.finally + (runIteratee (enumHandle h step)) + (IO.hClose h) + + +-- | Read text from a stream and write it to a handle. If an exception +-- occurs during file IO, enumeration will stop and 'Error' will be +-- returned. +-- +-- The handle should be opened with an appropriate text encoding, and +-- in 'IO.WriteMode' or 'IO.ReadWriteMode'. +-- +-- Since: 0.2 + +iterHandle :: MonadIO m => IO.Handle + -> Iteratee T.Text m () +iterHandle h = continue step where + step EOF = yield () EOF + step (Chunks []) = continue step + step (Chunks chunks) = do + tryIO (mapM_ (TIO.hPutStr h) chunks) + continue step + + data Codec = Codec { codecName :: T.Text , codecEncode @@ -545,3 +783,7 @@ maybeDecode (a, b) = case tryEvaluate a of Left _ -> Nothing Right _ -> Just (a, b) + + +toChunks :: TL.Text -> Stream T.Text +toChunks = Chunks . TL.toChunks diff --git a/hs/Data/Enumerator/Util.hs b/hs/Data/Enumerator/Util.hs --- a/hs/Data/Enumerator/Util.hs +++ b/hs/Data/Enumerator/Util.hs @@ -12,12 +12,12 @@ import qualified Control.Exception as Exc import Numeric (showIntAtBase) -tryStep :: MonadIO m => IO t -> (t -> Iteratee a m b) -> Iteratee a m b -tryStep get io = do - tried <- liftIO (Exc.try get) - case tried of - Right t -> io t - Left err -> throwError (err :: Exc.SomeException) +tryIO :: MonadIO m => IO b -> Iteratee a m b +tryIO io = Iteratee $ do + tried <- liftIO (Exc.try io) + return $ case tried of + Right b -> Yield b (Chunks []) + Left err -> Error err pad0 :: Int -> String -> String pad0 size str = padded where @@ -40,4 +40,11 @@ #else tSpanBy = T.spanBy tlSpanBy = TL.spanBy +#endif + +textToStrict :: TL.Text -> T.Text +#if MIN_VERSION_text(0,8,0) +textToStrict = TL.toStrict +#else +textToStrict = T.concat . TL.toChunks #endif diff --git a/scripts/cabal-dist b/scripts/cabal-dist deleted file mode 100644 --- a/scripts/cabal-dist +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -if [ ! -f 'enumerator.cabal' ]; then - echo -n "Can't find enumerator.cabal; please run this script as" - echo -n " ./scripts/cabal-dist from within the enumerator source" - echo " directory" - exit 1 -fi - -XZ=$(which xz) -XELATEX=$(which xelatex) -if [ -d 'cabal-dev' ]; then - CABAL=$(which cabal-dev) - PATH="$PATH:$PWD/cabal-dev/bin/" -else - CABAL=$(which cabal) -fi - -ANANSI=$(which anansi) -if [ -z "$ANANSI" ]; then - echo "Can't find 'anansi' executable; make sure it exists on your "'$PATH' - exit 1 -fi - -VERSION=$(awk '/^version:/{print $2}' enumerator.cabal) - -echo "Building dist for enumerator-$VERSION using $CABAL" - -rm -rf hs dist -anansi --noline -o hs src/enumerator.anansi || exit 1 -$CABAL configure || exit 1 -$CABAL build || exit 1 -$CABAL sdist || exit 1 -mv "dist/enumerator-$VERSION.tar.gz" "./enumerator_$VERSION.tar.gz" -if [ -n "$XZ" ]; then - gzip -dfc "enumerator_$VERSION.tar.gz" > "enumerator_$VERSION.tar" - xz -f -C sha256 -9 "enumerator_$VERSION.tar" -fi - -if [ -n "$XELATEX" ]; then - rm -f *.{aux,tex,idx,log,out,toc,pdf} - anansi -w -l latex-noweb -o enumerator.tex src/enumerator.anansi || exit 1 - xelatex enumerator.tex > /dev/null || exit 1 - xelatex enumerator.tex > /dev/null || exit 1 - - mv enumerator.pdf "enumerator_$VERSION.pdf" -fi - -echo "" -echo "============================================================" -if [ -n "$XELATEX" ]; then - echo " woven source : enumerator_$VERSION.pdf" -fi -echo " source tarball (gz) : enumerator_$VERSION.tar.gz" -if [ -n "$XZ" ]; then - echo " source archive (xz) : enumerator_$VERSION.tar.xz" -fi -echo "============================================================" diff --git a/scripts/common.bash b/scripts/common.bash new file mode 100644 --- /dev/null +++ b/scripts/common.bash @@ -0,0 +1,63 @@ +PATH="$PATH:$PWD/cabal-dev/bin/" + +VERSION=$(awk '/^version:/{print $2}' enumerator.cabal) + +CABAL_DEV=$(which cabal-dev) +ANANSI=$(which anansi) +XELATEX=$(which xelatex) +XZ=$(which xz) + +require_cabal_dev() +{ + if [ -z "$CABAL_DEV" ]; then + echo "Can't find 'cabal-dev' executable; make sure it exists on your "'$PATH' + echo "Cowardly refusing to fuck with the global package database" + exit 1 + fi +} + +require_anansi() +{ + if [ -z "$ANANSI" ]; then + echo "Can't find 'anansi' executable; running '$CABAL_DEV install anansi'" + require_cabal_dev + $CABAL_DEV install anansi &> /dev/null + if [ "$?" -ne "0" ]; then + echo "Installation failed; please install Anansi manually somehow" + exit 1 + fi + ANANSI=$(which anansi) + echo "Success; anansi = $ANANSI" + fi +} + +require_xelatex() +{ + if [ -z "$XELATEX" ]; then + echo "Can't find 'xelatex' executable; make sure it exists on your "'$PATH' + exit 1 + fi +} + +make_pdf() +{ + require_anansi + require_xelatex + + rm -f *.{aux,tex,idx,log,out,toc,pdf} + $ANANSI -w -l latex-noweb -o enumerator.tex src/enumerator.anansi || exit 1 + $XELATEX enumerator.tex > /dev/null || exit 1 + $XELATEX enumerator.tex > /dev/null || exit 1 + rm -f *.{aux,tex,idx,log,out,toc} + mv enumerator.pdf "enumerator_$VERSION.pdf" +} + +clean_dev_install() +{ + require_anansi + require_cabal_dev + + rm -rf hs dist + $ANANSI -o hs src/enumerator.anansi || exit 1 + $CABAL_DEV install || exit 1 +} diff --git a/scripts/dist b/scripts/dist new file mode 100644 --- /dev/null +++ b/scripts/dist @@ -0,0 +1,42 @@ +#!/bin/bash +if [ ! -f 'enumerator.cabal' ]; then + echo -n "Can't find enumerator.cabal; please run this script as" + echo -n " ./scripts/dist from within the enumerator source" + echo " directory" + exit 1 +fi + +. scripts/common.bash + +require_anansi +require_cabal_dev + +echo "Building dist for enumerator_$VERSION using $CABAL_DEV" + +rm -rf hs dist +$ANANSI --noline -o hs src/enumerator.anansi || exit 1 +$CABAL_DEV configure || exit 1 +$CABAL_DEV build || exit 1 +$CABAL_DEV sdist || exit 1 + +mv "dist/enumerator-$VERSION.tar.gz" "./enumerator_$VERSION.tar.gz" +ln -f "./enumerator_$VERSION.tar.gz" "./enumerator-$VERSION.tar.gz" +if [ -n "$XZ" ]; then + gzip -dfc "enumerator_$VERSION.tar.gz" > "enumerator_$VERSION.tar" + xz -f -C sha256 -9 "enumerator_$VERSION.tar" +fi + +if [ -n "$XELATEX" ]; then + make_pdf +fi + +echo "" +echo "============================================================" +if [ -n "$XELATEX" ]; then + echo " woven source : enumerator_$VERSION.pdf" +fi +echo " source tarball (gz) : enumerator_$VERSION.tar.gz" +if [ -n "$XZ" ]; then + echo " source archive (xz) : enumerator_$VERSION.tar.xz" +fi +echo "============================================================" diff --git a/scripts/haddock b/scripts/haddock new file mode 100644 --- /dev/null +++ b/scripts/haddock @@ -0,0 +1,17 @@ +#!/bin/bash +if [ ! -f 'enumerator.cabal' ]; then + echo -n "Can't find enumerator.cabal; please run this script as" + echo -n " ./scripts/haddock from within the enumerator source" + echo " directory" + exit 1 +fi + +. scripts/common.bash + +require_anansi +require_cabal_dev + +rm -rf hs dist +$ANANSI --noline -o hs src/enumerator.anansi || exit 1 +$CABAL_DEV configure || exit 1 +$CABAL_DEV haddock || exit 1 diff --git a/scripts/latex b/scripts/latex new file mode 100644 --- /dev/null +++ b/scripts/latex @@ -0,0 +1,11 @@ +#!/bin/bash +if [ ! -f 'enumerator.cabal' ]; then + echo -n "Can't find enumerator.cabal; please run this script as" + echo -n " ./scripts/latex from within the enumerator source" + echo " directory" + exit 1 +fi + +. scripts/common.bash + +make_pdf diff --git a/scripts/run-benchmarks b/scripts/run-benchmarks new file mode 100644 --- /dev/null +++ b/scripts/run-benchmarks @@ -0,0 +1,21 @@ +#!/bin/bash +if [ ! -f 'enumerator.cabal' ]; then + echo -n "Can't find enumerator.cabal; please run this script as" + echo -n " ./scripts/run-benchmarks from within the enumerator source" + echo " directory" + exit 1 +fi + +. scripts/common.bash + +require_anansi +require_cabal_dev + +clean_dev_install + +pushd tests +rm -rf dist +$CABAL_DEV -s ../cabal-dev install || exit 1 +popd + +cabal-dev/bin/enumerator_benchmarks $@ diff --git a/scripts/run-tests b/scripts/run-tests --- a/scripts/run-tests +++ b/scripts/run-tests @@ -1,31 +1,21 @@ #!/bin/bash if [ ! -f 'enumerator.cabal' ]; then echo -n "Can't find enumerator.cabal; please run this script as" - echo -n " ./scripts/make-tests from within the enumerator source" + echo -n " ./scripts/run-tests from within the enumerator source" echo " directory" exit 1 fi -CABAL=$(which cabal-dev) -if [ -z "$CABAL" ]; then - echo -n "Can't find 'cabal-dev'; cowardly refusing to fuck with the" - echo " global package database" -fi -PATH="$PATH:$PWD/cabal-dev/bin/" - -ANANSI=$(which anansi) -if [ -z "$ANANSI" ]; then - echo "Can't find 'anansi' executable; make sure it exists on your "'$PATH' - exit 1 -fi +. scripts/common.bash -rm -rf hs dist -anansi -o hs src/enumerator.anansi || exit 1 +require_anansi +require_cabal_dev -$CABAL install || exit 1 +clean_dev_install pushd tests -$CABAL -s ../cabal-dev install || exit 1 +rm -rf dist +$CABAL_DEV -s ../cabal-dev install || exit 1 popd cabal-dev/bin/enumerator_tests diff --git a/src/api-docs.anansi b/src/api-docs.anansi --- a/src/api-docs.anansi +++ b/src/api-docs.anansi @@ -103,7 +103,7 @@ : :d apidoc Data.Enumerator.($$) --- | @($$) = (==\<\<)@ +-- | @'($$)' = '(==\<\<)'@ -- -- This might be easier to read when passing a chain of iteratees to an -- enumerator. @@ -112,17 +112,17 @@ : :d apidoc Data.Enumerator.(<==<) --- | @(\<==\<) = flip (>==>)@ +-- | @'(\<==\<)' = flip '(>==>)'@ -- -- Since: 0.1.1 : :d apidoc Data.Enumerator.(==<<) --- | @(==\<\<) = flip (\>\>==)@ +-- | @'(==\<\<)' = flip '(\>\>==)'@ : :d apidoc Data.Enumerator.(>==>) --- | @(>==>) e1 e2 s = e1 s >>== e2@ +-- | @'(>==>)' e1 e2 s = e1 s '>>==' e2@ -- -- Since: 0.1.1 : @@ -200,7 +200,7 @@ : :d apidoc Data.Enumerator.checkDone --- | @checkDone = 'checkDoneEx' ('Chunks' [])@ +-- | @'checkDone' = 'checkDoneEx' ('Chunks' [])@ -- -- Use this for enumeratees which do not have an input buffer. : @@ -219,14 +219,13 @@ : :d apidoc Data.Enumerator.concatMap --- | @concatMap f = 'concatMapM' (return . f)@ +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.concatMap' instead -- -- Since: 0.4.3 : :d apidoc Data.Enumerator.concatMapM --- | @concatMapM f@ applies /f/ to each input element and feeds the --- resulting outputs to the inner iteratee. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.concatMapM' instead -- -- Since: 0.4.5 : @@ -236,7 +235,7 @@ : :d apidoc Data.Enumerator.continue --- | @continue k = 'returnI' ('Continue' k)@ +-- | @'continue' k = 'returnI' ('Continue' k)@ : :d apidoc Data.Enumerator.drop @@ -248,52 +247,49 @@ : :d apidoc Data.Enumerator.enumEOF --- | docs TODO +-- | Sends 'EOF' to its iteratee. Most clients should use 'run' or 'run_' +-- instead. : :d apidoc Data.Enumerator.enumList --- | @enumList n xs@ enumerates /xs/ as a stream, passing /n/ inputs per +-- | @'enumList' n xs@ enumerates /xs/ as a stream, passing /n/ inputs per -- chunk. -- -- Primarily useful for testing and debugging. : :d apidoc Data.Enumerator.filter --- | @filter p = 'concatMap' (\x -> 'Prelude.filter' p [x])@ +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.filter' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.filterM --- | @filterM p = 'concatMapM' (\x -> 'CM.filterM' p [x])@ +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.filterM' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.foldl --- | Run the entire input stream through a pure left fold, yielding when --- there is no more input. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.fold' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.foldl' --- | Run the entire input stream through a pure strict left fold, yielding --- when there is no more input. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.fold' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.foldM --- | Run the entire input stream through a monadic left fold, yielding --- when there is no more input. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.foldM' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.generateM --- | Like 'repeatM', except the computation may terminate the stream by --- returning 'Nothing'. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.generateM' instead -- -- Since: 0.4.5 : @@ -303,20 +299,18 @@ : :d apidoc Data.Enumerator.isEOF --- | docs TODO +-- | Check whether a stream has reached EOF. Most clients should use +-- 'Data.Enumerator.List.head' instead. : :d apidoc Data.Enumerator.iterate --- | @iterate f x@ enumerates an infinite stream of repeated applications --- of /f/ to /x/. --- --- Analogous to 'Prelude.iterate'. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.iterate' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.iterateM --- | Similar to 'iterate', except the iteration function is monadic. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.iterateM' instead -- -- Since: 0.4.5 : @@ -344,19 +338,19 @@ : :d apidoc Data.Enumerator.liftFoldL --- | Deprecated in 0.4.5: use 'Data.Enumerator.foldl' instead +-- | Deprecated in 0.4.5: use 'Data.Enumerator.List.fold' instead -- -- Since: 0.1.1 : :d apidoc Data.Enumerator.liftFoldL' --- | Deprecated in 0.4.5: use 'Data.Enumerator.foldl'' instead +-- | Deprecated in 0.4.5: use 'Data.Enumerator.List.fold' instead -- -- Since: 0.1.1 : :d apidoc Data.Enumerator.liftFoldM --- | Deprecated in 0.4.5: use 'Data.Enumerator.foldM' instead +-- | Deprecated in 0.4.5: use 'Data.Enumerator.List.foldM' instead -- -- Since: 0.1.1 : @@ -373,11 +367,11 @@ : :d apidoc Data.Enumerator.map --- | @map f = 'concatMap' (\x -> 'Prelude.map' f [x])@ +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.map' instead : :d apidoc Data.Enumerator.mapM --- | @mapM f = 'concatMapM' (\x -> 'Prelude.mapM' f [x])@ +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.mapM' instead -- -- Since: 0.4.3 : @@ -393,37 +387,31 @@ : :d apidoc Data.Enumerator.repeat --- | Enumerates an infinite stream of the provided value. --- --- Analogous to 'Prelude.repeat'. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.repeat' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.repeatM --- | Enumerates an infinite stream by running the provided computation and --- passing each result to the iteratee. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.repeatM' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.replicate --- | @replicate n x = 'replicateM' n (return x)@ --- --- Analogous to 'Prelude.replicate'. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.replicate' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.replicateM --- | @replicateM n m_x@ enumerates a stream of /n/ input elements; each --- element is generated by running the input computation /m_x/ once. +-- | Deprecated in 0.4.8: use 'Data.Enumerator.List.replicateM' instead -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.returnI --- | @returnI step = 'Iteratee' (return step)@ +-- | @'returnI' step = 'Iteratee' (return step)@ : :d apidoc Data.Enumerator.run @@ -448,38 +436,69 @@ : :d apidoc Data.Enumerator.throwError --- | @throwError exc = 'returnI' ('Error' ('Exc.toException' exc))@ +-- | @'throwError' exc = 'returnI' ('Error' ('Exc.toException' exc))@ : :d apidoc Data.Enumerator.yield --- | @yield x extra = 'returnI' ('Yield' x extra)@ +-- | @'yield' x extra = 'returnI' ('Yield' x extra)@ +-- +-- WARNING: due to the current encoding of iteratees in this library, +-- careless use of the 'yield' primitive may violate the monad laws. +-- To prevent this, always make sure that an iteratee never yields +-- extra data unless it has received at least one input element. +-- +-- More strictly, iteratees may not yield data that they did not +-- receive as input. Don't use 'yield' to “inject” elements +-- into the stream. : +:d apidoc Data.Enumerator.Binary.concatMap +-- | @'concatMap' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.concatMapM +-- | @'concatMapM' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Binary.consume --- | Read all remaining input from the stream, and return as a lazy --- ByteString. +-- | @'consume' = 'takeWhile' (const True)@ -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.Binary.drop --- | @drop n@ ignores /n/ bytes of input from the stream. +-- | @'drop' n@ ignores /n/ bytes of input from the stream. -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.Binary.dropWhile --- | @dropWhile p@ ignores input from the stream until the first byte which --- does not match the predicate. +-- | @'dropWhile' p@ ignores input from the stream until the first byte +-- which does not match the predicate. -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.Binary.enumFile --- | Opens a file path in binary mode, and passes the handle to 'enumHandle'. --- The file will be closed when the 'Iteratee' finishes. +-- | Opens a file path in binary mode, and passes the handle to +-- 'enumHandle'. The file will be closed when enumeration finishes. +-- +-- Since: 0.4.5 : +:d apidoc Data.Enumerator.Binary.enumFileRange +-- | Opens a file path in binary mode, and passes the handle to +-- 'enumHandleRange'. The file will be closed when enumeration finishes. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Binary.enumHandle -- | Read bytes (in chunks of the given buffer size) from the handle, and -- stream them to an 'Iteratee'. If an exception occurs during file IO, @@ -492,8 +511,68 @@ -- -- The handle should be opened with no encoding, and in 'IO.ReadMode' or -- 'IO.ReadWriteMode'. +-- +-- Since: 0.4.5 : +:d apidoc Data.Enumerator.Binary.enumHandleRange +-- | Read bytes (in chunks of the given buffer size) from the handle, and +-- stream them to an 'Iteratee'. If an exception occurs during file IO, +-- enumeration will stop and 'Error' will be returned. Exceptions from the +-- iteratee are not caught. +-- +-- This enumerator blocks until at least one byte is available from the +-- handle, and might read less than the maximum buffer size in some +-- cases. +-- +-- The handle should be opened with no encoding, and in 'IO.ReadMode' or +-- 'IO.ReadWriteMode'. +-- +-- If an offset is specified, the handle will be seeked to that offset +-- before reading. If the handle cannot be seeked, an error will be +-- thrown. +-- +-- If a maximum count is specified, the number of bytes read will not +-- exceed that count. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.filter +-- | Applies a predicate to the stream. The inner iteratee only receives +-- characters for which the predicate is @True@. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.filterM +-- | Applies a monadic predicate to the stream. The inner iteratee only +-- receives bytes for which the predicate returns @True@. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.fold +-- | Consume the entire input stream with a strict left fold, one byte +-- at a time. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.foldM +-- | Consume the entire input stream with a strict monadic left fold, one +-- byte at a time. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.generateM +-- | Like 'repeatM', except the computation may terminate the stream by +-- returning 'Nothing'. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Binary.head -- | Get the next byte from the stream, or 'Nothing' if the stream has -- ended. @@ -502,13 +581,28 @@ : :d apidoc Data.Enumerator.Binary.isolate --- | @isolate n@ reads at most /n/ bytes from the stream, and passes them +-- | @'isolate' n@ reads at most /n/ bytes from the stream, and passes them -- to its iteratee. If the iteratee finishes early, bytes continue to be -- consumed from the outer stream until /n/ have been consumed. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.Binary.iterate +-- | @'iterate' f x@ enumerates an infinite stream of repeated applications +-- of /f/ to /x/. +-- +-- Analogous to 'Prelude.iterate'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.iterateM +-- | Similar to 'iterate', except the iteration function is monadic. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Binary.iterHandle -- | Read bytes from a stream and write them to a handle. If an exception -- occurs during file IO, enumeration will stop and 'Error' will be @@ -516,60 +610,176 @@ -- -- The handle should be opened with no encoding, and in 'IO.WriteMode' or -- 'IO.ReadWriteMode'. +-- +-- Since: 0.4.5 : +:d apidoc Data.Enumerator.Binary.map +-- | @'map' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.mapM +-- | @'mapM' f@ applies /f/ to each input byte and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.repeat +-- | Enumerates an infinite stream of a single byte. +-- +-- Analogous to 'Prelude.repeat'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.repeatM +-- | Enumerates an infinite stream of byte. Each byte is computed by the +-- underlying monad. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.replicate +-- | @'replicate' n x@ enumerates a stream containing /n/ copies of /x/. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.replicateM +-- | @'replicateM' n m_x@ enumerates a stream of /n/ bytes, with each byte +-- computed by /m_x/. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Binary.require --- | @require n@ buffers input until at least /n/ bytes are available, or +-- | @'require' n@ buffers input until at least /n/ bytes are available, or -- throws an error if the stream ends early. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.Binary.splitWhen +-- | Split on bytes satisfying a given predicate. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Binary.take --- | @take n@ extracts the next /n/ bytes from the stream, as a lazy +-- | @'take' n@ extracts the next /n/ bytes from the stream, as a lazy -- ByteString. -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.Binary.takeWhile --- | @takeWhile p@ extracts input from the stream until the first byte which +-- | @'takeWhile' p@ extracts input from the stream until the first byte which -- does not match the predicate. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.Binary.unfold +-- | Enumerates a stream of bytes by repeatedly applying a function to +-- some state. +-- +-- Similar to 'iterate'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Binary.unfoldM +-- | Enumerates a stream of bytes by repeatedly applying a computation to +-- some state. +-- +-- Similar to 'iterateM'. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.IO.enumFile --- | Deprecated in 0.4.5: use 'Data.Enumerator.Binary.enumFile' instead +-- | Deprecated in 0.4.5: use 'EB.enumFile' instead : :d apidoc Data.Enumerator.IO.enumHandle --- | Deprecated in 0.4.5: use 'Data.Enumerator.Binary.enumHandle' instead +-- | Deprecated in 0.4.5: use 'EB.enumHandle' instead : :d apidoc Data.Enumerator.IO.iterHandle --- | Deprecated in 0.4.5: use 'Data.Enumerator.Binary.iterHandle' instead +-- | Deprecated in 0.4.5: use 'EB.iterHandle' instead : +:d apidoc Data.Enumerator.List.concatMap +-- | @'concatMap' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.concatMapM +-- | @'concatMapM' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: :d apidoc Data.Enumerator.List.consume --- | Read all remaining input elements from the stream, and return as a list. +-- | @'consume' = 'takeWhile' (const True)@ -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.List.drop --- | @drop n@ ignores /n/ input elements from the stream. +-- | @'drop' n@ ignores /n/ input elements from the stream. -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.List.dropWhile --- | @dropWhile p@ ignores input from the stream until the first element +-- | @'dropWhile' p@ ignores input from the stream until the first element -- which does not match the predicate. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.List.filter +-- | Applies a predicate to the stream. The inner iteratee only receives +-- elements for which the predicate is @True@. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.filterM +-- | Applies a monadic predicate to the stream. The inner iteratee only +-- receives elements for which the predicate returns @True@. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.fold +-- | Consume the entire input stream with a strict left fold, one element +-- at a time. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.foldM +-- | Consume the entire input stream with a strict monadic left fold, one +-- element at a time. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.generateM +-- | Like 'repeatM', except the computation may terminate the stream by +-- returning 'Nothing'. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.List.head -- | Get the next element from the stream, or 'Nothing' if the stream has -- ended. @@ -578,42 +788,136 @@ : :d apidoc Data.Enumerator.List.isolate --- | @isolate n@ reads at most /n/ elements from the stream, and passes them +-- | @'isolate' n@ reads at most /n/ elements from the stream, and passes them -- to its iteratee. If the iteratee finishes early, elements continue to be -- consumed from the outer stream until /n/ have been consumed. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.List.iterate +-- | @'iterate' f x@ enumerates an infinite stream of repeated applications +-- of /f/ to /x/. +-- +-- Analogous to 'Prelude.iterate'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.iterateM +-- | Similar to 'iterate', except the iteration function is monadic. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.map +-- | @'map' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.mapM +-- | @'mapM' f@ applies /f/ to each input element and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.repeat +-- | Enumerates an infinite stream of a single element. +-- +-- Analogous to 'Prelude.repeat'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.repeatM +-- | Enumerates an infinite stream of element. Each element is computed by +-- the underlying monad. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.replicate +-- | @'replicate' n x@ enumerates a stream containing /n/ copies of /x/. +-- +-- Analogous to 'Prelude.replicate'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.replicateM +-- | @'replicateM' n m_x@ enumerates a stream of /n/ elements, with each +-- element computed by /m_x/. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.List.require --- | @require n@ buffers input until at least /n/ elements are available, or +-- | @'require' n@ buffers input until at least /n/ elements are available, or -- throws an error if the stream ends early. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.List.splitWhen +-- | Split on elements satisfying a given predicate. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.List.take --- | @take n@ extracts the next /n/ elements from the stream, as a list. +-- | @'take' n@ extracts the next /n/ elements from the stream, as a list. -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.List.takeWhile --- | @takeWhile p@ extracts input from the stream until the first element +-- | @'takeWhile' p@ extracts input from the stream until the first element -- which does not match the predicate. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.List.unfold +-- | Enumerates a stream of elements by repeatedly applying a function to +-- some state. +-- +-- Similar to 'iterate'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.List.unfoldM +-- | Enumerates a stream of elements by repeatedly applying a computation to +-- some state. +-- +-- Similar to 'iterateM'. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Text.Codec --- | docs TODO +-- | Since: 0.2 +: + +:d apidoc Data.Enumerator.Text.concatMap +-- | @'concatMap' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. -- --- Since: 0.2 +-- Since: 0.4.8 : +:d apidoc Data.Enumerator.Text.concatMapM +-- | @'concatMapM' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Text.consume --- | Read all remaining input from the stream, and return as a lazy --- Text. +-- | @'consume' = 'takeWhile' (const True)@ -- -- Since: 0.4.5 : @@ -626,13 +930,13 @@ : :d apidoc Data.Enumerator.Text.drop --- | @drop n@ ignores /n/ characters of input from the stream. +-- | @'drop' n@ ignores /n/ characters of input from the stream. -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.Text.dropWhile --- | @dropWhile p@ ignores input from the stream until the first character +-- | @'dropWhile' p@ ignores input from the stream until the first character -- which does not match the predicate. -- -- Since: 0.4.5 @@ -663,6 +967,41 @@ -- Since: 0.2 : +:d apidoc Data.Enumerator.Text.filter +-- | Applies a predicate to the stream. The inner iteratee only receives +-- characters for which the predicate is @True@. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.filterM +-- | Applies a monadic predicate to the stream. The inner iteratee only +-- receives characters for which the predicate returns @True@. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.fold +-- | Consume the entire input stream with a strict left fold, one character +-- at a time. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.foldM +-- | Consume the entire input stream with a strict monadic left fold, one +-- character at a time. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.generateM +-- | Like 'repeatM', except the computation may terminate the stream by +-- returning 'Nothing'. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Text.head -- | Get the next character from the stream, or 'Nothing' if the stream has -- ended. @@ -671,13 +1010,28 @@ : :d apidoc Data.Enumerator.Text.isolate --- | @isolate n@ reads at most /n/ characters from the stream, and passes +-- | @'isolate' n@ reads at most /n/ characters from the stream, and passes -- them to its iteratee. If the iteratee finishes early, characters continue -- to be consumed from the outer stream until /n/ have been consumed. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.Text.iterate +-- | @'iterate' f x@ enumerates an infinite stream of repeated applications +-- of /f/ to /x/. +-- +-- Analogous to 'Prelude.iterate'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.iterateM +-- | Similar to 'iterate', except the iteration function is monadic. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Text.iterHandle -- | Read text from a stream and write it to a handle. If an exception -- occurs during file IO, enumeration will stop and 'Error' will be @@ -689,23 +1043,95 @@ -- Since: 0.2 : +:d apidoc Data.Enumerator.Text.lines +-- | @'lines' = 'splitWhen' (== '\n')@ +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.map +-- | @'map' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.mapM +-- | @'mapM' f@ applies /f/ to each input character and feeds the +-- resulting outputs to the inner iteratee. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.repeat +-- | Enumerates an infinite stream of a single character. +-- +-- Analogous to 'Prelude.repeat'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.repeatM +-- | Enumerates an infinite stream of characters. Each character is computed +-- by the underlying monad. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.replicate +-- | @'replicate' n x@ enumerates a stream containing /n/ copies of /x/. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.replicateM +-- | @'replicateM' n m_x@ enumerates a stream of /n/ characters, with each +-- character computed by /m_x/. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Text.require --- | @require n@ buffers input until at least /n/ characters are available, +-- | @'require' n@ buffers input until at least /n/ characters are available, -- or throws an error if the stream ends early. -- -- Since: 0.4.5 : +:d apidoc Data.Enumerator.Text.splitWhen +-- | Split on characters satisfying a given predicate. +-- +-- Since: 0.4.8 +: + :d apidoc Data.Enumerator.Text.take --- | @take n@ extracts the next /n/ characters from the stream, as a lazy +-- | @'take' n@ extracts the next /n/ characters from the stream, as a lazy -- Text. -- -- Since: 0.4.5 : :d apidoc Data.Enumerator.Text.takeWhile --- | @takeWhile p@ extracts input from the stream until the first character +-- | @'takeWhile' p@ extracts input from the stream until the first character -- which does not match the predicate. -- -- Since: 0.4.5 +: + +:d apidoc Data.Enumerator.Text.unfold +-- | Enumerates a stream of characters by repeatedly applying a function to +-- some state. +-- +-- Similar to 'iterate'. +-- +-- Since: 0.4.8 +: + +:d apidoc Data.Enumerator.Text.unfoldM +-- | Enumerates a stream of characters by repeatedly applying a computation +-- to some state. +-- +-- Similar to 'iterateM'. +-- +-- Since: 0.4.8 : diff --git a/src/binary.anansi b/src/binary.anansi deleted file mode 100644 --- a/src/binary.anansi +++ /dev/null @@ -1,222 +0,0 @@ -\section{Binary} - -:f Data/Enumerator/Binary.hs -|Data.Enumerator.Binary module header| -module Data.Enumerator.Binary ( - |Data.Enumerator.Binary exports| - ) where -import Prelude hiding (head, drop, takeWhile) -import Data.Enumerator hiding (head, drop) -import qualified Data.ByteString as B -|Data.Enumerator.Binary imports| -: - -\subsection{IO} - -{\tt enumHandle} and {\tt enumFile} are rough analogues of -{\tt hGetContents} and {\tt readFile} from the standard library, except -they operate only in binary mode. - -Any exceptions thrown while reading or writing data are caught and reported -using {\tt throwError}, so errors can be handled in pure iteratees. - -:d Data.Enumerator.Binary imports -import Data.Enumerator.Util (tryStep) -import qualified Control.Exception as Exc -import Control.Monad.IO.Class (MonadIO) -import qualified System.IO as IO -import System.IO.Error (isEOFError) -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.enumHandle| -enumHandle :: MonadIO m - => Integer -- ^ Buffer size - -> IO.Handle - -> Enumerator B.ByteString m b -enumHandle bufferSize h = loop where - loop (Continue k) = withBytes $ \bytes -> - if B.null bytes - then continue k - else k (Chunks [bytes]) >>== loop - - loop step = returnI step - - intSize = fromInteger bufferSize - withBytes = tryStep $ do - hasInput <- Exc.catch - (IO.hWaitForInput h (-1)) - (\err -> if isEOFError err - then return False - else Exc.throwIO err) - if hasInput - then B.hGetNonBlocking h intSize - else return B.empty -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.enumFile| -enumFile :: FilePath -> Enumerator B.ByteString IO b -enumFile path = enum where - withHandle = tryStep (IO.openBinaryFile path IO.ReadMode) - enum step = withHandle $ \h -> do - Iteratee $ Exc.finally - (runIteratee (enumHandle 4096 h step)) - (IO.hClose h) -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.iterHandle| -iterHandle :: MonadIO m => IO.Handle - -> Iteratee B.ByteString m () -iterHandle h = continue step where - step EOF = yield () EOF - step (Chunks []) = continue step - step (Chunks bytes) = let - put = mapM_ (B.hPut h) bytes - in tryStep put (\_ -> continue step) -: - -:d Data.Enumerator.Binary exports - -- * Binary IO - enumHandle -, enumFile -, iterHandle -: - -\subsection{List analogues} - -:d Data.Enumerator.Binary imports -import Data.Word (Word8) -import qualified Data.ByteString.Lazy as BL -: - -:f Data/Enumerator/Binary.hs -toChunks :: BL.ByteString -> Stream B.ByteString -toChunks = Chunks . BL.toChunks -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.head| -head :: Monad m => Iteratee B.ByteString m (Maybe Word8) -head = continue loop where - loop (Chunks xs) = case BL.uncons (BL.fromChunks xs) of - Just (char, extra) -> yield (Just char) (toChunks extra) - Nothing -> head - loop EOF = yield Nothing EOF -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.drop| -drop :: Monad m => Integer -> Iteratee B.ByteString m () -drop n | n <= 0 = return () -drop n = continue (loop n) where - loop n' (Chunks xs) = iter where - lazy = BL.fromChunks xs - len = toInteger (BL.length lazy) - iter = if len < n' - then drop (n' - len) - else yield () (toChunks (BL.drop (fromInteger n') lazy)) - loop _ EOF = yield () EOF -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.dropWhile| -dropWhile :: Monad m => (Word8 -> Bool) -> Iteratee B.ByteString m () -dropWhile p = continue loop where - loop (Chunks xs) = iter where - lazy = BL.dropWhile p (BL.fromChunks xs) - iter = if BL.null lazy - then continue loop - else yield () (toChunks lazy) - loop EOF = yield () EOF -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.take| -take :: Monad m => Integer -> Iteratee B.ByteString m BL.ByteString -take n | n <= 0 = return BL.empty -take n = continue (loop id n) where - loop acc n' (Chunks xs) = iter where - lazy = BL.fromChunks xs - len = toInteger (BL.length lazy) - - iter = if len < n' - then continue (loop (acc . (BL.append lazy)) (n' - len)) - else let - (xs', extra) = BL.splitAt (fromInteger n') lazy - in yield (acc xs') (toChunks extra) - loop acc _ EOF = yield (acc BL.empty) EOF -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.takeWhile| -takeWhile :: Monad m => (Word8 -> Bool) -> Iteratee B.ByteString m BL.ByteString -takeWhile p = continue (loop id) where - loop acc (Chunks []) = continue (loop acc) - loop acc (Chunks xs) = iter where - lazy = BL.fromChunks xs - (xs', extra) = BL.span p lazy - iter = if BL.null extra - then continue (loop (acc . (BL.append lazy))) - else yield (acc xs') (toChunks extra) - loop acc EOF = yield (acc BL.empty) EOF -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.consume| -consume :: Monad m => Iteratee B.ByteString m BL.ByteString -consume = continue (loop id) where - loop acc (Chunks []) = continue (loop acc) - loop acc (Chunks xs) = iter where - lazy = BL.fromChunks xs - iter = continue (loop (acc . (BL.append lazy))) - loop acc EOF = yield (acc BL.empty) EOF -: - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.require| -require :: Monad m => Integer -> Iteratee B.ByteString m () -require n | n <= 0 = return () -require n = continue (loop id n) where - loop acc n' (Chunks xs) = iter where - lazy = BL.fromChunks xs - len = toInteger (BL.length lazy) - iter = if len < n' - then continue (loop (acc . (BL.append lazy)) (n' - len)) - else yield () (toChunks (acc lazy)) - loop _ _ EOF = throwError (Exc.ErrorCall "require: Unexpected EOF") -: - -Same caveats as {\tt Data.Enumerator.List.isolate} - -:f Data/Enumerator/Binary.hs -|apidoc Data.Enumerator.Binary.isolate| -isolate :: Monad m => Integer -> Enumeratee B.ByteString B.ByteString m b -isolate n step | n <= 0 = return step -isolate n (Continue k) = continue loop where - loop (Chunks []) = continue loop - loop (Chunks xs) = iter where - lazy = BL.fromChunks xs - len = toInteger (BL.length lazy) - - iter = if len <= n - then k (Chunks xs) >>== isolate (n - len) - else let - (s1, s2) = BL.splitAt (fromInteger n) lazy - in k (toChunks s1) >>== (\step -> yield step (toChunks s2)) - loop EOF = k EOF >>== (\step -> yield step EOF) -isolate n step = drop n >> return step -: - -:d Data.Enumerator.Binary exports --- * List analogues -, Data.Enumerator.Binary.head -, Data.Enumerator.Binary.drop -, Data.Enumerator.Binary.dropWhile -, Data.Enumerator.Binary.take -, Data.Enumerator.Binary.takeWhile -, Data.Enumerator.Binary.consume -, require -, isolate -: diff --git a/src/compat.anansi b/src/compat.anansi deleted file mode 100644 --- a/src/compat.anansi +++ /dev/null @@ -1,237 +0,0 @@ -\section{Compatibility} - -Version 0.4.5 of this library introduced some substantial reorganization -and renamings; this section implements compatibility shims, so the API -remains stable. - -:d Data.Enumerator exports --- * Compatibility -: - -\subsection{Obsolete functions} - -These are functions which seemed like good ideas, or were defined by other -enumerator/iteratee libraries, but turned out to be basically useless. At -least, I've never figured out what they're good for. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.liftTrans| -liftTrans :: (Monad m, MonadTrans t, Monad (t m)) => - Iteratee a m b -> Iteratee a (t m) b -liftTrans iter = Iteratee $ do - step <- lift (runIteratee iter) - return $ case step of - Yield x cs -> Yield x cs - Error err -> Error err - Continue k -> Continue (liftTrans . k) -: - -:f Data/Enumerator.hs -{-# DEPRECATED liftI - "Use 'Data.Enumerator.continue' instead" #-} -|apidoc Data.Enumerator.liftI| -liftI :: Monad m => (Stream a -> Step a m b) - -> Iteratee a m b -liftI k = continue (returnI . k) -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.peek| -peek :: Monad m => Iteratee a m (Maybe a) -peek = continue loop where - loop (Chunks []) = continue loop - loop chunk@(Chunks (x:_)) = yield (Just x) chunk - loop EOF = yield Nothing EOF -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.last| -last :: Monad m => Iteratee a m (Maybe a) -last = continue (loop Nothing) where - loop ret (Chunks xs) = continue . loop $ case xs of - [] -> ret - _ -> Just (Prelude.last xs) - loop ret EOF = yield ret EOF -: - -:d Data.Enumerator imports -import Data.List (genericLength) -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.length| -length :: Monad m => Iteratee a m Integer -length = continue (loop 0) where - len = genericLength - loop n (Chunks xs) = continue (loop (n + len xs)) - loop n EOF = yield n EOF -: - -:d Data.Enumerator exports --- ** Obsolete functions -, liftTrans -, liftI -, peek -, Data.Enumerator.last -, Data.Enumerator.length -: - -\subsection{Aliases} - -In previous library versions, several list-based iteratees were defined in -{\tt Data.Enumerator}. They are now defined in {\tt Data.Enumerator.List}; -because these functions use core enumerator types, a bit of module -gymnastics is required to get everything compiling properly. - -:f Data/Enumerator.hs-boot -module Data.Enumerator where -import qualified Control.Exception as Exc -data Stream a -data Step a m b - = Continue (Stream a -> Iteratee a m b) - | Yield b (Stream a) - | Error Exc.SomeException -newtype Iteratee a m b = Iteratee - { runIteratee :: m (Step a m b) - } -: - -:f Data/Enumerator/List.hs-boot -module Data.Enumerator.List where -import {-# SOURCE #-} Data.Enumerator -head :: Monad m => Iteratee a m (Maybe a) -drop :: Monad m => Integer -> Iteratee a m () -dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () -takeWhile :: Monad m => (a -> Bool) -> Iteratee a m [a] -consume :: Monad m => Iteratee a m [a] -: - -:d Data.Enumerator imports -import {-# SOURCE #-} qualified Data.Enumerator.List as EL -: - -These {\tt .hs-boot} files are enough for {\tt Data.Enumerator} to re-export -the list functions under old names, with appropriate deprecation warnings. - -:f Data/Enumerator.hs -{-# DEPRECATED head - "Use 'Data.Enumerator.List.head' instead" #-} -|apidoc Data.Enumerator.head| -head :: Monad m => Iteratee a m (Maybe a) -head = EL.head - -{-# DEPRECATED drop - "Use 'Data.Enumerator.List.drop' instead" #-} -|apidoc Data.Enumerator.drop| -drop :: Monad m => Integer -> Iteratee a m () -drop = EL.drop - -{-# DEPRECATED dropWhile - "Use 'Data.Enumerator.List.dropWhile' instead" #-} -|apidoc Data.Enumerator.dropWhile| -dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () -dropWhile = EL.dropWhile - -{-# DEPRECATED span - "Use 'Data.Enumerator.List.takeWhile' instead" #-} -|apidoc Data.Enumerator.span| -span :: Monad m => (a -> Bool) -> Iteratee a m [a] -span = EL.takeWhile - -{-# DEPRECATED break - "Use 'Data.Enumerator.List.takeWhile' instead" #-} -|apidoc Data.Enumerator.break| -break :: Monad m => (a -> Bool) -> Iteratee a m [a] -break p = EL.takeWhile (not . p) - -{-# DEPRECATED consume - "Use 'Data.Enumerator.List.consume' instead" #-} -|apidoc Data.Enumerator.consume| -consume :: Monad m => Iteratee a m [a] -consume = EL.consume -: - -:d Data.Enumerator exports --- ** Deprecated aliases -, Data.Enumerator.head -, Data.Enumerator.drop -, Data.Enumerator.dropWhile -, Data.Enumerator.span -, Data.Enumerator.break -, Data.Enumerator.consume -: - -0.4.5 also saw the pure-fold enumerators renamed, to match other functions -based on {\tt Prelude} names. - -:f Data/Enumerator.hs -{-# DEPRECATED liftFoldL - "Use 'Data.Enumerator.foldl' instead" #-} -|apidoc Data.Enumerator.liftFoldL| -liftFoldL :: Monad m => (b -> a -> b) -> b - -> Iteratee a m b -liftFoldL = Data.Enumerator.foldl - -{-# DEPRECATED liftFoldL' - "Use 'Data.Enumerator.foldl' ' instead" #-} -|apidoc Data.Enumerator.liftFoldL'| -liftFoldL' :: Monad m => (b -> a -> b) -> b - -> Iteratee a m b -liftFoldL' = Data.Enumerator.foldl' - -{-# DEPRECATED liftFoldM - "Use 'Data.Enumerator.foldM' instead" #-} -|apidoc Data.Enumerator.liftFoldM| -liftFoldM :: Monad m => (b -> a -> m b) -> b - -> Iteratee a m b -liftFoldM = Data.Enumerator.foldM -: - -:d Data.Enumerator exports -, liftFoldL -, liftFoldL' -, liftFoldM -: - -\clearpage -Finally, the {\tt Data.Enumerator.IO} module was moved to -{\tt Data.Enumerator.Binary}, and altered to include many more functions -related to binary and {\tt ByteString} processing. - -:f Data/Enumerator/IO.hs -|Data.Enumerator.IO module header| -module Data.Enumerator.IO - {-# DEPRECATED - "Use 'Data.Enumerator.Binary' instead" #-} - ( enumHandle - , enumFile - , iterHandle - ) where -import qualified Data.Enumerator as E -import qualified Data.Enumerator.Binary as EB -import Control.Monad.IO.Class (MonadIO) -import qualified Data.ByteString as B -import qualified System.IO as IO - -{-# DEPRECATED enumHandle - "Use 'Data.Enumerator.Binary.enumHandle' instead" #-} -|apidoc Data.Enumerator.IO.enumHandle| -enumHandle :: MonadIO m - => Integer - -> IO.Handle - -> E.Enumerator B.ByteString m b -enumHandle = EB.enumHandle - -{-# DEPRECATED enumFile - "Use 'Data.Enumerator.Binary.enumFile' instead" #-} -|apidoc Data.Enumerator.IO.enumFile| -enumFile :: FilePath -> E.Enumerator B.ByteString IO b -enumFile = EB.enumFile - -{-# DEPRECATED iterHandle - "Use 'Data.Enumerator.Binary.iterHandle' instead" #-} -|apidoc Data.Enumerator.IO.iterHandle| -iterHandle :: MonadIO m => IO.Handle - -> E.Iteratee B.ByteString m () -iterHandle = EB.iterHandle -: diff --git a/src/compatibility.anansi b/src/compatibility.anansi new file mode 100644 --- /dev/null +++ b/src/compatibility.anansi @@ -0,0 +1,287 @@ +\section{Legacy compatibility} + +Version 0.4.5 of this library introduced some substantial reorganization +and renamings; this section implements compatibility shims, so the API +remains stable. + +\subsection{Obsolete functions} + +These are functions which seemed like good ideas, or were defined by other +enumerator/iteratee libraries, but turned out to be basically useless. At +least, I've never figured out what they're good for. + +:d compatibility: obsolete +|apidoc Data.Enumerator.liftTrans| +liftTrans :: (Monad m, MonadTrans t, Monad (t m)) => + Iteratee a m b -> Iteratee a (t m) b +liftTrans iter = Iteratee $ do + step <- lift (runIteratee iter) + return $ case step of + Yield x cs -> Yield x cs + Error err -> Error err + Continue k -> Continue (liftTrans . k) +: + +:d compatibility: obsolete +{-# DEPRECATED liftI "Use 'Data.Enumerator.continue' instead" #-} +|apidoc Data.Enumerator.liftI| +liftI :: Monad m => (Stream a -> Step a m b) + -> Iteratee a m b +liftI k = continue (returnI . k) +: + +:d compatibility: obsolete +|apidoc Data.Enumerator.peek| +peek :: Monad m => Iteratee a m (Maybe a) +peek = continue loop where + loop (Chunks []) = continue loop + loop chunk@(Chunks (x:_)) = yield (Just x) chunk + loop EOF = yield Nothing EOF +: + +:d compatibility: obsolete +|apidoc Data.Enumerator.last| +last :: Monad m => Iteratee a m (Maybe a) +last = continue (loop Nothing) where + loop ret (Chunks xs) = continue . loop $ case xs of + [] -> ret + _ -> Just (Prelude.last xs) + loop ret EOF = yield ret EOF +: + +:d compatibility: obsolete +|apidoc Data.Enumerator.length| +length :: Monad m => Iteratee a m Integer +length = continue (loop 0) where + len = genericLength + loop n (Chunks xs) = continue (loop (n + len xs)) + loop n EOF = yield n EOF +: + +\subsection{Aliases} + +In previous library versions, several list-based iteratees were defined in +{\tt Data.Enumerator}. They are now defined in {\tt Data.Enumerator.List}; +because these functions use core enumerator types, a bit of module +gymnastics is required to get everything compiling properly. + +:f Data/Enumerator.hs-boot +module Data.Enumerator where +import qualified Control.Exception as Exc +data Stream a +data Step a m b + = Continue (Stream a -> Iteratee a m b) + | Yield b (Stream a) + | Error Exc.SomeException +newtype Iteratee a m b = Iteratee + { runIteratee :: m (Step a m b) + } +type Enumerator a m b = Step a m b -> Iteratee a m b +type Enumeratee ao ai m b = Step ai m b -> Iteratee ao m (Step ai m b) +: + +:f Data/Enumerator/List.hs-boot +module Data.Enumerator.List where +import {-# SOURCE #-} Data.Enumerator +head :: Monad m => Iteratee a m (Maybe a) +drop :: Monad m => Integer -> Iteratee a m () +dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () +takeWhile :: Monad m => (a -> Bool) -> Iteratee a m [a] +consume :: Monad m => Iteratee a m [a] +fold :: Monad m => (b -> a -> b) -> b -> Iteratee a m b +foldM :: Monad m => (b -> a -> m b) -> b -> Iteratee a m b +iterate :: Monad m => (a -> a) -> a -> Enumerator a m b +iterateM :: Monad m => (a -> m a) -> a -> Enumerator a m b +repeat :: Monad m => a -> Enumerator a m b +repeatM :: Monad m => m a -> Enumerator a m b +replicateM :: Monad m => Integer -> m a -> Enumerator a m b +replicate :: Monad m => Integer -> a -> Enumerator a m b +generateM :: Monad m => m (Maybe a) -> Enumerator a m b +map :: Monad m => (ao -> ai) -> Enumeratee ao ai m b +mapM :: Monad m => (ao -> m ai) -> Enumeratee ao ai m b +concatMap :: Monad m => (ao -> [ai]) -> Enumeratee ao ai m b +concatMapM :: Monad m => (ao -> m [ai]) -> Enumeratee ao ai m b +filter :: Monad m => (a -> Bool) -> Enumeratee a a m b +filterM :: Monad m => (a -> m Bool) -> Enumeratee a a m b +: + +These {\tt .hs-boot} files are enough for {\tt Data.Enumerator} to re-export +the list functions under old names, with appropriate deprecation warnings. + +:d compatibility: aliases +{-# DEPRECATED head "Use 'Data.Enumerator.List.head' instead" #-} +|apidoc Data.Enumerator.head| +head :: Monad m => Iteratee a m (Maybe a) +head = EL.head + +{-# DEPRECATED drop "Use 'Data.Enumerator.List.drop' instead" #-} +|apidoc Data.Enumerator.drop| +drop :: Monad m => Integer -> Iteratee a m () +drop = EL.drop + +{-# DEPRECATED dropWhile "Use 'Data.Enumerator.List.dropWhile' instead" #-} +|apidoc Data.Enumerator.dropWhile| +dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () +dropWhile = EL.dropWhile + +{-# DEPRECATED span "Use 'Data.Enumerator.List.takeWhile' instead" #-} +|apidoc Data.Enumerator.span| +span :: Monad m => (a -> Bool) -> Iteratee a m [a] +span = EL.takeWhile + +{-# DEPRECATED break "Use 'Data.Enumerator.List.takeWhile' instead" #-} +|apidoc Data.Enumerator.break| +break :: Monad m => (a -> Bool) -> Iteratee a m [a] +break p = EL.takeWhile (not . p) + +{-# DEPRECATED consume "Use 'Data.Enumerator.List.consume' instead" #-} +|apidoc Data.Enumerator.consume| +consume :: Monad m => Iteratee a m [a] +consume = EL.consume + +{-# DEPRECATED foldl "Use Data.Enumerator.List.fold instead" #-} +|apidoc Data.Enumerator.foldl| +foldl :: Monad m => (b -> a -> b) -> b -> Iteratee a m b +foldl step = continue . loop where + fold = Prelude.foldl step + loop acc stream = case stream of + Chunks [] -> continue (loop acc) + Chunks xs -> continue (loop (fold acc xs)) + EOF -> yield acc EOF + +{-# DEPRECATED foldl' "Use Data.Enumerator.List.fold instead" #-} +|apidoc Data.Enumerator.foldl'| +foldl' :: Monad m => (b -> a -> b) -> b -> Iteratee a m b +foldl' = EL.fold + +{-# DEPRECATED foldM "Use Data.Enumerator.List.foldM instead" #-} +|apidoc Data.Enumerator.foldM| +foldM :: Monad m => (b -> a -> m b) -> b -> Iteratee a m b +foldM = EL.foldM + +{-# DEPRECATED iterate "Use Data.Enumerator.List.iterate instead" #-} +|apidoc Data.Enumerator.iterate| +iterate :: Monad m => (a -> a) -> a -> Enumerator a m b +iterate = EL.iterate + +{-# DEPRECATED iterateM "Use Data.Enumerator.List.iterateM instead" #-} +|apidoc Data.Enumerator.iterateM| +iterateM :: Monad m => (a -> m a) -> a -> Enumerator a m b +iterateM = EL.iterateM + +{-# DEPRECATED repeat "Use Data.Enumerator.List.repeat instead" #-} +|apidoc Data.Enumerator.repeat| +repeat :: Monad m => a -> Enumerator a m b +repeat = EL.repeat + +{-# DEPRECATED repeatM "Use Data.Enumerator.List.repeatM instead" #-} +|apidoc Data.Enumerator.repeatM| +repeatM :: Monad m => m a -> Enumerator a m b +repeatM = EL.repeatM + +{-# DEPRECATED replicate "Use Data.Enumerator.List.replicate instead" #-} +|apidoc Data.Enumerator.replicate| +replicate :: Monad m => Integer -> a -> Enumerator a m b +replicate = EL.replicate + +{-# DEPRECATED replicateM "Use Data.Enumerator.List.replicateM instead" #-} +|apidoc Data.Enumerator.replicateM| +replicateM :: Monad m => Integer -> m a -> Enumerator a m b +replicateM = EL.replicateM + +{-# DEPRECATED generateM "Use Data.Enumerator.List.generateM instead" #-} +|apidoc Data.Enumerator.generateM| +generateM :: Monad m => m (Maybe a) -> Enumerator a m b +generateM = EL.generateM + +{-# DEPRECATED map "Use Data.Enumerator.List.map instead" #-} +|apidoc Data.Enumerator.map| +map :: Monad m => (ao -> ai) -> Enumeratee ao ai m b +map = EL.map + +{-# DEPRECATED mapM "Use Data.Enumerator.List.mapM instead" #-} +|apidoc Data.Enumerator.mapM| +mapM :: Monad m => (ao -> m ai) -> Enumeratee ao ai m b +mapM = EL.mapM + +{-# DEPRECATED concatMap "Use Data.Enumerator.List.concatMap instead" #-} +|apidoc Data.Enumerator.concatMap| +concatMap :: Monad m => (ao -> [ai]) -> Enumeratee ao ai m b +concatMap = EL.concatMap + +{-# DEPRECATED concatMapM "Use Data.Enumerator.List.concatMapM instead" #-} +|apidoc Data.Enumerator.concatMapM| +concatMapM :: Monad m => (ao -> m [ai]) -> Enumeratee ao ai m b +concatMapM = EL.concatMapM + +{-# DEPRECATED filter "Use Data.Enumerator.List.filter instead" #-} +|apidoc Data.Enumerator.filter| +filter :: Monad m => (a -> Bool) -> Enumeratee a a m b +filter = EL.filter + +{-# DEPRECATED filterM "Use Data.Enumerator.List.filterM instead" #-} +|apidoc Data.Enumerator.filterM| +filterM :: Monad m => (a -> m Bool) -> Enumeratee a a m b +filterM = EL.filterM +: + +0.4.5 also saw the pure-fold enumerators renamed, to match other functions +based on {\tt Prelude} names. + +:d compatibility: aliases +{-# DEPRECATED liftFoldL "Use Data.Enumerator.List.fold instead" #-} +|apidoc Data.Enumerator.liftFoldL| +liftFoldL :: Monad m => (b -> a -> b) -> b + -> Iteratee a m b +liftFoldL = Data.Enumerator.foldl + +{-# DEPRECATED liftFoldL' "Use Data.Enumerator.List.fold instead" #-} +|apidoc Data.Enumerator.liftFoldL'| +liftFoldL' :: Monad m => (b -> a -> b) -> b + -> Iteratee a m b +liftFoldL' = EL.fold + +{-# DEPRECATED liftFoldM "Use Data.Enumerator.List.foldM instead" #-} +|apidoc Data.Enumerator.liftFoldM| +liftFoldM :: Monad m => (b -> a -> m b) -> b + -> Iteratee a m b +liftFoldM = EL.foldM +: + +Finally, the {\tt Data.Enumerator.IO} module was moved to +{\tt Data.Enumerator.Binary}, and altered to include many more functions +related to binary and {\tt ByteString} processing. + +:f Data/Enumerator/IO.hs +|Data.Enumerator.IO module header| +module Data.Enumerator.IO + {-# DEPRECATED "Use 'Data.Enumerator.Binary' instead" #-} + ( enumHandle + , enumFile + , iterHandle + ) where +import qualified Data.Enumerator as E +import qualified Data.Enumerator.Binary as EB +import Control.Monad.IO.Class (MonadIO) +import qualified Data.ByteString as B +import qualified System.IO as IO + +{-# DEPRECATED enumHandle "Use 'Data.Enumerator.Binary.enumHandle' instead" #-} +|apidoc Data.Enumerator.IO.enumHandle| +enumHandle :: MonadIO m + => Integer + -> IO.Handle + -> E.Enumerator B.ByteString m b +enumHandle = EB.enumHandle + +{-# DEPRECATED enumFile "Use 'Data.Enumerator.Binary.enumFile' instead" #-} +|apidoc Data.Enumerator.IO.enumFile| +enumFile :: FilePath -> E.Enumerator B.ByteString IO b +enumFile = EB.enumFile + +{-# DEPRECATED iterHandle "Use 'Data.Enumerator.Binary.iterHandle' instead" #-} +|apidoc Data.Enumerator.IO.iterHandle| +iterHandle :: MonadIO m => IO.Handle + -> E.Iteratee B.ByteString m () +iterHandle = EB.iterHandle +: diff --git a/src/enumerator.anansi b/src/enumerator.anansi --- a/src/enumerator.anansi +++ b/src/enumerator.anansi @@ -2,22 +2,25 @@ :# :# See license.txt for details -:option tab-size 2 +:option tab-size 8 \documentclass{article} \usepackage{color} \usepackage{hyperref} -\usepackage{noweb} \usepackage{indentfirst} \usepackage{amsmath} \usepackage{multicol} +\usepackage{comment} -\noweboptions{smallcode} +\usepackage{latex/noweb} +\usepackage[margin=3cm]{latex/geometry} -% Smaller margins -\usepackage[left=1cm,top=1cm,right=1cm]{geometry} +\usepackage{titlesec} +\newcommand{\subsectionbreak}{\clearpage} +% \noweboptions{smallcode} + % Remove boxes from hyperlinks \hypersetup{ colorlinks, @@ -27,59 +30,23 @@ \newcommand{\io}{{\sc i/o}} -\title{enumerator\_0.4.6} +\title{enumerator\_0.4.8} \author{John Millikin\\ \href{mailto:"John Millikin" }{\tt jmillikin@gmail.com}} -\date{February 03, 2011} +\date{March 19, 2011} \begin{document} +\newgeometry{left=1.1cm,top=1cm,right=1.1cm} + \maketitle \setlength{\parskip}{5pt plus 1pt} +\setlength{\columnsep}{0.8cm} \begin{multicols}{2} -\section*{Abstract} -Typical buffer-based incremental \io{} is based around a single loop, which -reads data from some source (such as a socket or file), transforms it, and -generates one or more outputs (such as a line count, {\sc http} responses, -or modified file). Although efficient and safe, these loops are all -single-purpose; it is difficult or impossible to compose buffer-based -processing loops. - -Haskell's concept of ``lazy \io{}'' allows pure code to operate on data from -an external source. However, lazy \io{} has several shortcomings. Most notably, -resources such as memory and file handles can be retained for arbitrarily -long periods of time, causing unpredictable performance and error conditions. - -Enumerators are an efficient, predictable, and safe alternative to lazy \io{}. -Discovered by Oleg \mbox{Kiselyov}, they allow large datasets to be processed -in near-constant space by pure code. Although somewhat more complex to write, -using enumerators instead of lazy \io{} produces more correct programs. - -This library contains an enumerator implementation for Haskell, designed to -be both simple and efficient. Three core types are defined, along with -numerous helper functions: - -\begin{itemize} - -\item {\it Iteratee\/}: Data sinks, analogous to left folds. Iteratees consume -a sequence of \emph{input} values, and generate a single \emph{output} value. -Many iteratees are designed to perform side effects (such as printing to -{\tt stdout}), so they can also be used as monad transformers. - -\item {\it Enumerator\/}: Data sources, which generate input sequences. Typical -enumerators read from a file handle, socket, random number generator, or -other external stream. To operate, enumerators are passed an iteratee, and -provide that iteratee with input until either the iteratee has completed its -computation, or {\sc eof}. - -\item {\it Enumeratee\/}: Data transformers, which operate as both enumerators and -iteratees. Enumeratees read from an \emph{outer} enumerator, and provide the -transformed data to an \emph{inner} iteratee. - -\end{itemize} +:include summary.anansi \noindent Homepage: \href{http://john-millikin.com/software/enumerator/} {\small \tt http://john-millikin.com/software/enumerator/} @@ -89,31 +56,37 @@ \setlength{\parskip}{4pt plus 1pt} \end{multicols} +\restoregeometry + \newpage -\begin{multicols*}{2} :include types.anansi \newpage :include primitives.anansi -\end{multicols*} \newpage -:include list.anansi +:include list-analogues.anansi \newpage -:include binary.anansi +:include io.anansi \newpage -:include text.anansi +:include text-codecs.anansi \newpage -:include util.anansi +:include utilities.anansi +\appendix + \newpage -\begin{multicols*}{2} -:include compat.anansi -\end{multicols*} +:include public-interface.anansi +\newpage +:include compatibility.anansi + +% exclude API docs from LaTeX output, since they're generally uninteresting +\begin{comment} :include api-docs.anansi +\end{comment} \end{document} diff --git a/src/io.anansi b/src/io.anansi new file mode 100644 --- /dev/null +++ b/src/io.anansi @@ -0,0 +1,148 @@ +\section{IO} + +\subsection{Binary IO} + +{\tt enumHandle} and {\tt enumFile} are rough analogues of +{\tt hGetContents} and {\tt readFile} from the standard library, except +they operate only in binary mode. + +Any exceptions thrown while reading or writing data are caught and reported +using {\tt throwError}, so errors can be handled in pure iteratees. + +:d binary IO +|apidoc Data.Enumerator.Binary.enumHandle| +enumHandle :: MonadIO m + => Integer -- ^ Buffer size + -> IO.Handle + -> Enumerator B.ByteString m b +enumHandle bufferSize h = do + let intSize = fromInteger bufferSize + + fix $ \loop step -> case step of + Continue k -> do + bytes <- tryIO (getBytes h intSize) + if B.null bytes + then continue k + else k (Chunks [bytes]) >>== loop + _ -> returnI step +: + +:d binary IO +|apidoc Data.Enumerator.Binary.enumHandleRange| +enumHandleRange :: MonadIO m + => Integer -- ^ Buffer size + -> Maybe Integer -- ^ Offset + -> Maybe Integer -- ^ Maximum count + -> IO.Handle + -> Enumerator B.ByteString m b +enumHandleRange bufferSize offset count h s = seek >> enum where + seek = case offset of + Nothing -> return () + Just off -> tryIO (IO.hSeek h IO.AbsoluteSeek off) + + enum = case count of + Just n -> loop n s + Nothing -> enumHandle bufferSize h s + + loop n (Continue k) = + let rem = fromInteger (min bufferSize n) in + if rem <= 0 + then continue k + else do + bytes <- tryIO (getBytes h rem) + if B.null bytes + then continue k + else k (Chunks [bytes]) >>== loop (n - (toInteger (B.length bytes))) + loop _ step = returnI step +: + +:d binary IO +getBytes :: IO.Handle -> Int -> IO B.ByteString +getBytes h n = do + hasInput <- Exc.catch + (IO.hWaitForInput h (-1)) + (\err -> if isEOFError err + then return False + else Exc.throwIO err) + if hasInput + then B.hGetNonBlocking h n + else return B.empty +: + +:d binary IO +|apidoc Data.Enumerator.Binary.enumFile| +enumFile :: FilePath -> Enumerator B.ByteString IO b +enumFile path = enumFileRange path Nothing Nothing +: + +:d binary IO +|apidoc Data.Enumerator.Binary.enumFileRange| +enumFileRange :: FilePath + -> Maybe Integer -- ^ Offset + -> Maybe Integer -- ^ Maximum count + -> Enumerator B.ByteString IO b +enumFileRange path offset count step = do + h <- tryIO (IO.openBinaryFile path IO.ReadMode) + let iter = enumHandleRange 4096 offset count h step + Iteratee (Exc.finally (runIteratee iter) (IO.hClose h)) +: + +:d binary IO +|apidoc Data.Enumerator.Binary.iterHandle| +iterHandle :: MonadIO m => IO.Handle + -> Iteratee B.ByteString m () +iterHandle h = continue step where + step EOF = yield () EOF + step (Chunks []) = continue step + step (Chunks bytes) = do + tryIO (mapM_ (B.hPut h) bytes) + continue step +: + +\subsection{Text IO} + +Reading text is similar to reading bytes, but the enumerators have slightly +different behavior -- instead of reading in fixed-size chunks of data, the +text enumerators read in lines. This matches similar text-based {\sc api}s, +such as Python's {\tt xreadlines()}. + +:d text IO +|apidoc Data.Enumerator.Text.enumHandle| +enumHandle :: MonadIO m => IO.Handle + -> Enumerator T.Text m b +enumHandle h = loop where + loop (Continue k) = do + maybeText <- tryIO getText + case maybeText of + Nothing -> continue k + Just text -> k (Chunks [text]) >>== loop + + loop step = returnI step + getText = Exc.catch + (Just `fmap` TIO.hGetLine h) + (\err -> if isEOFError err + then return Nothing + else Exc.throwIO err) +: + +:d text IO +|apidoc Data.Enumerator.Text.enumFile| +enumFile :: FilePath -> Enumerator T.Text IO b +enumFile path step = do + h <- tryIO (IO.openFile path IO.ReadMode) + Iteratee $ Exc.finally + (runIteratee (enumHandle h step)) + (IO.hClose h) +: + +:d text IO +|apidoc Data.Enumerator.Text.iterHandle| +iterHandle :: MonadIO m => IO.Handle + -> Iteratee T.Text m () +iterHandle h = continue step where + step EOF = yield () EOF + step (Chunks []) = continue step + step (Chunks chunks) = do + tryIO (mapM_ (TIO.hPutStr h) chunks) + continue step +: diff --git a/src/list-analogues.anansi b/src/list-analogues.anansi new file mode 100644 --- /dev/null +++ b/src/list-analogues.anansi @@ -0,0 +1,812 @@ +\section{List analogues} + +\subsection{Folds} + +Since iteratees are semantically a left-fold, there are many existing +folds that can be lifted to iteratees. The {\tt foldl}, {\tt foldl'}, and +{\tt foldM} functions work like their standard library namesakes, but +construct iteratees instead. These iteratees are not as complex as what can +be created using {\tt Yield} and {\tt Continue}, but cover many common cases. + +Each fold consumes input from the stream until {\sc eof}, when it yields its +current accumulator. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.fold| +fold :: Monad m => (b -> a -> b) -> b + -> Iteratee a m b +fold step = continue . loop where + f = L.foldl' step + loop acc stream = case stream of + Chunks [] -> continue (loop acc) + Chunks xs -> continue (loop (f acc xs)) + EOF -> yield acc EOF +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.foldM| +foldM :: Monad m => (b -> a -> m b) -> b + -> Iteratee a m b +foldM step = continue . loop where + f = CM.foldM step + + loop acc stream = acc `seq` case stream of + Chunks [] -> continue (loop acc) + Chunks xs -> lift (f acc xs) >>= continue . loop + EOF -> yield acc EOF +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.fold| +fold :: Monad m => (b -> Word8 -> b) -> b + -> Iteratee B.ByteString m b +fold step = EL.fold (B.foldl' step) + +|apidoc Data.Enumerator.Binary.foldM| +foldM :: Monad m => (b -> Word8 -> m b) -> b + -> Iteratee B.ByteString m b +foldM step = EL.foldM (\b bytes -> CM.foldM step b (B.unpack bytes)) +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.fold| +fold :: Monad m => (b -> Char -> b) -> b + -> Iteratee T.Text m b +fold step = EL.fold (T.foldl' step) + +|apidoc Data.Enumerator.Text.foldM| +foldM :: Monad m => (b -> Char -> m b) -> b + -> Iteratee T.Text m b +foldM step = EL.foldM (\b txt -> CM.foldM step b (T.unpack txt)) +: + +\subsection{Unfolds} + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.unfold| +unfold :: Monad m => (s -> Maybe (a, s)) -> s -> Enumerator a m b +unfold f = loop where + loop s (Continue k) = case f s of + Nothing -> continue k + Just (a, s') -> k (Chunks [a]) >>== loop s' + loop _ step = returnI step +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.unfold| +unfold :: Monad m => (s -> Maybe (Word8, s)) -> s -> Enumerator B.ByteString m b +unfold f = loop where + loop s (Continue k) = case f s of + Nothing -> continue k + Just (b, s') -> k (Chunks [B.singleton b]) >>== loop s' + loop _ step = returnI step +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.unfold| +unfold :: Monad m => (s -> Maybe (Char, s)) -> s -> Enumerator T.Text m b +unfold f = loop where + loop s (Continue k) = case f s of + Nothing -> continue k + Just (c, s') -> k (Chunks [T.singleton c]) >>== loop s' + loop _ step = returnI step +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.unfoldM| +unfoldM :: Monad m => (s -> m (Maybe (a, s))) -> s -> Enumerator a m b +unfoldM f = loop where + loop s (Continue k) = do + fs <- lift (f s) + case fs of + Nothing -> continue k + Just (a, s') -> k (Chunks [a]) >>== loop s' + loop _ step = returnI step +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.unfoldM| +unfoldM :: Monad m => (s -> m (Maybe (Word8, s))) -> s -> Enumerator B.ByteString m b +unfoldM f = loop where + loop s (Continue k) = do + fs <- lift (f s) + case fs of + Nothing -> continue k + Just (b, s') -> k (Chunks [B.singleton b]) >>== loop s' + loop _ step = returnI step +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.unfoldM| +unfoldM :: Monad m => (s -> m (Maybe (Char, s))) -> s -> Enumerator T.Text m b +unfoldM f = loop where + loop s (Continue k) = do + fs <- lift (f s) + case fs of + Nothing -> continue k + Just (c, s') -> k (Chunks [T.singleton c]) >>== loop s' + loop _ step = returnI step +: + +\subsection{Maps} + +Enumeratees are conceptually similar to a monadic {\tt concatMap}; each +outer input element is converted to a list of inner inputs, which are passed +to the inner iteratee. Error handling and performance considerations +make most real-life enumeratees more complex, but some don't need the extra +design. + +The {\tt checkDone} and {\tt checkDoneEx} functions referenced here are +defined later, with other utilities. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.concatMapM| +concatMapM :: Monad m => (ao -> m [ai]) + -> Enumeratee ao ai m b +concatMapM f = checkDone (continue . step) where + step k EOF = yield (Continue k) EOF + step k (Chunks xs) = loop k xs + + loop k [] = continue (step k) + loop k (x:xs) = do + fx <- lift (f x) + k (Chunks fx) >>== + checkDoneEx (Chunks xs) (\k' -> loop k' xs) +: + +Once {\tt concatMapM} is defined, similar enumeratees can be easily created +via small wrappers. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.concatMap| +concatMap :: Monad m => (ao -> [ai]) + -> Enumeratee ao ai m b +concatMap f = concatMapM (return . f) + +|apidoc Data.Enumerator.List.map| +map :: Monad m => (ao -> ai) + -> Enumeratee ao ai m b +map f = Data.Enumerator.List.concatMap (\x -> [f x]) + +|apidoc Data.Enumerator.List.mapM| +mapM :: Monad m => (ao -> m ai) + -> Enumeratee ao ai m b +mapM f = concatMapM (\x -> Prelude.mapM f [x]) +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.map| +map :: Monad m => (Word8 -> Word8) -> Enumeratee B.ByteString B.ByteString m b +map f = Data.Enumerator.Binary.concatMap (\x -> B.singleton (f x)) + +|apidoc Data.Enumerator.Binary.mapM| +mapM :: Monad m => (Word8 -> m Word8) -> Enumeratee B.ByteString B.ByteString m b +mapM f = Data.Enumerator.Binary.concatMapM (\x -> liftM B.singleton (f x)) + +|apidoc Data.Enumerator.Binary.concatMap| +concatMap :: Monad m => (Word8 -> B.ByteString) -> Enumeratee B.ByteString B.ByteString m b +concatMap f = Data.Enumerator.Binary.concatMapM (return . f) + +|apidoc Data.Enumerator.Binary.concatMapM| +concatMapM :: Monad m => (Word8 -> m B.ByteString) -> Enumeratee B.ByteString B.ByteString m b +concatMapM f = checkDone (continue . step) where + step k EOF = yield (Continue k) EOF + step k (Chunks xs) = loop k (BL.unpack (BL.fromChunks xs)) + + loop k [] = continue (step k) + loop k (x:xs) = do + fx <- lift (f x) + k (Chunks [fx]) >>== + checkDoneEx (Chunks [B.pack xs]) (\k' -> loop k' xs) +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.map| +map :: Monad m => (Char -> Char) -> Enumeratee T.Text T.Text m b +map f = Data.Enumerator.Text.concatMap (\x -> T.singleton (f x)) + +|apidoc Data.Enumerator.Text.mapM| +mapM :: Monad m => (Char -> m Char) -> Enumeratee T.Text T.Text m b +mapM f = Data.Enumerator.Text.concatMapM (\x -> liftM T.singleton (f x)) + +|apidoc Data.Enumerator.Text.concatMap| +concatMap :: Monad m => (Char -> T.Text) -> Enumeratee T.Text T.Text m b +concatMap f = Data.Enumerator.Text.concatMapM (return . f) + +|apidoc Data.Enumerator.Text.concatMapM| +concatMapM :: Monad m => (Char -> m T.Text) -> Enumeratee T.Text T.Text m b +concatMapM f = checkDone (continue . step) where + step k EOF = yield (Continue k) EOF + step k (Chunks xs) = loop k (TL.unpack (TL.fromChunks xs)) + + loop k [] = continue (step k) + loop k (x:xs) = do + fx <- lift (f x) + k (Chunks [fx]) >>== + checkDoneEx (Chunks [T.pack xs]) (\k' -> loop k' xs) +: + +\subsection{Infinite streams} + +{\tt iterate} and {\tt iterateM} apply a function repeatedly to the base +input, passing the results through as a stream. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.iterate| +iterate :: Monad m => (a -> a) -> a -> Enumerator a m b +iterate f = loop where + loop a (Continue k) = k (Chunks [a]) >>== loop (f a) + loop _ step = returnI step +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.iterate| +iterate :: Monad m => (Word8 -> Word8) -> Word8 -> Enumerator B.ByteString m b +iterate f = loop where + loop byte (Continue k) = k (Chunks [B.singleton byte]) >>== loop (f byte) + loop _ step = returnI step +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.iterate| +iterate :: Monad m => (Char -> Char) -> Char -> Enumerator T.Text m b +iterate f = loop where + loop char (Continue k) = k (Chunks [T.singleton char]) >>== loop (f char) + loop _ step = returnI step +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.iterateM| +iterateM :: Monad m => (a -> m a) -> a + -> Enumerator a m b +iterateM f base = loop (return base) where + loop m_a (Continue k) = do + a <- lift m_a + k (Chunks [a]) >>== loop (f a) + loop _ step = returnI step +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.iterateM| +iterateM :: Monad m => (Word8 -> m Word8) -> Word8 -> Enumerator B.ByteString m b +iterateM f base = loop (return base) where + loop m_byte (Continue k) = do + byte <- lift m_byte + k (Chunks [B.singleton byte]) >>== loop (f byte) + loop _ step = returnI step +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.iterateM| +iterateM :: Monad m => (Char -> m Char) -> Char -> Enumerator T.Text m b +iterateM f base = loop (return base) where + loop m_char (Continue k) = do + char <- lift m_char + k (Chunks [T.singleton char]) >>== loop (f char) + loop _ step = returnI step +: + +{\tt repeat} and {\tt repeatM} create infinite streams, where each input +is a single value. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.repeat| +repeat :: Monad m => a -> Enumerator a m b +repeat a = Data.Enumerator.List.iterate (const a) a +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.repeatM| +repeatM :: Monad m => m a -> Enumerator a m b +repeatM m_a step = do + a <- lift m_a + iterateM (const m_a) a step +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.repeat| +repeat :: Monad m => Word8 -> Enumerator B.ByteString m b +repeat byte = EL.repeat (B.singleton byte) + +|apidoc Data.Enumerator.Binary.repeatM| +repeatM :: Monad m => m Word8 -> Enumerator B.ByteString m b +repeatM next = EL.repeatM (liftM B.singleton next) +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.repeat| +repeat :: Monad m => Char -> Enumerator T.Text m b +repeat char = EL.repeat (T.singleton char) + +|apidoc Data.Enumerator.Text.repeatM| +repeatM :: Monad m => m Char -> Enumerator T.Text m b +repeatM next = EL.repeatM (liftM T.singleton next) +: + +\subsection{Bounded streams} + +{\tt replicate} and {\tt replicateM} create streams containing a given +quantity of the input value. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.replicateM| +replicateM :: Monad m => Integer -> m a + -> Enumerator a m b +replicateM maxCount getNext = loop maxCount where + loop 0 step = returnI step + loop n (Continue k) = do + next <- lift getNext + k (Chunks [next]) >>== loop (n - 1) + loop _ step = returnI step +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.replicate| +replicate :: Monad m => Integer -> a + -> Enumerator a m b +replicate maxCount a = replicateM maxCount (return a) +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.replicate| +replicate :: Monad m => Integer -> Word8 -> Enumerator B.ByteString m b +replicate n byte = EL.replicate n (B.singleton byte) + +|apidoc Data.Enumerator.Binary.replicateM| +replicateM :: Monad m => Integer -> m Word8 -> Enumerator B.ByteString m b +replicateM n next = EL.replicateM n (liftM B.singleton next) +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.replicate| +replicate :: Monad m => Integer -> Char -> Enumerator T.Text m b +replicate n byte = EL.replicate n (T.singleton byte) + +|apidoc Data.Enumerator.Text.replicateM| +replicateM :: Monad m => Integer -> m Char -> Enumerator T.Text m b +replicateM n next = EL.replicateM n (liftM T.singleton next) +: + +{\tt generateM} runs a monadic computation until it returns {\tt Nothing}, +which signals the end of enumeration. + +Note that when the enumerator is finished, it does not send {\tt EOF} to +the iteratee. Instead, it returns a continuation, so additional enumerators +may add their own input to the stream. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.generateM| +generateM :: Monad m => m (Maybe a) + -> Enumerator a m b +generateM getNext = loop where + loop (Continue k) = do + next <- lift getNext + case next of + Nothing -> continue k + Just x -> k (Chunks [x]) >>== loop + loop step = returnI step +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.generateM| +generateM :: Monad m => m (Maybe Word8) -> Enumerator B.ByteString m b +generateM next = EL.generateM (liftM (liftM B.singleton) next) +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.generateM| +generateM :: Monad m => m (Maybe Char) -> Enumerator T.Text m b +generateM next = EL.generateM (liftM (liftM T.singleton) next) +: + +\subsection{Filters} + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.filter| +filter :: Monad m => (a -> Bool) + -> Enumeratee a a m b +filter p = Data.Enumerator.List.concatMap (\x -> [x | p x]) +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.filterM| +filterM :: Monad m => (a -> m Bool) + -> Enumeratee a a m b +filterM p = concatMapM (\x -> CM.filterM p [x]) +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.filter| +filter :: Monad m => (Word8 -> Bool) -> Enumeratee B.ByteString B.ByteString m b +filter p = Data.Enumerator.Binary.concatMap (\x -> B.pack [x | p x]) + +|apidoc Data.Enumerator.Binary.filterM| +filterM :: Monad m => (Word8 -> m Bool) -> Enumeratee B.ByteString B.ByteString m b +filterM p = Data.Enumerator.Binary.concatMapM (\x -> liftM B.pack (CM.filterM p [x])) +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.filter| +filter :: Monad m => (Char -> Bool) -> Enumeratee T.Text T.Text m b +filter p = Data.Enumerator.Text.concatMap (\x -> T.pack [x | p x]) + +|apidoc Data.Enumerator.Text.filterM| +filterM :: Monad m => (Char -> m Bool) -> Enumeratee T.Text T.Text m b +filterM p = Data.Enumerator.Text.concatMapM (\x -> liftM T.pack (CM.filterM p [x])) +: + +\subsection{Consumers} + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.take| +take :: Monad m => Integer -> Iteratee a m [a] +take n | n <= 0 = return [] +take n = continue (loop id n) where + len = L.genericLength + loop acc n' (Chunks xs) + | len xs < n' = continue (loop (acc . (xs ++)) (n' - len xs)) + | otherwise = let + (xs', extra) = L.genericSplitAt n' xs + in yield (acc xs') (Chunks extra) + loop acc _ EOF = yield (acc []) EOF +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.take| +take :: Monad m => Integer -> Iteratee B.ByteString m BL.ByteString +take n | n <= 0 = return BL.empty +take n = continue (loop id n) where + loop acc n' (Chunks xs) = iter where + lazy = BL.fromChunks xs + len = toInteger (BL.length lazy) + + iter = if len < n' + then continue (loop (acc . (BL.append lazy)) (n' - len)) + else let + (xs', extra) = BL.splitAt (fromInteger n') lazy + in yield (acc xs') (toChunks extra) + loop acc _ EOF = yield (acc BL.empty) EOF +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.take| +take :: Monad m => Integer -> Iteratee T.Text m TL.Text +take n | n <= 0 = return TL.empty +take n = continue (loop id n) where + loop acc n' (Chunks xs) = iter where + lazy = TL.fromChunks xs + len = toInteger (TL.length lazy) + + iter = if len < n' + then continue (loop (acc . (TL.append lazy)) (n' - len)) + else let + (xs', extra) = TL.splitAt (fromInteger n') lazy + in yield (acc xs') (toChunks extra) + loop acc _ EOF = yield (acc TL.empty) EOF +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.takeWhile| +takeWhile :: Monad m => (a -> Bool) -> Iteratee a m [a] +takeWhile p = continue (loop id) where + loop acc (Chunks []) = continue (loop acc) + loop acc (Chunks xs) = case Prelude.span p xs of + (_, []) -> continue (loop (acc . (xs ++))) + (xs', extra) -> yield (acc xs') (Chunks extra) + loop acc EOF = yield (acc []) EOF +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.takeWhile| +takeWhile :: Monad m => (Word8 -> Bool) -> Iteratee B.ByteString m BL.ByteString +takeWhile p = continue (loop id) where + loop acc (Chunks []) = continue (loop acc) + loop acc (Chunks xs) = iter where + lazy = BL.fromChunks xs + (xs', extra) = BL.span p lazy + iter = if BL.null extra + then continue (loop (acc . (BL.append lazy))) + else yield (acc xs') (toChunks extra) + loop acc EOF = yield (acc BL.empty) EOF +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.takeWhile| +takeWhile :: Monad m => (Char -> Bool) -> Iteratee T.Text m TL.Text +takeWhile p = continue (loop id) where + loop acc (Chunks []) = continue (loop acc) + loop acc (Chunks xs) = iter where + lazy = TL.fromChunks xs + (xs', extra) = tlSpanBy p lazy + iter = if TL.null extra + then continue (loop (acc . (TL.append lazy))) + else yield (acc xs') (toChunks extra) + loop acc EOF = yield (acc TL.empty) EOF +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.consume| +consume :: Monad m => Iteratee a m [a] +consume = continue (loop id) where + loop acc (Chunks []) = continue (loop acc) + loop acc (Chunks xs) = continue (loop (acc . (xs ++))) + loop acc EOF = yield (acc []) EOF +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.consume| +consume :: Monad m => Iteratee B.ByteString m BL.ByteString +consume = continue (loop id) where + loop acc (Chunks []) = continue (loop acc) + loop acc (Chunks xs) = iter where + lazy = BL.fromChunks xs + iter = continue (loop (acc . (BL.append lazy))) + loop acc EOF = yield (acc BL.empty) EOF +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.consume| +consume :: Monad m => Iteratee T.Text m TL.Text +consume = continue (loop id) where + loop acc (Chunks []) = continue (loop acc) + loop acc (Chunks xs) = iter where + lazy = TL.fromChunks xs + iter = continue (loop (acc . (TL.append lazy))) + loop acc EOF = yield (acc TL.empty) EOF +: + +\subsection{Unsorted} + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.head| +head :: Monad m => Iteratee a m (Maybe a) +head = continue loop where + loop (Chunks []) = head + loop (Chunks (x:xs)) = yield (Just x) (Chunks xs) + loop EOF = yield Nothing EOF +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.head| +head :: Monad m => Iteratee B.ByteString m (Maybe Word8) +head = continue loop where + loop (Chunks xs) = case BL.uncons (BL.fromChunks xs) of + Just (char, extra) -> yield (Just char) (toChunks extra) + Nothing -> head + loop EOF = yield Nothing EOF +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.head| +head :: Monad m => Iteratee T.Text m (Maybe Char) +head = continue loop where + loop (Chunks xs) = case TL.uncons (TL.fromChunks xs) of + Just (char, extra) -> yield (Just char) (toChunks extra) + Nothing -> head + loop EOF = yield Nothing EOF +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.drop| +drop :: Monad m => Integer -> Iteratee a m () +drop n | n <= 0 = return () +drop n = continue (loop n) where + loop n' (Chunks xs) = iter where + len = L.genericLength xs + iter = if len < n' + then drop (n' - len) + else yield () (Chunks (L.genericDrop n' xs)) + loop _ EOF = yield () EOF +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.drop| +drop :: Monad m => Integer -> Iteratee B.ByteString m () +drop n | n <= 0 = return () +drop n = continue (loop n) where + loop n' (Chunks xs) = iter where + lazy = BL.fromChunks xs + len = toInteger (BL.length lazy) + iter = if len < n' + then drop (n' - len) + else yield () (toChunks (BL.drop (fromInteger n') lazy)) + loop _ EOF = yield () EOF +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.drop| +drop :: Monad m => Integer -> Iteratee T.Text m () +drop n | n <= 0 = return () +drop n = continue (loop n) where + loop n' (Chunks xs) = iter where + lazy = TL.fromChunks xs + len = toInteger (TL.length lazy) + iter = if len < n' + then drop (n' - len) + else yield () (toChunks (TL.drop (fromInteger n') lazy)) + loop _ EOF = yield () EOF +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.dropWhile| +dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () +dropWhile p = continue loop where + loop (Chunks xs) = case L.dropWhile p xs of + [] -> continue loop + xs' -> yield () (Chunks xs') + loop EOF = yield () EOF +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.dropWhile| +dropWhile :: Monad m => (Word8 -> Bool) -> Iteratee B.ByteString m () +dropWhile p = continue loop where + loop (Chunks xs) = iter where + lazy = BL.dropWhile p (BL.fromChunks xs) + iter = if BL.null lazy + then continue loop + else yield () (toChunks lazy) + loop EOF = yield () EOF +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.dropWhile| +dropWhile :: Monad m => (Char -> Bool) -> Iteratee T.Text m () +dropWhile p = continue loop where + loop (Chunks xs) = iter where + lazy = TL.dropWhile p (TL.fromChunks xs) + iter = if TL.null lazy + then continue loop + else yield () (toChunks lazy) + loop EOF = yield () EOF +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.require| +require :: Monad m => Integer -> Iteratee a m () +require n | n <= 0 = return () +require n = continue (loop id n) where + len = L.genericLength + loop acc n' (Chunks xs) + | len xs < n' = continue (loop (acc . (xs ++)) (n' - len xs)) + | otherwise = yield () (Chunks (acc xs)) + loop _ _ EOF = throwError (ErrorCall "require: Unexpected EOF") +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.require| +require :: Monad m => Integer -> Iteratee B.ByteString m () +require n | n <= 0 = return () +require n = continue (loop id n) where + loop acc n' (Chunks xs) = iter where + lazy = BL.fromChunks xs + len = toInteger (BL.length lazy) + iter = if len < n' + then continue (loop (acc . (BL.append lazy)) (n' - len)) + else yield () (toChunks (acc lazy)) + loop _ _ EOF = throwError (Exc.ErrorCall "require: Unexpected EOF") +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.require| +require :: Monad m => Integer -> Iteratee T.Text m () +require n | n <= 0 = return () +require n = continue (loop id n) where + loop acc n' (Chunks xs) = iter where + lazy = TL.fromChunks xs + len = toInteger (TL.length lazy) + iter = if len < n' + then continue (loop (acc . (TL.append lazy)) (n' - len)) + else yield () (toChunks (acc lazy)) + loop _ _ EOF = throwError (Exc.ErrorCall "require: Unexpected EOF") +: + +Note: {\tt isolate} has some odd behavior regarding extra input in the +inner iteratee. Depending on how large the chunks are, extra input might +be returned in the {\tt Step}, or dropped. + +This doesn't matter if {\tt joinI} is used, but might if a user is poking +around inside the {\tt Step}. Eventually, enumeratees will be modified to +avoid exposing its internal iteratee state. + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.isolate| +isolate :: Monad m => Integer -> Enumeratee a a m b +isolate n step | n <= 0 = return step +isolate n (Continue k) = continue loop where + len = L.genericLength + + loop (Chunks []) = continue loop + loop (Chunks xs) + | len xs <= n = k (Chunks xs) >>== isolate (n - len xs) + | otherwise = let + (s1, s2) = L.genericSplitAt n xs + in k (Chunks s1) >>== (\step -> yield step (Chunks s2)) + loop EOF = k EOF >>== (\step -> yield step EOF) +isolate n step = drop n >> return step +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.isolate| +isolate :: Monad m => Integer -> Enumeratee B.ByteString B.ByteString m b +isolate n step | n <= 0 = return step +isolate n (Continue k) = continue loop where + loop (Chunks []) = continue loop + loop (Chunks xs) = iter where + lazy = BL.fromChunks xs + len = toInteger (BL.length lazy) + + iter = if len <= n + then k (Chunks xs) >>== isolate (n - len) + else let + (s1, s2) = BL.splitAt (fromInteger n) lazy + in k (toChunks s1) >>== (\step -> yield step (toChunks s2)) + loop EOF = k EOF >>== (\step -> yield step EOF) +isolate n step = drop n >> return step +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.isolate| +isolate :: Monad m => Integer -> Enumeratee T.Text T.Text m b +isolate n step | n <= 0 = return step +isolate n (Continue k) = continue loop where + loop (Chunks []) = continue loop + loop (Chunks xs) = iter where + lazy = TL.fromChunks xs + len = toInteger (TL.length lazy) + + iter = if len <= n + then k (Chunks xs) >>== isolate (n - len) + else let + (s1, s2) = TL.splitAt (fromInteger n) lazy + in k (toChunks s1) >>== (\step -> yield step (toChunks s2)) + loop EOF = k EOF >>== (\step -> yield step EOF) +isolate n step = drop n >> return step +: + +:d element-oriented list analogues +|apidoc Data.Enumerator.List.splitWhen| +splitWhen :: Monad m => (a -> Bool) -> Enumeratee a [a] m b +splitWhen p = sequence $ do + as <- takeWhile (not . p) + drop 1 + return as +: + +:d byte-oriented list analogues +|apidoc Data.Enumerator.Binary.splitWhen| +splitWhen :: Monad m => (Word8 -> Bool) -> Enumeratee B.ByteString B.ByteString m b +splitWhen p = loop where + loop = checkDone step + step k = isEOF >>= \eof -> if eof + then yield (Continue k) EOF + else do + lazy <- takeWhile (not . p) + let bytes = B.concat (BL.toChunks lazy) + eof <- isEOF + drop 1 + if BL.null lazy && eof + then yield (Continue k) EOF + else k (Chunks [bytes]) >>== loop +: + +:d text-oriented list analogues +|apidoc Data.Enumerator.Text.splitWhen| +splitWhen :: Monad m => (Char -> Bool) -> Enumeratee T.Text T.Text m b +splitWhen p = loop where + loop = checkDone step + step k = isEOF >>= \eof -> if eof + then yield (Continue k) EOF + else do + lazy <- takeWhile (not . p) + let text = textToStrict lazy + eof <- isEOF + drop 1 + if TL.null lazy && eof + then yield (Continue k) EOF + else k (Chunks [text]) >>== loop + +|apidoc Data.Enumerator.Text.lines| +lines :: Monad m => Enumeratee T.Text T.Text m b +lines = splitWhen (== '\n') +: diff --git a/src/list.anansi b/src/list.anansi deleted file mode 100644 --- a/src/list.anansi +++ /dev/null @@ -1,155 +0,0 @@ -\section{Lists} - -:f Data/Enumerator/List.hs -|Data.Enumerator.List module header| -module Data.Enumerator.List ( - |Data.Enumerator.List exports| - ) where -import Data.Enumerator hiding (consume, head, peek, drop, dropWhile) -import Control.Exception (ErrorCall(..)) -import Prelude hiding (head, drop, dropWhile, take, takeWhile) -import qualified Data.List as L -: - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.head| -head :: Monad m => Iteratee a m (Maybe a) -head = continue loop where - loop (Chunks []) = head - loop (Chunks (x:xs)) = yield (Just x) (Chunks xs) - loop EOF = yield Nothing EOF -: - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.drop| -drop :: Monad m => Integer -> Iteratee a m () -drop n | n <= 0 = return () -drop n = continue (loop n) where - loop n' (Chunks xs) = iter where - len = L.genericLength xs - iter = if len < n' - then drop (n' - len) - else yield () (Chunks (L.genericDrop n' xs)) - loop _ EOF = yield () EOF -: - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.dropWhile| -dropWhile :: Monad m => (a -> Bool) -> Iteratee a m () -dropWhile p = continue loop where - loop (Chunks xs) = case L.dropWhile p xs of - [] -> continue loop - xs' -> yield () (Chunks xs') - loop EOF = yield () EOF -: - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.take| -take :: Monad m => Integer -> Iteratee a m [a] -take n | n <= 0 = return [] -take n = continue (loop id n) where - len = L.genericLength - loop acc n' (Chunks xs) - | len xs < n' = continue (loop (acc . (xs ++)) (n' - len xs)) - | otherwise = let - (xs', extra) = L.genericSplitAt n' xs - in yield (acc xs') (Chunks extra) - loop acc _ EOF = yield (acc []) EOF -: - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.takeWhile| -takeWhile :: Monad m => (a -> Bool) -> Iteratee a m [a] -takeWhile p = continue (loop id) where - loop acc (Chunks []) = continue (loop acc) - loop acc (Chunks xs) = case Prelude.span p xs of - (_, []) -> continue (loop (acc . (xs ++))) - (xs', extra) -> yield (acc xs') (Chunks extra) - loop acc EOF = yield (acc []) EOF -: - -:# NOTE: peeking properly is currently impossible with the current design of -:# 'Stream'. Once it's updated to support EOF with "final data", peek can be -:# re-enabled -:# -:# :d Data/Enumerator/List.hs -:# |apidoc Data.Enumerator.List.peek| -:# peek :: Monad m => Integer -> Iteratee a m [a] -:# peek n | n <= 0 = return [] -:# peek n = continue (loop id n) where -:# len = L.genericLength -:# loop acc n' (Chunks xs) -:# | len xs < n' = continue (loop (acc . (xs ++)) (n' - len xs)) -:# | otherwise = let -:# xs' = L.genericTake n' xs -:# in yield (acc xs') (Chunks (acc xs)) -:# loop acc _ EOF = yield (acc []) EOF -:# : -:# -:# :d Data/Enumerator/List.hs (disabled) -:# |apidoc Data.Enumerator.List.peekWhile| -:# peekWhile :: Monad m => (a -> Bool) -> Iteratee a m [a] -:# peekWhile p = continue (loop id) where -:# loop acc (Chunks []) = continue (loop acc) -:# loop acc (Chunks xs) = case Prelude.span p xs of -:# (_, []) -> continue (loop (acc . (xs ++))) -:# (xs', _) -> yield (acc xs') (Chunks (acc xs)) -:# loop acc EOF = yield (acc []) EOF -:# : - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.consume| -consume :: Monad m => Iteratee a m [a] -consume = continue (loop id) where - loop acc (Chunks []) = continue (loop acc) - loop acc (Chunks xs) = continue (loop (acc . (xs ++))) - loop acc EOF = yield (acc []) EOF -: - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.require| -require :: Monad m => Integer -> Iteratee a m () -require n | n <= 0 = return () -require n = continue (loop id n) where - len = L.genericLength - loop acc n' (Chunks xs) - | len xs < n' = continue (loop (acc . (xs ++)) (n' - len xs)) - | otherwise = yield () (Chunks (acc xs)) - loop _ _ EOF = throwError (ErrorCall "require: Unexpected EOF") -: - -Note: {\tt isolate} has some odd behavior regarding extra input in the -inner iteratee. Depending on how large the chunks are, extra input might -be returned in the {\tt Step}, or dropped. - -This doesn't matter if {\tt joinI} is used, but might if a user is poking -around inside the {\tt Step}. Eventually, enumeratees will be modified to -avoid exposing its internal iteratee state. - -:f Data/Enumerator/List.hs -|apidoc Data.Enumerator.List.isolate| -isolate :: Monad m => Integer -> Enumeratee a a m b -isolate n step | n <= 0 = return step -isolate n (Continue k) = continue loop where - len = L.genericLength - - loop (Chunks []) = continue loop - loop (Chunks xs) - | len xs <= n = k (Chunks xs) >>== isolate (n - len xs) - | otherwise = let - (s1, s2) = L.genericSplitAt n xs - in k (Chunks s1) >>== (\step -> yield step (Chunks s2)) - loop EOF = k EOF >>== (\step -> yield step EOF) -isolate n step = drop n >> return step -: - -:d Data.Enumerator.List exports - head -, drop -, dropWhile -, take -, takeWhile -, consume -, require -, isolate -: diff --git a/src/primitives.anansi b/src/primitives.anansi --- a/src/primitives.anansi +++ b/src/primitives.anansi @@ -1,9 +1,85 @@ \section{Primitives} -:d Data.Enumerator exports --- * Primitives +\subsection{Operators} + +Because {\tt Iteratee a m b} is semantically equivalent to +{\tt m (Step a m b)}, several of the monadic combinators ({\tt (>>=)}, +{\tt (>=>)}, etc) are useful to save typing when constructing enumerators +and enumeratees. {\tt (>>==)} corresponds to {\tt (>>=)}, {\tt (>==>)} to +{\tt (>=>)}, and so on. + +:d iteratee operators +infixl 1 >>== +infixr 1 ==<< +infixr 0 $$ +infixr 1 >==> +infixr 1 <==< + +|apidoc Data.Enumerator.(>>==)| +(>>==) :: Monad m + => Iteratee a m b + -> (Step a m b -> Iteratee a' m b') + -> Iteratee a' m b' +i >>== f = Iteratee (runIteratee i >>= runIteratee . f) + +|apidoc Data.Enumerator.(==<<)| +(==<<) :: Monad m + => (Step a m b -> Iteratee a' m b') + -> Iteratee a m b + -> Iteratee a' m b' +(==<<) = flip (>>==) + +|apidoc Data.Enumerator.($$)| +($$) :: Monad m + => (Step a m b -> Iteratee a' m b') + -> Iteratee a m b + -> Iteratee a' m b' +($$) = (==<<) + +|apidoc Data.Enumerator.(>==>)| +(>==>) :: Monad m + => Enumerator a m b + -> (Step a m b -> Iteratee a' m b') + -> Step a m b + -> Iteratee a' m b' +(>==>) e1 e2 s = e1 s >>== e2 + +|apidoc Data.Enumerator.(<==<)| +(<==<) :: Monad m + => (Step a m b -> Iteratee a' m b') + -> Enumerator a m b + -> Step a m b + -> Iteratee a' m b' +(<==<) = flip (>==>) : +\subsection{Running iteratees} + +To simplify running iteratees, {\tt run} sends {\tt EOF} and then examines +the result. It is not possible for the result to be {\tt Continue}, because +{\tt enumEOF} calls {\tt error} for divergent iteratees. + +:d primitives +|apidoc Data.Enumerator.run| +run :: Monad m => Iteratee a m b + -> m (Either Exc.SomeException b) +run i = do + mStep <- runIteratee $ enumEOF ==<< i + case mStep of + Error err -> return $ Left err + Yield x _ -> return $ Right x + Continue _ -> error "run: divergent iteratee" +: + +{\tt run\_} is even more simplified; it's used in simple scripts, where the +user doesn't care about error handling. + +:d primitives +|apidoc Data.Enumerator.run_| +run_ :: Monad m => Iteratee a m b -> m b +run_ i = run i >>= either Exc.throw return +: + \subsection{Error handling} Most real-world applications have to deal with error conditions; however, @@ -16,10 +92,9 @@ Instances for the {\tt MonadError} class are provided in auxiliary libraries, to avoid extraneous dependencies. -:f Data/Enumerator.hs +:d primitives |apidoc Data.Enumerator.throwError| -throwError :: (Monad m, Exc.Exception e) => e - -> Iteratee a m b +throwError :: (Monad m, Exc.Exception e) => e -> Iteratee a m b throwError exc = returnI (Error (Exc.toException exc)) : @@ -30,287 +105,14 @@ This limitation means that {\tt catchError} is mostly only useful for transforming or logging errors, not ignoring them. -:f Data/Enumerator.hs +:d primitives |apidoc Data.Enumerator.catchError| -catchError :: Monad m => Iteratee a m b +catchError :: Monad m + => Iteratee a m b -> (Exc.SomeException -> Iteratee a m b) -> Iteratee a m b catchError iter h = iter >>== step where step (Yield b as) = yield b as step (Error err) = h err step (Continue k) = continue (\s -> k s >>== step) -: - -:d Data.Enumerator exports --- ** Error handling -, throwError -, catchError -: - -\subsection{Iteratees} - -Since iteratees are semantically a left-fold, there are many existing -folds that can be lifted to iteratees. The {\tt foldl}, {\tt foldl'}, and -{\tt foldM} functions work like their standard library namesakes, but -construct iteratees instead. These iteratees are not as complex as what can -be created using {\tt Yield} and {\tt Continue}, but cover many common cases. - -Each fold consumes input from the stream until {\sc eof}, when it yields its -current accumulator. - -:d Data.Enumerator imports -import Data.List (foldl') -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.foldl| -foldl :: Monad m => (b -> a -> b) -> b - -> Iteratee a m b -foldl step = continue . loop where - fold = Prelude.foldl step - loop acc stream = case stream of - Chunks [] -> continue (loop acc) - Chunks xs -> continue (loop (fold acc xs)) - EOF -> yield acc EOF -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.foldl'| -foldl' :: Monad m => (b -> a -> b) -> b - -> Iteratee a m b -foldl' step = continue . loop where - fold = Data.List.foldl' step - loop acc stream = case stream of - Chunks [] -> continue (loop acc) - Chunks xs -> continue (loop (fold acc xs)) - EOF -> yield acc EOF -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.foldM| -foldM :: Monad m => (b -> a -> m b) -> b - -> Iteratee a m b -foldM step = continue . loop where - fold acc = lift . CM.foldM step acc - - loop acc stream = case stream of - Chunks [] -> continue (loop acc) - Chunks xs -> fold acc xs >>= continue . loop - EOF -> yield acc EOF -: - -:d Data.Enumerator exports --- ** Iteratees -, Data.Enumerator.foldl -, Data.Enumerator.foldl' -, Data.Enumerator.foldM -: - -\subsection{Enumerators} - -At their simplest, enumerators just check to see whether their received step -can accept any more input. If so, input is generated somehow, fed to the step, -and its result checked again. Most enumerators are defined using a -worker/wrapper pair, for efficiency and readability. - -Here we define a number of enumerators based on functions from -{\tt Data.List}. Each generator has a monadic and non-monadic form, to -demonstrate how side effects might be ordered with respect to the iteratee's -processing. - -{\tt iterate} and {\tt iterateM} apply a function repeatedly to the base -input, passing the results through as a stream. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.iterate| -iterate :: Monad m => (a -> a) -> a -> Enumerator a m b -iterate f = loop where - loop a (Continue k) = k (Chunks [a]) >>== loop (f a) - loop _ step = returnI step -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.iterateM| -iterateM :: Monad m => (a -> m a) -> a - -> Enumerator a m b -iterateM f base = loop (return base) where - loop m_a (Continue k) = do - a <- lift m_a - k (Chunks [a]) >>== loop (f a) - loop _ step = returnI step -: - -{\tt repeat} and {\tt repeatM} create infinite streams, where each input -is a single value. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.repeat| -repeat :: Monad m => a -> Enumerator a m b -repeat a = Data.Enumerator.iterate (const a) a -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.repeatM| -repeatM :: Monad m => m a -> Enumerator a m b -repeatM m_a step = do - a <- lift m_a - iterateM (const m_a) a step -: - -{\tt replicate} and {\tt replicateM} create streams containing a given -quantity of the input value. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.replicateM| -replicateM :: Monad m => Integer -> m a - -> Enumerator a m b -replicateM maxCount getNext = loop maxCount where - loop 0 step = returnI step - loop n (Continue k) = do - next <- lift getNext - k (Chunks [next]) >>== loop (n - 1) - loop _ step = returnI step -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.replicate| -replicate :: Monad m => Integer -> a - -> Enumerator a m b -replicate maxCount a = replicateM maxCount (return a) -: - -{\tt generateM} runs a monadic computation until it returns {\tt Nothing}, -which signals the end of enumeration. - -Note that when the enumerator is finished, it does not send {\tt EOF} to -the iteratee. Instead, it returns a continuation, so additional enumerators -may add their own input to the stream. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.generateM| -generateM :: Monad m => m (Maybe a) - -> Enumerator a m b -generateM getNext = loop where - loop (Continue k) = do - next <- lift getNext - case next of - Nothing -> continue k - Just x -> k (Chunks [x]) >>== loop - loop step = returnI step -: - -:d Data.Enumerator exports --- ** Enumerators -, Data.Enumerator.iterate -, iterateM -, Data.Enumerator.repeat -, repeatM -, Data.Enumerator.replicate -, replicateM -, generateM -: - -\subsection{Enumeratees} - -Enumeratees are conceptually similar to a monadic {\tt concatMap}; each -outer input element is converted to a list of inner inputs, which are passed -to the inner iteratee. Error handling and performance considerations -make most real-life enumeratees more complex, but some don't need the extra -design. - -The {\tt checkDone} and {\tt checkDoneEx} functions referenced here are -defined later, with other utilities. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.concatMapM| -concatMapM :: Monad m => (ao -> m [ai]) - -> Enumeratee ao ai m b -concatMapM f = checkDone (continue . step) where - step k EOF = yield (Continue k) EOF - step k (Chunks xs) = loop k xs - - loop k [] = continue (step k) - loop k (x:xs) = do - fx <- lift (f x) - k (Chunks fx) >>== - checkDoneEx (Chunks xs) (\k' -> loop k' xs) -: - -Once {\tt concatMapM} is defined, similar enumeratees can be easily created -via small wrappers. - -:d excluded Prelude imports -concatMap, -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.concatMap| -concatMap :: Monad m => (ao -> [ai]) - -> Enumeratee ao ai m b -concatMap f = concatMapM (return . f) -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.map| -map :: Monad m => (ao -> ai) - -> Enumeratee ao ai m b -map f = concatMap (\x -> Prelude.map f [x]) -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.filter| -filter :: Monad m => (a -> Bool) - -> Enumeratee a a m b -filter p = concatMap (\x -> Prelude.filter p [x]) -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.mapM| -mapM :: Monad m => (ao -> m ai) - -> Enumeratee ao ai m b -mapM f = concatMapM (\x -> Prelude.mapM f [x]) -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.filterM| -filterM :: Monad m => (a -> m Bool) - -> Enumeratee a a m b -filterM p = concatMapM (\x -> CM.filterM p [x]) -: - -:d Data.Enumerator exports --- ** Enumeratees -, Data.Enumerator.map -, Data.Enumerator.concatMap -, Data.Enumerator.filter -, Data.Enumerator.mapM -, concatMapM -, Data.Enumerator.filterM -: - -\subsection{Debugging} - -Debugging enumerator-based code is mostly a question of what inputs are -being passed around. {\tt printChunks} prints out exactly what chunks are -being sent from an enumerator. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.printChunks| -printChunks :: (MonadIO m, Show a) - => Bool -- ^ Print empty chunks - -> Iteratee a m () -printChunks printEmpty = continue loop where - loop (Chunks xs) = do - let hide = null xs && not printEmpty - CM.unless hide (liftIO (print xs)) - continue loop - - loop EOF = do - liftIO (putStrLn "EOF") - yield () EOF -: - -:d Data.Enumerator exports --- ** Debugging -, printChunks : diff --git a/src/public-interface.anansi b/src/public-interface.anansi new file mode 100644 --- /dev/null +++ b/src/public-interface.anansi @@ -0,0 +1,362 @@ +\section{Public interface} + +:f Data/Enumerator.hs +|Data.Enumerator module header| +module Data.Enumerator ( + |Data.Enumerator exports| + ) where +|Data.Enumerator imports| + +|types and instances| +|supplemental instances| +|primitives| +|iteratee operators| +|utilities for testing and debugging| +|unsorted utilities| +|compatibility: obsolete| +|compatibility: aliases| +: + +:f Data/Enumerator/Binary.hs +|Data.Enumerator.Binary module header| +module Data.Enumerator.Binary ( + |Data.Enumerator.Binary exports| + ) where +|Data.Enumerator.Binary imports| +|byte-oriented list analogues| +|binary IO| + +toChunks :: BL.ByteString -> Stream B.ByteString +toChunks = Chunks . BL.toChunks +: + +:f Data/Enumerator/List.hs +|Data.Enumerator.List module header| +module Data.Enumerator.List ( + |Data.Enumerator.List exports| + ) where +|Data.Enumerator.List imports| +|element-oriented list analogues| +: + +:f Data/Enumerator/Text.hs +|Data.Enumerator.Text module header| +module Data.Enumerator.Text ( + |Data.Enumerator.Text exports| + ) where +|Data.Enumerator.Text imports| +|text-oriented list analogues| +|text IO| +|text codecs| + +toChunks :: TL.Text -> Stream T.Text +toChunks = Chunks . TL.toChunks +: + +:d Data.Enumerator imports +import qualified Control.Exception as Exc +import Data.Monoid (Monoid, mempty, mappend, mconcat) +import Control.Monad.Trans.Class (MonadTrans, lift) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Control.Applicative as A +import qualified Control.Monad as CM +import Data.Function (fix) +import {-# SOURCE #-} qualified Data.Enumerator.List as EL +import Data.List (genericLength) +: + +:d Data.Enumerator.Binary imports +import Prelude hiding (head, drop, takeWhile) +import Data.Enumerator hiding ( head, drop, iterateM, repeatM, replicateM + , generateM, filterM, consume, foldM + , concatMapM) +import Data.Enumerator.Util (tryIO) +import Control.Monad.IO.Class (MonadIO) +import qualified Data.ByteString as B +import qualified System.IO as IO +import Data.Function (fix) +import qualified Control.Exception as Exc +import System.IO.Error (isEOFError) +import Data.Word (Word8) +import qualified Data.Enumerator.List as EL +import qualified Control.Monad as CM +import qualified Data.ByteString.Lazy as BL +import Control.Monad.Trans.Class (lift) +import Control.Monad (liftM) +: + +:d Data.Enumerator.List imports +import Prelude hiding (head, drop, sequence, takeWhile) +import Data.Enumerator hiding ( concatMapM, iterateM, replicateM, head, drop + , foldM, repeatM, generateM, filterM, consume) +import Control.Monad.Trans.Class (lift) +import qualified Control.Monad as CM +import qualified Data.List as L +import Control.Exception (ErrorCall(..)) +: + +:d Data.Enumerator.Text imports +import Prelude hiding (head, drop, takeWhile, lines) +import qualified Prelude +import Data.Enumerator hiding ( head, drop, generateM, filterM, consume + , concatMapM, iterateM, repeatM, replicateM + , foldM) +import Data.Enumerator.Util (tryIO, tSpanBy, tlSpanBy, reprWord, reprChar, textToStrict) +import Control.Monad.IO.Class (MonadIO) +import qualified Control.Exception as Exc +import Control.Arrow (first) +import Data.Maybe (catMaybes) +import qualified Data.Text as T +import qualified Data.ByteString as B +import qualified Data.ByteString.Char8 as B8 +import qualified Data.Text.Encoding as TE +import Data.Word (Word8, Word16) +import Data.Bits ((.&.), (.|.), shiftL) +import qualified System.IO as IO +import System.IO.Error (isEOFError) +import qualified Data.Text.IO as TIO +import Data.Char (ord) +import System.IO.Unsafe (unsafePerformIO) +import qualified Data.Text.Lazy as TL +import qualified Data.Enumerator.List as EL +import qualified Control.Monad as CM +import Control.Monad.Trans.Class (lift) +import Control.Monad (liftM) +: + +:d Data.Enumerator exports +-- * Types + Stream (..) +, Iteratee (..) +, Step (..) +, Enumerator +, Enumeratee + +-- * Primitives +, returnI +, continue +, yield + +-- ** Operators +, (>>==) +, (==<<) +, ($$) +, (>==>) +, (<==<) + +-- ** Running iteratees +, run +, run_ + +-- ** Error handling +, throwError +, catchError + +-- * Miscellaneous +, concatEnums +, joinI +, joinE +, Data.Enumerator.sequence +, enumEOF +, checkDoneEx +, checkDone +, isEOF + +-- ** Testing and debugging +, printChunks +, enumList + +-- * Legacy compatibility + +-- ** Obsolete +, liftTrans +, liftI +, peek +, Data.Enumerator.last +, Data.Enumerator.length + +-- ** Aliases +, Data.Enumerator.head +, Data.Enumerator.drop +, Data.Enumerator.dropWhile +, Data.Enumerator.span +, Data.Enumerator.break +, consume +, Data.Enumerator.foldl +, Data.Enumerator.foldl' +, foldM +, Data.Enumerator.iterate +, iterateM +, Data.Enumerator.repeat +, repeatM +, Data.Enumerator.replicate +, replicateM +, generateM +, Data.Enumerator.map +, Data.Enumerator.mapM +, Data.Enumerator.concatMap +, concatMapM +, Data.Enumerator.filter +, filterM +, liftFoldL +, liftFoldL' +, liftFoldM +: + +:d Data.Enumerator.Binary exports +-- * IO + enumHandle +, enumHandleRange +, enumFile +, enumFileRange +, iterHandle + +-- * List analogues + +-- ** Folds +, fold +, foldM + +-- ** Maps +, Data.Enumerator.Binary.map +, Data.Enumerator.Binary.mapM +, Data.Enumerator.Binary.concatMap +, concatMapM + +-- ** Infinite streams +, Data.Enumerator.Binary.iterate +, iterateM +, Data.Enumerator.Binary.repeat +, repeatM + +-- ** Bounded streams +, Data.Enumerator.Binary.replicate +, replicateM +, generateM +, unfold +, unfoldM + +-- ** Filters +, Data.Enumerator.Binary.filter +, filterM + +-- ** Consumers +, Data.Enumerator.Binary.take +, takeWhile +, consume + +-- ** Unsorted +, Data.Enumerator.Binary.head +, Data.Enumerator.Binary.drop +, Data.Enumerator.Binary.dropWhile +, require +, isolate +, splitWhen + +: + +:d Data.Enumerator.List exports +-- * List analogues + +-- ** Folds + fold +, foldM + +-- ** Maps +, Data.Enumerator.List.map +, Data.Enumerator.List.mapM +, Data.Enumerator.List.concatMap +, concatMapM + +-- ** Infinite streams +, Data.Enumerator.List.iterate +, iterateM +, Data.Enumerator.List.repeat +, repeatM + +-- ** Bounded streams +, Data.Enumerator.List.replicate +, replicateM +, generateM +, unfold +, unfoldM + +-- ** Filters +, Data.Enumerator.List.filter +, filterM + +-- ** Consumers +, Data.Enumerator.List.take +, takeWhile +, consume + +-- ** Unsorted +, head +, drop +, Data.Enumerator.List.dropWhile +, require +, isolate +, splitWhen +: + +:d Data.Enumerator.Text exports +-- * IO + enumHandle +, enumFile +, iterHandle + +-- * List analogues + +-- ** Folds +, fold +, foldM + +-- ** Maps +, Data.Enumerator.Text.map +, Data.Enumerator.Text.mapM +, Data.Enumerator.Text.concatMap +, concatMapM + +-- ** Infinite streams +, Data.Enumerator.Text.iterate +, iterateM +, Data.Enumerator.Text.repeat +, repeatM + +-- ** Bounded streams +, Data.Enumerator.Text.replicate +, replicateM +, generateM +, unfold +, unfoldM + +-- ** Filters +, Data.Enumerator.Text.filter +, filterM + +-- ** Consumers +, Data.Enumerator.Text.take +, takeWhile +, consume + +-- ** Unsorted +, Data.Enumerator.Text.head +, Data.Enumerator.Text.drop +, Data.Enumerator.Text.dropWhile +, require +, isolate +, splitWhen +, lines + +-- * Text codecs +, Codec +, encode +, decode +, utf8 +, utf16_le +, utf16_be +, utf32_le +, utf32_be +, ascii +, iso8859_1 +: diff --git a/src/summary.anansi b/src/summary.anansi new file mode 100644 --- /dev/null +++ b/src/summary.anansi @@ -0,0 +1,41 @@ +\section*{Summary} + +Typical buffer-based incremental \io{} is based around a single loop, which +reads data from some source (such as a socket or file), transforms it, and +generates one or more outputs (such as a line count, {\sc http} responses, +or modified file). Although efficient and safe, these loops are all +single-purpose; it is difficult or impossible to compose buffer-based +processing loops. + +Haskell's concept of ``lazy \io{}'' allows pure code to operate on data from +an external source. However, lazy \io{} has several shortcomings. Most notably, +resources such as memory and file handles can be retained for arbitrarily +long periods of time, causing unpredictable performance and error conditions. + +Enumerators are an efficient, predictable, and safe alternative to lazy \io{}. +Discovered by Oleg \mbox{Kiselyov}, they allow large datasets to be processed +in near-constant space by pure code. Although somewhat more complex to write, +using enumerators instead of lazy \io{} produces more correct programs. + +This library contains an enumerator implementation for Haskell, designed to +be both simple and efficient. Three core types are defined, along with +numerous helper functions: + +\begin{itemize} + +\item {\it Iteratee\/}: Data sinks, analogous to left folds. Iteratees consume +a sequence of \emph{input} values, and generate a single \emph{output} value. +Many iteratees are designed to perform side effects (such as printing to +{\tt stdout}), so they can also be used as monad transformers. + +\item {\it Enumerator\/}: Data sources, which generate input sequences. Typical +enumerators read from a file handle, socket, random number generator, or +other external stream. To operate, enumerators are passed an iteratee, and +provide that iteratee with input until either the iteratee has completed its +computation, or {\sc eof}. + +\item {\it Enumeratee\/}: Data transformers, which operate as both enumerators and +iteratees. Enumeratees read from an \emph{outer} enumerator, and provide the +transformed data to an \emph{inner} iteratee. + +\end{itemize} diff --git a/src/text-codecs.anansi b/src/text-codecs.anansi new file mode 100644 --- /dev/null +++ b/src/text-codecs.anansi @@ -0,0 +1,339 @@ +\section{Text Codecs} + +Many protocols need the non-blocking input behavior of binary \io{}, but +are defined in terms of unicode characters. The {\tt encode} and +{\tt decode} enumeratees allow text-based protocols to be easily parsed +from a binary input source. + +Most common codecs ({\sc utf-8}, {\sc iso-8859-1}, {\sc ascii}) are +supported; more complex codecs can be implemented by bindings to libraries +such as libicu. + +All of the codecs here are incremental; that is, they try to read as much +data as possible, but no more. This allows iteratees to read partial data +if the input stream contains invalid data. + +:d text codecs +data Codec = Codec + { codecName :: T.Text + , codecEncode + :: T.Text + -> (B.ByteString, Maybe (Exc.SomeException, T.Text)) + , codecDecode + :: B.ByteString + -> (T.Text, Either + (Exc.SomeException, B.ByteString) + B.ByteString) + } + +instance Show Codec where + showsPrec d c = showParen (d > 10) $ + showString "Codec " . shows (codecName c) +: + +:d text codecs +|apidoc Data.Enumerator.Text.encode| +encode :: Monad m => Codec + -> Enumeratee T.Text B.ByteString m b +encode codec = checkDone (continue . step) where + step k EOF = yield (Continue k) EOF + step k (Chunks xs) = loop k xs + + loop k [] = continue (step k) + loop k (x:xs) = let + (bytes, extra) = codecEncode codec x + extraChunks = Chunks $ case extra of + Nothing -> xs + Just (_, text) -> text:xs + + checkError k' = case extra of + Nothing -> loop k' xs + Just (exc, _) -> throwError exc + + in if B.null bytes + then checkError k + else k (Chunks [bytes]) >>== + checkDoneEx extraChunks checkError +: + +:d text codecs +|apidoc Data.Enumerator.Text.decode| +decode :: Monad m => Codec + -> Enumeratee B.ByteString T.Text m b +decode codec = checkDone (continue . step B.empty) where + step _ k EOF = yield (Continue k) EOF + step acc k (Chunks xs) = loop acc k xs + + loop acc k [] = continue (step acc k) + loop acc k (x:xs) = let + (text, extra) = codecDecode codec (B.append acc x) + extraChunks = Chunks (either snd id extra : xs) + + checkError k' = case extra of + Left (exc, _) -> throwError exc + Right bytes -> loop bytes k' xs + + in if T.null text + then checkError k + else k (Chunks [text]) >>== + checkDoneEx extraChunks checkError +: + +The variable-width decoders all follow the same basic pattern. First, +they examine their input to calculate how many bytes the decoder +function should accept. Next they try to decode it -- if the input +is valid, decoding is finished. + +If the input is invalid, trying to decode the full input will throw +an exception. When an exception is caught, decoding is passed off to +{\tt splitSlowly} for a more careful parse. The input is reduced until +the decoder can parse something, and the rest of the bytes are stored +for later. An error will only be thrown if the iteratee requires input, +but there are no valid bytes remaining. + +:d text codecs +byteSplits :: B.ByteString + -> [(B.ByteString, B.ByteString)] +byteSplits bytes = loop (B.length bytes) where + loop 0 = [(B.empty, bytes)] + loop n = B.splitAt n bytes : loop (n - 1) +: + +:d text codecs +splitSlowly :: (B.ByteString -> T.Text) + -> B.ByteString + -> (T.Text, Either + (Exc.SomeException, B.ByteString) + B.ByteString) +splitSlowly dec bytes = valid where + valid = firstValid (Prelude.map decFirst splits) + splits = byteSplits bytes + firstValid = Prelude.head . catMaybes + tryDec = tryEvaluate . dec + + decFirst (a, b) = case tryDec a of + Left _ -> Nothing + Right text -> Just (text, case tryDec b of + Left exc -> Left (exc, b) + + -- this case shouldn't occur, since splitSlowly + -- is only called when parsing failed somewhere + Right _ -> Right B.empty) +: + +\subsection{UTF-8} + +:d text codecs +utf8 :: Codec +utf8 = Codec name enc dec where + name = T.pack "UTF-8" + enc text = (TE.encodeUtf8 text, Nothing) + dec bytes = case splitQuickly bytes of + Just (text, extra) -> (text, Right extra) + Nothing -> splitSlowly TE.decodeUtf8 bytes + |utf8 split bytes| +: + +:d utf8 split bytes +splitQuickly bytes = loop 0 >>= maybeDecode where + |utf8 required bytes count| + maxN = B.length bytes + + loop n | n == maxN = Just (TE.decodeUtf8 bytes, B.empty) + loop n = let + req = required (B.index bytes n) + tooLong = first TE.decodeUtf8 (B.splitAt n bytes) + decodeMore = loop $! n + req + in if req == 0 + then Nothing + else if n + req > maxN + then Just tooLong + else decodeMore +: + +:d utf8 required bytes count +required x0 + | x0 .&. 0x80 == 0x00 = 1 + | x0 .&. 0xE0 == 0xC0 = 2 + | x0 .&. 0xF0 == 0xE0 = 3 + | x0 .&. 0xF8 == 0xF0 = 4 + + -- Invalid input; let Text figure it out + | otherwise = 0 +: + +\subsection{UTF-16} + +:d text codecs +utf16_le :: Codec +utf16_le = Codec name enc dec where + name = T.pack "UTF-16-LE" + enc text = (TE.encodeUtf16LE text, Nothing) + dec bytes = case splitQuickly bytes of + Just (text, extra) -> (text, Right extra) + Nothing -> splitSlowly TE.decodeUtf16LE bytes + |utf16-le split bytes| +: + +:d text codecs +utf16_be :: Codec +utf16_be = Codec name enc dec where + name = T.pack "UTF-16-BE" + enc text = (TE.encodeUtf16BE text, Nothing) + dec bytes = case splitQuickly bytes of + Just (text, extra) -> (text, Right extra) + Nothing -> splitSlowly TE.decodeUtf16BE bytes + |utf16-be split bytes| +: + +:d utf16-le split bytes +splitQuickly bytes = maybeDecode (loop 0) where + maxN = B.length bytes + + loop n | n == maxN = decodeAll + | (n + 1) == maxN = decodeTo n + loop n = let + req = utf16Required + (B.index bytes 0) + (B.index bytes 1) + decodeMore = loop $! n + req + in if n + req > maxN + then decodeTo n + else decodeMore + + decodeTo n = first TE.decodeUtf16LE (B.splitAt n bytes) + decodeAll = (TE.decodeUtf16LE bytes, B.empty) +: + +:d utf16-be split bytes +splitQuickly bytes = maybeDecode (loop 0) where + maxN = B.length bytes + + loop n | n == maxN = decodeAll + | (n + 1) == maxN = decodeTo n + loop n = let + req = utf16Required + (B.index bytes 1) + (B.index bytes 0) + decodeMore = loop $! n + req + in if n + req > maxN + then decodeTo n + else decodeMore + + decodeTo n = first TE.decodeUtf16BE (B.splitAt n bytes) + decodeAll = (TE.decodeUtf16BE bytes, B.empty) +: + +:d text codecs +utf16Required :: Word8 -> Word8 -> Int +utf16Required x0 x1 = required where + required = if x >= 0xD800 && x <= 0xDBFF + then 4 + else 2 + x :: Word16 + x = (fromIntegral x1 `shiftL` 8) .|. fromIntegral x0 +: + +\subsection{UTF-32} + +:d text codecs +utf32_le :: Codec +utf32_le = Codec name enc dec where + name = T.pack "UTF-32-LE" + enc text = (TE.encodeUtf32LE text, Nothing) + dec bs = case utf32SplitBytes TE.decodeUtf32LE bs of + Just (text, extra) -> (text, Right extra) + Nothing -> splitSlowly TE.decodeUtf32LE bs + +utf32_be :: Codec +utf32_be = Codec name enc dec where + name = T.pack "UTF-32-BE" + enc text = (TE.encodeUtf32BE text, Nothing) + dec bs = case utf32SplitBytes TE.decodeUtf32BE bs of + Just (text, extra) -> (text, Right extra) + Nothing -> splitSlowly TE.decodeUtf32BE bs +: + +:d text codecs +utf32SplitBytes :: (B.ByteString -> T.Text) + -> B.ByteString + -> Maybe (T.Text, B.ByteString) +utf32SplitBytes dec bytes = split where + split = maybeDecode (dec toDecode, extra) + len = B.length bytes + lenExtra = mod len 4 + + lenToDecode = len - lenExtra + (toDecode, extra) = if lenExtra == 0 + then (bytes, B.empty) + else B.splitAt lenToDecode bytes +: + +\subsection{ASCII} + +:d text codecs +ascii :: Codec +ascii = Codec name enc dec where + name = T.pack "ASCII" + enc text = (bytes, extra) where + (safe, unsafe) = tSpanBy (\c -> ord c <= 0x7F) text + bytes = B8.pack (T.unpack safe) + extra = if T.null unsafe + then Nothing + else Just (illegalEnc name (T.head unsafe), unsafe) + + dec bytes = (text, extra) where + (safe, unsafe) = B.span (<= 0x7F) bytes + text = T.pack (B8.unpack safe) + extra = if B.null unsafe + then Right B.empty + else Left (illegalDec name (B.head unsafe), unsafe) +: + +\subsection{ISO 8859-1} + +:d text codecs +iso8859_1 :: Codec +iso8859_1 = Codec name enc dec where + name = T.pack "ISO-8859-1" + enc text = (bytes, extra) where + (safe, unsafe) = tSpanBy (\c -> ord c <= 0xFF) text + bytes = B8.pack (T.unpack safe) + extra = if T.null unsafe + then Nothing + else Just (illegalEnc name (T.head unsafe), unsafe) + + dec bytes = (T.pack (B8.unpack bytes), Right B.empty) +: + +\subsection*{Encoding Utilities} + +:d text codecs +illegalEnc :: T.Text -> Char -> Exc.SomeException +illegalEnc name c = Exc.toException . Exc.ErrorCall $ + concat [ "Codec " + , show name + , " can't encode character " + , reprChar c + ] +: + +:d text codecs +illegalDec :: T.Text -> Word8 -> Exc.SomeException +illegalDec name w = Exc.toException . Exc.ErrorCall $ + concat [ "Codec " + , show name + , " can't decode byte " + , reprWord w + ] +: + +:d text codecs +tryEvaluate :: a -> Either Exc.SomeException a +tryEvaluate = unsafePerformIO . Exc.try . Exc.evaluate + +maybeDecode:: (a, b) -> Maybe (a, b) +maybeDecode (a, b) = case tryEvaluate a of + Left _ -> Nothing + Right _ -> Just (a, b) +: diff --git a/src/text.anansi b/src/text.anansi deleted file mode 100644 --- a/src/text.anansi +++ /dev/null @@ -1,608 +0,0 @@ -\section{Text} - -:f Data/Enumerator/Text.hs -|Data.Enumerator.Text module header| -module Data.Enumerator.Text ( - |Data.Enumerator.Text exports| - ) where -import qualified Prelude -import Prelude hiding (head, drop, takeWhile) -import Data.Enumerator hiding (head, drop) -import qualified Data.Text as T -|Data.Enumerator.Text imports| -: - -\subsection{IO} - -Reading text is similar to reading bytes, but the enumerators have slightly -different behavior -- instead of reading in fixed-size chunks of data, the -text enumerators read in lines. This matches similar text-based {\sc api}s, -such as Python's {\tt xreadlines()}. - -:d Data.Enumerator.Text imports -import Data.Enumerator.Util (tryStep) -import qualified Data.Text.IO as TIO - -import qualified Control.Exception as Exc -import Control.Monad.IO.Class (MonadIO) -import qualified System.IO as IO -import System.IO.Error (isEOFError) -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.enumHandle| -enumHandle :: MonadIO m => IO.Handle - -> Enumerator T.Text m b -enumHandle h = loop where - loop (Continue k) = withText $ \maybeText -> - case maybeText of - Nothing -> continue k - Just text -> k (Chunks [text]) >>== loop - - loop step = returnI step - withText = tryStep $ Exc.catch - (Just `fmap` TIO.hGetLine h) - (\err -> if isEOFError err - then return Nothing - else Exc.throwIO err) -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.enumFile| -enumFile :: FilePath -> Enumerator T.Text IO b -enumFile path = enum where - withHandle = tryStep (IO.openFile path IO.ReadMode) - enum step = withHandle $ \h -> Iteratee $ Exc.finally - (runIteratee (enumHandle h step)) - (IO.hClose h) -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.iterHandle| -iterHandle :: MonadIO m => IO.Handle - -> Iteratee T.Text m () -iterHandle h = continue step where - step EOF = yield () EOF - step (Chunks []) = continue step - step (Chunks chunks) = let - put = mapM_ (TIO.hPutStr h) chunks - in tryStep put (\_ -> continue step) -: - -:d Data.Enumerator.Text exports - -- * Text IO - enumHandle -, enumFile -, iterHandle -: - -\subsection{List analogues} - -:d Data.Enumerator.Text imports -import qualified Data.Text.Lazy as TL -: - -:f Data/Enumerator/Text.hs -toChunks :: TL.Text -> Stream T.Text -toChunks = Chunks . TL.toChunks -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.head| -head :: Monad m => Iteratee T.Text m (Maybe Char) -head = continue loop where - loop (Chunks xs) = case TL.uncons (TL.fromChunks xs) of - Just (char, extra) -> yield (Just char) (toChunks extra) - Nothing -> head - loop EOF = yield Nothing EOF -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.drop| -drop :: Monad m => Integer -> Iteratee T.Text m () -drop n | n <= 0 = return () -drop n = continue (loop n) where - loop n' (Chunks xs) = iter where - lazy = TL.fromChunks xs - len = toInteger (TL.length lazy) - iter = if len < n' - then drop (n' - len) - else yield () (toChunks (TL.drop (fromInteger n') lazy)) - loop _ EOF = yield () EOF -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.dropWhile| -dropWhile :: Monad m => (Char -> Bool) -> Iteratee T.Text m () -dropWhile p = continue loop where - loop (Chunks xs) = iter where - lazy = TL.dropWhile p (TL.fromChunks xs) - iter = if TL.null lazy - then continue loop - else yield () (toChunks lazy) - loop EOF = yield () EOF -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.take| -take :: Monad m => Integer -> Iteratee T.Text m TL.Text -take n | n <= 0 = return TL.empty -take n = continue (loop id n) where - loop acc n' (Chunks xs) = iter where - lazy = TL.fromChunks xs - len = toInteger (TL.length lazy) - - iter = if len < n' - then continue (loop (acc . (TL.append lazy)) (n' - len)) - else let - (xs', extra) = TL.splitAt (fromInteger n') lazy - in yield (acc xs') (toChunks extra) - loop acc _ EOF = yield (acc TL.empty) EOF -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.takeWhile| -takeWhile :: Monad m => (Char -> Bool) -> Iteratee T.Text m TL.Text -takeWhile p = continue (loop id) where - loop acc (Chunks []) = continue (loop acc) - loop acc (Chunks xs) = iter where - lazy = TL.fromChunks xs - (xs', extra) = tlSpanBy p lazy - iter = if TL.null extra - then continue (loop (acc . (TL.append lazy))) - else yield (acc xs') (toChunks extra) - loop acc EOF = yield (acc TL.empty) EOF -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.consume| -consume :: Monad m => Iteratee T.Text m TL.Text -consume = continue (loop id) where - loop acc (Chunks []) = continue (loop acc) - loop acc (Chunks xs) = iter where - lazy = TL.fromChunks xs - iter = continue (loop (acc . (TL.append lazy))) - loop acc EOF = yield (acc TL.empty) EOF -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.require| -require :: Monad m => Integer -> Iteratee T.Text m () -require n | n <= 0 = return () -require n = continue (loop id n) where - loop acc n' (Chunks xs) = iter where - lazy = TL.fromChunks xs - len = toInteger (TL.length lazy) - iter = if len < n' - then continue (loop (acc . (TL.append lazy)) (n' - len)) - else yield () (toChunks (acc lazy)) - loop _ _ EOF = throwError (Exc.ErrorCall "require: Unexpected EOF") -: - -Same caveats as {\tt Data.Enumerator.List.isolate} - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.isolate| -isolate :: Monad m => Integer -> Enumeratee T.Text T.Text m b -isolate n step | n <= 0 = return step -isolate n (Continue k) = continue loop where - loop (Chunks []) = continue loop - loop (Chunks xs) = iter where - lazy = TL.fromChunks xs - len = toInteger (TL.length lazy) - - iter = if len <= n - then k (Chunks xs) >>== isolate (n - len) - else let - (s1, s2) = TL.splitAt (fromInteger n) lazy - in k (toChunks s1) >>== (\step -> yield step (toChunks s2)) - loop EOF = k EOF >>== (\step -> yield step EOF) -isolate n step = drop n >> return step -: - -:d Data.Enumerator.Text exports --- * List analogues -, Data.Enumerator.Text.head -, Data.Enumerator.Text.drop -, Data.Enumerator.Text.dropWhile -, Data.Enumerator.Text.take -, Data.Enumerator.Text.takeWhile -, Data.Enumerator.Text.consume -, require -, isolate -: - -\subsection{Codecs} - -Many protocols need the non-blocking input behavior of binary \io{}, but -are defined in terms of unicode characters. The {\tt encode} and -{\tt decode} enumeratees allow text-based protocols to be easily parsed -from a binary input source. - -Most common codecs ({\sc utf-8}, {\sc iso-8859-1}, {\sc ascii}) are -supported; more complex codecs can be implemented by bindings to libraries -such as libicu. - -All of the codecs here are incremental; that is, they try to read as much -data as possible, but no more. This allows iteratees to read partial data -if the input stream contains invalid data. - -:d Data.Enumerator.Text imports -import qualified Data.ByteString as B -import Data.Enumerator.Util (tSpanBy, tlSpanBy, reprWord, reprChar) -: - -:d Data.Enumerator.Text exports - -- * Codecs -, Codec -, encode -, decode -|text codec exports| -: - -:f Data/Enumerator/Text.hs -data Codec = Codec - { codecName :: T.Text - , codecEncode - :: T.Text - -> (B.ByteString, Maybe (Exc.SomeException, T.Text)) - , codecDecode - :: B.ByteString - -> (T.Text, Either - (Exc.SomeException, B.ByteString) - B.ByteString) - } - -instance Show Codec where - showsPrec d c = showParen (d > 10) $ - showString "Codec " . shows (codecName c) -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.encode| -encode :: Monad m => Codec - -> Enumeratee T.Text B.ByteString m b -encode codec = checkDone (continue . step) where - step k EOF = yield (Continue k) EOF - step k (Chunks xs) = loop k xs - - loop k [] = continue (step k) - loop k (x:xs) = let - (bytes, extra) = codecEncode codec x - extraChunks = Chunks $ case extra of - Nothing -> xs - Just (_, text) -> text:xs - - checkError k' = case extra of - Nothing -> loop k' xs - Just (exc, _) -> throwError exc - - in if B.null bytes - then checkError k - else k (Chunks [bytes]) >>== - checkDoneEx extraChunks checkError -: - -:f Data/Enumerator/Text.hs -|apidoc Data.Enumerator.Text.decode| -decode :: Monad m => Codec - -> Enumeratee B.ByteString T.Text m b -decode codec = checkDone (continue . step B.empty) where - step _ k EOF = yield (Continue k) EOF - step acc k (Chunks xs) = loop acc k xs - - loop acc k [] = continue (step acc k) - loop acc k (x:xs) = let - (text, extra) = codecDecode codec (B.append acc x) - extraChunks = Chunks (either snd id extra : xs) - - checkError k' = case extra of - Left (exc, _) -> throwError exc - Right bytes -> loop bytes k' xs - - in if T.null text - then checkError k - else k (Chunks [text]) >>== - checkDoneEx extraChunks checkError -: - -Most of the codecs here need to perform at least basic bitbashing, to -calculate how many input bytes will be needed for the next character. - -:d Data.Enumerator.Text imports -import Control.Arrow (first) -import Data.Bits ((.&.), (.|.), shiftL) -import Data.Char (ord) -import Data.Word (Word8, Word16) -import qualified Data.ByteString.Char8 as B8 -import qualified Data.Text.Encoding as TE -: - -The variable-width decoders all follow the same basic pattern. First, -they examine their input to calculate how many bytes the decoder -function should accept. Next they try to decode it -- if the input -is valid, decoding is finished. - -If the input is invalid, trying to decode the full input will throw -an exception. When an exception is caught, decoding is passed off to -{\tt splitSlowly} for a more careful parse. The input is reduced until -the decoder can parse something, and the rest of the bytes are stored -for later. An error will only be thrown if the iteratee requires input, -but there are no valid bytes remaining. - -:f Data/Enumerator/Text.hs -byteSplits :: B.ByteString - -> [(B.ByteString, B.ByteString)] -byteSplits bytes = loop (B.length bytes) where - loop 0 = [(B.empty, bytes)] - loop n = B.splitAt n bytes : loop (n - 1) -: - -:d Data.Enumerator.Text imports -import Data.Maybe (catMaybes) -: - -:f Data/Enumerator/Text.hs -splitSlowly :: (B.ByteString -> T.Text) - -> B.ByteString - -> (T.Text, Either - (Exc.SomeException, B.ByteString) - B.ByteString) -splitSlowly dec bytes = valid where - valid = firstValid (Prelude.map decFirst splits) - splits = byteSplits bytes - firstValid = Prelude.head . catMaybes - tryDec = tryEvaluate . dec - - decFirst (a, b) = case tryDec a of - Left _ -> Nothing - Right text -> Just (text, case tryDec b of - Left exc -> Left (exc, b) - - -- this case shouldn't occur, since splitSlowly - -- is only called when parsing failed somewhere - Right _ -> Right B.empty) -: - -\subsubsection{UTF-8} - -:d text codec exports -, utf8 -: - -:f Data/Enumerator/Text.hs -utf8 :: Codec -utf8 = Codec name enc dec where - name = T.pack "UTF-8" - enc text = (TE.encodeUtf8 text, Nothing) - dec bytes = case splitQuickly bytes of - Just (text, extra) -> (text, Right extra) - Nothing -> splitSlowly TE.decodeUtf8 bytes - |utf8 split bytes| -: - -:d utf8 split bytes -splitQuickly bytes = loop 0 >>= maybeDecode where - |utf8 required bytes count| - maxN = B.length bytes - - loop n | n == maxN = Just (TE.decodeUtf8 bytes, B.empty) - loop n = let - req = required (B.index bytes n) - tooLong = first TE.decodeUtf8 (B.splitAt n bytes) - decodeMore = loop $! n + req - in if req == 0 - then Nothing - else if n + req > maxN - then Just tooLong - else decodeMore -: - -:d utf8 required bytes count -required x0 - | x0 .&. 0x80 == 0x00 = 1 - | x0 .&. 0xE0 == 0xC0 = 2 - | x0 .&. 0xF0 == 0xE0 = 3 - | x0 .&. 0xF8 == 0xF0 = 4 - - -- Invalid input; let Text figure it out - | otherwise = 0 -: - -\subsubsection{UTF-16} - -:d text codec exports -, utf16_le -, utf16_be -: - -:f Data/Enumerator/Text.hs -utf16_le :: Codec -utf16_le = Codec name enc dec where - name = T.pack "UTF-16-LE" - enc text = (TE.encodeUtf16LE text, Nothing) - dec bytes = case splitQuickly bytes of - Just (text, extra) -> (text, Right extra) - Nothing -> splitSlowly TE.decodeUtf16LE bytes - |utf16-le split bytes| -: - -:f Data/Enumerator/Text.hs -utf16_be :: Codec -utf16_be = Codec name enc dec where - name = T.pack "UTF-16-BE" - enc text = (TE.encodeUtf16BE text, Nothing) - dec bytes = case splitQuickly bytes of - Just (text, extra) -> (text, Right extra) - Nothing -> splitSlowly TE.decodeUtf16BE bytes - |utf16-be split bytes| -: - -:d utf16-le split bytes -splitQuickly bytes = maybeDecode (loop 0) where - maxN = B.length bytes - - loop n | n == maxN = decodeAll - | (n + 1) == maxN = decodeTo n - loop n = let - req = utf16Required - (B.index bytes 0) - (B.index bytes 1) - decodeMore = loop $! n + req - in if n + req > maxN - then decodeTo n - else decodeMore - - decodeTo n = first TE.decodeUtf16LE (B.splitAt n bytes) - decodeAll = (TE.decodeUtf16LE bytes, B.empty) -: - -:d utf16-be split bytes -splitQuickly bytes = maybeDecode (loop 0) where - maxN = B.length bytes - - loop n | n == maxN = decodeAll - | (n + 1) == maxN = decodeTo n - loop n = let - req = utf16Required - (B.index bytes 1) - (B.index bytes 0) - decodeMore = loop $! n + req - in if n + req > maxN - then decodeTo n - else decodeMore - - decodeTo n = first TE.decodeUtf16BE (B.splitAt n bytes) - decodeAll = (TE.decodeUtf16BE bytes, B.empty) -: - -:f Data/Enumerator/Text.hs -utf16Required :: Word8 -> Word8 -> Int -utf16Required x0 x1 = required where - required = if x >= 0xD800 && x <= 0xDBFF - then 4 - else 2 - x :: Word16 - x = (fromIntegral x1 `shiftL` 8) .|. fromIntegral x0 -: - -\subsubsection{UTF-32} - -:d text codec exports -, utf32_le -, utf32_be -: - -:f Data/Enumerator/Text.hs -utf32_le :: Codec -utf32_le = Codec name enc dec where - name = T.pack "UTF-32-LE" - enc text = (TE.encodeUtf32LE text, Nothing) - dec bs = case utf32SplitBytes TE.decodeUtf32LE bs of - Just (text, extra) -> (text, Right extra) - Nothing -> splitSlowly TE.decodeUtf32LE bs - -utf32_be :: Codec -utf32_be = Codec name enc dec where - name = T.pack "UTF-32-BE" - enc text = (TE.encodeUtf32BE text, Nothing) - dec bs = case utf32SplitBytes TE.decodeUtf32BE bs of - Just (text, extra) -> (text, Right extra) - Nothing -> splitSlowly TE.decodeUtf32BE bs -: - -:f Data/Enumerator/Text.hs -utf32SplitBytes :: (B.ByteString -> T.Text) - -> B.ByteString - -> Maybe (T.Text, B.ByteString) -utf32SplitBytes dec bytes = split where - split = maybeDecode (dec toDecode, extra) - len = B.length bytes - lenExtra = mod len 4 - - lenToDecode = len - lenExtra - (toDecode, extra) = if lenExtra == 0 - then (bytes, B.empty) - else B.splitAt lenToDecode bytes -: - -\subsubsection{ASCII} - -:d text codec exports -, ascii -: - -:f Data/Enumerator/Text.hs -ascii :: Codec -ascii = Codec name enc dec where - name = T.pack "ASCII" - enc text = (bytes, extra) where - (safe, unsafe) = tSpanBy (\c -> ord c <= 0x7F) text - bytes = B8.pack (T.unpack safe) - extra = if T.null unsafe - then Nothing - else Just (illegalEnc name (T.head unsafe), unsafe) - - dec bytes = (text, extra) where - (safe, unsafe) = B.span (<= 0x7F) bytes - text = T.pack (B8.unpack safe) - extra = if B.null unsafe - then Right B.empty - else Left (illegalDec name (B.head unsafe), unsafe) -: - -\subsubsection{ISO 8859-1} - -:d text codec exports -, iso8859_1 -: - -:f Data/Enumerator/Text.hs -iso8859_1 :: Codec -iso8859_1 = Codec name enc dec where - name = T.pack "ISO-8859-1" - enc text = (bytes, extra) where - (safe, unsafe) = tSpanBy (\c -> ord c <= 0xFF) text - bytes = B8.pack (T.unpack safe) - extra = if T.null unsafe - then Nothing - else Just (illegalEnc name (T.head unsafe), unsafe) - - dec bytes = (T.pack (B8.unpack bytes), Right B.empty) -: - -\subsection{Encoding Utilities} - -:f Data/Enumerator/Text.hs -illegalEnc :: T.Text -> Char -> Exc.SomeException -illegalEnc name c = Exc.toException . Exc.ErrorCall $ - concat [ "Codec " - , show name - , " can't encode character " - , reprChar c - ] -: - -:f Data/Enumerator/Text.hs -illegalDec :: T.Text -> Word8 -> Exc.SomeException -illegalDec name w = Exc.toException . Exc.ErrorCall $ - concat [ "Codec " - , show name - , " can't decode byte " - , reprWord w - ] -: - -:d Data.Enumerator.Text imports -import System.IO.Unsafe (unsafePerformIO) -: - -:f Data/Enumerator/Text.hs -tryEvaluate :: a -> Either Exc.SomeException a -tryEvaluate = unsafePerformIO . Exc.try . Exc.evaluate - -maybeDecode:: (a, b) -> Maybe (a, b) -maybeDecode (a, b) = case tryEvaluate a of - Left _ -> Nothing - Right _ -> Just (a, b) -: diff --git a/src/types.anansi b/src/types.anansi --- a/src/types.anansi +++ b/src/types.anansi @@ -1,35 +1,4 @@ -\section{Core types} - -Most of this library's types and functions are exported from the -{\tt Data.Enumerator} module. - -:f Data/Enumerator.hs -|Data.Enumerator module header| -module Data.Enumerator ( - |Data.Enumerator exports| - ) where -|Data.Enumerator imports| -: - -\noindent A few utility functions share names with functions from the Prelude, so -those are removed from the default namespace. - -:d Data.Enumerator imports -import qualified Prelude as Prelude -import Prelude hiding ( - |excluded Prelude imports| - ) -: - -:d Data.Enumerator exports --- * Core --- ** Types - Stream (..) -, Iteratee (..) -, Step (..) -, Enumerator -, Enumeratee -: +\section{Types and instances} \subsection{Input streams} @@ -43,7 +12,7 @@ no data is currently available. Iteratees and enumeratees often special-case empty chunks for performance reasons, though they're not required to. -:f Data/Enumerator.hs +:d types and instances |apidoc Data.Enumerator.Stream| data Stream a = Chunks [a] @@ -54,15 +23,6 @@ return = Chunks . return Chunks xs >>= f = mconcat (fmap f xs) EOF >>= _ = EOF - -instance Functor Stream where - fmap f (Chunks xs) = Chunks (fmap f xs) - fmap _ EOF = EOF - --- | Since: 0.4.5 -instance A.Applicative Stream where - pure = return - (<*>) = CM.ap : The {\tt Monoid} instance deserves some special attention, because it has @@ -70,11 +30,7 @@ it's reasonable that appending chunks to an {\sc eof} stream should provide a valid stream, such behavior would violate the monoid laws. -:d Data.Enumerator imports -import Data.Monoid (Monoid, mempty, mappend, mconcat) -: - -:f Data/Enumerator.hs +:d types and instances instance Monoid (Stream a) where mempty = Chunks mempty mappend (Chunks xs) (Chunks ys) = Chunks (xs ++ ys) @@ -105,11 +61,7 @@ proceeding further. \end{itemize} -:d Data.Enumerator imports -import qualified Control.Exception as Exc -: - -:f Data/Enumerator.hs +:d types and instances data Step a m b |apidoc Data.Enumerator.Continue| = Continue (Stream a -> Iteratee a m b) @@ -126,10 +78,10 @@ } : -Users often need to construct iteratees which only yield or continue, -so we define some helper functions to save typing: +The pattern {\tt Iteratee (return (} \ldots{\tt ))} shows up a lot, so I define a +couple simple wrappers to save typing: -:f Data/Enumerator.hs +:d primitives |apidoc Data.Enumerator.returnI| returnI :: Monad m => Step a m b -> Iteratee a m b returnI step = Iteratee (return step) @@ -139,17 +91,64 @@ yield x extra = returnI (Yield x extra) |apidoc Data.Enumerator.continue| -continue :: Monad m => (Stream a -> Iteratee a m b) - -> Iteratee a m b +continue :: Monad m => (Stream a -> Iteratee a m b) -> Iteratee a m b continue k = returnI (Continue k) : -:d Data.Enumerator exports -, returnI -, yield -, continue +\subsection*{Monad instances} + +Iteratees are monads; by sequencing iteratees, very complex processing may +be applied to arbitrary input streams. Iteratees are also applicative +functors and monad transformers. + +:d types and instances +instance Monad m => Monad (Iteratee a m) where + return x = yield x (Chunks []) + |optimized iteratee bind| : +Because iteratees are often used for high-performance software, it is +important that the {\tt (>>=)} method be very efficient. We use a couple +magic-ish features here: + +\begin{itemize} +\item First, the whole {\tt (>>=)} is worker-wrapper transformed so {\tt f} +can be cached when working with {\tt Continue}. This prevents a potential +space leak when working with infinite streams. + +\item Second, the worker is rendered anonymous with {\tt fix}, so it doesn't +incur any additional time overhead. +\end{itemize} + +The end result is a bind implementation with time performance equivalent to +the standard definition, but with significantly reduced memory allocation +rates. + +:d optimized iteratee bind +m0 >>= f = ($ m0) $ fix $ + \bind m -> Iteratee $ runIteratee m >>= \r1 -> + case r1 of + Continue k -> return (Continue (bind . k)) + Error err -> return (Error err) + Yield x (Chunks []) -> runIteratee (f x) + Yield x extra -> runIteratee (f x) >>= \r2 -> + case r2 of + Continue k -> runIteratee (k extra) + Error err -> return (Error err) + Yield x' _ -> return (Yield x' extra) +: + +Most iteratees are used to wrap \io{} operations, so it's sensible to define +instances for typeclasses from {\tt transformers}. + +:d types and instances +instance MonadTrans (Iteratee a) where + lift m = Iteratee (m >>= runIteratee . return) + +instance MonadIO m => MonadIO (Iteratee a m) where + liftIO = lift . liftIO +: + \subsection{Enumerators} Enumerators typically read from an external source (parser, handle, random @@ -161,7 +160,7 @@ also be considered step transformers of type {\tt Step a m b -> m (Step a m b)}. -:f Data/Enumerator.hs +:d types and instances |apidoc Data.Enumerator.Enumerator| type Enumerator a m b = Step a m b -> Iteratee a m b : @@ -197,163 +196,7 @@ to an inner iteratee. This model allows a single outer input to generate many inner inputs, and vice-versa. -:f Data/Enumerator.hs +:d types and instances |apidoc Data.Enumerator.Enumeratee| -type Enumeratee ao ai m b = Step ai m b - -> Iteratee ao m (Step ai m b) -: - -\subsection{Operators} - -Because {\tt Iteratee a m b} is semantically equivalent to -{\tt m (Step a m b)}, several of the monadic combinators ({\tt (>>=)}, -{\tt (>=>)}, etc) are useful to save typing when constructing enumerators -and enumeratees. {\tt (>>==)} corresponds to {\tt (>>=)}, {\tt (>==>)} to -{\tt (>=>)}, and so on. - -For compatibility, {\tt (==<<)} is aliased to {\tt (\$\$)}. - -:f Data/Enumerator.hs -infixl 1 >>== - -|apidoc Data.Enumerator.(>>==)| -(>>==) :: Monad m - => Iteratee a m b - -> (Step a m b -> Iteratee a' m b') - -> Iteratee a' m b' -i >>== f = Iteratee (runIteratee i >>= runIteratee . f) -: - -:f Data/Enumerator.hs -infixr 1 ==<< - -|apidoc Data.Enumerator.(==<<)| -(==<<) :: Monad m - => (Step a m b -> Iteratee a' m b') - -> Iteratee a m b - -> Iteratee a' m b' -(==<<) = flip (>>==) -: - -:f Data/Enumerator.hs -infixr 0 $$ - -|apidoc Data.Enumerator.($$)| -($$) :: Monad m - => (Step a m b -> Iteratee a' m b') - -> Iteratee a m b - -> Iteratee a' m b' -($$) = (==<<) -: - -:f Data/Enumerator.hs -infixr 1 >==> - -|apidoc Data.Enumerator.(>==>)| -(>==>) :: Monad m - => Enumerator a m b - -> (Step a m b -> Iteratee a' m b') - -> Step a m b - -> Iteratee a' m b' -(>==>) e1 e2 s = e1 s >>== e2 -: - -:f Data/Enumerator.hs -infixr 1 <==< - -|apidoc Data.Enumerator.(<==<)| -(<==<) :: Monad m - => (Step a m b -> Iteratee a' m b') - -> Enumerator a m b - -> Step a m b - -> Iteratee a' m b' -(<==<) = flip (>==>) -: - -:d Data.Enumerator exports --- ** Operators -, (>>==) -, (==<<) -, ($$) -, (>==>) -, (<==<) -: - -\subsection{Iteratees as Monads} - -Iteratees are monads; by sequencing iteratees, very complex processing may -be applied to arbitrary input streams. Iteratees are also applicative -functors and monad transformers. - -:d Data.Enumerator imports -import Data.Function (fix) -: - -:f Data/Enumerator.hs -instance Monad m => Monad (Iteratee a m) where - return x = yield x (Chunks []) - - m0 >>= f = ($ m0) $ fix $ \bind m -> Iteratee $ runIteratee m >>= - \r1 -> case r1 of - Continue k -> return (Continue (bind . k)) - Error err -> return (Error err) - Yield x (Chunks []) -> runIteratee (f x) - Yield x extra -> runIteratee (f x) >>= - \r2 -> case r2 of - Continue k -> runIteratee (k extra) - Error err -> return (Error err) - Yield x' _ -> return (Yield x' extra) -: - -Most iteratees are used to wrap \io{} operations, so it's sensible to define -instances for typeclasses from {\tt transformers}. - -:d Data.Enumerator imports -import Control.Monad.Trans.Class (MonadTrans, lift) -import Control.Monad.IO.Class (MonadIO, liftIO) -: - -:f Data/Enumerator.hs -instance MonadTrans (Iteratee a) where - lift m = Iteratee (m >>= runIteratee . return) - -instance MonadIO m => MonadIO (Iteratee a m) where - liftIO = lift . liftIO -: - -It's probably possible to define {\tt Functor} and {\tt Applicative} -instances for {\tt Iteratee} without a {\tt Monad} constraint, but I haven't -bothered, since every useful operation requires {\tt m} to be a Monad anyway. - -:d Data.Enumerator imports -import qualified Control.Applicative as A -import qualified Control.Monad as CM -: - -:f Data/Enumerator.hs -instance Monad m => Functor (Iteratee a m) where - fmap = CM.liftM -: - -:f Data/Enumerator.hs -instance Monad m => A.Applicative (Iteratee a m) where - pure = return - (<*>) = CM.ap -: - -:d Data.Enumerator imports -import Data.Typeable ( Typeable, typeOf - , Typeable1, typeOf1 - , mkTyConApp, mkTyCon) -: - -:f Data/Enumerator.hs --- | Since: 0.4.6 -instance (Typeable a, Typeable1 m) => Typeable1 (Iteratee a m) where - typeOf1 i = mkTyConApp tyCon [typeOf a, typeOf1 m] where - tyCon = mkTyCon "Data.Enumerator.Iteratee" - (a, m) = peel i - - peel :: Iteratee a m b -> (a, m ()) - peel = undefined +type Enumeratee ao ai m b = Step ai m b -> Iteratee ao m (Step ai m b) : diff --git a/src/util.anansi b/src/util.anansi deleted file mode 100644 --- a/src/util.anansi +++ /dev/null @@ -1,250 +0,0 @@ -\section{Misc. utilities} - -A few special-case utilities that are used by similar libraries, or were -present in previous versions of {\tt enumerator}, or otherwise don't have a -good place to go. - -:d Data.Enumerator exports --- * Misc. utilities -: - -\subsection{Enumeratees} - -Sequencing a fixed set of enumerators is easy, but for more complex -cases, it's useful to have a small utility wrapper. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.concatEnums| -concatEnums :: Monad m => [Enumerator a m b] - -> Enumerator a m b -concatEnums = Prelude.foldl (>==>) returnI -: - -:d Data.Enumerator exports -, concatEnums -: - -{\tt joinI} is used to ``flatten'' enumeratees, to transform them into an -{\tt Iteratee}. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.joinI| -joinI :: Monad m => Iteratee a m (Step a' m b) - -> Iteratee a m b -joinI outer = outer >>= check where - check (Continue k) = k EOF >>== \s -> case s of - Continue _ -> error "joinI: divergent iteratee" - _ -> check s - check (Yield x _) = return x - check (Error e) = throwError e -: - -{\tt joinE} is similar, except it flattens an enumerator/enumeratee pair -into a single enumerator. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.joinE| -joinE :: Monad m - => Enumerator ao m (Step ai m b) - -> Enumeratee ao ai m b - -> Enumerator ai m b -joinE enum enee s = Iteratee $ do - step <- runIteratee (enumEOF $$ enum $$ enee s) - case step of - Error err -> return (Error err) - Yield x _ -> return x - Continue _ -> error "joinE: divergent iteratee" -: - -{\tt sequence} repeatedly runs its parameter to transform the stream. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.sequence| -sequence :: Monad m => Iteratee ao m ai - -> Enumeratee ao ai m b -sequence i = loop where - loop = checkDone check - check k = isEOF >>= \f -> if f - then yield (Continue k) EOF - else step k - step k = i >>= \v -> k (Chunks [v]) >>== loop -: - -Another small, useful enumerator separates an input list into chunks, and -sends them to the iteratee. This is useful for testing iteratees in pure -code. - -:d Data.Enumerator imports -import Data.List (genericSplitAt) -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.enumList| -enumList :: Monad m => Integer -> [a] -> Enumerator a m b -enumList n = loop where - loop xs (Continue k) | not (null xs) = let - (s1, s2) = genericSplitAt n xs - in k (Chunks s1) >>== loop s2 - loop _ step = returnI step -: - -:d Data.Enumerator exports -, joinI -, joinE -, Data.Enumerator.sequence -, enumList -: - -\subsection{Running iteratees} - -To simplify running iteratees, {\tt run} sends {\tt EOF} and then examines -the result. It is not possible for the result to be {\tt Continue}, because -{\tt enumEOF} calls {\tt error} for divergent iteratees. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.run| -run :: Monad m => Iteratee a m b - -> m (Either Exc.SomeException b) -run i = do - mStep <- runIteratee $ enumEOF ==<< i - case mStep of - Error err -> return $ Left err - Yield x _ -> return $ Right x - Continue _ -> error "run: divergent iteratee" -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.enumEOF| -enumEOF :: Monad m => Enumerator a m b -enumEOF (Yield x _) = yield x EOF -enumEOF (Error err) = throwError err -enumEOF (Continue k) = k EOF >>== check where - check (Continue _) = error "enumEOF: divergent iteratee" - check s = enumEOF s -: - -{\tt run\_} is even more simplified; it's used in simple scripts, where the -user doesn't care about error handling. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.run_| -run_ :: Monad m => Iteratee a m b -> m b -run_ i = run i >>= either Exc.throw return -: - -:d Data.Enumerator exports -, enumEOF -, run -, run_ -: - -\subsection{{\tt checkDone} and {\tt checkDoneEx}} - -A common pattern in {\tt Enumeratee} implementations is to check whether -the inner {\tt Iteratee} has finished, and if so, to return its output. -{\tt checkDone} passes its parameter a continuation if the {\tt Iteratee} -can still consume input, or yields otherwise. - -Oleg's version of {\tt checkDone} has a problem---when the enumeratee has -some sort of input buffer, but the underlying iteratee enters {\tt Yield}, -it will discard the output buffer. {\tt checkDoneEx} corrects this; for -backwards compatibility, {\tt checkDone} remains. - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.checkDoneEx| -checkDoneEx :: Monad m => - Stream a' -> - ((Stream a -> Iteratee a m b) -> Iteratee a' m (Step a m b)) -> - Enumeratee a' a m b -checkDoneEx _ f (Continue k) = f k -checkDoneEx extra _ step = yield step extra - -|apidoc Data.Enumerator.checkDone| -checkDone :: Monad m => - ((Stream a -> Iteratee a m b) -> Iteratee a' m (Step a m b)) -> - Enumeratee a' a m b -checkDone = checkDoneEx (Chunks []) -: - -:d Data.Enumerator exports -, checkDone -, checkDoneEx -: - -:f Data/Enumerator.hs -|apidoc Data.Enumerator.isEOF| -isEOF :: Monad m => Iteratee a m Bool -isEOF = continue $ \s -> case s of - EOF -> yield True s - _ -> yield False s -: - -:d Data.Enumerator exports -, isEOF -: - -{\tt Data.Enumerator.Util} is a hidden module for functions used by several -public modules, but not logically part of the {\tt enumerator} API. - -:f Data/Enumerator/Util.hs -{-# LANGUAGE CPP #-} -module Data.Enumerator.Util where -import Data.Enumerator - -import Data.Char (toUpper, intToDigit, ord) -import Data.Word (Word8) -import qualified Data.Text as T -import qualified Data.Text.Lazy as TL - -import Control.Monad.IO.Class (MonadIO, liftIO) -import qualified Control.Exception as Exc -import Numeric (showIntAtBase) -: - -:f Data/Enumerator/Util.hs -tryStep :: MonadIO m => IO t -> (t -> Iteratee a m b) -> Iteratee a m b -tryStep get io = do - tried <- liftIO (Exc.try get) - case tried of - Right t -> io t - Left err -> throwError (err :: Exc.SomeException) -: - -:f Data/Enumerator/Util.hs -pad0 :: Int -> String -> String -pad0 size str = padded where - len = Prelude.length str - padded = if len >= size - then str - else Prelude.replicate (size - len) '0' ++ str -: - -:f Data/Enumerator/Util.hs -reprChar :: Char -> String -reprChar c = "U+" ++ (pad0 4 (showIntAtBase 16 (toUpper . intToDigit) (ord c) "")) -: - -:f Data/Enumerator/Util.hs -reprWord :: Word8 -> String -reprWord w = "0x" ++ (pad0 2 (showIntAtBase 16 (toUpper . intToDigit) w "")) -: - -{\tt text-0.11} changed some function names to appease a few bikeshedding -idiots in -cafe; to support it, a bit of compatibility code is needed. - -I had a choice between using the preprocessor, or a separate module plus -some Cabal magic. It turns out that {\tt cabal sdist} doesn't properly -handle multiple source directories selected by flags, so the preprocessor -is used for now. - -:f Data/Enumerator/Util.hs -tSpanBy :: (Char -> Bool) -> T.Text -> (T.Text, T.Text) -tlSpanBy :: (Char -> Bool) -> TL.Text -> (TL.Text, TL.Text) -#if MIN_VERSION_text(0,11,0) -tSpanBy = T.span -tlSpanBy = TL.span -#else -tSpanBy = T.spanBy -tlSpanBy = TL.spanBy -#endif -: diff --git a/src/utilities.anansi b/src/utilities.anansi new file mode 100644 --- /dev/null +++ b/src/utilities.anansi @@ -0,0 +1,301 @@ +\section{Miscellaneous} + +A few special-case utilities that are used by similar libraries, or were +present in previous versions of {\tt enumerator}, or otherwise don't have a +good place to go. + +Sequencing a fixed set of enumerators is easy, but for more complex +cases, it's useful to have a small utility wrapper. + +:d unsorted utilities +|apidoc Data.Enumerator.concatEnums| +concatEnums :: Monad m => [Enumerator a m b] + -> Enumerator a m b +concatEnums = Prelude.foldl (>==>) returnI +: + +{\tt joinI} is used to ``flatten'' enumeratees, to transform them into an +{\tt Iteratee}. + +:d unsorted utilities +|apidoc Data.Enumerator.joinI| +joinI :: Monad m => Iteratee a m (Step a' m b) + -> Iteratee a m b +joinI outer = outer >>= check where + check (Continue k) = k EOF >>== \s -> case s of + Continue _ -> error "joinI: divergent iteratee" + _ -> check s + check (Yield x _) = return x + check (Error e) = throwError e +: + +{\tt joinE} is similar, except it flattens an enumerator/enumeratee pair +into a single enumerator. + +:d unsorted utilities +|apidoc Data.Enumerator.joinE| +joinE :: Monad m + => Enumerator ao m (Step ai m b) + -> Enumeratee ao ai m b + -> Enumerator ai m b +joinE enum enee s = Iteratee $ do + step <- runIteratee (enumEOF $$ enum $$ enee s) + case step of + Error err -> return (Error err) + Yield x _ -> return x + Continue _ -> error "joinE: divergent iteratee" +: + +{\tt sequence} repeatedly runs its parameter to transform the stream. + +:d unsorted utilities +|apidoc Data.Enumerator.sequence| +sequence :: Monad m => Iteratee ao m ai + -> Enumeratee ao ai m b +sequence i = loop where + loop = checkDone check + check k = isEOF >>= \f -> if f + then yield (Continue k) EOF + else step k + step k = i >>= \v -> k (Chunks [v]) >>== loop +: + +:d unsorted utilities +|apidoc Data.Enumerator.enumEOF| +enumEOF :: Monad m => Enumerator a m b +enumEOF (Yield x _) = yield x EOF +enumEOF (Error err) = throwError err +enumEOF (Continue k) = k EOF >>== check where + check (Continue _) = error "enumEOF: divergent iteratee" + check s = enumEOF s +: + +A common pattern in {\tt Enumeratee} implementations is to check whether +the inner {\tt Iteratee} has finished, and if so, to return its output. +{\tt checkDone} passes its parameter a continuation if the {\tt Iteratee} +can still consume input, or yields otherwise. + +Oleg's version of {\tt checkDone} has a problem---when the enumeratee has +some sort of input buffer, but the underlying iteratee enters {\tt Yield}, +it will discard the output buffer. {\tt checkDoneEx} corrects this; for +backwards compatibility, {\tt checkDone} remains. + +:d unsorted utilities +|apidoc Data.Enumerator.checkDoneEx| +checkDoneEx :: Monad m => + Stream a' -> + ((Stream a -> Iteratee a m b) -> Iteratee a' m (Step a m b)) -> + Enumeratee a' a m b +checkDoneEx _ f (Continue k) = f k +checkDoneEx extra _ step = yield step extra + +|apidoc Data.Enumerator.checkDone| +checkDone :: Monad m => + ((Stream a -> Iteratee a m b) -> Iteratee a' m (Step a m b)) -> + Enumeratee a' a m b +checkDone = checkDoneEx (Chunks []) +: + +:d unsorted utilities +|apidoc Data.Enumerator.isEOF| +isEOF :: Monad m => Iteratee a m Bool +isEOF = continue $ \s -> case s of + EOF -> yield True s + _ -> yield False s +: + +{\tt Data.Enumerator.Util} is a hidden module for functions used by several +public modules, but not logically part of the {\tt enumerator} API. + +:f Data/Enumerator/Util.hs +{-# LANGUAGE CPP #-} +module Data.Enumerator.Util where +import Data.Enumerator + +import Data.Char (toUpper, intToDigit, ord) +import Data.Word (Word8) +import qualified Data.Text as T +import qualified Data.Text.Lazy as TL + +import Control.Monad.IO.Class (MonadIO, liftIO) +import qualified Control.Exception as Exc +import Numeric (showIntAtBase) +: + +:f Data/Enumerator/Util.hs +tryIO :: MonadIO m => IO b -> Iteratee a m b +tryIO io = Iteratee $ do + tried <- liftIO (Exc.try io) + return $ case tried of + Right b -> Yield b (Chunks []) + Left err -> Error err +: + +:f Data/Enumerator/Util.hs +pad0 :: Int -> String -> String +pad0 size str = padded where + len = Prelude.length str + padded = if len >= size + then str + else Prelude.replicate (size - len) '0' ++ str +: + +:f Data/Enumerator/Util.hs +reprChar :: Char -> String +reprChar c = "U+" ++ (pad0 4 (showIntAtBase 16 (toUpper . intToDigit) (ord c) "")) +: + +:f Data/Enumerator/Util.hs +reprWord :: Word8 -> String +reprWord w = "0x" ++ (pad0 2 (showIntAtBase 16 (toUpper . intToDigit) w "")) +: + +{\tt text-0.11} changed some function names to appease a few bikeshedding +idiots in -cafe; to support it, a bit of compatibility code is needed. + +I had a choice between using the preprocessor, or a separate module plus +some Cabal magic. It turns out that {\tt cabal sdist} doesn't properly +handle multiple source directories selected by flags, so the preprocessor +is used for now. + +:f Data/Enumerator/Util.hs +tSpanBy :: (Char -> Bool) -> T.Text -> (T.Text, T.Text) +tlSpanBy :: (Char -> Bool) -> TL.Text -> (TL.Text, TL.Text) +#if MIN_VERSION_text(0,11,0) +tSpanBy = T.span +tlSpanBy = TL.span +#else +tSpanBy = T.spanBy +tlSpanBy = TL.spanBy +#endif +: + +{\tt text-0.8} added the useful {\tt toStrict} function; this wrapper +lets {\tt enumerator} work with {\tt text-0.7}. + +:f Data/Enumerator/Util.hs +textToStrict :: TL.Text -> T.Text +#if MIN_VERSION_text(0,8,0) +textToStrict = TL.toStrict +#else +textToStrict = T.concat . TL.toChunks +#endif +: + +\subsection{Supplemental instances} + +It can be pretty useful to define {\tt Typeable} instances for iteratees +and streams. For example, they allow iteratee-based libraries to be loaded +dynamically as plugins. + +Normally I'd use the {\tt DeriveDataTypeable} language extension, but +many users have said they find {\tt enumerator} useful in large part +because it doesn't rely on extensions. So instead, the instances are +derived manually. + +:d Data.Enumerator imports +import Data.Typeable ( Typeable, typeOf + , Typeable1, typeOf1 + , mkTyConApp, mkTyCon) +: + +:d supplemental instances +-- | Since: 0.4.8 +instance Typeable1 Stream where + typeOf1 _ = mkTyConApp tyCon [] where + tyCon = mkTyCon "Data.Enumerator.Stream" +: + +:d supplemental instances +-- | Since: 0.4.6 +instance (Typeable a, Typeable1 m) => + Typeable1 (Iteratee a m) where + typeOf1 i = let + tyCon = mkTyCon "Data.Enumerator.Iteratee" + (a, m) = peel i + + peel :: Iteratee a m b -> (a, m ()) + peel = undefined + + in mkTyConApp tyCon [typeOf a, typeOf1 m] +: + +:d supplemental instances +-- | Since: 0.4.8 +instance (Typeable a, Typeable1 m) => + Typeable1 (Step a m) where + typeOf1 s = let + tyCon = mkTyCon "Data.Enumerator.Step" + (a, m) = peel s + + peel :: Step a m b -> (a, m ()) + peel = undefined + + in mkTyConApp tyCon [typeOf a, typeOf1 m] +: + +It's probably possible to define {\tt Functor} and {\tt Applicative} +instances for {\tt Iteratee} without a {\tt Monad} constraint, but I haven't +bothered, since every useful operation requires {\tt m} to be a Monad anyway. + +:d supplemental instances +instance Monad m => Functor (Iteratee a m) where + fmap = CM.liftM +: + +:d supplemental instances +instance Monad m => A.Applicative (Iteratee a m) where + pure = return + (<*>) = CM.ap +: + +:d supplemental instances +instance Functor Stream where + fmap f (Chunks xs) = Chunks (fmap f xs) + fmap _ EOF = EOF + +-- | Since: 0.4.5 +instance A.Applicative Stream where + pure = return + (<*>) = CM.ap +: + +\subsection{Testing and debugging} + +Debugging enumerator-based code is mostly a question of what inputs are +being passed around. {\tt printChunks} prints out exactly what chunks are +being sent from an enumerator. + +:d utilities for testing and debugging +|apidoc Data.Enumerator.printChunks| +printChunks :: (MonadIO m, Show a) + => Bool -- ^ Print empty chunks + -> Iteratee a m () +printChunks printEmpty = continue loop where + loop (Chunks xs) = do + let hide = null xs && not printEmpty + CM.unless hide (liftIO (print xs)) + continue loop + + loop EOF = do + liftIO (putStrLn "EOF") + yield () EOF +: + +Another small, useful enumerator separates an input list into chunks, and +sends them to the iteratee. This is useful for testing iteratees in pure +code. + +:d Data.Enumerator imports +import Data.List (genericSplitAt) +: + +:d utilities for testing and debugging +|apidoc Data.Enumerator.enumList| +enumList :: Monad m => Integer -> [a] -> Enumerator a m b +enumList n = loop where + loop xs (Continue k) | not (null xs) = let + (s1, s2) = genericSplitAt n xs + in k (Chunks s1) >>== loop s2 + loop _ step = returnI step +: diff --git a/tests/Benchmarks.hs b/tests/Benchmarks.hs new file mode 100644 --- /dev/null +++ b/tests/Benchmarks.hs @@ -0,0 +1,98 @@ +-- Copyright (C) 2010-2011 John Millikin +-- +-- See license.txt for details +module Main where + +import Criterion.Types +import qualified Criterion.Config as C +import qualified Criterion.Main as C +import qualified Progression.Config as P +import qualified Progression.Main as P + +import qualified Data.ByteString as B +import qualified Data.ByteString.Char8 as B8 +import qualified Data.ByteString.Lazy as BL + +import qualified Data.Text as T +import qualified Data.Text as TL + +import Data.Enumerator hiding (map, replicate) +import qualified Data.Enumerator as E +import qualified Data.Enumerator.List as EL +import qualified Data.Enumerator.Binary as EB +import qualified Data.Enumerator.Text as ET + +import Control.DeepSeq +import Data.Functor.Identity +import System.Environment +import System.Exit +import System.IO + +instance NFData B.ByteString + +instance NFData BL.ByteString where + rnf a = rnf (BL.toChunks a) + +bytes_100 :: B.ByteString +bytes_100 = B.replicate 100 0x61 + +chars_100 :: T.Text +chars_100 = T.replicate 100 (T.singleton 'a') + +bench_binary :: Iteratee B.ByteString Identity b -> b +bench_binary iter = runIdentity (run_ (enum $$ iter)) where + enum = enumList 2 (replicate 1000 bytes_100) + +bench_text :: Iteratee T.Text Identity b -> b +bench_text iter = runIdentity (run_ (enum $$ iter)) where + enum = enumList 2 (replicate 1000 chars_100) + +bench_bind :: Iteratee Int Identity b -> b +bench_bind iter = runIdentity (run_ (enum 10000 $$ iter)) where + enum 0 step = returnI step + enum n (Continue k) = k (Chunks [n]) >>== enum (n - 1) + enum _ step = returnI step + +bench_enumFile :: Maybe Integer -> Iteratee B.ByteString IO b -> IO b +bench_enumFile limit iter = run_ (EB.enumFileRange "/dev/zero" Nothing limit $$ iter) + +iterUnit :: Monad m => Iteratee a m () +iterUnit = continue loop where + loop EOF = yield () EOF + loop (Chunks _) = continue loop + +iterUnitTo :: Monad m => Int -> Iteratee a m () +iterUnitTo n | n <= 0 = yield () EOF +iterUnitTo n = continue check where + check EOF = yield () EOF + check (Chunks _) = iterUnitTo (n - 1) + +benchmarks :: [Benchmark] +benchmarks = + [ bgroup "general" + [ bench "bind" (nf bench_bind iterUnit) + ] + , bgroup "binary" + [ bench "takeWhile" (nf bench_binary (EB.takeWhile (const True))) + , bench "consume" (nf bench_binary EB.consume) + , bench "enumFile-nolimit" (nfIO (bench_enumFile Nothing (iterUnitTo 10000))) + , bench "enumFile-limit" (nfIO (bench_enumFile (Just 1000000000) (iterUnitTo 10000))) + ] + , bgroup "text" + [ bench "takeWhile" (nf bench_text (ET.takeWhile (const True))) + , bench "consume" (nf bench_text ET.consume) + ] + ] + +main :: IO () +main = do + args <- getArgs + case args of + "progression":extra -> withArgs extra $ P.defaultMain (bgroup "all" benchmarks) + "criterion":extra -> withArgs extra $ let + config = C.defaultConfig { C.cfgPerformGC = C.ljust True } + in C.defaultMainWith config (return ()) benchmarks + _ -> do + name <- getProgName + hPutStrLn stderr $ concat ["Usage: ", name, " "] + exitFailure diff --git a/tests/Properties.hs b/tests/Properties.hs --- a/tests/Properties.hs +++ b/tests/Properties.hs @@ -3,7 +3,17 @@ -- See license.txt for details module Main (tests, main) where -import Data.Enumerator (($$)) +import qualified Control.Exception as Exc +import Data.Bits ((.&.)) +import Data.Char (chr) +import qualified Data.List as L +import qualified Data.List.Split as LS +import Data.Monoid (mappend, mempty, mconcat) +import Data.Functor.Identity (Identity, runIdentity) +import Data.String (IsString, fromString) +import Data.Word (Word8) + +import Data.Enumerator (($$)) import qualified Data.Enumerator as E import qualified Data.Enumerator.Binary as EB import qualified Data.Enumerator.Text as ET @@ -16,19 +26,10 @@ import qualified Data.Text.Lazy as TL import qualified Data.Text.Encoding as TE -import Test.QuickCheck hiding ((.&.)) -import Test.QuickCheck.Poly +import Test.QuickCheck hiding ((.&.)) +import Test.QuickCheck.Poly (A, B, C) import qualified Test.Framework as F -import Test.Framework.Providers.QuickCheck2 (testProperty) - -import Control.Applicative -import Control.Exception -import Control.Monad -import Data.Bits -import Data.Char (chr) -import Data.Monoid -import Data.Functor.Identity -import Data.String +import Test.Framework.Providers.QuickCheck2 (testProperty) tests :: [F.Test] tests = @@ -48,7 +49,6 @@ test_StreamInstances = F.testGroup "Stream Instances" [ test_StreamMonoid , test_StreamFunctor - , test_StreamApplicative , test_StreamMonad ] @@ -84,30 +84,6 @@ prop_law2 :: E.Stream A -> Blind (B -> C) -> Blind (A -> B) -> Bool prop_law2 x (Blind f) (Blind g) = fmap (f . g) x == (fmap f . fmap g) x -test_StreamApplicative :: F.Test -test_StreamApplicative = F.testGroup "Applicative Stream" props where - props = [ testProperty "law 1" prop_law1 - , testProperty "law 2" prop_law2 - , testProperty "law 3" prop_law3 - , testProperty "law 4" prop_law4 - , testProperty "law 5" prop_law5 - ] - - prop_law1 :: E.Stream A -> Bool - prop_law1 v = (pure id <*> v) == v - - prop_law2 :: Blind (E.Stream (B -> C)) -> Blind (E.Stream (A -> B)) -> E.Stream A -> Bool - prop_law2 (Blind u) (Blind v) w = (pure (.) <*> u <*> v <*> w) == (u <*> (v <*> w)) - - prop_law3 :: Blind (A -> B) -> A -> Bool - prop_law3 (Blind f) x = (pure f <*> pure x) == (pure (f x) `asTypeOf` E.Chunks [B 0]) - - prop_law4 :: Blind (E.Stream (A -> B)) -> A -> Bool - prop_law4 (Blind u) y = (u <*> pure y) == (pure ($ y) <*> u) - - prop_law5 :: Blind (A -> B) -> E.Stream A -> Bool - prop_law5 (Blind f) x = (fmap f x) == (pure f <*> x) - test_StreamMonad :: F.Test test_StreamMonad = F.testGroup "Monad Stream" props where props = [ testProperty "law 1" prop_law1 @@ -149,7 +125,7 @@ result = runIdentity (E.run_ iter) iter = E.enumList n xs $$ do - a <- enee $$ E.throwError (ErrorCall "") + _ <- enee $$ E.throwError (Exc.ErrorCall "") EL.consume in result == xs @@ -169,22 +145,22 @@ ] test_Map :: F.Test -test_Map = test_Enumeratee "map" (E.map id) +test_Map = test_Enumeratee "map" (EL.map id) test_ConcatMap :: F.Test -test_ConcatMap = test_Enumeratee "concatMap" (E.concatMap (:[])) +test_ConcatMap = test_Enumeratee "concatMap" (EL.concatMap (:[])) test_MapM :: F.Test -test_MapM = test_Enumeratee "mapM" (E.mapM return) +test_MapM = test_Enumeratee "mapM" (EL.mapM return) test_ConcatMapM :: F.Test -test_ConcatMapM = test_Enumeratee "concatMapM" (E.concatMapM (\x -> return [x])) +test_ConcatMapM = test_Enumeratee "concatMapM" (EL.concatMapM (\x -> return [x])) test_Filter :: F.Test -test_Filter = test_Enumeratee "filter" (E.filter (\_ -> True)) +test_Filter = test_Enumeratee "filter" (EL.filter (\_ -> True)) test_FilterM :: F.Test -test_FilterM = test_Enumeratee "filterM" (E.filterM (\_ -> return True)) +test_FilterM = test_Enumeratee "filterM" (EL.filterM (\_ -> return True)) -- }}} @@ -452,262 +428,194 @@ test_ListAnalogues :: F.Test test_ListAnalogues = F.testGroup "list analogues" - [ test_ListConsume - , test_ListHead - , test_ListDrop - , test_ListTake - -- , test_ListPeek - , test_ListRequire - , test_ListIsolate - - , test_BinaryConsume - , test_BinaryHead - , test_BinaryDrop - , test_BinaryTake - , test_BinaryRequire - , test_BinaryIsolate - - , test_TextConsume - , test_TextHead - , test_TextDrop - , test_TextTake - , test_TextRequire - , test_TextIsolate + [ test_Consume + , test_Head + , test_Drop + , test_Take + , test_Require + , test_Isolate + , test_SplitWhen ] -test_ListConsume :: F.Test -test_ListConsume = testProperty "List.consume" prop where - prop :: [A] -> Bool - prop xs = result == xs where - result = runIdentity (E.run_ iter) - iter = E.enumList 1 xs $$ EL.consume - -test_ListHead :: F.Test -test_ListHead = testProperty "List.head" prop where - prop :: [A] -> Bool - prop xs = result == expected where - result = runIdentity (E.run_ iter) - expected = case xs of - [] -> (Nothing, []) - (x:xs') -> (Just x, xs') - - iter = E.enumList 1 xs $$ do - x <- EL.head - extra <- EL.consume - return (x, extra) - -test_ListDrop :: F.Test -test_ListDrop = testProperty "List.drop" prop where - prop :: Positive Integer -> [A] -> Bool - prop (Positive n) xs = result == expected where - result = runIdentity (E.run_ iter) - expected = drop (fromInteger n) xs - - iter = E.enumList 1 xs $$ do - EL.drop n - EL.consume - -test_ListTake :: F.Test -test_ListTake = testProperty "List.take" prop where - prop :: Positive Integer -> [A] -> Bool - prop (Positive n) xs = result == expected where - result = runIdentity (E.run_ iter) - expected = splitAt (fromInteger n) xs - - iter = E.enumList 1 xs $$ do - xs <- EL.take n - extra <- EL.consume - return (xs, extra) - -{- -test_ListPeek :: F.Test -test_ListPeek = testProperty "List.peek" prop where - prop :: Positive Integer -> [A] -> Bool - prop (Positive n) xs = result == expected where - result = runIdentity (E.run_ iter) - expected = (take (fromInteger n) xs, xs) - - iter = E.enumList 1 xs $$ do - xs <- EL.peek n - extra <- EL.consume - return (xs, extra) --} - -test_ListRequire :: F.Test -test_ListRequire = testProperty "List.require" prop where - prop :: Positive Integer -> [A] -> Bool - prop (Positive n) xs = result == expected where - result = case runIdentity (E.run iter) of - Left exc -> Left (show exc) - Right x -> Right x - expected = if n > toInteger (length xs) - then Left "require: Unexpected EOF" - else Right xs - - iter = E.enumList 1 xs $$ do - EL.require n - EL.consume - -test_ListIsolate :: F.Test -test_ListIsolate = testProperty "List.isolate" prop where - prop :: Positive Integer -> [A] -> Bool - prop (Positive n) xs = result == expected where - result = runIdentity (E.run_ iter) - expected = case xs of - [] -> (Nothing, []) - (x:[]) -> (Just x, []) - (x:_:xs') -> (Just x, xs') - - iter = E.enumList 1 xs $$ do - x <- E.joinI (EL.isolate 2 $$ EL.head) - extra <- EL.consume - return (x, extra) - -test_BinaryConsume :: F.Test -test_BinaryConsume = testProperty "Binary.consume" prop where - prop ts = result == BL.fromChunks ts where - result = runIdentity (E.run_ iter) - iter = E.enumList 1 ts $$ EB.consume - -test_BinaryHead :: F.Test -test_BinaryHead = testProperty "Binary.head" prop where - prop ts = result == expected where - result = runIdentity (E.run_ iter) - expected = case BL.uncons (BL.fromChunks ts) of - Nothing -> (Nothing, BL.empty) - Just (x, extra) -> (Just x, extra) - - iter = E.enumList 1 ts $$ do - x <- EB.head - extra <- EB.consume - return (x, extra) +check :: Eq b => E.Iteratee a Identity b -> ([a] -> Either Exc.ErrorCall b) -> [a] -> Bool +check iter plain xs = expected == run iter xs where + expected = case plain xs of + Left exc -> Left (Just exc) + Right x -> Right x + + run iter xs = case runIdentity (E.run (E.enumList 1 xs $$ iter)) of + Left exc -> Left (Exc.fromException exc) + Right x -> Right x -test_BinaryDrop :: F.Test -test_BinaryDrop = testProperty "Binary.drop" prop where - prop :: Positive Integer -> [B.ByteString] -> Bool - prop (Positive n) ts = result == expected where - result = runIdentity (E.run_ iter) - expected = BL.drop (fromInteger n) (BL.fromChunks ts) - - iter = E.enumList 1 ts $$ do - EB.drop n - EB.consume +testListAnalogue name iterList plainList iterText plainText iterBytes plainBytes = F.testGroup name tests where + tests = [ testProperty "list" prop_List + , testProperty "text" prop_Text + , testProperty "bytes" prop_Bytes + ] + + prop_List :: [A] -> Bool + prop_List xs = check iterList plainList xs + + prop_Text xs = check iterText (plainText . TL.fromChunks) xs + prop_Bytes xs = check iterBytes (plainBytes . BL.fromChunks) xs -test_BinaryTake :: F.Test -test_BinaryTake = testProperty "Binary.take" prop where - prop :: Positive Integer -> [B.ByteString] -> Bool - prop (Positive n) ts = result == expected where - result = runIdentity (E.run_ iter) - expected = BL.splitAt (fromInteger n) (BL.fromChunks ts) - - iter = E.enumList 1 ts $$ do - xs <- EB.take n - extra <- EB.consume - return (xs, extra) +testListAnalogueN name iterList plainList iterText plainText iterBytes plainBytes = F.testGroup name tests where + tests = [ testProperty "list" prop_List + , testProperty "text" prop_Text + , testProperty "bytes" prop_Bytes + ] + + prop_List :: Positive Integer -> [A] -> Bool + prop_List (Positive n) xs = check (iterList n) (plainList n) xs + + prop_Text (Positive n) xs = check (iterText n) (plainText n . TL.fromChunks) xs + prop_Bytes (Positive n) xs = check (iterBytes n) (plainBytes n . BL.fromChunks) xs -test_BinaryRequire :: F.Test -test_BinaryRequire = testProperty "Binary.require" prop where - prop :: Positive Integer -> [B.ByteString] -> Bool - prop (Positive n) ts = result == expected where - result = case runIdentity (E.run iter) of - Left exc -> Left (show exc) - Right x -> Right x - lazy = BL.fromChunks ts - expected = if n > toInteger (BL.length lazy) - then Left "require: Unexpected EOF" - else Right lazy - - iter = E.enumList 1 ts $$ do - EB.require n - EB.consume +testListAnalogueX name iterList plainList iterText plainText iterBytes plainBytes = F.testGroup name tests where + tests = [ testProperty "list" prop_List + , testProperty "text" prop_Text + , testProperty "bytes" prop_Bytes + ] + + prop_List :: A -> [A] -> Bool + prop_List x xs = check (iterList x) (plainList x) xs + + prop_Text x xs = check (iterText x) (plainText x . TL.fromChunks) xs + prop_Bytes x xs = check (iterBytes x) (plainBytes x . BL.fromChunks) xs -test_BinaryIsolate :: F.Test -test_BinaryIsolate = testProperty "Binary.isolate" prop where - prop :: Positive Integer -> [B.ByteString] -> Bool - prop (Positive n) ts = result == expected where - result = runIdentity (E.run_ iter) - expected = case BL.unpack (BL.fromChunks ts) of - [] -> (Nothing, BL.empty) - (x:[]) -> (Just x, BL.empty) - (x:_:xs') -> (Just x, BL.pack xs') - - iter = E.enumList 1 ts $$ do - x <- E.joinI (EB.isolate 2 $$ EB.head) - extra <- EB.consume - return (x, extra) +test_Consume :: F.Test +test_Consume = testListAnalogue "consume" + EL.consume Right + ET.consume Right + EB.consume Right -test_TextConsume :: F.Test -test_TextConsume = testProperty "Text.consume" prop where - prop ts = result == TL.fromChunks ts where - result = runIdentity (E.run_ iter) - iter = E.enumList 1 ts $$ ET.consume +test_Head :: F.Test +test_Head = testListAnalogue "head" + (do + x <- EL.head + extra <- EL.consume + return (x, extra) + ) + (\xs -> Right $ case xs of + [] -> (Nothing, []) + (x:xs') -> (Just x, xs')) + (do + x <- ET.head + extra <- ET.consume + return (x, extra) + ) + (\text -> Right $ case TL.uncons text of + Nothing -> (Nothing, TL.empty) + Just (x, extra) -> (Just x, extra)) + (do + x <- EB.head + extra <- EB.consume + return (x, extra) + ) + (\bytes -> Right $ case BL.uncons bytes of + Nothing -> (Nothing, BL.empty) + Just (x, extra) -> (Just x, extra)) -test_TextHead :: F.Test -test_TextHead = testProperty "Text.head" prop where - prop ts = result == expected where - result = runIdentity (E.run_ iter) - expected = case TL.uncons (TL.fromChunks ts) of - Nothing -> (Nothing, TL.empty) - Just (x, extra) -> (Just x, extra) - - iter = E.enumList 1 ts $$ do - x <- ET.head - extra <- ET.consume - return (x, extra) +test_Drop :: F.Test +test_Drop = testListAnalogueN "drop" + (\n -> EL.drop n >> EL.consume) + (\n -> Right . L.genericDrop n) + (\n -> ET.drop n >> ET.consume) + (\n -> Right . TL.drop (fromInteger n)) + (\n -> EB.drop n >> EB.consume) + (\n -> Right . BL.drop (fromInteger n)) -test_TextDrop :: F.Test -test_TextDrop = testProperty "Text.drop" prop where - prop :: Positive Integer -> [T.Text] -> Bool - prop (Positive n) ts = result == expected where - result = runIdentity (E.run_ iter) - expected = TL.drop (fromInteger n) (TL.fromChunks ts) - - iter = E.enumList 1 ts $$ do - ET.drop n - ET.consume +test_Take :: F.Test +test_Take = testListAnalogueN "take" + (\n -> do + xs <- EL.take n + extra <- EL.consume + return (xs, extra)) + (\n -> Right . L.genericSplitAt n) + (\n -> do + xs <- ET.take n + extra <- ET.consume + return (xs, extra)) + (\n -> Right . TL.splitAt (fromInteger n)) + (\n -> do + xs <- EB.take n + extra <- EB.consume + return (xs, extra)) + (\n -> Right . BL.splitAt (fromInteger n)) -test_TextTake :: F.Test -test_TextTake = testProperty "Text.take" prop where - prop :: Positive Integer -> [T.Text] -> Bool - prop (Positive n) ts = result == expected where - result = runIdentity (E.run_ iter) - expected = TL.splitAt (fromInteger n) (TL.fromChunks ts) - - iter = E.enumList 1 ts $$ do - xs <- ET.take n - extra <- ET.consume - return (xs, extra) +test_Require :: F.Test +test_Require = testListAnalogueN "require" + (\n -> do + EL.require n + EL.consume) + (\n xs -> if n > toInteger (length xs) + then Left (Exc.ErrorCall "require: Unexpected EOF") + else Right xs) + (\n -> do + ET.require n + ET.consume) + (\n xs -> if n > toInteger (TL.length xs) + then Left (Exc.ErrorCall "require: Unexpected EOF") + else Right xs) + (\n -> do + EB.require n + EB.consume) + (\n xs -> if n > toInteger (BL.length xs) + then Left (Exc.ErrorCall "require: Unexpected EOF") + else Right xs) -test_TextRequire :: F.Test -test_TextRequire = testProperty "Text.require" prop where - prop :: Positive Integer -> [T.Text] -> Bool - prop (Positive n) ts = result == expected where - result = case runIdentity (E.run iter) of - Left exc -> Left (show exc) - Right x -> Right x - lazy = TL.fromChunks ts - expected = if n > toInteger (TL.length lazy) - then Left "require: Unexpected EOF" - else Right lazy - - iter = E.enumList 1 ts $$ do - ET.require n - ET.consume +test_Isolate :: F.Test +test_Isolate = testListAnalogue "isolate" + (do + x <- E.joinI (EL.isolate 2 $$ EL.head) + extra <- EL.consume + return (x, extra)) + (\xs -> Right $ case xs of + [] -> (Nothing, []) + (x:[]) -> (Just x, []) + (x:_:xs') -> (Just x, xs')) + (do + x <- E.joinI (ET.isolate 2 $$ ET.head) + extra <- ET.consume + return (x, extra)) + (\text -> Right $ case TL.unpack text of + [] -> (Nothing, TL.empty) + (x:[]) -> (Just x, TL.empty) + (x:_:xs') -> (Just x, TL.pack xs')) + (do + x <- E.joinI (EB.isolate 2 $$ EB.head) + extra <- EB.consume + return (x, extra)) + (\bytes -> Right $ case BL.unpack bytes of + [] -> (Nothing, BL.empty) + (x:[]) -> (Just x, BL.empty) + (x:_:xs) -> (Just x, BL.pack xs)) -test_TextIsolate :: F.Test -test_TextIsolate = testProperty "Text.isolate" prop where - prop :: Positive Integer -> [T.Text] -> Bool - prop (Positive n) ts = result == expected where - result = runIdentity (E.run_ iter) - expected = case TL.unpack (TL.fromChunks ts) of - [] -> (Nothing, TL.empty) - (x:[]) -> (Just x, TL.empty) - (x:_:xs') -> (Just x, TL.pack xs') - - iter = E.enumList 1 ts $$ do - x <- E.joinI (ET.isolate 2 $$ ET.head) - extra <- ET.consume - return (x, extra) +test_SplitWhen :: F.Test +test_SplitWhen = testListAnalogueX "splitWhen" + (\x -> do + xs <- E.joinI (EL.splitWhen (== x) $$ EL.consume) + extra <- EL.consume + return (xs, extra)) + (\x xs -> let + split = LS.split . LS.dropFinalBlank . LS.dropDelims . LS.whenElt + in Right (split (== x) xs, [])) + (\c -> do + xs <- E.joinI (ET.splitWhen (== c) $$ EL.consume) + extra <- EL.consume + return (xs, extra)) + (\c text -> let + split = LS.split . LS.dropFinalBlank . LS.dropDelims . LS.whenElt + chars = TL.unpack text + in Right (map T.pack (split (== c) chars), [])) + (\x -> do + xs <- E.joinI (EB.splitWhen (== x) $$ EL.consume) + extra <- EL.consume + return (xs, extra)) + (\x bytes -> let + split = LS.split . LS.dropFinalBlank . LS.dropDelims . LS.whenElt + words = BL.unpack bytes + in Right (map B.pack (split (== x) words), [])) -- }}} @@ -735,7 +643,7 @@ result = runIdentity (E.run_ iter) expected = map (* 10) xs - iter = (E.joinE (E.enumList 1 xs) (E.map (* 10))) $$ EL.consume + iter = (E.joinE (E.enumList 1 xs) (EL.map (* 10))) $$ EL.consume -- misc @@ -814,3 +722,6 @@ instance Arbitrary B.ByteString where arbitrary = genUnicode + +instance Eq Exc.ErrorCall where + (Exc.ErrorCall s1) == (Exc.ErrorCall s2) = s1 == s2 diff --git a/tests/enumerator-tests.cabal b/tests/enumerator-tests.cabal --- a/tests/enumerator-tests.cabal +++ b/tests/enumerator-tests.cabal @@ -5,6 +5,7 @@ executable enumerator_tests main-is: Properties.hs + ghc-options: -Wall -O2 build-depends: base > 3 && < 5 @@ -12,6 +13,21 @@ , bytestring , text , enumerator + , split , QuickCheck == 2.4.* , test-framework >= 0.2 && < 0.4 , test-framework-quickcheck2 == 0.2.9 + +executable enumerator_benchmarks + main-is: Benchmarks.hs + ghc-options: -Wall -O2 + + build-depends: + base > 3 && < 5 + , transformers + , bytestring + , text + , enumerator + , criterion + , progression + , deepseq