diff --git a/Data/Conduit/List.hs b/Data/Conduit/List.hs --- a/Data/Conduit/List.hs +++ b/Data/Conduit/List.hs @@ -32,16 +32,21 @@ -- ** Pure , map , mapMaybe + , mapFoldable , catMaybes + , concat , concatMap , concatMapAccum + , scanl , groupBy , isolate , filter -- ** Monadic , mapM , iterM + , scanlM , mapMaybeM + , mapFoldableM , concatMapM , concatMapAccumM -- * Misc @@ -62,6 +67,7 @@ , either ) import Data.Monoid (Monoid, mempty, mappend) +import qualified Data.Foldable as F import Data.Conduit import Control.Monad (when, (<=<)) import Control.Monad.Trans.Class (lift) @@ -279,6 +285,13 @@ catMaybes :: Monad m => Conduit (Maybe a) m a catMaybes = awaitForever $ maybe (return ()) yield +-- | Generalization of 'catMaybes'. It puts all values from +-- 'F.Foldable' into stream. +-- +-- Since 1.0.6 +concat :: (Monad m, F.Foldable f) => Conduit (f a) m a +concat = awaitForever $ F.mapM_ yield + -- | Apply a transformation to all values in a stream, concatenating the output -- values. -- @@ -297,31 +310,53 @@ -- -- Since 0.3.0 concatMapAccum :: Monad m => (a -> accum -> (accum, [b])) -> accum -> Conduit a m b -concatMapAccum f = +concatMapAccum f x0 = scanl f x0 =$= concat + +-- | Analog of 'Prelude.scanl' for lists. +-- +-- Since 1.0.6 +scanl :: Monad m => (a -> s -> (s,b)) -> s -> Conduit a m b +scanl f = loop where - loop accum = - await >>= maybe (return ()) go + loop s = await >>= F.mapM_ go where - go a = do - let (accum', bs) = f a accum - Prelude.mapM_ yield bs - loop accum' + go a = case f a s of + (s',b) -> yield b >> loop s' --- | 'concatMapM' with an accumulator. +-- | Monadic scanl. -- --- Since 0.3.0 -concatMapAccumM :: Monad m => (a -> accum -> m (accum, [b])) -> accum -> Conduit a m b -concatMapAccumM f = +-- Since 1.0.6 +scanlM :: Monad m => (a -> s -> m (s,b)) -> s -> Conduit a m b +scanlM f = loop where - loop accum = do - await >>= maybe (return ()) go + loop s = await >>= F.mapM_ go where - go a = do - (accum', bs) <- lift $ f a accum - Prelude.mapM_ yield bs - loop accum' + go a = do (s',b) <- lift $ f a s + yield b >> loop s' + +-- | 'concatMapM' with an accumulator. +-- +-- Since 0.3.0 +concatMapAccumM :: Monad m => (a -> accum -> m (accum, [b])) -> accum -> Conduit a m b +concatMapAccumM f x0 = scanlM f x0 =$= concat + + +-- | Generalization of 'mapMaybe' and 'concatMap'. It applies function +-- to all values in a stream and send values inside resulting +-- 'Foldable' downstream. +-- +-- Since 1.0.6 +mapFoldable :: (Monad m, F.Foldable f) => (a -> f b) -> Conduit a m b +mapFoldable f = awaitForever $ F.mapM_ yield . f + +-- | Monadic variant of 'mapFoldable'. +-- +-- Since 1.0.6 +mapFoldableM :: (Monad m, F.Foldable f) => (a -> m (f b)) -> Conduit a m b +mapFoldableM f = awaitForever $ F.mapM_ yield <=< lift . f + -- | Consume all values from the stream and return as a list. Note that this -- will pull all values into memory. For a lazy variant, see diff --git a/conduit.cabal b/conduit.cabal --- a/conduit.cabal +++ b/conduit.cabal @@ -1,5 +1,5 @@ Name: conduit -Version: 1.0.5.1 +Version: 1.0.6 Synopsis: Streaming data processing library. Description: @conduit@ is a solution to the streaming data problem, allowing for production, transformation, and consumption of streams of data in constant memory. It is an alternative to lazy I\/O which guarantees deterministic resource handling, and fits in the same general solution space as @enumerator@\/@iteratee@ and @pipes@. For a tutorial, please visit . diff --git a/test/main.hs b/test/main.hs --- a/test/main.hs +++ b/test/main.hs @@ -12,7 +12,7 @@ import qualified Data.Conduit.Binary as CB import qualified Data.Conduit.Text as CT import Data.Conduit (runResourceT) -import Data.Maybe (fromMaybe) +import Data.Maybe (fromMaybe,catMaybes) import qualified Data.List as DL import Control.Monad.ST (runST) import Data.Monoid @@ -31,7 +31,7 @@ import Control.Monad.Trans.Writer (execWriter, tell, runWriterT) import Control.Monad.Trans.State (evalStateT, get, put) import Control.Applicative (pure, (<$>), (<*>)) -import Data.Functor.Identity (runIdentity) +import Data.Functor.Identity (Identity,runIdentity) import Control.Monad (forever) import Data.Void (Void) import qualified Control.Concurrent.MVar as M @@ -39,6 +39,13 @@ (@=?) :: (Eq a, Show a) => a -> a -> IO () (@=?) = flip shouldBe +-- Quickcheck property for testing equivalence of list processing +-- functions and their conduit counterparts +equivToList :: Eq b => ([a] -> [b]) -> CI.Conduit a Identity b -> [a] -> Bool +equivToList f conduit xs = + f xs == runIdentity (CL.sourceList xs C.$$ conduit C.=$= CL.consume) + + main :: IO () main = hspec $ do describe "data loss rules" $ do @@ -60,6 +67,28 @@ it "even" $ do x <- runResourceT $ CL.sourceList [1..10] C.$$ CL.filter even C.=$ CL.consume x `shouldBe` filter even [1..10 :: Int] + + prop "concat" $ equivToList (concat :: [[Int]]->[Int]) CL.concat + + describe "mapFoldable" $ do + prop "list" $ + equivToList (concatMap (:[]) :: [Int]->[Int]) (CL.mapFoldable (:[])) + let f x = if odd x then Just x else Nothing + prop "Maybe" $ + equivToList (catMaybes . map f :: [Int]->[Int]) (CL.mapFoldable f) + + prop "scanl" $ equivToList (tail . scanl (+) 0 :: [Int]->[Int]) (CL.scanl (\a s -> (a+s,a+s)) 0) + + -- mapFoldableM and scanlM are fully polymorphic in type of monad + -- so it suffice to check only with Identity. + describe "mapFoldableM" $ do + prop "list" $ + equivToList (concatMap (:[]) :: [Int]->[Int]) (CL.mapFoldableM (return . (:[]))) + let f x = if odd x then Just x else Nothing + prop "Maybe" $ + equivToList (catMaybes . map f :: [Int]->[Int]) (CL.mapFoldableM (return . f)) + + prop "scanl" $ equivToList (tail . scanl (+) 0 :: [Int]->[Int]) (CL.scanlM (\a s -> return (a+s,a+s)) 0) describe "ResourceT" $ do it "resourceForkIO" $ do