conduit-extra 1.1.6.2 → 1.1.7.0

raw patch · 6 files changed

+247/−131 lines, 6 filesdep ~blaze-builderdep ~streaming-commonsPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: blaze-builder, streaming-commons

API changes (from Hackage documentation)

Files

• ChangeLog.md +4/−0
• Data/Conduit/Blaze.hs +26/−65
• Data/Conduit/ByteString/Builder.hs +147/−0
• conduit-extra.cabal +6/−3
• test/Data/Conduit/BlazeSpec.hs +0/−63
• test/Data/Conduit/ByteString/BuilderSpec.hs +64/−0

@@ -1,3 +1,7 @@+## 1.1.7++Add `Data.Conduit.ByteString.Builder`+ ## 1.1.6  Generalized return type in `runGeneralTCPServer`.

@@ -1,12 +1,14 @@-{-# LANGUAGE FlexibleContexts #-}-{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE RankNTypes #-} -- | Convert a stream of blaze-builder @Builder@s into a stream of @ByteString@s. -- -- Adapted from blaze-builder-enumerator, written by myself and Simon Meier. -- -- Note that the functions here can work in any monad built on top of @IO@ or -- @ST@.+--+-- Since 1.1.7.0, the functions here call their counterparts in+-- "Data.Conduit.ByteString.Builder", which work with both+-- 'Data.ByteString.Builder.Builder' and blaze-builder 0.3's+-- 'Blaze.ByteString.Builder.Builder'. module Data.Conduit.Blaze     ( @@ -20,57 +22,50 @@   , builderToByteStringWithFlush    -- * Buffers-  , Buffer+  , B.Buffer    -- ** Status information-  , freeSize-  , sliceSize-  , bufferSize+  , B.freeSize+  , B.sliceSize+  , B.bufferSize    -- ** Creation and modification-  , allocBuffer-  , reuseBuffer-  , nextSlice+  , B.allocBuffer+  , B.reuseBuffer+  , B.nextSlice    -- ** Conversion to bytestings-  , unsafeFreezeBuffer-  , unsafeFreezeNonEmptyBuffer+  , B.unsafeFreezeBuffer+  , B.unsafeFreezeNonEmptyBuffer    -- * Buffer allocation strategies-  , BufferAllocStrategy-  , allNewBuffersStrategy-  , reuseBufferStrategy+  , B.BufferAllocStrategy+  , B.allNewBuffersStrategy+  , B.reuseBufferStrategy     ) where  import Data.Conduit-import Control.Monad (unless, liftM)-import Control.Monad.Trans.Class (lift, MonadTrans)  import qualified Data.ByteString                   as S -import Blaze.ByteString.Builder.Internal-import Blaze.ByteString.Builder.Internal.Types-import Blaze.ByteString.Builder.Internal.Buffer-import Control.Monad.Primitive (PrimMonad, unsafePrimToPrim)-import Control.Monad.Base (MonadBase, liftBase)+import Blaze.ByteString.Builder (Builder)+import Control.Monad.Primitive (PrimMonad)+import Control.Monad.Base (MonadBase) import Data.Streaming.Blaze -unsafeLiftIO :: (MonadBase base m, PrimMonad base) => IO a -> m a-unsafeLiftIO = liftBase . unsafePrimToPrim+import qualified Data.Conduit.ByteString.Builder as B  -- | Incrementally execute builders and pass on the filled chunks as -- bytestrings. builderToByteString :: (MonadBase base m, PrimMonad base) => Conduit Builder m S.ByteString-builderToByteString =-  builderToByteStringWith defaultStrategy+builderToByteString = B.builderToByteString {-# INLINE builderToByteString #-}  -- | -- -- Since 0.0.2 builderToByteStringFlush :: (MonadBase base m, PrimMonad base) => Conduit (Flush Builder) m (Flush S.ByteString)-builderToByteStringFlush =-  builderToByteStringWithFlush defaultStrategy+builderToByteStringFlush = B.builderToByteStringFlush {-# INLINE builderToByteStringFlush #-}  -- | Incrementally execute builders on the given buffer and pass on the filled@@ -83,7 +78,7 @@ unsafeBuilderToByteString :: (MonadBase base m, PrimMonad base)                           => IO Buffer  -- action yielding the inital buffer.                           -> Conduit Builder m S.ByteString-unsafeBuilderToByteString = builderToByteStringWith . reuseBufferStrategy+unsafeBuilderToByteString = B.unsafeBuilderToByteString {-# INLINE unsafeBuilderToByteString #-}  @@ -94,11 +89,7 @@ builderToByteStringWith :: (MonadBase base m, PrimMonad base)                         => BufferAllocStrategy                         -> Conduit Builder m S.ByteString-builderToByteStringWith =-    helper (liftM (fmap Chunk) await) yield'-  where-    yield' Flush = return ()-    yield' (Chunk bs) = yield bs+builderToByteStringWith = B.builderToByteStringWith {-# INLINE builderToByteStringWith #-}  -- |@@ -108,35 +99,5 @@     :: (MonadBase base m, PrimMonad base)     => BufferAllocStrategy     -> Conduit (Flush Builder) m (Flush S.ByteString)-builderToByteStringWithFlush = helper await yield+builderToByteStringWithFlush = B.builderToByteStringWithFlush {-# INLINE builderToByteStringWithFlush #-}--helper :: (MonadBase base m, PrimMonad base, Monad (t m), MonadTrans t)-       => t m (Maybe (Flush Builder))-       -> (Flush S.ByteString -> t m ())-       -> BufferAllocStrategy-       -> t m ()-helper await' yield' strat = do-    (recv, finish) <- lift $ unsafeLiftIO $ newBlazeRecv strat-    let loop = await' >>= maybe finish' cont-        finish' = do-            mbs <- lift $ unsafeLiftIO finish-            maybe (return ()) (yield' . Chunk) mbs-        cont fbuilder = do-            let builder =-                    case fbuilder of-                        Flush -> flush-                        Chunk b -> b-            popper <- lift $ unsafeLiftIO $ recv builder-            let cont' = do-                    bs <- lift $ unsafeLiftIO popper-                    unless (S.null bs) $ do-                        yield' (Chunk bs)-                        cont'-            cont'-            case fbuilder of-                Flush -> yield' Flush-                Chunk _ -> return ()-            loop-    loop-{-# INLINE helper #-}

@@ -0,0 +1,147 @@+{-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE RankNTypes #-}+-- | Convert a stream of blaze-builder @Builder@s into a stream of @ByteString@s.+--+-- Works with both blaze-builder < 0.4's @Builder@s and+-- 'Data.ByteString.Builder.Builder'.+--+-- Adapted from blaze-builder-enumerator, written by myself and Simon Meier.+--+-- Note that the functions here can work in any monad built on top of @IO@ or+-- @ST@.+--+-- Since 1.1.7.0+--+module Data.Conduit.ByteString.Builder+    (++  -- * Conduits from builders to bytestrings+    builderToByteString+  , unsafeBuilderToByteString+  , builderToByteStringWith++  -- ** Flush+  , builderToByteStringFlush+  , builderToByteStringWithFlush++  -- * Buffers+  , Buffer++  -- ** Status information+  , freeSize+  , sliceSize+  , bufferSize++  -- ** Creation and modification+  , allocBuffer+  , reuseBuffer+  , nextSlice++  -- ** Conversion to bytestings+  , unsafeFreezeBuffer+  , unsafeFreezeNonEmptyBuffer++  -- * Buffer allocation strategies+  , BufferAllocStrategy+  , allNewBuffersStrategy+  , reuseBufferStrategy+    ) where++import Data.Conduit+import Control.Monad (unless, liftM)+import Control.Monad.Trans.Class (lift, MonadTrans)++import qualified Data.ByteString                   as S++import Control.Monad.Primitive (PrimMonad, unsafePrimToPrim)+import Control.Monad.Base (MonadBase, liftBase)+import Data.Streaming.ByteString.Builder.Class++unsafeLiftIO :: (MonadBase base m, PrimMonad base) => IO a -> m a+unsafeLiftIO = liftBase . unsafePrimToPrim++-- | Incrementally execute builders and pass on the filled chunks as+-- bytestrings.+builderToByteString :: (MonadBase base m, PrimMonad base, StreamingBuilder b)+                    => Conduit b m S.ByteString+builderToByteString =+  builderToByteStringWith defaultStrategy+{-# INLINE builderToByteString #-}++-- |+--+-- Since 0.0.2+builderToByteStringFlush :: (MonadBase base m, PrimMonad base, StreamingBuilder b)+                         => Conduit (Flush b) m (Flush S.ByteString)+builderToByteStringFlush =+  builderToByteStringWithFlush defaultStrategy+{-# INLINE builderToByteStringFlush #-}++-- | Incrementally execute builders on the given buffer and pass on the filled+-- chunks as bytestrings. Note that, if the given buffer is too small for the+-- execution of a build step, a larger one will be allocated.+--+-- WARNING: This conduit yields bytestrings that are NOT+-- referentially transparent. Their content will be overwritten as soon+-- as control is returned from the inner sink!+unsafeBuilderToByteString :: (MonadBase base m, PrimMonad base, StreamingBuilder b)+                          => IO Buffer  -- action yielding the inital buffer.+                          -> Conduit b m S.ByteString+unsafeBuilderToByteString = builderToByteStringWith . reuseBufferStrategy+{-# INLINE unsafeBuilderToByteString #-}+++-- | A conduit that incrementally executes builders and passes on the+-- filled chunks as bytestrings to an inner sink.+--+-- INV: All bytestrings passed to the inner sink are non-empty.+builderToByteStringWith :: (MonadBase base m, PrimMonad base, StreamingBuilder b)+                        => BufferAllocStrategy+                        -> Conduit b m S.ByteString+builderToByteStringWith =+    helper (liftM (fmap Chunk) await) yield'+  where+    yield' Flush = return ()+    yield' (Chunk bs) = yield bs+{-# INLINE builderToByteStringWith #-}++-- |+--+-- Since 0.0.2+builderToByteStringWithFlush+    :: (MonadBase base m, PrimMonad base, StreamingBuilder b)+    => BufferAllocStrategy+    -> Conduit (Flush b) m (Flush S.ByteString)+builderToByteStringWithFlush = helper await yield+{-# INLINE builderToByteStringWithFlush #-}++helper :: (MonadBase base m, PrimMonad base, Monad (t m), MonadTrans t, StreamingBuilder b)+       => t m (Maybe (Flush b))+       -> (Flush S.ByteString -> t m ())+       -> BufferAllocStrategy+       -> t m ()+helper await' yield' strat = do+    (recv, finish) <- lift $ unsafeLiftIO $ newBuilderRecv strat+    let loop = await' >>= maybe finish' cont+        finish' = do+            mbs <- lift $ unsafeLiftIO finish+            maybe (return ()) (yield' . Chunk) mbs+        cont fbuilder = do+            let builder =+                    case fbuilder of+                        Flush -> builderFlush+                        Chunk b -> b+            popper <- lift $ unsafeLiftIO $ recv builder+            let cont' = do+                    bs <- lift $ unsafeLiftIO popper+                    unless (S.null bs) $ do+                        yield' (Chunk bs)+                        cont'+            cont'+            case fbuilder of+                Flush -> yield' Flush+                Chunk _ -> return ()+            loop+    loop+{-# INLINE helper #-}

@@ -1,5 +1,5 @@ Name:                conduit-extra-Version:             1.1.6.2+Version:             1.1.7.0 Synopsis:            Batteries included conduit: adapters for common libraries. Description:     The conduit package itself maintains relative small dependencies. The purpose of this package is to collect commonly used utility functions wrapping other library dependencies, without depending on heavier-weight dependencies. The basic idea is that this package should only depend on haskell-platform packages and conduit.@@ -21,6 +21,7 @@   Exposed-modules:     Data.Conduit.Attoparsec                        Data.Conduit.Binary                        Data.Conduit.Blaze+                       Data.Conduit.ByteString.Builder                        Data.Conduit.Filesystem                        Data.Conduit.Lazy                        Data.Conduit.Network@@ -51,7 +52,7 @@                      , process                      , resourcet                >= 1.1                      , stm-                     , streaming-commons        >= 0.1.8+                     , streaming-commons        >= 0.1.10    ghc-options:     -Wall @@ -69,11 +70,13 @@                    , async                    , attoparsec                    , blaze-builder+                   , bytestring-builder                    , bytestring                    , exceptions                    , process                    , resourcet                    , stm+                   , streaming-commons                    , text                    , transformers                    , transformers-base@@ -82,7 +85,7 @@         cpp-options: -DWINDOWS     other-modules:   Data.Conduit.AttoparsecSpec                      Data.Conduit.BinarySpec-                     Data.Conduit.BlazeSpec+                     Data.Conduit.ByteString.BuilderSpec                      Data.Conduit.ExtraSpec                      Data.Conduit.FilesystemSpec                      Data.Conduit.LazySpec

@@ -1,63 +0,0 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE CPP #-}-module Data.Conduit.BlazeSpec (spec) where--import Test.Hspec-import Test.Hspec.QuickCheck (prop)--import qualified Data.Conduit as C-import qualified Data.Conduit.List as CL-import Data.ByteString.Char8 ()-import Data.Conduit.Blaze (builderToByteString, builderToByteStringFlush)-import Control.Monad.ST (runST)-import Data.Monoid-import qualified Data.ByteString as S-import Blaze.ByteString.Builder (fromByteString, toLazyByteString, insertLazyByteString, flush)-import qualified Data.ByteString.Lazy as L-import Data.ByteString.Lazy.Char8 ()--spec :: Spec-spec =-    describe "Data.Conduit.Blaze" $ do-        prop "idempotent to toLazyByteString" $ \bss' -> runST $ do-            let bss = map S.pack bss'-            let builders = map fromByteString bss-            let lbs = toLazyByteString $ mconcat builders-            let src = mconcat $ map (CL.sourceList . return) builders-            outBss <- src C.$= builderToByteString C.$$ CL.consume-            return $ lbs == L.fromChunks outBss--        it "works for large input" $ do-            let builders = replicate 10000 (fromByteString "hello world!")-            let lbs = toLazyByteString $ mconcat builders-            let src = mconcat $ map (CL.sourceList . return) builders-            outBss <- src C.$= builderToByteString C.$$ CL.consume-            lbs `shouldBe` L.fromChunks outBss--        it "works for lazy bytestring insertion" $ do-            let builders = replicate 10000 (insertLazyByteString "hello world!")-            let lbs = toLazyByteString $ mconcat builders-            let src = mconcat $ map (CL.sourceList . return) builders-            outBss <- src C.$= builderToByteString C.$$ CL.consume-            lbs `shouldBe` L.fromChunks outBss--        it "flush shouldn't bring in empty strings." $ do-            let dat = ["hello", "world"]-                src = CL.sourceList dat C.$= CL.map ((`mappend` flush) . fromByteString)-            out <- src C.$= builderToByteString C.$$ CL.consume-            dat `shouldBe` out--        prop "flushing" $ \bss' -> runST $ do-            let bss = concatMap (\bs -> [C.Chunk $ S.pack bs, C.Flush]) $ filter (not . null) bss'-            let src = CL.sourceList $ map (fmap fromByteString) bss-            outBss <- src C.$= builderToByteStringFlush C.$$ CL.consume-            if bss == outBss then return () else error (show (bss, outBss))-            return $ bss == outBss-        it "large flush input" $ do-            let lbs = L.pack $ concat $ replicate 100000 [0..255]-                chunks = map (C.Chunk . fromByteString) (L.toChunks lbs)-                src = CL.sourceList chunks-            bss <- src C.$$ builderToByteStringFlush C.=$ CL.consume-            let unFlush (C.Chunk x) = [x]-                unFlush C.Flush = []-            L.fromChunks (concatMap unFlush bss) `shouldBe` lbs

@@ -0,0 +1,64 @@+{-# LANGUAGE OverloadedStrings #-}+module Data.Conduit.ByteString.BuilderSpec (spec) where++import Test.Hspec+import Test.Hspec.QuickCheck (prop)++import qualified Data.Conduit as C+import qualified Data.Conduit.List as CL+import Data.ByteString.Char8 ()+import Data.Conduit.ByteString.Builder (builderToByteString, builderToByteStringFlush)+import Control.Monad.ST (runST)+import Data.Monoid+import qualified Data.ByteString as S+import Data.ByteString.Builder (byteString, toLazyByteString)+import Data.ByteString.Builder.Internal (lazyByteStringInsert, flush)+import qualified Data.ByteString.Lazy as L+import Data.ByteString.Lazy.Char8 ()++spec :: Spec+spec =+    describe "Data.Conduit.ByteString.Builder" $ do+        prop "idempotent to toLazyByteString" $ \bss' -> runST $ do+            let bss = map S.pack bss'+            let builders = map byteString bss+            let lbs = toLazyByteString $ mconcat builders+            let src = mconcat $ map (CL.sourceList . return) builders+            outBss <- src C.$= builderToByteString C.$$ CL.consume+            return $ lbs == L.fromChunks outBss++        it "works for large input" $ do+            let builders = replicate 10000 (byteString "hello world!")+            let lbs = toLazyByteString $ mconcat builders+            let src = mconcat $ map (CL.sourceList . return) builders+            outBss <- src C.$= builderToByteString C.$$ CL.consume+            lbs `shouldBe` L.fromChunks outBss++        it "works for lazy bytestring insertion" $ do+            let builders = replicate 10000 (lazyByteStringInsert "hello world!")+            let lbs = toLazyByteString $ mconcat builders+            let src = mconcat $ map (CL.sourceList . return) builders+            outBss <- src C.$= builderToByteString C.$$ CL.consume+            lbs `shouldBe` L.fromChunks outBss++        it "flush shouldn't bring in empty strings." $ do+            let dat = ["hello", "world"]+                src = CL.sourceList dat C.$= CL.map ((`mappend` flush) . byteString)+            out <- src C.$= builderToByteString C.$$ CL.consume+            dat `shouldBe` out++        prop "flushing" $ \bss' -> runST $ do+            let bss = concatMap (\bs -> [C.Chunk $ S.pack bs, C.Flush]) $ filter (not . null) bss'+            let chunks = map (fmap byteString) bss+            let src = CL.sourceList chunks+            outBss <- src C.$= builderToByteStringFlush C.$$ CL.consume+            if bss == outBss then return () else error (show (bss, outBss))+            return $ bss == outBss+        it "large flush input" $ do+            let lbs = L.pack $ concat $ replicate 100000 [0..255]+            let chunks = map (C.Chunk . byteString) (L.toChunks lbs)+            let src = CL.sourceList chunks+            bss <- src C.$$ builderToByteStringFlush C.=$ CL.consume+            let unFlush (C.Chunk x) = [x]+                unFlush C.Flush = []+            L.fromChunks (concatMap unFlush bss) `shouldBe` lbs