diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +## 0.1.10 + +Support blaze-builder >= 0.4. Add `newByteStringBuilderRecv` to Data.Streaming.ByteString.Builder; add modules Data.Streaming.ByteString.Builder.Buffer and Data.Streaming.ByteString.Builder.Class. + ## 0.1.9 Add Data.Streaming.ByteString.Builder diff --git a/Data/Streaming/Blaze.hs b/Data/Streaming/Blaze.hs --- a/Data/Streaming/Blaze.hs +++ b/Data/Streaming/Blaze.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE BangPatterns #-} {-# LANGUAGE RankNTypes #-} @@ -5,6 +6,9 @@ -- -- Adapted from blaze-builder-enumerator, written by myself and Simon Meier. -- +-- Note: if you have blaze-builder >= 0.4, 'newBlazeRecv' just calls +-- 'Data.Streaming.ByteString.Builder.newByteStringBuilderRecv' + -- Note that the functions here can work in any monad built on top of @IO@ or -- @ST@. module Data.Streaming.Blaze @@ -37,26 +41,25 @@ , defaultStrategy ) where -import Data.IORef +import Blaze.ByteString.Builder +import qualified Data.ByteString as S -import qualified Data.ByteString as S +#if MIN_VERSION_blaze_builder(0,4,0) -import Blaze.ByteString.Builder.Internal -import Blaze.ByteString.Builder.Internal.Types -import Blaze.ByteString.Builder.Internal.Buffer +import Data.Streaming.ByteString.Builder --- | Provides a series of @ByteString@s until empty, at which point it provides --- an empty @ByteString@. --- --- Since 0.1.2 -type BlazePopper = IO S.ByteString +newBlazeRecv :: BufferAllocStrategy -> IO (BlazeRecv, BlazeFinish) +newBlazeRecv = newByteStringBuilderRecv +{-# INLINE newBlazeRecv #-} -type BlazeRecv = Builder -> IO BlazePopper +#else /* !MIN_VERSION_blaze_builder(0,4,0) */ -type BlazeFinish = IO (Maybe S.ByteString) +import Blaze.ByteString.Builder.Internal hiding (insertByteString) +import Blaze.ByteString.Builder.Internal.Types hiding (insertByteString) +import Blaze.ByteString.Builder.Internal.Buffer (execBuildStep) +import Data.IORef -defaultStrategy :: BufferAllocStrategy -defaultStrategy = allNewBuffersStrategy defaultBufferSize +import Data.Streaming.ByteString.Builder.Buffer newBlazeRecv :: BufferAllocStrategy -> IO (BlazeRecv, BlazeFinish) newBlazeRecv (ioBufInit, nextBuf) = do @@ -133,3 +136,15 @@ buf <- lift $ unsafeLiftIO $ ioBuf maybe (return ()) (yield' . Chunk) (unsafeFreezeNonEmptyBuffer buf) -} + +#endif /* !MIN_VERSION_blaze_builder(0,4,0) */ + +-- | Provides a series of @ByteString@s until empty, at which point it provides +-- an empty @ByteString@. +-- +-- Since 0.1.2 +type BlazePopper = IO S.ByteString + +type BlazeRecv = Builder -> IO BlazePopper + +type BlazeFinish = IO (Maybe S.ByteString) diff --git a/Data/Streaming/ByteString/Builder.hs b/Data/Streaming/ByteString/Builder.hs --- a/Data/Streaming/ByteString/Builder.hs +++ b/Data/Streaming/ByteString/Builder.hs @@ -1,23 +1,139 @@ --- | Provides @toByteStringIO*@ like "Blaze.ByteString.Builder"s, for "Data.ByteString.Builder". +{-# LANGUAGE CPP #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE RankNTypes #-} + +-- | Convert a stream of bytestring @Builder@s into a stream of @ByteString@s. -- +-- Adapted from blaze-builder-enumerator, written by Michael Snoyman and Simon Meier. +-- +-- Note that the functions here can work in any monad built on top of @IO@ or +-- @ST@. +-- +-- Also provides @toByteStringIO*@ like "Blaze.ByteString.Builder"s, for +-- "Data.ByteString.Builder". +-- -- Since 0.1.9 +-- module Data.Streaming.ByteString.Builder - ( toByteStringIO + ( BuilderRecv + , BuilderPopper + , BuilderFinish + , newByteStringBuilderRecv + + -- * toByteStringIO + , toByteStringIO , toByteStringIOWith - , toByteStringIOWithBuffer ) + , toByteStringIOWithBuffer + + -- * Buffers + , Buffer + + -- ** Status information + , freeSize + , sliceSize + , bufferSize + + -- ** Creation and modification + , allocBuffer + , reuseBuffer + , nextSlice + + -- ** Conversion to bytestings + , unsafeFreezeBuffer + , unsafeFreezeNonEmptyBuffer + + -- * Buffer allocation strategies + , BufferAllocStrategy + , allNewBuffersStrategy + , reuseBufferStrategy + , defaultStrategy + ) where -import Control.Monad (when) +import Control.Monad (when,unless) +import qualified Data.ByteString as S import Data.ByteString.Builder (Builder) import Data.ByteString.Builder.Extra (runBuilder, BufferWriter, Next(Done, More, Chunk)) import Data.ByteString.Internal (mallocByteString, ByteString(PS)) import Data.ByteString.Lazy.Internal (defaultChunkSize) +import Data.IORef (newIORef, writeIORef, readIORef) import Data.Word (Word8) import Foreign.ForeignPtr (ForeignPtr, withForeignPtr) +import Foreign.Ptr (plusPtr, minusPtr) +import Data.Streaming.ByteString.Builder.Buffer + +-- | Provides a series of @ByteString@s until empty, at which point it provides +-- an empty @ByteString@. +-- +-- Since 0.1.10.0 +-- +type BuilderPopper = IO S.ByteString + +type BuilderRecv = Builder -> IO BuilderPopper + +type BuilderFinish = IO (Maybe S.ByteString) + +newByteStringBuilderRecv :: BufferAllocStrategy -> IO (BuilderRecv, BuilderFinish) +newByteStringBuilderRecv (ioBufInit, nextBuf) = do + refBuf <- newIORef ioBufInit + return (push refBuf, finish refBuf) + where + finish refBuf = do + ioBuf <- readIORef refBuf + buf <- ioBuf + return $ unsafeFreezeNonEmptyBuffer buf + + push refBuf builder = do + refWri <- newIORef $ Left $ runBuilder builder + return $ popper refBuf refWri + + popper refBuf refWri = do + ioBuf <- readIORef refBuf + ebWri <- readIORef refWri + case ebWri of + Left bWri -> do + !buf@(Buffer _ _ op ope) <- ioBuf + (bytes, next) <- bWri op (ope `minusPtr` op) + let op' = op `plusPtr` bytes + case next of + Done -> do + writeIORef refBuf $ return $ updateEndOfSlice buf op' + return S.empty + More minSize bWri' -> do + let buf' = updateEndOfSlice buf op' + {-# INLINE cont #-} + cont mbs = do + -- sequencing the computation of the next buffer + -- construction here ensures that the reference to the + -- foreign pointer `fp` is lost as soon as possible. + ioBuf' <- nextBuf minSize buf' + writeIORef refBuf ioBuf' + writeIORef refWri $ Left bWri' + case mbs of + Just bs | not $ S.null bs -> return bs + _ -> popper refBuf refWri + cont $ unsafeFreezeNonEmptyBuffer buf' + Chunk bs bWri' -> do + let buf' = updateEndOfSlice buf op' + let yieldBS = do + nextBuf 1 buf' >>= writeIORef refBuf + writeIORef refWri $ Left bWri' + if S.null bs + then popper refBuf refWri + else return bs + case unsafeFreezeNonEmptyBuffer buf' of + Nothing -> yieldBS + Just bs' -> do + writeIORef refWri $ Right yieldBS + return bs' + Right action -> action + -- | Use a pre-existing buffer to 'toByteStringIOWith'. -- -- Since 0.1.9 +-- toByteStringIOWithBuffer :: Int -> (ByteString -> IO ()) -> Builder @@ -31,7 +147,7 @@ loop :: BufferWriter -> IO () loop wr = do (len, next) <- withForeignPtr buf (flip wr bufSize) - when (len > 0) (io (PS buf 0 len)) + when (len > 0) (io $! PS buf 0 len) case next of Done -> return () More newBufSize nextWr @@ -40,7 +156,7 @@ go newBufSize newBuf nextWr | otherwise -> loop nextWr Chunk s nextWr -> do - io s + unless (S.null s) (io s) loop nextWr -- | @toByteStringIOWith bufSize io b@ runs the builder @b@ with a buffer of @@ -67,6 +183,7 @@ -> IO () toByteStringIOWith bufSize io b = toByteStringIOWithBuffer bufSize io b =<< mallocByteString bufSize +{-# INLINE toByteStringIOWith #-} -- | Run the builder with a 'defaultChunkSize'd buffer and execute the given -- 'IO' action whenever the buffer is full or gets flushed. diff --git a/Data/Streaming/ByteString/Builder/Buffer.hs b/Data/Streaming/ByteString/Builder/Buffer.hs new file mode 100644 --- /dev/null +++ b/Data/Streaming/ByteString/Builder/Buffer.hs @@ -0,0 +1,208 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE Rank2Types #-} + +-- | Buffers for 'Builder's. This is a partial copy of blaze-builder-0.3.3.4's +-- "Blaze.ByteString.Builder.Internal.Buffer" module, which was removed in +-- blaze-builder-0.4. +-- +-- If you are using blaze-builder 0.3.*, this module just re-exports from +-- "Blaze.ByteString.Builder.Internal.Buffer". +-- +-- Since 0.1.10.0 +-- +module Data.Streaming.ByteString.Builder.Buffer + ( + -- * Buffers + Buffer (..) + + -- ** Status information + , freeSize + , sliceSize + , bufferSize + + -- ** Creation and modification + , allocBuffer + , reuseBuffer + , nextSlice + , updateEndOfSlice + + -- ** Conversion to bytestings + , unsafeFreezeBuffer + , unsafeFreezeNonEmptyBuffer + + -- * Buffer allocation strategies + , BufferAllocStrategy + , allNewBuffersStrategy + , reuseBufferStrategy + , defaultStrategy + ) where + +import Data.ByteString.Lazy.Internal (defaultChunkSize) + +#if MIN_VERSION_blaze_builder(0,4,0) + +import qualified Data.ByteString as S +import qualified Data.ByteString.Internal as S +import Foreign (Word8, ForeignPtr, Ptr, plusPtr, minusPtr) +import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr) + +------------------------------------------------------------------------------ +-- Buffers +------------------------------------------------------------------------------ + +-- | A buffer @Buffer fpbuf p0 op ope@ describes a buffer with the underlying +-- byte array @fpbuf..ope@, the currently written slice @p0..op@ and the free +-- space @op..ope@. +-- +-- Since 0.1.10.0 +-- +data Buffer = Buffer {-# UNPACK #-} !(ForeignPtr Word8) -- underlying pinned array + {-# UNPACK #-} !(Ptr Word8) -- beginning of slice + {-# UNPACK #-} !(Ptr Word8) -- next free byte + {-# UNPACK #-} !(Ptr Word8) -- first byte after buffer + +-- | The size of the free space of the buffer. +-- +-- Since 0.1.10.0 +-- +freeSize :: Buffer -> Int +freeSize (Buffer _ _ op ope) = ope `minusPtr` op + +-- | The size of the written slice in the buffer. +-- +-- Since 0.1.10.0 +-- +sliceSize :: Buffer -> Int +sliceSize (Buffer _ p0 op _) = op `minusPtr` p0 + +-- | The size of the whole byte array underlying the buffer. +-- +-- Since 0.1.10.0 +-- +bufferSize :: Buffer -> Int +bufferSize (Buffer fpbuf _ _ ope) = + ope `minusPtr` unsafeForeignPtrToPtr fpbuf + +-- | @allocBuffer size@ allocates a new buffer of size @size@. +-- +-- Since 0.1.10.0 +-- +{-# INLINE allocBuffer #-} +allocBuffer :: Int -> IO Buffer +allocBuffer size = do + fpbuf <- S.mallocByteString size + let !pbuf = unsafeForeignPtrToPtr fpbuf + return $! Buffer fpbuf pbuf pbuf (pbuf `plusPtr` size) + +-- | Resets the beginning of the next slice and the next free byte such that +-- the whole buffer can be filled again. +-- +-- Since 0.1.10.0 +-- +{-# INLINE reuseBuffer #-} +reuseBuffer :: Buffer -> Buffer +reuseBuffer (Buffer fpbuf _ _ ope) = Buffer fpbuf p0 p0 ope + where + p0 = unsafeForeignPtrToPtr fpbuf + +-- | Convert the buffer to a bytestring. This operation is unsafe in the sense +-- that created bytestring shares the underlying byte array with the buffer. +-- Hence, depending on the later use of this buffer (e.g., if it gets reset and +-- filled again) referential transparency may be lost. +-- +-- Since 0.1.10.0 +-- +{-# INLINE unsafeFreezeBuffer #-} +unsafeFreezeBuffer :: Buffer -> S.ByteString +unsafeFreezeBuffer (Buffer fpbuf p0 op _) = + S.PS fpbuf (p0 `minusPtr` unsafeForeignPtrToPtr fpbuf) (op `minusPtr` p0) + +-- | Convert a buffer to a non-empty bytestring. See 'unsafeFreezeBuffer' for +-- the explanation of why this operation may be unsafe. +-- +-- Since 0.1.10.0 +-- +{-# INLINE unsafeFreezeNonEmptyBuffer #-} +unsafeFreezeNonEmptyBuffer :: Buffer -> Maybe S.ByteString +unsafeFreezeNonEmptyBuffer buf + | sliceSize buf <= 0 = Nothing + | otherwise = Just $ unsafeFreezeBuffer buf + +-- | Update the end of slice pointer. +-- +-- Since 0.1.10.0 +-- +{-# INLINE updateEndOfSlice #-} +updateEndOfSlice :: Buffer -- Old buffer + -> Ptr Word8 -- New end of slice + -> Buffer -- Updated buffer +updateEndOfSlice (Buffer fpbuf p0 _ ope) op' = Buffer fpbuf p0 op' ope + +-- | Move the beginning of the slice to the next free byte such that the +-- remaining free space of the buffer can be filled further. This operation +-- is safe and can be used to fill the remaining part of the buffer after a +-- direct insertion of a bytestring or a flush. +-- +-- Since 0.1.10.0 +-- +{-# INLINE nextSlice #-} +nextSlice :: Int -> Buffer -> Maybe Buffer +nextSlice minSize (Buffer fpbuf _ op ope) + | ope `minusPtr` op <= minSize = Nothing + | otherwise = Just (Buffer fpbuf op op ope) + +------------------------------------------------------------------------------ +-- Buffer allocation strategies +------------------------------------------------------------------------------ + +-- | A buffer allocation strategy @(buf0, nextBuf)@ specifies the initial +-- buffer to use and how to compute a new buffer @nextBuf minSize buf@ with at +-- least size @minSize@ from a filled buffer @buf@. The double nesting of the +-- @IO@ monad helps to ensure that the reference to the filled buffer @buf@ is +-- lost as soon as possible, but the new buffer doesn't have to be allocated +-- too early. +-- +-- Since 0.1.10.0 +-- +type BufferAllocStrategy = (IO Buffer, Int -> Buffer -> IO (IO Buffer)) + +-- | The simplest buffer allocation strategy: whenever a buffer is requested, +-- allocate a new one that is big enough for the next build step to execute. +-- +-- NOTE that this allocation strategy may spill quite some memory upon direct +-- insertion of a bytestring by the builder. Thats no problem for garbage +-- collection, but it may lead to unreasonably high memory consumption in +-- special circumstances. +-- +-- Since 0.1.10.0 +-- +allNewBuffersStrategy :: Int -- Minimal buffer size. + -> BufferAllocStrategy +allNewBuffersStrategy bufSize = + ( allocBuffer bufSize + , \reqSize _ -> return (allocBuffer (max reqSize bufSize)) ) + +-- | An unsafe, but possibly more efficient buffer allocation strategy: +-- reuse the buffer, if it is big enough for the next build step to execute. +-- +-- Since 0.1.10.0 +-- +reuseBufferStrategy :: IO Buffer + -> BufferAllocStrategy +reuseBufferStrategy buf0 = + (buf0, tryReuseBuffer) + where + tryReuseBuffer reqSize buf + | bufferSize buf >= reqSize = return $ return (reuseBuffer buf) + | otherwise = return $ allocBuffer reqSize + + +#else /* !MIN_VERSION_blaze_builder(0,4,0) */ + +import Blaze.ByteString.Builder.Internal.Buffer + +#endif /* !MIN_VERSION_blaze_builder(0,4,0) */ + +defaultStrategy :: BufferAllocStrategy +defaultStrategy = allNewBuffersStrategy defaultChunkSize diff --git a/Data/Streaming/ByteString/Builder/Class.hs b/Data/Streaming/ByteString/Builder/Class.hs new file mode 100644 --- /dev/null +++ b/Data/Streaming/ByteString/Builder/Class.hs @@ -0,0 +1,43 @@ +{-# LANGUAGE CPP #-} +-- | Typeclass to stream blaze-builder and bytestring(-builder) @Builder@s. +-- +-- Since 0.1.10.0 +-- +module Data.Streaming.ByteString.Builder.Class + ( StreamingBuilder (..) + , module Data.Streaming.ByteString.Builder + ) where + +import qualified Data.ByteString.Builder +import qualified Data.ByteString.Builder.Internal +import Data.Monoid (Monoid) + +import Data.Streaming.ByteString.Builder hiding (newByteStringBuilderRecv) +import qualified Data.Streaming.ByteString.Builder + +#if !MIN_VERSION_blaze_builder(0,4,0) + +import qualified Blaze.ByteString.Builder + +import Data.Streaming.Blaze + +instance StreamingBuilder Blaze.ByteString.Builder.Builder where + newBuilderRecv = newBlazeRecv + builderFlush = Blaze.ByteString.Builder.flush + +#endif /* !MIN_VERSION_blaze_builder(0,4,0) */ + +-- | Typeclass to stream blaze-builder (< 0.4) and bytestring(-builder) @Builder@s. +-- This is primarily to aid the transition from blaze-builder to bytestring @Builder@s +-- (if using blaze-builder >= 0.4, there is only one instance, since the @Builder@ +-- type is shared). +-- +-- Since 0.1.10.0 +-- +class Monoid b => StreamingBuilder b where + newBuilderRecv :: BufferAllocStrategy -> IO (b -> IO BuilderPopper, BuilderFinish) + builderFlush :: b + +instance StreamingBuilder Data.ByteString.Builder.Builder where + newBuilderRecv = Data.Streaming.ByteString.Builder.newByteStringBuilderRecv + builderFlush = Data.ByteString.Builder.Internal.flush diff --git a/streaming-commons.cabal b/streaming-commons.cabal --- a/streaming-commons.cabal +++ b/streaming-commons.cabal @@ -1,11 +1,11 @@ name: streaming-commons -version: 0.1.9.1 +version: 0.1.10.0 synopsis: Common lower-level functions needed by various streaming data libraries description: Provides low-dependency functionality commonly needed by various streaming data libraries, such as conduit and pipes. homepage: https://github.com/fpco/streaming-commons license: MIT license-file: LICENSE -author: Michael Snoyman +author: Michael Snoyman, Emanuel Borsboom maintainer: michael@snoyman.com -- copyright: category: Data @@ -27,6 +27,8 @@ library exposed-modules: Data.Streaming.Blaze Data.Streaming.ByteString.Builder + Data.Streaming.ByteString.Builder.Buffer + Data.Streaming.ByteString.Builder.Class Data.Streaming.FileRead Data.Streaming.Filesystem Data.Streaming.Network @@ -47,7 +49,7 @@ build-depends: base >= 4.4 && < 5 , array - , blaze-builder >= 0.3 && < 0.4 + , blaze-builder >= 0.3 && < 0.5 , bytestring , directory , network @@ -82,6 +84,7 @@ type: exitcode-stdio-1.0 ghc-options: -Wall -threaded other-modules: Data.Streaming.ByteString.BuilderSpec + Data.Streaming.BlazeSpec Data.Streaming.FileReadSpec Data.Streaming.FilesystemSpec Data.Streaming.NetworkSpec diff --git a/test/Data/Streaming/BlazeSpec.hs b/test/Data/Streaming/BlazeSpec.hs new file mode 100644 --- /dev/null +++ b/test/Data/Streaming/BlazeSpec.hs @@ -0,0 +1,16 @@ +{-# LANGUAGE OverloadedStrings #-} +module Data.Streaming.BlazeSpec (spec) where + +import Test.Hspec +import qualified Blaze.ByteString.Builder as B +import Data.Streaming.ByteString.BuilderSpec hiding (spec) + +spec :: Spec +spec = do + describe "Data.Streaming.Blaze" $ builderSpec BuilderFunctions + { bfFromByteString = B.fromByteString + , bfInsertLazyByteString = B.insertLazyByteString + , bfToLazyByteString = B.toLazyByteString + , bfInsertByteString = B.insertByteString + , bfCopyByteString = B.copyByteString + } diff --git a/test/Data/Streaming/ByteString/BuilderSpec.hs b/test/Data/Streaming/ByteString/BuilderSpec.hs --- a/test/Data/Streaming/ByteString/BuilderSpec.hs +++ b/test/Data/Streaming/ByteString/BuilderSpec.hs @@ -1,20 +1,136 @@ {-# LANGUAGE OverloadedStrings #-} -module Data.Streaming.ByteString.BuilderSpec (spec) where +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} +module Data.Streaming.ByteString.BuilderSpec + ( spec + , builderSpec + , BuilderFunctions(..) + ) where -import qualified Data.ByteString.Char8 as S +import qualified Data.ByteString as S +import Data.ByteString.Char8 () import qualified Data.ByteString.Unsafe as S import qualified Data.ByteString.Builder as B -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString.Builder.Internal as B +import qualified Data.ByteString.Lazy as L +import Data.ByteString.Lazy.Char8 () import Data.IORef +import Data.Maybe import Data.Monoid import Test.Hspec import Test.Hspec.QuickCheck (prop) import Data.Streaming.ByteString.Builder +import Data.Streaming.ByteString.Builder.Class +data BuilderFunctions b = BuilderFunctions + { bfFromByteString :: S.ByteString -> b + , bfInsertLazyByteString :: L.ByteString -> b + , bfToLazyByteString :: b -> L.ByteString + , bfInsertByteString :: S.ByteString -> b + , bfCopyByteString :: S.ByteString -> b + } + +tester :: StreamingBuilder b => BufferAllocStrategy -> [b] -> IO [S.ByteString] +tester strat builders0 = do + (recv, finish) <- newBuilderRecv strat + let loop front [] = do + mbs <- finish + return $ front $ maybe [] return mbs + loop front0 (bu:bus) = do + popper <- recv bu + let go front = do + bs <- popper + if S.null bs + then loop front bus + else go (front . (bs:)) + go front0 + loop id builders0 + +testerFlush :: StreamingBuilder b + => BufferAllocStrategy -> [Maybe b] -> IO [Maybe S.ByteString] +testerFlush strat builders0 = do + (recv, finish) <- newBuilderRecv strat + let loop front [] = do + mbs <- finish + return $ front $ maybe [] (return . Just) mbs + loop front0 (mbu:bus) = do + popper <- recv $ fromMaybe builderFlush mbu + let go front = do + bs <- popper + if S.null bs + then + case mbu of + Nothing -> loop (front . (Nothing:)) bus + Just _ -> loop front bus + else go (front . (Just bs:)) + go front0 + loop id builders0 + +builderSpec :: forall b. StreamingBuilder b => BuilderFunctions b -> Spec +builderSpec BuilderFunctions{..} = do + prop "idempotent to toLazyByteString" $ \bss' -> do + let bss = map S.pack bss' + let builders :: [b] + builders = map bfFromByteString bss + let lbs = bfToLazyByteString $ mconcat builders + outBss <- tester defaultStrategy builders + L.fromChunks outBss `shouldBe` lbs + + it "works for large input" $ do + let builders :: [b] + builders = replicate 10000 (bfFromByteString "hello world!" :: b) + let lbs = bfToLazyByteString $ mconcat builders + outBss <- tester defaultStrategy builders + L.fromChunks outBss `shouldBe` lbs + + it "works for lazy bytestring insertion" $ do + let builders :: [b] + builders = replicate 10000 (bfInsertLazyByteString "hello world!") + let lbs = bfToLazyByteString $ mconcat builders + outBss <- tester defaultStrategy builders + L.fromChunks outBss `shouldBe` lbs + + prop "works for strict bytestring insertion" $ \bs' -> do + let bs = S.pack bs' + let builders :: [b] + builders = replicate 10000 (bfCopyByteString bs `mappend` bfInsertByteString bs) + let lbs = bfToLazyByteString $ mconcat builders + outBss <- tester defaultStrategy builders + L.fromChunks outBss `shouldBe` lbs + + it "flush shouldn't bring in empty strings." $ do + let dat = ["hello", "world"] + builders :: [b] + builders = map ((`mappend` builderFlush) . bfFromByteString) dat + out <- tester defaultStrategy builders + dat `shouldBe` out + + prop "flushing" $ \bss' -> do + let bss = concatMap (\bs -> [Just $ S.pack bs, Nothing]) $ filter (not . null) bss' + let builders :: [Maybe b] + builders = map (fmap bfFromByteString) bss + outBss <- testerFlush defaultStrategy builders + outBss `shouldBe` bss + it "large flush input" $ do + let lbs = L.pack $ concat $ replicate 100000 [0..255] + chunks :: [Maybe b] + chunks = map (Just . bfFromByteString) (L.toChunks lbs) + bss <- testerFlush defaultStrategy chunks + L.fromChunks (catMaybes bss) `shouldBe` lbs + spec :: Spec spec = describe "Data.Streaming.ByteString.Builder" $ do + + builderSpec BuilderFunctions + { bfFromByteString = B.byteString + , bfInsertLazyByteString = B.lazyByteStringInsert + , bfToLazyByteString = B.toLazyByteString + , bfInsertByteString = B.byteStringInsert + , bfCopyByteString = B.byteStringCopy + } + prop "toByteStringIO idempotent to toLazyByteString" $ \bss' -> do let bss = mconcat (map (B.byteString . S.pack) bss') ior <- newIORef []