streaming-commons 0.1.8.1 → 0.1.9
raw patch · 5 files changed
+188/−2 lines, 5 filesdep +bytestring-builderdep ~basedep ~blaze-builderdep ~bytestring
Dependencies added: bytestring-builder
Dependency ranges changed: base, blaze-builder, bytestring
Files
- ChangeLog.md +4/−0
- Data/Streaming/ByteString/Builder.hs +82/−0
- bench/builder-to-bytestring-io.hs +39/−0
- streaming-commons.cabal +37/−2
- test/Data/Streaming/ByteString/BuilderSpec.hs +26/−0
ChangeLog.md view
@@ -1,3 +1,7 @@+## 0.1.9++Add Data.Streaming.ByteString.Builder+ ## 0.1.8 Generalise types of run\*Server which never cleanly return [#13](https://github.com/fpco/streaming-commons/pull/13)
+ Data/Streaming/ByteString/Builder.hs view
@@ -0,0 +1,82 @@+-- | Provides @toByteStringIO*@ like "Blaze.ByteString.Builder"s, for "Data.ByteString.Builder".+--+-- Since 0.1.9+module Data.Streaming.ByteString.Builder+ ( toByteStringIO+ , toByteStringIOWith+ , toByteStringIOWithBuffer )+ where++import Control.Monad (when)+import Data.ByteString.Builder (Builder)+import Data.ByteString.Builder.Extra (runBuilder, BufferWriter, Next(Done, More, Chunk))+import Data.ByteString.Internal (mallocByteString, ByteString(PS))+import Data.ByteString.Lazy.Internal (defaultChunkSize)+import Data.Word (Word8)+import Foreign.ForeignPtr (ForeignPtr, withForeignPtr)++-- | Use a pre-existing buffer to 'toByteStringIOWith'.+--+-- Since 0.1.9+toByteStringIOWithBuffer :: Int+ -> (ByteString -> IO ())+ -> Builder+ -> ForeignPtr Word8+ -> IO ()+toByteStringIOWithBuffer initBufSize io b initBuf = do+ go initBufSize initBuf (runBuilder b)+ where+ go bufSize buf = loop+ where+ loop :: BufferWriter -> IO ()+ loop wr = do+ (len, next) <- withForeignPtr buf (flip wr bufSize)+ when (len > 0) (io (PS buf 0 len))+ case next of+ Done -> return ()+ More newBufSize nextWr+ | newBufSize > bufSize -> do+ newBuf <- mallocByteString newBufSize+ go newBufSize newBuf nextWr+ | otherwise -> loop nextWr+ Chunk s nextWr -> do+ io s+ loop nextWr++-- | @toByteStringIOWith bufSize io b@ runs the builder @b@ with a buffer of+-- at least the size @bufSize@ and executes the 'IO' action @io@ whenever the+-- buffer is full.+--+-- Compared to 'toLazyByteStringWith' this function requires less allocation,+-- as the output buffer is only allocated once at the start of the+-- serialization and whenever something bigger than the current buffer size has+-- to be copied into the buffer, which should happen very seldomly for the+-- default buffer size of 32kb. Hence, the pressure on the garbage collector is+-- reduced, which can be an advantage when building long sequences of bytes.+--+-- Since 0.1.9+--+toByteStringIOWith :: Int -- ^ Buffer size (upper bounds+ -- the number of bytes forced+ -- per call to the 'IO' action).+ -> (ByteString -> IO ()) -- ^ 'IO' action to execute per+ -- full buffer, which is+ -- referenced by a strict+ -- 'S.ByteString'.+ -> Builder -- ^ 'Builder' to run.+ -> IO ()+toByteStringIOWith bufSize io b =+ toByteStringIOWithBuffer bufSize io b =<< mallocByteString bufSize++-- | Run the builder with a 'defaultChunkSize'd buffer and execute the given+-- 'IO' action whenever the buffer is full or gets flushed.+--+-- @ 'toByteStringIO' = 'toByteStringIOWith' 'defaultChunkSize'@+--+-- Since 0.1.9+--+toByteStringIO :: (ByteString -> IO ())+ -> Builder+ -> IO ()+toByteStringIO = toByteStringIOWith defaultChunkSize+{-# INLINE toByteStringIO #-}
+ bench/builder-to-bytestring-io.hs view
@@ -0,0 +1,39 @@+{-# LANGUAGE ScopedTypeVariables #-}++import qualified Blaze.ByteString.Builder as ZB+import Criterion.Main+import qualified Data.ByteString.Char8 as S+import qualified Data.ByteString.Builder as BB+import Data.Monoid (mconcat, Monoid)+import qualified Data.Streaming.ByteString.Builder as BB++main :: IO ()+main = defaultMain [ bgroup "Data.Streaming.ByteString.Builder.toByteStringIO"+ (benchmarks bIO b100_10000 b10000_100 b10000_10000)+ , bgroup "Blaze.ByteString.Builder.toByteStringIO"+ (benchmarks zIO z100_10000 z10000_100 z10000_10000)+ , bgroup "Data.ByteString.Builder.toLazyByteString"+ (benchmarks bLazy b100_10000 b10000_100 b10000_10000)+ , bgroup "Blaze.ByteString.Builder.toLazyByteString"+ (benchmarks zLazy z100_10000 z10000_100 z10000_10000)+ ]+ where+ bIO = whnfIO . BB.toByteStringIO (const (return ()))+ zIO = whnfIO . ZB.toByteStringIO (const (return ()))+ bLazy = nf BB.toLazyByteString+ zLazy = nf ZB.toLazyByteString+ benchmarks run bld100_10000 bld10000_100 bld10000_10000 =+ [ bench' run bld100_10000 100 10000+ , bench' run bld10000_100 10000 100+ , bench' run bld10000_10000 10000 10000+ ]+ bench' :: (b -> Benchmarkable) -> b -> Int -> Int -> Benchmark+ bench' run bld' len reps = bench (show len ++ "/" ++ show reps) (run bld')+ b100_10000 = bld BB.byteString 100 10000+ b10000_100 = bld BB.byteString 10000 100+ b10000_10000 = bld BB.byteString 10000 10000+ z100_10000 = bld ZB.fromByteString 100 10000+ z10000_100 = bld ZB.fromByteString 10000 100+ z10000_10000 = bld ZB.fromByteString 10000 10000+ bld :: Monoid a => (S.ByteString -> a) -> Int -> Int -> a+ bld f len reps = mconcat (replicate reps (f (S.replicate len 'x')))
streaming-commons.cabal view
@@ -1,5 +1,5 @@ name: streaming-commons-version: 0.1.8.1+version: 0.1.9 synopsis: Common lower-level functions needed by various streaming data libraries description: Provides low-dependency functionality commonly needed by various streaming data libraries, such as conduit and pipes. homepage: https://github.com/fpco/streaming-commons@@ -20,8 +20,12 @@ ChangeLog.md README.md +flag use-bytestring-builder+ description: Use bytestring-builder package+ library exposed-modules: Data.Streaming.Blaze+ Data.Streaming.ByteString.Builder Data.Streaming.FileRead Data.Streaming.Filesystem Data.Streaming.Network@@ -65,12 +69,19 @@ else build-depends: unix + if flag(use-bytestring-builder)+ build-depends: bytestring < 0.10.2.0+ , bytestring-builder+ else+ build-depends: bytestring >= 0.10.2.0+ test-suite test hs-source-dirs: test main-is: Spec.hs type: exitcode-stdio-1.0 ghc-options: -Wall -threaded- other-modules: Data.Streaming.FileReadSpec+ other-modules: Data.Streaming.ByteString.BuilderSpec+ Data.Streaming.FileReadSpec Data.Streaming.FilesystemSpec Data.Streaming.NetworkSpec Data.Streaming.TextSpec@@ -89,6 +100,12 @@ , text , zlib + if flag(use-bytestring-builder)+ build-depends: bytestring < 0.10.2.0+ , bytestring-builder+ else+ build-depends: bytestring >= 0.10.2.0+ if os(windows) cpp-options: -DWINDOWS else@@ -112,6 +129,24 @@ , text main-is: decode-memory-usage.hs ghc-options: -Wall -O2 -with-rtsopts=-s++benchmark builder-to-bytestring-io+ type: exitcode-stdio-1.0+ hs-source-dirs: bench+ main-is: builder-to-bytestring-io.hs+ ghc-options: -Wall -O2+ build-depends: base+ , blaze-builder+ , bytestring+ , criterion+ , deepseq+ , streaming-commons++ if flag(use-bytestring-builder)+ build-depends: bytestring < 0.10.2.0+ , bytestring-builder+ else+ build-depends: bytestring >= 0.10.2.0 source-repository head type: git
+ test/Data/Streaming/ByteString/BuilderSpec.hs view
@@ -0,0 +1,26 @@+{-# LANGUAGE OverloadedStrings #-}+module Data.Streaming.ByteString.BuilderSpec (spec) where++import qualified Data.ByteString.Char8 as S+import qualified Data.ByteString.Unsafe as S+import qualified Data.ByteString.Builder as B+import qualified Data.ByteString.Lazy.Char8 as L+import Data.IORef+import Data.Monoid+import Test.Hspec+import Test.Hspec.QuickCheck (prop)++import Data.Streaming.ByteString.Builder++spec :: Spec+spec =+ describe "Data.Streaming.ByteString.Builder" $ do+ prop "toByteStringIO idempotent to toLazyByteString" $ \bss' -> do+ let bss = mconcat (map (B.byteString . S.pack) bss')+ ior <- newIORef []+ toByteStringIOWith 16+ (\s -> do s' <- S.useAsCStringLen s S.unsafePackCStringLen+ modifyIORef ior (s' :))+ bss+ chunks <- readIORef ior+ L.fromChunks (reverse chunks) `shouldBe` B.toLazyByteString bss