streamly-bytestring 0.1.0.1 → 0.1.1
raw patch · 7 files changed
+194/−128 lines, 7 filesPVP ok
version bump matches the API change (PVP)
API changes (from Hackage documentation)
+ Streamly.External.ByteString.Lazy: fromChunksIO :: SerialT IO (Array Word8) -> IO ByteString
Files
- Changelog.md +3/−1
- README.md +13/−39
- benchmark/Main.hs +25/−19
- src/Streamly/External/ByteString.hs +4/−4
- src/Streamly/External/ByteString/Lazy.hs +59/−3
- streamly-bytestring.cabal +3/−3
- test/Main.hs +87/−59
Changelog.md view
@@ -1,6 +1,8 @@ # Changelog for streamly-bytestring -## Unreleased+## 0.1.1++* Add `fromChunksIO` function to `Streamly.External.ByteString.Lazy` module. ## 0.1.0.1
README.md view
@@ -1,35 +1,19 @@-# `streamly-bytestring`+# streamly-bytestring Library for streamly and bytestring interoperation. -## Description--This package provides `Streamly.External.ByteString` and-`Streamly.External.ByteString.Lazy`.--### Strict ByteString--`Streamly.External.ByteString` provides functions to for-interoperation between streamly and strict bytestring.--`fromArray` and `toArray` are used to efficiently convert between-streamly's pinned array type (`Streamly.Memory.Array`) and bytestring.--`read`, `writeN` and `write` are `Unfold`s & `Fold`s provided by streamly-that are used to create and consume a stream of `Word8`. `writeN` is more-efficient than `write` and should be preferred over `write` when possible.--### Lazy Bytestring+If you are writing code from scratch, please use `Streamly.Memory.Array` which+is a generalization of `ByteString` and better integrated with streamly. -`Streamly.External.ByteString.Lazy` provides functions to for-interoperation between streamly and lazy bytestring.+This library is to enable interoperation of streamly with existing code that+uses `ByteString`. -`readChunks` and `read` are `Unfold`s. `unfold` from `Streamly.Prelude` can be-used to create a stream of `Array Word8` or a stream of `Word8` from a-lazy `ByteString`.+The package provides APIs to interconvert between strict `Bytestring` and+streamly `Array Word8` and between lazy `Bytestring` and stream of `Array Word8`. -`toChunks` is defined as `unfold readChunks`. `fromChunks` can be used to create a-lazy `ByteString` from a stream of `Array Word8` chunks.+The interconversion in the case of strict `Bytestring` and streamly `Array+Word8` has no overhead as the underlying representation of ByteString and Array+are the same, we just need to rewrap the data in a different type. ## Usage @@ -52,22 +36,12 @@ lazyByteStringSize :: BSL.ByteString -> IO Int lazyByteStringSize bsl = S.foldl' (+) 0- $ S.mapM strictByteStringSize- $ S.map Strict.fromArray- $ Lazy.toChunks bsl+ $ S.mapM strictByteStringSize+ $ S.map Strict.fromArray+ $ Lazy.toChunks bsl fileSize :: FilePath -> IO Int fileSize path = do bsl <- BSL.readFile path lazyByteStringSize bsl ```----------
benchmark/Main.hs view
@@ -21,24 +21,30 @@ {-# INLINE benchIO #-} benchIO :: NFData b => String -> (Int -> IO a) -> (a -> b) -> Benchmark-benchIO name src f = bench name $ nfIO $- randomRIO (1,1) >>= src >>= return . f+benchIO name src f = bench name $ nfIO $ randomRIO (1, 1) >>= src >>= return . f {-# INLINE benchIO' #-} benchIO' :: NFData b => String -> (Int -> IO a) -> (a -> IO b) -> Benchmark-benchIO' name src f = bench name $ nfIO $- randomRIO (1,1) >>= src >>= f+benchIO' name src f = bench name $ nfIO $ randomRIO (1, 1) >>= src >>= f {-# INLINE fromChunks #-} fromChunks :: Monad m => Int -> m BSL.ByteString fromChunks n =- Lazy.fromChunks- $ S.map Strict.toArray- $ S.map BS.singleton- $ S.map fromIntegral- $ S.map (\x -> x `mod` 256)- $ S.enumerateFromTo n (n + numChunks)+ Lazy.fromChunks $+ S.map Strict.toArray $+ S.map BS.singleton $+ S.map fromIntegral $+ S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numChunks) +{-# INLINE fromChunksIO #-}+fromChunksIO :: Int -> IO BSL.ByteString+fromChunksIO n =+ Lazy.fromChunksIO $+ S.map Strict.toArray $+ S.map BS.singleton $+ S.map fromIntegral $+ S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numChunks)+ {-# INLINE toChunks #-} toChunks :: Monad m => BSL.ByteString -> m () toChunks = S.drain . Lazy.toChunks@@ -46,18 +52,16 @@ {-# INLINE strictWrite #-} strictWrite :: MonadIO m => Int -> m BS.ByteString strictWrite n =- S.fold Strict.write- $ S.map fromIntegral- $ S.map (\x -> x `mod` 256)- $ S.enumerateFromTo n (n + numElements)+ S.fold Strict.write $+ S.map fromIntegral $+ S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numElements) {-# INLINE strictWriteN #-} strictWriteN :: MonadIO m => Int -> m BS.ByteString strictWriteN n =- S.fold (Strict.writeN numElements)- $ S.map fromIntegral- $ S.map (\x -> x `mod` 256)- $ S.enumerateFromTo n (n + numElements)+ S.fold (Strict.writeN numElements) $+ S.map fromIntegral $+ S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numElements) {-# INLINE strictRead #-} strictRead :: MonadIO m => BS.ByteString -> m ()@@ -68,11 +72,13 @@ lazyRead = S.drain . S.unfold Lazy.read main :: IO ()-main = defaultMain+main =+ defaultMain [ benchIO "Strict Write" strictWrite id , benchIO "Strict WriteN" strictWriteN id , benchIO' "Strict Read" strictWrite strictRead , benchIO' "Lazy Read" fromChunks lazyRead , benchIO "fromChunks" fromChunks id+ , benchIO "fromChunksIO" fromChunksIO id , benchIO' "toChunks" fromChunks toChunks ]
src/Streamly/External/ByteString.hs view
@@ -36,7 +36,7 @@ {-# INLINE toArray #-} toArray :: ByteString -> Array Word8 toArray (PS fp off len) = Array nfp endPtr endPtr- where+ where nfp = fp `plusForeignPtr` off endPtr = unsafeForeignPtrToPtr fp `plusPtr` len @@ -48,9 +48,9 @@ fromArray Array {..} | aLen == 0 = mempty | otherwise = PS aStart 0 aLen- where- aStartPtr = unsafeForeignPtrToPtr aStart- aLen = aEnd `minusPtr` aStartPtr+ where+ aStartPtr = unsafeForeignPtrToPtr aStart+ aLen = aEnd `minusPtr` aStartPtr -- | Unfold a strict ByteString to a stream of Word8. {-# INLINE read #-}
src/Streamly/External/ByteString/Lazy.hs view
@@ -1,9 +1,12 @@+{-# LANGUAGE GeneralizedNewtypeDeriving #-}+ module Streamly.External.ByteString.Lazy ( readChunks , read- + , toChunks , fromChunks+ , fromChunksIO ) where @@ -17,8 +20,10 @@ import qualified Streamly.External.ByteString as Strict import qualified Streamly.Internal.Memory.Array as A +import System.IO.Unsafe (unsafeInterleaveIO)+ import Streamly-import qualified Streamly.Prelude as S+import qualified Streamly.Internal.Prelude as S import qualified Data.ByteString.Lazy.Internal as BSLI @@ -28,7 +33,7 @@ {-# INLINE readChunks #-} readChunks :: Monad m => Unfold m ByteString (Array Word8) readChunks = Unfold step seed- where+ where seed = return step (Chunk bs bl) = return $ Yield (Strict.toArray bs) bl step Empty = return $ Stop@@ -43,7 +48,58 @@ toChunks :: Monad m => ByteString -> SerialT m (Array Word8) toChunks = S.unfold readChunks +{-+newtype LazyIO a = LazyIO { runLazy :: IO a } deriving (Functor, Applicative)++liftToLazy :: IO a -> LazyIO a+liftToLazy = LazyIO++instance Monad LazyIO where+ return = pure+ LazyIO a >>= f = LazyIO (unsafeInterleaveIO a >>= unsafeInterleaveIO . runLazy . f)+-}+ -- | Convert a serial stream of 'Array' 'Word8' to a lazy 'ByteString'.+--+-- IMPORTANT NOTE: This function is lazy only for lazy monads+-- (e.g. Identity). For strict monads (e.g. /IO/) it consumes the entire input+-- before generating the output. For /IO/ monad please use fromChunksIO+-- instead.+--+-- For strict monads like /IO/ you could create a newtype wrapper to make the+-- monad bind operation lazy and lift the stream to that type using hoist, then+-- you can use this function to generate the bytestring lazily. For example you+-- can wrap the /IO/ type to make the bind lazy like this:+--+-- @+-- newtype LazyIO a = LazyIO { runLazy :: IO a } deriving (Functor, Applicative)+--+-- liftToLazy :: IO a -> LazyIO a+-- liftToLazy = LazyIO+--+-- instance Monad LazyIO where+-- return = pure+-- LazyIO a >>= f = LazyIO (unsafeInterleaveIO a >>= unsafeInterleaveIO . runLazy . f)+-- @+--+-- /fromChunks/ can then be used as,+-- @+-- {-# INLINE fromChunksIO #-}+-- fromChunksIO :: SerialT IO (Array Word8) -> IO ByteString+-- fromChunksIO str = runLazy (fromChunks (S.hoist liftToLazy str))+-- @ {-# INLINE fromChunks #-} fromChunks :: Monad m => SerialT m (Array Word8) -> m ByteString fromChunks = S.foldr BSLI.chunk Empty . S.map Strict.fromArray++-- | Convert a serial stream of 'Array' 'Word8' to a lazy 'ByteString' in the+-- /IO/ monad.+{-# INLINE fromChunksIO #-}+fromChunksIO :: SerialT IO (Array Word8) -> IO ByteString+fromChunksIO =+-- Although the /IO/ monad is strict in nature we emulate laziness using+-- 'unsafeInterleaveIO'.+ S.foldrM+ (\x b -> unsafeInterleaveIO b >>= pure . BSLI.chunk x)+ (pure BSLI.Empty) .+ S.map Strict.fromArray
streamly-bytestring.cabal view
@@ -1,13 +1,13 @@ cabal-version: 1.12 --- This file has been generated from package.yaml by hpack version 0.33.0.+-- This file has been generated from package.yaml by hpack version 0.31.2. -- -- see: https://github.com/sol/hpack ----- hash: 025212f67560e558928440828becf41e3f1e28c9e66cb82ac3e415918168f8e3+-- hash: af662f24c97fa6b630e84ef660885ad0de001dc9face27ef4698f5361e4e5484 name: streamly-bytestring-version: 0.1.0.1+version: 0.1.1 synopsis: Library for streamly and bytestring interoperation. description: Please see the README on GitHub at <https://github.com/psibi/streamly-bytestring#readme> category: Streamly, Stream, ByteString
test/Main.hs view
@@ -21,84 +21,112 @@ import qualified Streamly.External.ByteString.Lazy as Lazy import qualified Streamly.Prelude as S -streamToByteString :: MonadAsync m => SerialT m (Array Word8)-> m ByteString+streamToByteString :: MonadAsync m => SerialT m (Array Word8) -> m ByteString streamToByteString stream = S.foldl' (<>) mempty $ S.map Strict.fromArray stream checkFileContent :: FilePath -> Handle -> IO () checkFileContent filename handle' = do- print $ "Checking " <> filename- bsContent <- BS.hGetContents handle'- handle <- openFile filename ReadMode- bsStreamly <- streamToByteString $ S.unfold readChunks handle- bsContent `shouldBe` bsStreamly+ print $ "Checking " <> filename+ bsContent <- BS.hGetContents handle'+ handle <- openFile filename ReadMode+ bsStreamly <- streamToByteString $ S.unfold readChunks handle+ bsContent `shouldBe` bsStreamly word8List :: Int -> IO [Word8]-word8List l = S.toList- $ S.take l- $ S.map fromIntegral- $ S.map (\x -> abs x `mod` 256)- $ S.repeatM (randomIO :: IO Int)+word8List l =+ S.toList $+ S.take l $+ S.map fromIntegral $+ S.map (\x -> abs x `mod` 256) $ S.repeatM (randomIO :: IO Int) propFoldTestStrict :: [Word8] -> Spec propFoldTestStrict bl =- prop ("Strict: fold write . fromList = pack" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do- bl' <- fmap BS.unpack $ S.fold Strict.write $ S.fromList bl- bl' `shouldBe` bl+ prop+ ("Strict: fold write . fromList = pack" +++ " -- Size: " ++ show (length bl) ++ " bytes") $ do+ bl' <- fmap BS.unpack $ S.fold Strict.write $ S.fromList bl+ bl' `shouldBe` bl propNFoldTestStrict :: Int -> [Word8] -> Spec propNFoldTestStrict n bl =- prop ("Strict: fold (writeN n) . fromList = pack . take n" ++ " -- Size: " ++ show n ++ " bytes") $ do- bl' <- fmap BS.unpack $ S.fold (Strict.writeN n) $ S.fromList bl- bl' `shouldBe` take n bl+ prop+ ("Strict: fold (writeN n) . fromList = pack . take n" +++ " -- Size: " ++ show n ++ " bytes") $ do+ bl' <- fmap BS.unpack $ S.fold (Strict.writeN n) $ S.fromList bl+ bl' `shouldBe` take n bl propUnfoldTestStrict :: [Word8] -> Spec propUnfoldTestStrict bl =- prop ("Strict: toList . unfold read . pack = id" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do- bl' <- S.toList (S.unfold Strict.read (BS.pack bl))- bl' `shouldBe` bl+ prop+ ("Strict: toList . unfold read . pack = id" +++ " -- Size: " ++ show (length bl) ++ " bytes") $ do+ bl' <- S.toList (S.unfold Strict.read (BS.pack bl))+ bl' `shouldBe` bl propUnfoldTestLazy :: [Word8] -> Spec propUnfoldTestLazy bl =- prop ("Lazy: toList . unfold read . pack = id" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do- bl' <- S.toList (S.unfold Lazy.read (BSL.pack bl))- bl' `shouldBe` bl+ prop+ ("Lazy: toList . unfold read . pack = id" +++ " -- Size: " ++ show (length bl) ++ " bytes") $ do+ bl' <- S.toList (S.unfold Lazy.read (BSL.pack bl))+ bl' `shouldBe` bl -testFromChunksLaziness :: Word8 -> IO Word8-testFromChunksLaziness h = do- lbs <- Lazy.fromChunks $ S.fromList [Strict.toArray (BS.singleton h), undefined]- return $ BSL.head lbs+propFromChunks :: Spec+propFromChunks =+ prop ("Lazy.fromChunks = BSL.fromChunks") $ \aL -> do+ x1 <- Lazy.fromChunks $ S.map Strict.toArray $ S.fromList aL+ let x2 = BSL.fromChunks aL+ x1 `shouldBe` x2 +propFromChunksIO :: Spec+propFromChunksIO =+ prop ("Lazy.fromChunks = BSL.fromChunks") $ \aL -> do+ x1 <- Lazy.fromChunksIO $ S.map Strict.toArray $ S.fromList aL+ let x2 = BSL.fromChunks aL+ x1 `shouldBe` x2++testFromChunksIOLaziness :: Word8 -> IO Word8+testFromChunksIOLaziness h = do+ lbs <-+ Lazy.fromChunksIO $+ S.fromList (Strict.toArray (BS.singleton h) : undefined)+ return $ BSL.head lbs+ main :: IO () main =- hspec $ do- describe "Array tests" $ do- it "Strict fromArray" $- mapM_ (flip withSystemTempFile checkFileContent . show) ([1..100] :: [Int])- prop "Strict Identity" $ \bs ->- bs `shouldBe` Strict.fromArray (Strict.toArray bs)- prop "Lazy Identity" $ \bs -> do- bs2 <- Lazy.fromChunks . Lazy.toChunks $ bs- bs `shouldBe` bs2- wl0 <- runIO $ word8List 0- wlM <- runIO $ word8List 900- wlL <- runIO $ word8List 73700- describe "Strict: Fold tests" $ do- propFoldTestStrict wl0- propFoldTestStrict wlM- propFoldTestStrict wlL- describe "Strict: N Fold tests" $ do- propNFoldTestStrict 0 wl0- propNFoldTestStrict 900 wlM- propNFoldTestStrict 73700 wlL- describe "Strict: Unfold tests" $ do- propUnfoldTestStrict wl0- propUnfoldTestStrict wlM- propUnfoldTestStrict wlL- describe "Lazy: Unfold tests" $ do- propUnfoldTestLazy wl0- propUnfoldTestLazy wlM- propUnfoldTestLazy wlL- describe "Laziness of fromChunks" $ do- it "Should not fail" $ do- w <- testFromChunksLaziness 100- w `shouldBe` 100+ hspec $ do+ describe "Array tests" $ do+ it "Strict fromArray" $+ mapM_+ (flip withSystemTempFile checkFileContent . show)+ ([1 .. 100] :: [Int])+ prop "Strict Identity" $ \bs ->+ bs `shouldBe` Strict.fromArray (Strict.toArray bs)+ prop "Lazy Identity" $ \bs -> do+ bs2 <- Lazy.fromChunks . Lazy.toChunks $ bs+ bs `shouldBe` bs2+ wl0 <- runIO $ word8List 0+ wlM <- runIO $ word8List 900+ wlL <- runIO $ word8List 73700+ describe "Strict: Fold tests" $ do+ propFoldTestStrict wl0+ propFoldTestStrict wlM+ propFoldTestStrict wlL+ describe "Strict: N Fold tests" $ do+ propNFoldTestStrict 0 wl0+ propNFoldTestStrict 900 wlM+ propNFoldTestStrict 73700 wlL+ describe "Strict: Unfold tests" $ do+ propUnfoldTestStrict wl0+ propUnfoldTestStrict wlM+ propUnfoldTestStrict wlL+ describe "Lazy: Unfold tests" $ do+ propUnfoldTestLazy wl0+ propUnfoldTestLazy wlM+ propUnfoldTestLazy wlL+ describe "Laziness of fromChunksIO" $ do+ it "Should not fail" $ do+ w <- testFromChunksIOLaziness 100+ w `shouldBe` 100+ describe "Correctness of fromChunks" propFromChunks+ describe "Correctness of fromChunksIO" propFromChunksIO