streamly-bytestring 0.1.4 → 0.2.0
raw patch · 7 files changed
+187/−123 lines, 7 filesdep +QuickCheckdep +streamly-coredep −base-compatdep −streamlydep −transformersdep ~bytestringnew-uploader
Dependencies added: QuickCheck, streamly-core
Dependencies removed: base-compat, streamly, transformers
Dependency ranges changed: bytestring
Files
- Changelog.md +6/−0
- README.md +13/−11
- benchmark/Main.hs +17/−16
- src/Streamly/External/ByteString.hs +70/−28
- src/Streamly/External/ByteString/Lazy.hs +44/−26
- streamly-bytestring.cabal +18/−23
- test/Main.hs +19/−19
Changelog.md view
@@ -1,5 +1,11 @@ # Changelog for streamly-bytestring +## 0.2.0 (Mar 2023)++* Support streamly-core-0.1.0+* Support bytestring >= 0.3.0+* Rename read to reader+ ## 0.1.4 (Dec 2021) * Support streamly-0.8.1
README.md view
@@ -2,8 +2,8 @@ Library for streamly and bytestring interoperation. -If you are writing code from scratch, please use `Streamly.Data.Array.Foreign`-which is a generalization of `ByteString` and better integrated with streamly.+If you are writing code from scratch, please use `Streamly.Data.Array` which is+a generalization of `ByteString` and better integrated with streamly. This library is to enable interoperation of streamly with existing code that uses `ByteString`.@@ -13,15 +13,16 @@ Word8`. The interconversion in the case of strict `Bytestring` and streamly `Array-Word8` has no overhead as the underlying representation of ByteString and Array-are the same, we just need to rewrap the data in a different type.+Word8` has no overhead for GHC allocated memory. For foreign allocator allocated+memory there is a copy involved. ## Usage This is a dumb program that counts the number of bytes in a file. -```-import qualified Streamly.Prelude as S+```haskell+import qualified Streamly.Data.Stream as S+import qualified Streamly.Data.Fold as FL import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BSL@@ -32,13 +33,14 @@ import System.IO (FilePath) strictByteStringSize :: BS.ByteString -> IO Int-strictByteStringSize bs = S.length $ S.unfold Strict.read bs+strictByteStringSize bs = S.fold FL.length $ S.unfold Strict.reader bs lazyByteStringSize :: BSL.ByteString -> IO Int-lazyByteStringSize bsl = S.foldl' (+) 0- $ S.mapM strictByteStringSize- $ S.map Strict.fromArray- $ Lazy.toChunks bsl+lazyByteStringSize bsl =+ S.fold (FL.foldl' (+) 0)+ $ S.mapM strictByteStringSize+ $ fmap Strict.fromArray+ $ Lazy.toChunks bsl fileSize :: FilePath -> IO Int fileSize path = do
benchmark/Main.hs view
@@ -4,7 +4,8 @@ import qualified Streamly.External.ByteString.Lazy as Lazy import Control.Monad.IO.Class (MonadIO) -import qualified Streamly.Prelude as S+import qualified Streamly.Data.Stream as S+import qualified Streamly.Data.Fold as Fold import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BSL @@ -31,45 +32,45 @@ fromChunks :: Monad m => Int -> m BSL.ByteString fromChunks n = Lazy.fromChunks $- S.map Strict.toArray $- S.map BS.singleton $- S.map fromIntegral $- S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numChunks)+ fmap Strict.toArray $+ fmap BS.singleton $+ fmap fromIntegral $+ fmap (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numChunks) {-# INLINE fromChunksIO #-} fromChunksIO :: Int -> IO BSL.ByteString fromChunksIO n = Lazy.fromChunksIO $- S.map Strict.toArray $- S.map BS.singleton $- S.map fromIntegral $- S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numChunks)+ fmap Strict.toArray $+ fmap BS.singleton $+ fmap fromIntegral $+ fmap (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numChunks) {-# INLINE toChunks #-} toChunks :: Monad m => BSL.ByteString -> m ()-toChunks = S.drain . Lazy.toChunks+toChunks = S.fold Fold.drain . Lazy.toChunks {-# INLINE strictWrite #-} strictWrite :: MonadIO m => Int -> m BS.ByteString strictWrite n = S.fold Strict.write $- S.map fromIntegral $- S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numElements)+ fmap fromIntegral $+ fmap (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numElements) {-# INLINE strictWriteN #-} strictWriteN :: MonadIO m => Int -> m BS.ByteString strictWriteN n = S.fold (Strict.writeN numElements) $- S.map fromIntegral $- S.map (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numElements)+ fmap fromIntegral $+ fmap (\x -> x `mod` 256) $ S.enumerateFromTo n (n + numElements) {-# INLINE strictRead #-} strictRead :: MonadIO m => BS.ByteString -> m ()-strictRead = S.drain . S.unfold Strict.read+strictRead = S.fold Fold.drain . S.unfold Strict.reader {-# INLINE lazyRead #-} lazyRead :: MonadIO m => BSL.ByteString -> m ()-lazyRead = S.drain . S.unfold Lazy.read+lazyRead = S.fold Fold.drain . S.unfold Lazy.reader main :: IO () main =
src/Streamly/External/ByteString.hs view
@@ -6,47 +6,80 @@ ( toArray , fromArray - , read+ , reader+ , writeN , write++ -- Deprecated+ , read ) where import Control.Monad.IO.Class (MonadIO) import Data.Word (Word8)-import GHC.ForeignPtr (ForeignPtr(..))-import GHC.Ptr (Ptr(..), minusPtr, nullPtr, plusPtr)-import Streamly.Data.Unfold (Unfold, lmap)+import Foreign.ForeignPtr (withForeignPtr)+import Foreign.Storable (peek)+import GHC.Exts+ ( Addr#+ , MutableByteArray#+ , RealWorld+ , byteArrayContents#+ , minusAddr#+ , plusAddr#+ , unsafeCoerce#+ )+import GHC.ForeignPtr (ForeignPtr(..), ForeignPtrContents(..))+import GHC.Int (Int(..))+import GHC.Ptr (Ptr(..), nullPtr, plusPtr) import Streamly.Data.Fold (Fold)+import Streamly.Data.Unfold (Unfold, lmap) -- Internal imports import Data.ByteString.Internal (ByteString(..))-import Streamly.Internal.Data.Array.Foreign.Type (Array(..))-import Streamly.Internal.Data.Array.Foreign.Mut.Type- (ArrayContents, arrayToFptrContents, fptrToArrayContents, nilArrayContents)+import Streamly.Internal.Data.Array.Type (Array(..))+import Streamly.Internal.Data.Unboxed (MutableByteArray(..))+import Streamly.Internal.System.IO (unsafeInlineIO) -import qualified Streamly.Data.Array.Foreign as A+import qualified Streamly.Data.Array as Array+import qualified Streamly.Internal.Data.Unfold as Unfold (fold, mkUnfoldrM)+import qualified Streamly.Internal.Data.Unboxed as Unboxed (nil)+import qualified Streamly.Internal.Data.Stream.StreamD as StreamD (Step(Yield)) import Prelude hiding (read) +{-# INLINE mutableByteArrayContents# #-}+mutableByteArrayContents# :: MutableByteArray# RealWorld -> Addr#+mutableByteArrayContents# marr# = byteArrayContents# (unsafeCoerce# marr#)+ -- | Helper function that creates a ForeignPtr-makeForeignPtr :: ArrayContents -> Ptr a -> ForeignPtr a-makeForeignPtr contents (Ptr addr#) =- ForeignPtr addr# (arrayToFptrContents contents)+{-# INLINE makeForeignPtr #-}+makeForeignPtr :: MutableByteArray -> Int -> ForeignPtr a+makeForeignPtr (MutableByteArray marr#) (I# off#) =+ ForeignPtr+ (mutableByteArrayContents# marr# `plusAddr#` off#)+ (PlainPtr marr#) --- | Convert a 'ByteString' to an array of 'Word8'. This function unwraps the--- 'ByteString' and wraps it with 'Array' constructors and hence the operation--- is performed in constant time.+-- | Convert a 'ByteString' to an array of 'Word8'. It can be done in constant+-- time only for GHC allocated memory. For foreign allocator allocated memory+-- there is a copy involved. {-# INLINE toArray #-} toArray :: ByteString -> Array Word8-toArray (PS (ForeignPtr addr# _) _ _)- | Ptr addr# == nullPtr = Array nilArrayContents nullPtr nullPtr-toArray (PS (ForeignPtr addr# fpcontents) off len) =- Array (fptrToArrayContents fpcontents) startPtr endPtr- where- startPtr = Ptr addr# `plusPtr` off- endPtr = startPtr `plusPtr` len+toArray (BS (ForeignPtr addr# _) _)+ | Ptr addr# == nullPtr = Array Unboxed.nil 0 0+toArray (BS (ForeignPtr addr# (PlainPtr marr#)) len) =+ let off = I# (addr# `minusAddr#` mutableByteArrayContents# marr#)+ in Array (MutableByteArray marr#) off (off + len)+toArray (BS fptr len) =+ unsafeInlineIO+ $ withForeignPtr fptr $ Unfold.fold (Array.writeN len) generator + where++ generator =+ Unfold.mkUnfoldrM+ (\ptr -> flip StreamD.Yield (ptr `plusPtr` 1) <$> peek ptr)+ -- | Convert an array of 'Word8' to a 'ByteString'. This function unwraps the -- 'Array' and wraps it with 'ByteString' constructors and hence the operation -- is performed in constant time.@@ -54,21 +87,30 @@ fromArray :: Array Word8 -> ByteString fromArray Array {..} | aLen == 0 = mempty- | otherwise = PS (makeForeignPtr arrContents arrStart) 0 aLen+ | otherwise = BS (makeForeignPtr arrContents arrStart) aLen where- aLen = aEnd `minusPtr` arrStart+ aLen = arrEnd - arrStart -- | Unfold a strict ByteString to a stream of Word8.-{-# INLINE read #-}-read :: Monad m => Unfold m ByteString Word8-read = lmap toArray A.read+{-# INLINE reader #-}+reader :: Monad m => Unfold m ByteString Word8+reader = lmap toArray Array.reader -- | Fold a stream of Word8 to a strict ByteString of given size in bytes. {-# INLINE writeN #-} writeN :: MonadIO m => Int -> Fold m Word8 ByteString-writeN i = fromArray <$> A.writeN i+writeN i = fromArray <$> Array.writeN i -- | Fold a stream of Word8 to a strict ByteString of appropriate size. {-# INLINE write #-} write :: MonadIO m => Fold m Word8 ByteString-write = fromArray <$> A.write+write = fromArray <$> Array.write++--------------------------------------------------------------------------------+-- Deprecated+--------------------------------------------------------------------------------++{-# DEPRECATED read "Please use reader instead." #-}+{-# INLINE read #-}+read :: Monad m => Unfold m ByteString Word8+read = reader
src/Streamly/External/ByteString/Lazy.hs view
@@ -1,17 +1,21 @@ module Streamly.External.ByteString.Lazy- ( readChunks- , read+ ( chunkReader+ , reader , toChunks , fromChunks , fromChunksIO++ -- Deprecated+ , read+ , readChunks ) where import Data.Word (Word8)-import Streamly.Data.Unfold (many)-import Streamly.Data.Array.Foreign (Array)+import Streamly.Data.Array (Array) import System.IO.Unsafe (unsafeInterleaveIO)+import Streamly.Data.Stream (Stream) -- Internal imports import Data.ByteString.Lazy.Internal (ByteString(..), chunk)@@ -19,29 +23,31 @@ import Streamly.Internal.Data.Unfold.Type (Unfold(..)) import qualified Streamly.External.ByteString as Strict-import qualified Streamly.Data.Array.Foreign as A-import qualified Streamly.Prelude as S+import qualified Streamly.Data.Array as Array+import qualified Streamly.Data.Unfold as Unfold+import qualified Streamly.Data.Stream as Stream -import Prelude hiding (concat, read)+import Prelude hiding (read) -- | Unfold a lazy ByteString to a stream of 'Array' 'Words'.-{-# INLINE readChunks #-}-readChunks :: Monad m => Unfold m ByteString (Array Word8)-readChunks = Unfold step seed+{-# INLINE chunkReader #-}+chunkReader :: Monad m => Unfold m ByteString (Array Word8)+chunkReader = Unfold step seed where seed = return step (Chunk bs bl) = return $ Yield (Strict.toArray bs) bl step Empty = return Stop -- | Unfold a lazy ByteString to a stream of Word8-{-# INLINE read #-}-read :: Monad m => Unfold m ByteString Word8-read = many readChunks A.read+{-# INLINE reader #-}+reader :: Monad m => Unfold m ByteString Word8+reader = Unfold.many Array.reader readChunks +-- TODO: "toChunks" should be called "read" instead -- | Convert a lazy 'ByteString' to a serial stream of 'Array' 'Word8'. {-# INLINE toChunks #-}-toChunks :: Monad m => ByteString -> S.SerialT m (Array Word8)-toChunks = S.unfold readChunks+toChunks :: Monad m => ByteString -> Stream m (Array Word8)+toChunks = Stream.unfold readChunks {- newtype LazyIO a = LazyIO { runLazy :: IO a } deriving (Functor, Applicative)@@ -80,21 +86,33 @@ -- /fromChunks/ can then be used as, -- @ -- {-# INLINE fromChunksIO #-}--- fromChunksIO :: SerialT IO (Array Word8) -> IO ByteString--- fromChunksIO str = runLazy (fromChunks (S.hoist liftToLazy str))+-- fromChunksIO :: Stream IO (Array Word8) -> IO ByteString+-- fromChunksIO str = runLazy (fromChunks (Stream.hoist liftToLazy str)) -- @ {-# INLINE fromChunks #-}-fromChunks :: Monad m => S.SerialT m (Array Word8) -> m ByteString-fromChunks = S.foldr chunk Empty . S.map Strict.fromArray+fromChunks :: Monad m => Stream m (Array Word8) -> m ByteString+fromChunks = Stream.foldr chunk Empty . fmap Strict.fromArray -- | Convert a serial stream of 'Array' 'Word8' to a lazy 'ByteString' in the -- /IO/ monad. {-# INLINE fromChunksIO #-}-fromChunksIO :: S.SerialT IO (Array Word8) -> IO ByteString+fromChunksIO :: Stream IO (Array Word8) -> IO ByteString fromChunksIO =--- Although the /IO/ monad is strict in nature we emulate laziness using--- 'unsafeInterleaveIO'.- S.foldrM- (\x b -> chunk x <$> unsafeInterleaveIO b)- (pure Empty) .- S.map Strict.fromArray+ -- Although the /IO/ monad is strict in nature we emulate laziness using+ -- 'unsafeInterleaveIO'.+ Stream.foldrM (\x b -> chunk x <$> unsafeInterleaveIO b) (pure Empty)+ . fmap Strict.fromArray++--------------------------------------------------------------------------------+-- Deprecated+--------------------------------------------------------------------------------++{-# DEPRECATED readChunks "Please use chunkReader instead." #-}+{-# INLINE readChunks #-}+readChunks :: Monad m => Unfold m ByteString (Array Word8)+readChunks = chunkReader++{-# DEPRECATED read "Please use reader instead." #-}+{-# INLINE read #-}+read :: Monad m => Unfold m ByteString Word8+read = reader
streamly-bytestring.cabal view
@@ -1,6 +1,6 @@ cabal-version: 1.12 name: streamly-bytestring-version: 0.1.4+version: 0.2.0 synopsis: Library for streamly and bytestring interoperation. description: Please see the README on GitHub at <https://github.com/psibi/streamly-bytestring#readme> category: Streamly, Stream, ByteString@@ -15,6 +15,12 @@ extra-source-files: README.md Changelog.md+tested-with: GHC==8.6.5+ , GHC==8.8.4+ , GHC==8.10.7+ , GHC==9.0.1+ , GHC==9.2.7+ , GHC==9.4.4 source-repository head type: git@@ -31,14 +37,14 @@ ghc-options: -Wall -O2 build-depends: base >=4.7 && <5- , bytestring >=0.10.0 && <=0.11.1.0- , streamly >= 0.8.1 && < 0.8.2- if impl(ghc < 8.1)- build-depends:- base-compat >=0.11- if impl(ghc < 8)- build-depends:- transformers >=0.4+ , bytestring == 0.11.0.*+ || == 0.11.1.*+ || == 0.11.2.*+ -- bytestring-0.11.3.0 causes panics and other issues on+ -- windows.+ || ( >= 0.11.3.1 && < 0.11.4 )+ || == 0.11.4.*+ , streamly-core == 0.1.0.* default-language: Haskell2010 test-suite sb-test@@ -58,15 +64,10 @@ , hspec-discover , quickcheck-instances , random- , streamly+ , streamly-core , streamly-bytestring , temporary- if impl(ghc < 8.1)- build-depends:- base-compat >=0.11- if impl(ghc < 8)- build-depends:- transformers >=0.4+ , QuickCheck default-language: Haskell2010 benchmark sb-benchmark@@ -83,12 +84,6 @@ , deepseq , gauge , random- , streamly+ , streamly-core , streamly-bytestring- if impl(ghc < 8.1)- build-depends:- base-compat >=0.11- if impl(ghc < 8)- build-depends:- transformers >=0.4 default-language: Haskell2010
test/Main.hs view
@@ -5,9 +5,8 @@ import Data.ByteString (ByteString) import Data.Word (Word8) import GHC.IO.Handle (Handle)-import GHC.Ptr (minusPtr) import System.Random (randomIO)-import Streamly.FileSystem.Handle (readChunks)+import Streamly.FileSystem.Handle (chunkReader) import System.IO (openFile, IOMode(ReadMode)) import System.IO.Temp (withSystemTempFile) import Test.Hspec@@ -15,31 +14,33 @@ import Test.QuickCheck.Instances.ByteString () -- Internal imports-import Streamly.Internal.Data.Array.Foreign.Type (Array(..))+import Streamly.Internal.Data.Array.Type (Array(..)) import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BSL import qualified Streamly.External.ByteString as Strict import qualified Streamly.External.ByteString.Lazy as Lazy-import qualified Streamly.Prelude as S+import qualified Streamly.Data.Stream as S+import qualified Streamly.Data.Fold as Fold -streamToByteString :: S.MonadAsync m => S.SerialT m (Array Word8) -> m ByteString-streamToByteString stream = S.foldl' (<>) mempty $ S.map Strict.fromArray stream+streamToByteString :: Monad m => S.Stream m (Array Word8) -> m ByteString+streamToByteString stream =+ S.fold (Fold.foldl' (<>) mempty) $ fmap Strict.fromArray stream checkFileContent :: FilePath -> Handle -> IO () checkFileContent filename handle' = do print $ "Checking " <> filename bsContent <- BS.hGetContents handle' handle <- openFile filename ReadMode- bsStreamly <- streamToByteString $ S.unfold readChunks handle+ bsStreamly <- streamToByteString $ S.unfold chunkReader handle bsContent `shouldBe` bsStreamly word8List :: Int -> IO [Word8] word8List l =- S.toList $+ S.fold Fold.toList $ S.take l $- S.map fromIntegral $- S.map (\x -> abs x `mod` 256) $ S.repeatM (randomIO :: IO Int)+ fmap fromIntegral $+ fmap (\x -> abs x `mod` 256) $ S.repeatM (randomIO :: IO Int) propFoldTestStrict :: [Word8] -> Spec propFoldTestStrict bl =@@ -62,7 +63,7 @@ prop ("Strict: toList . unfold read . pack = id" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do- bl' <- S.toList (S.unfold Strict.read (BS.pack bl))+ bl' <- S.fold Fold.toList (S.unfold Strict.reader (BS.pack bl)) bl' `shouldBe` bl propUnfoldTestLazy :: [Word8] -> Spec@@ -70,20 +71,20 @@ prop ("Lazy: toList . unfold read . pack = id" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do- bl' <- S.toList (S.unfold Lazy.read (BSL.pack bl))+ bl' <- S.fold Fold.toList (S.unfold Lazy.reader (BSL.pack bl)) bl' `shouldBe` bl propFromChunks :: Spec propFromChunks = prop "Lazy.fromChunks = BSL.fromChunks" $ \aL -> do- x1 <- Lazy.fromChunks $ S.map Strict.toArray $ S.fromList aL+ x1 <- Lazy.fromChunks $ fmap Strict.toArray $ S.fromList aL let x2 = BSL.fromChunks aL x1 `shouldBe` x2 propFromChunksIO :: Spec propFromChunksIO = prop "Lazy.fromChunks = BSL.fromChunks" $ \aL -> do- x1 <- Lazy.fromChunksIO $ S.map Strict.toArray $ S.fromList aL+ x1 <- Lazy.fromChunksIO $ fmap Strict.toArray $ S.fromList aL let x2 = BSL.fromChunks aL x1 `shouldBe` x2 @@ -94,7 +95,6 @@ S.fromList (Strict.toArray (BS.singleton h) : undefined) return $ BSL.head lbs - main :: IO () main = hspec $ do@@ -106,12 +106,12 @@ prop "Strict Identity" $ \bs -> bs `shouldBe` Strict.fromArray (Strict.toArray bs) prop "Strict Identity (with offset)" $ \bs ->- let bs' = BS.drop 5 bs- in bs' `shouldBe` Strict.fromArray (Strict.toArray bs')+ let bs1 = BS.drop 5 bs+ in bs1 `shouldBe` Strict.fromArray (Strict.toArray bs1) prop "toArray never produces negative length" $ \bs -> -- 'BS.drop 5' to trigger non-zero offset- let (Array _ sPtr ePtr) = Strict.toArray (BS.drop 5 bs)- in (ePtr `minusPtr` sPtr) >= 0 `shouldBe` True+ let (Array _ startI endI) = Strict.toArray (BS.drop 5 bs)+ in (endI - startI) >= 0 `shouldBe` True prop "Lazy Identity" $ \bs -> do bs2 <- Lazy.fromChunks . Lazy.toChunks $ bs bs `shouldBe` bs2