streamly-bytestring (empty) → 0.1.0.0

raw patch · 9 files changed

+482/−0 lines, 9 filesdep +basedep +bytestringdep +deepseqsetup-changed

Dependencies added: base, bytestring, deepseq, directory, filepath, gauge, hspec, hspec-discover, quickcheck-instances, random, streamly, streamly-bytestring, temporary

Files

Changelog.md +0/−0
LICENSE +30/−0
README.md +73/−0
Setup.hs +2/−0
benchmark/Main.hs +78/−0
src/Streamly/External/ByteString.hs +63/−0
src/Streamly/External/ByteString/Lazy.hs +49/−0
streamly-bytestring.cabal +83/−0
test/Main.hs +104/−0

+ Changelog.md view

+ LICENSE view

@@ -0,0 +1,30 @@+Copyright Sibi Prabakaran (c) 2019++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of Sibi Prabakaran nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

+ README.md view

@@ -0,0 +1,73 @@+# `streamly-bytestring`++Library for streamly and bytestring interoperation.++## Description++This package provides `Streamly.External.ByteString` and+`Streamly.External.ByteString.Lazy`.++### Strict ByteString++`Streamly.External.ByteString` provides functions to for+interoperation between streamly and strict bytestring.++`fromArray` and `toArray` are used to efficiently convert between+streamly's pinned array type (`Streamly.Memory.Array`) and bytestring.++`read`, `writeN` and `write` are `Unfold`s & `Fold`s provided by streamly+that are used to create and consume a stream of `Word8`. `writeN` is more+efficient than `write` and should be preferred over `write` when possible.++### Lazy Bytestring++`Streamly.External.ByteString.Lazy` provides functions to for+interoperation between streamly and lazy bytestring.++`readChunks` and `read` are `Unfold`s. `unfold` from `Streamly.Prelude` can be+used to create a stream of `Array Word8` or a stream of `Word8` from a+lazy `ByteString`.++`toChunks` is defined as `unfold readChunks`. `fromChunks` can be used to create a+lazy `ByteString` from a stream of `Array Word8` chunks.++## Usage++This is a dumb program that counts the number of bytes in a file.++```+import Streamly+import qualified Streamly.Prelude as S++import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BSL++import qualified Streamly.External.ByteString as Strict+import qualified Streamly.External.ByteString.Lazy as Lazy++import System.IO (FilePath)++strictByteStringSize :: BS.ByteString -> IO Int+strictByteStringSize bs = S.length $ S.unfold Strict.read bs++lazyByteStringSize :: BSL.ByteString -> IO Int+lazyByteStringSize bsl = S.foldl' (+) 0+		       $ S.mapM strictByteStringSize+		       $ S.map Strict.fromArray+		       $ Lazy.toChunks bsl++fileSize :: FilePath -> IO Int+fileSize path = do+    bsl <- BSL.readFile path+    lazyByteStringSize bsl+```++++++++++

+ Setup.hs view

@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain

+ benchmark/Main.hs view

@@ -0,0 +1,78 @@+module Main where++import qualified Streamly.External.ByteString as Strict+import qualified Streamly.External.ByteString.Lazy as Lazy+import Control.Monad.IO.Class (MonadIO)++import qualified Streamly.Prelude as S+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BSL++import Control.DeepSeq (NFData(..))+import System.Random (randomRIO)++import Gauge++numElements :: Int+numElements = 100000++numChunks :: Int+numChunks = 100000++{-# INLINE benchIO #-}+benchIO :: NFData b => String -> (Int -> IO a) -> (a -> b) -> Benchmark+benchIO name src f = bench name $ nfIO $+    randomRIO (1,1) >>= src >>= return . f++{-# INLINE benchIO' #-}+benchIO' :: NFData b => String -> (Int -> IO a) -> (a -> IO b) -> Benchmark+benchIO' name src f = bench name $ nfIO $+    randomRIO (1,1) >>= src >>= f++{-# INLINE fromChunks #-}+fromChunks :: Monad m => Int -> m BSL.ByteString+fromChunks n =+    Lazy.fromChunks+  $ S.map Strict.toArray+  $ S.map BS.singleton+  $ S.map fromIntegral+  $ S.map (\x -> x `mod` 256)+  $ S.enumerateFromTo n (n + numChunks)++{-# INLINE toChunks #-}+toChunks :: Monad m => BSL.ByteString -> m ()+toChunks = S.drain . Lazy.toChunks++{-# INLINE strictWrite #-}+strictWrite :: MonadIO m => Int -> m BS.ByteString+strictWrite n =+    S.fold Strict.write+  $ S.map fromIntegral+  $ S.map (\x -> x `mod` 256)+  $ S.enumerateFromTo n (n + numElements)++{-# INLINE strictWriteN #-}+strictWriteN :: MonadIO m => Int -> m BS.ByteString+strictWriteN n =+    S.fold (Strict.writeN numElements)+  $ S.map fromIntegral+  $ S.map (\x -> x `mod` 256)+  $ S.enumerateFromTo n (n + numElements)++{-# INLINE strictRead #-}+strictRead :: MonadIO m => BS.ByteString -> m ()+strictRead = S.drain . S.unfold Strict.read++{-# INLINE lazyRead #-}+lazyRead :: MonadIO m => BSL.ByteString -> m ()+lazyRead = S.drain . S.unfold Lazy.read++main :: IO ()+main = defaultMain+        [ benchIO "Strict Write" strictWrite id+        , benchIO "Strict WriteN" strictWriteN id+        , benchIO' "Strict Read" strictWrite strictRead+        , benchIO' "Lazy Read" fromChunks lazyRead+        , benchIO "fromChunks" fromChunks id+        , benchIO' "toChunks" fromChunks toChunks+        ]

+ src/Streamly/External/ByteString.hs view

@@ -0,0 +1,63 @@+{-# LANGUAGE RecordWildCards #-}++module Streamly.External.ByteString+  ( toArray+  , fromArray++  , read+  , writeN+  , write+  )+where++import Control.Monad.IO.Class (MonadIO)+import Data.ByteString.Internal (ByteString(..))+import Data.Word (Word8)+import Foreign.ForeignPtr (plusForeignPtr)+import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)+import GHC.Ptr (minusPtr, plusPtr)+import Streamly.Internal.Memory.Array.Types (Array(..))+import Streamly.Internal.Data.Unfold.Types (Unfold(..))+import Streamly.Internal.Data.Unfold (lmap)+import Streamly.Internal.Data.Fold.Types (Fold(..))++import qualified Streamly.Internal.Memory.Array as A++import Prelude hiding (read)++-- | Convert a 'ByteString' to an array of 'Word8'. This function unwraps the+-- 'ByteString' and wraps it with 'Array' constructors and hence the operation+-- is performed in constant time.+{-# INLINE toArray #-}+toArray :: ByteString -> Array Word8+toArray (PS fp off len) = Array nfp endPtr endPtr+    where+    nfp = fp `plusForeignPtr` off+    endPtr = unsafeForeignPtrToPtr fp `plusPtr` len++-- | Convert an array of 'Word8' to a 'ByteString'. This function unwraps the+-- 'Array' and wraps it with 'ByteString' constructors and hence the operation+-- is performed in constant time.+{-# INLINE fromArray #-}+fromArray :: Array Word8 -> ByteString+fromArray Array {..}+    | aLen == 0 = mempty+    | otherwise = PS aStart 0 aLen+        where+        aStartPtr = unsafeForeignPtrToPtr aStart+        aLen = aEnd `minusPtr` aStartPtr++-- | Unfold a strict ByteString to a stream of Word8.+{-# INLINE read #-}+read :: Monad m => Unfold m ByteString Word8+read = lmap toArray A.read++-- | Fold a stream of Word8 to a strict ByteString of given size in bytes.+{-# INLINE writeN #-}+writeN :: MonadIO m => Int -> Fold m Word8 ByteString+writeN i = fromArray <$> (A.writeN i)++-- | Fold a stream of Word8 to a strict ByteString of appropriate size.+{-# INLINE write #-}+write :: MonadIO m => Fold m Word8 ByteString+write = fromArray <$> A.write

+ src/Streamly/External/ByteString/Lazy.hs view

@@ -0,0 +1,49 @@+module Streamly.External.ByteString.Lazy+  ( readChunks+  , read+  +  , toChunks+  , fromChunks+  )+where++import Data.ByteString.Lazy.Internal (ByteString(..))+import Data.Word (Word8)+import Streamly.Internal.Memory.Array.Types (Array(..))+import Streamly.Internal.Data.Stream.StreamD.Type (Step(..))+import Streamly.Internal.Data.Unfold (concat)+import Streamly.Internal.Data.Unfold.Types (Unfold(..))++import qualified Streamly.External.ByteString as Strict+import qualified Streamly.Internal.Memory.Array as A++import Streamly+import qualified Streamly.Prelude as S++import qualified Data.ByteString.Lazy.Internal as BSLI++import Prelude hiding (concat, read)++-- | Unfold a lazy ByteString to a stream of 'Array' 'Words'.+{-# INLINE  readChunks #-}+readChunks :: Monad m => Unfold m ByteString (Array Word8)+readChunks = Unfold step seed+    where+    seed = return+    step (Chunk bs bl) = return $ Yield (Strict.toArray bs) bl+    step Empty = return $ Stop++-- | Unfold a lazy ByteString to a stream of Word8+{-# INLINE read #-}+read :: Monad m => Unfold m ByteString Word8+read = concat readChunks A.read++-- | Convert a lazy 'ByteString' to a serial stream of 'Array' 'Word8'.+{-# INLINE toChunks #-}+toChunks :: Monad m => ByteString -> SerialT m (Array Word8)+toChunks = S.unfold readChunks++-- | Convert a serial stream of 'Array' 'Word8' to a lazy 'ByteString'.+{-# INLINE fromChunks #-}+fromChunks :: Monad m => SerialT m (Array Word8) -> m ByteString+fromChunks = S.foldr BSLI.chunk Empty . S.map Strict.fromArray

+ streamly-bytestring.cabal view

@@ -0,0 +1,83 @@+cabal-version: 1.12++-- This file has been generated from package.yaml by hpack version 0.31.1.+--+-- see: https://github.com/sol/hpack+--+-- hash: a905fe13f29d409f8faa63a7f3cb9bb337977bbc17c040e304969cba1c02a275++name:           streamly-bytestring+version:        0.1.0.0+synopsis:       Library for streamly and bytestring interoperation.+description:    Please see the README on GitHub at <https://github.com/psibi/streamly-bytestring#readme>+category:       Streamly, Stream, ByteString+homepage:       https://github.com/psibi/streamly-bytestring#readme+bug-reports:    https://github.com/psibi/streamly-bytestring/issues+author:         Sibi Prabakaran+maintainer:     sibi@psibi.in+copyright:      Sibi Prabakaran+license:        BSD3+license-file:   LICENSE+build-type:     Simple+extra-source-files:+    README.md+    Changelog.md++source-repository head+  type: git+  location: https://github.com/psibi/streamly-bytestring++library+  exposed-modules:+      Streamly.External.ByteString+      Streamly.External.ByteString.Lazy+  other-modules:+      Paths_streamly_bytestring+  hs-source-dirs:+      src+  ghc-options: -Wall -O2+  build-depends:+      base >=4.7 && <5+    , bytestring >=0.10.0 && <0.11+    , streamly >=0.7.0 && <0.8+  default-language: Haskell2010++test-suite sb-test+  type: exitcode-stdio-1.0+  main-is: Main.hs+  other-modules:+      Paths_streamly_bytestring+  hs-source-dirs:+      test+  ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N+  build-depends:+      base >=4.7 && <5+    , bytestring+    , directory+    , filepath+    , hspec+    , hspec-discover+    , quickcheck-instances+    , random+    , streamly+    , streamly-bytestring+    , temporary+  default-language: Haskell2010++benchmark sb-benchmark+  type: exitcode-stdio-1.0+  main-is: Main.hs+  other-modules:+      Paths_streamly_bytestring+  hs-source-dirs:+      benchmark+  ghc-options: -Wall -O2+  build-depends:+      base >=4.7 && <5+    , bytestring+    , deepseq+    , gauge+    , random+    , streamly+    , streamly-bytestring+  default-language: Haskell2010

+ test/Main.hs view

@@ -0,0 +1,104 @@+{-# LANGUAGE FlexibleContexts #-}++module Main where++import Data.ByteString (ByteString)+import Data.Word (Word8)+import GHC.IO.Handle (Handle)+import System.Random (randomIO)+import Streamly+import Streamly.FileSystem.Handle (readChunks)+import Streamly.Memory.Array (Array)+import System.IO (openFile, IOMode(ReadMode))+import System.IO.Temp (withSystemTempFile)+import Test.Hspec+import Test.Hspec.QuickCheck+import Test.QuickCheck.Instances.ByteString ()++import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BSL+import qualified Streamly.External.ByteString as Strict+import qualified Streamly.External.ByteString.Lazy as Lazy+import qualified Streamly.Prelude as S++streamToByteString :: MonadAsync m => SerialT m (Array Word8)-> m ByteString+streamToByteString stream = S.foldl' (<>) mempty $ S.map Strict.fromArray stream++checkFileContent :: FilePath -> Handle -> IO ()+checkFileContent filename handle' = do+  print $ "Checking " <> filename+  bsContent <- BS.hGetContents handle'+  handle <- openFile filename ReadMode+  bsStreamly <- streamToByteString $ S.unfold readChunks handle+  bsContent `shouldBe` bsStreamly++word8List :: Int -> IO [Word8]+word8List l = S.toList+       $ S.take l+       $ S.map fromIntegral+       $ S.map (\x -> abs x `mod` 256)+       $ S.repeatM (randomIO :: IO Int)++propFoldTestStrict :: [Word8] -> Spec+propFoldTestStrict bl =+  prop ("Strict: fold write . fromList = pack" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do+  bl' <- fmap BS.unpack $ S.fold Strict.write $ S.fromList bl+  bl' `shouldBe` bl++propNFoldTestStrict :: Int -> [Word8] -> Spec+propNFoldTestStrict n bl =+  prop ("Strict: fold (writeN n) . fromList = pack . take n" ++ " -- Size: " ++ show n ++ " bytes") $ do+  bl' <- fmap BS.unpack $ S.fold (Strict.writeN n) $ S.fromList bl+  bl' `shouldBe` take n bl++propUnfoldTestStrict :: [Word8] -> Spec+propUnfoldTestStrict bl =+  prop ("Strict: toList . unfold read . pack = id" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do+  bl' <- S.toList (S.unfold Strict.read (BS.pack bl))+  bl' `shouldBe` bl++propUnfoldTestLazy :: [Word8] -> Spec+propUnfoldTestLazy bl =+  prop ("Lazy: toList . unfold read . pack = id" ++ " -- Size: " ++ show (length bl) ++ " bytes") $ do+  bl' <- S.toList (S.unfold Lazy.read (BSL.pack bl))+  bl' `shouldBe` bl++testFromChunksLaziness :: Word8 -> IO Word8+testFromChunksLaziness h = do+  lbs <- Lazy.fromChunks $ S.fromList [Strict.toArray (BS.singleton h), undefined]+  return $ BSL.head lbs++main :: IO ()+main =+  hspec $ do+    describe "Array tests" $ do+      it "Strict fromArray" $+        mapM_ (flip withSystemTempFile checkFileContent . show) ([1..100] :: [Int])+      prop "Strict Identity" $ \bs ->+        bs `shouldBe` Strict.fromArray (Strict.toArray bs)+      prop "Lazy Identity" $ \bs -> do+        bs2 <- Lazy.fromChunks . Lazy.toChunks $ bs+        bs `shouldBe` bs2+    wl0 <- runIO $ word8List 0+    wlM <- runIO $ word8List 900+    wlL <- runIO $ word8List 73700+    describe "Strict: Fold tests" $ do+      propFoldTestStrict wl0+      propFoldTestStrict wlM+      propFoldTestStrict wlL+    describe "Strict: N Fold tests" $ do+      propNFoldTestStrict 0 wl0+      propNFoldTestStrict 900 wlM+      propNFoldTestStrict 73700 wlL+    describe "Strict: Unfold tests" $ do+      propUnfoldTestStrict wl0+      propUnfoldTestStrict wlM+      propUnfoldTestStrict wlL+    describe "Lazy: Unfold tests" $ do+      propUnfoldTestLazy wl0+      propUnfoldTestLazy wlM+      propUnfoldTestLazy wlL+    describe "Laziness of fromChunks" $ do+      it "Should not fail" $ do+        w <- testFromChunksLaziness 100+        w `shouldBe` 100