streamly-bytestring 0.2.1 → 0.2.2
raw patch · 4 files changed
+95/−18 lines, 4 filesdep ~bytestringdep ~streamly-core
Dependency ranges changed: bytestring, streamly-core
Files
- Changelog.md +4/−0
- src/Streamly/External/ByteString.hs +64/−15
- streamly-bytestring.cabal +4/−3
- test/Main.hs +23/−0
Changelog.md view
@@ -1,5 +1,9 @@ # Changelog for streamly-bytestring +## 0.2.2 (Jul 2024)++* Fix a bug that uses an unpinned array in the foreign pointer+ ## 0.2.1 (Dec 2023) * Support streamly-core-0.2.0
src/Streamly/External/ByteString.hs view
@@ -42,15 +42,20 @@ import qualified Streamly.Data.Array as Array import qualified Streamly.Internal.Data.Unfold as Unfold (fold, mkUnfoldrM) +#if !(MIN_VERSION_bytestring(0,11,0))+import GHC.ForeignPtr (plusForeignPtr)+#endif+ #if MIN_VERSION_streamly_core(0,2,0) import Streamly.Internal.Data.Array (Array(..)) import Streamly.Internal.Data.MutByteArray (MutByteArray(..))-import qualified Streamly.Internal.Data.MutByteArray as MutBA (nil)+import qualified Streamly.Internal.Data.Array as Array+import qualified Streamly.Internal.Data.MutByteArray as MutBA import qualified Streamly.Internal.Data.Stream as StreamD (Step(Yield)) #else import Streamly.Internal.Data.Array.Type (Array(..)) import Streamly.Internal.Data.Unboxed (MutableByteArray(..))-import qualified Streamly.Internal.Data.Unboxed as MutBA (nil)+import qualified Streamly.Internal.Data.Unboxed as MutBA import qualified Streamly.Internal.Data.Stream.StreamD as StreamD (Step(Yield)) #endif @@ -62,6 +67,42 @@ #define MUT_BYTE_ARRAY MutableByteArray #endif +#if MIN_VERSION_streamly_core(0,2,2)+#define NIL MutBA.empty+#else+#define NIL MutBA.nil+#endif++#if MIN_VERSION_bytestring(0,11,0)+#define CONSTRUCTOR(a, b, c) BS a c+#define WHEN_0_10_12(x)+#else+#define CONSTRUCTOR(a, b, c) PS a b c+#define WHEN_0_10_12(x) x+#endif+++{-# INLINE ensurePinned #-}+ensurePinned :: Array a -> IO (Array a)+{-# INLINE pinnedCreateOf #-}+pinnedCreateOf :: MonadIO m => Int -> Fold m Word8 (Array Word8)+{-# INLINE pinnedCreate #-}+pinnedCreate :: MonadIO m => Fold m Word8 (Array Word8)++#if MIN_VERSION_streamly_core(0,2,2)+ensurePinned = Array.pin+pinnedCreateOf = Array.pinnedCreateOf+pinnedCreate = Array.pinnedCreate+#elif MIN_VERSION_streamly_core(0,2,0)+ensurePinned = Array.pin+pinnedCreateOf = Array.pinnedWriteN+pinnedCreate = Array.pinnedWrite+#else+ensurePinned = pure+pinnedCreateOf = Array.writeN+pinnedCreate = Array.write+#endif+ {-# INLINE mutableByteArrayContents# #-} mutableByteArrayContents# :: MutableByteArray# RealWorld -> Addr# mutableByteArrayContents# marr# = byteArrayContents# (unsafeCoerce# marr#)@@ -79,14 +120,16 @@ -- there is a copy involved. {-# INLINE toArray #-} toArray :: ByteString -> Array Word8-toArray (BS (ForeignPtr addr# _) _)- | Ptr addr# == nullPtr = Array MutBA.nil 0 0-toArray (BS (ForeignPtr addr# (PlainPtr marr#)) len) =+toArray (CONSTRUCTOR((ForeignPtr addr# _), _, _))+ | Ptr addr# == nullPtr = Array NIL 0 0+toArray (CONSTRUCTOR((ForeignPtr addr# (PlainPtr marr#)), off0, len)) = let off = I# (addr# `minusAddr#` mutableByteArrayContents# marr#)+ WHEN_0_10_12(+ off0) in Array (MUT_BYTE_ARRAY marr#) off (off + len)-toArray (BS fptr len) =+toArray (CONSTRUCTOR(fptr, off, len)) = unsafeInlineIO- $ withForeignPtr fptr $ Unfold.fold (Array.writeN len) generator+ $ withForeignPtr (fptr WHEN_0_10_12(`plusForeignPtr` off))+ $ Unfold.fold (Array.writeN len) generator where @@ -94,18 +137,24 @@ Unfold.mkUnfoldrM (\ptr -> flip StreamD.Yield (ptr `plusPtr` 1) <$> peek ptr) --- | Convert an array of 'Word8' to a 'ByteString'. This function unwraps the--- 'Array' and wraps it with 'ByteString' constructors and hence the operation--- is performed in constant time.+-- | Convert an array of 'Word8' to a 'ByteString'.+--+-- Please ensure that the array is pinned when using this function.++-- If the array is pinned, the operation is performed in constant time. Whereas+-- for an unpinned array a copy is involved to pin it.+-- {-# INLINE fromArray #-} fromArray :: Array Word8 -> ByteString-fromArray (Array {..})+fromArray arr | aLen == 0 = mempty- | otherwise = BS (makeForeignPtr arrContents arrStart) aLen+ | otherwise = unsafeInlineIO $ do+ Array{..} <- ensurePinned arr+ pure $ CONSTRUCTOR((makeForeignPtr arrContents arrStart), 0, aLen) where - aLen = arrEnd - arrStart+ aLen = arrEnd arr - arrStart arr -- | Unfold a strict ByteString to a stream of Word8. {-# INLINE reader #-}@@ -115,12 +164,12 @@ -- | Fold a stream of Word8 to a strict ByteString of given size in bytes. {-# INLINE writeN #-} writeN :: MonadIO m => Int -> Fold m Word8 ByteString-writeN i = fromArray <$> Array.writeN i+writeN i = fromArray <$> pinnedCreateOf i -- | Fold a stream of Word8 to a strict ByteString of appropriate size. {-# INLINE write #-} write :: MonadIO m => Fold m Word8 ByteString-write = fromArray <$> Array.write+write = fromArray <$> pinnedCreate -------------------------------------------------------------------------------- -- Deprecated
streamly-bytestring.cabal view
@@ -1,6 +1,6 @@ cabal-version: 1.12 name: streamly-bytestring-version: 0.2.1+version: 0.2.2 synopsis: Library for streamly and bytestring interoperation. description: Please see the README on GitHub at <https://github.com/psibi/streamly-bytestring#readme> category: Streamly, Stream, ByteString@@ -38,7 +38,8 @@ ghc-options: -Wall -O2 build-depends: base >=4.7 && <5- , bytestring == 0.11.0.*+ , bytestring == 0.10.12.*+ || == 0.11.0.* || == 0.11.1.* || == 0.11.2.* -- bytestring-0.11.3.0 causes panics and other issues on@@ -47,7 +48,7 @@ || == 0.11.4.* || == 0.11.5.* || == 0.12.0.*- , streamly-core >= 0.1.0 && < 0.2.2+ , streamly-core >= 0.1.0 && < 0.2.3 default-language: Haskell2010 test-suite sb-test
test/Main.hs view
@@ -1,8 +1,12 @@ {-# LANGUAGE CPP #-} {-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE ScopedTypeVariables #-} module Main where +import Control.Monad (when)+import System.Mem (performMajorGC)+import Control.Concurrent (threadDelay) import Data.ByteString (ByteString) import Data.Word (Word8) import GHC.IO.Handle (Handle)@@ -28,6 +32,7 @@ import qualified Streamly.External.ByteString.Lazy as Lazy import qualified Streamly.Data.Stream as S import qualified Streamly.Data.Fold as Fold+import qualified Streamly.Internal.Data.Array as Array streamToByteString :: Monad m => S.Stream m (Array Word8) -> m ByteString streamToByteString stream =@@ -101,10 +106,28 @@ S.fromList (Strict.toArray (BS.singleton h) : undefined) return $ BSL.head lbs +checkPinnedNature :: IO ()+checkPinnedNature = do+ (arr :: Array Word8) <-+ Array.fromStream (S.fromList (take 1000 (cycle [0..255])))+ performMajorGC+ threadDelay 50000+ threadDelay 50000+ performMajorGC+ threadDelay 50000+ threadDelay 50000+ (_ :: Array Word8) <-+ Array.fromStream (S.fromList (take 10000 (cycle [0..255])))+ let bs = Strict.fromArray arr+ lst1 = BS.unpack bs+ lst2 = Array.toList arr+ when (lst1 /= lst2) $ error "Pinned nature isn't ensured"+ main :: IO () main = hspec $ do describe "Array tests" $ do+ it "Pinned in nature" $ checkPinnedNature it "Strict fromArray" $ mapM_ (flip withSystemTempFile checkFileContent . show)