streaming-bytestring 0.1.7 → 0.2.0
raw patch · 6 files changed
+361/−270 lines, 6 filesdep +ghc-prim
Dependencies added: ghc-prim
Files
- CHANGELOG.md +36/−0
- lib/Streaming/ByteString.hs +159/−137
- lib/Streaming/ByteString/Char8.hs +104/−102
- lib/Streaming/ByteString/Internal.hs +47/−30
- streaming-bytestring.cabal +2/−1
- tests/Test.hs +13/−0
CHANGELOG.md view
@@ -1,3 +1,39 @@+## 0.2.0 (2020-10-26)++**Note:** The deprecations added in `0.1.7` have *not* been removed in this+version. Instead of `0.1.7`, that release should have been `0.2` in the first+place.++#### Added++- Add missing exports of `zipWithStream`, `materialize`, and `dematerialize`.++#### Changed++- **Breaking:** Switched names of `fold` and `fold_` in the non-`Char8` modules.+ The corresponding `Char8` functions and the rest of the library uses `_` for+ the variant that forgets the `r` value.+- **Breaking:** Unified `nextByte`/`nextChar` with `uncons`. The old `uncons`+ returned `Maybe` instead of the more natural `Either r`.+- **Breaking:** Similarly, `unconsChunk` and `nextChunk` have been unified.+- `nextByte`, `nextChar`, and `nextChunk` have been deprecated.+- Relaxed signature of `toStrict_` to allow any `r`, not just `()`.+- Permance improvements for `packChars` and `denull`.+- Various documentation improvements.+- Improved performance of `w8IsSpace` to more quickly filter out non-whitespace+ characters, and updated `words` to use it instead of the internal function+ `isSpaceWord8` from the `bytestring` package. See also+ [bytestring#315](https://github.com/haskell/bytestring/pull/315).++#### Fixed++- An edge case involving overflow in `readInt`.+- A potential crash in `uncons`.+- `intersperse` now ignores any initial empty chunks.+- `intercalate` now does not insert anything between the final substream and the+ outer stream end.+- `unlines` now correctly handles `Chunk "" (Empty r)` and `Empty r`.+ ## 0.1.7 (2020-10-14) Thanks to Viktor Dukhovni and Colin Woodbury for their contributions to this release.
lib/Streaming/ByteString.hs view
@@ -1,7 +1,9 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE CPP #-} {-# LANGUAGE GADTs #-}+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE RankNTypes #-}+{-# LANGUAGE ViewPatterns #-} -- | -- Module : Streaming.ByteString@@ -51,79 +53,80 @@ , ByteString -- * Introducing and eliminating 'ByteStream's- , empty -- empty :: ByteStream m ()- , singleton -- singleton :: Monad m => Word8 -> ByteStream m ()- , pack -- pack :: Monad m => Stream (Of Word8) m r -> ByteStream m r- , unpack -- unpack :: Monad m => ByteStream m r -> Stream (Of Word8) m r- , fromLazy -- fromLazy :: Monad m => ByteString -> ByteStream m ()- , toLazy -- toLazy :: Monad m => ByteStream m () -> m ByteString- , toLazy_ -- toLazy' :: Monad m => ByteStream m () -> m (Of ByteString r)- , fromChunks -- fromChunks :: Monad m => Stream (Of ByteString) m r -> ByteStream m r- , toChunks -- toChunks :: Monad m => ByteStream m r -> Stream (Of ByteString) m r- , fromStrict -- fromStrict :: ByteString -> ByteStream m ()- , toStrict -- toStrict :: Monad m => ByteStream m () -> m ByteString- , toStrict_ -- toStrict_ :: Monad m => ByteStream m r -> m (Of ByteString r)+ , empty+ , singleton+ , pack+ , unpack+ , fromLazy+ , toLazy+ , toLazy_+ , fromChunks+ , toChunks+ , fromStrict+ , toStrict+ , toStrict_ , effects , copy , drained , mwrap- , distribute -- distribute :: ByteStream (t m) a -> t (ByteStream m) a -- * Transforming ByteStreams- , map -- map :: Monad m => (Word8 -> Word8) -> ByteStream m r -> ByteStream m r- , intercalate -- intercalate :: Monad m => ByteStream m () -> Stream (ByteStream m) m r -> ByteStream m r- , intersperse -- intersperse :: Monad m => Word8 -> ByteStream m r -> ByteStream m r+ , map+ , intercalate+ , intersperse -- * Basic interface- , cons -- cons :: Monad m => Word8 -> ByteStream m r -> ByteStream m r- , cons' -- cons' :: Word8 -> ByteStream m r -> ByteStream m r+ , cons+ , cons' , snoc- , append -- append :: Monad m => ByteStream m r -> ByteStream m s -> ByteStream m s- , filter -- filter :: (Word8 -> Bool) -> ByteStream m r -> ByteStream m r- , uncons -- uncons :: Monad m => ByteStream m r -> m (Either r (Word8, ByteStream m r))- , nextByte -- nextByte :: Monad m => ByteStream m r -> m (Either r (Word8, ByteStream m r))- , denull+ , append+ , filter+ , uncons+ , nextByte -- * Substrings -- ** Breaking strings- , break -- break :: Monad m => (Word8 -> Bool) -> ByteStream m r -> ByteStream m (ByteStream m r)- , drop -- drop :: Monad m => GHC.Int.Int64 -> ByteStream m r -> ByteStream m r+ , break+ , drop , dropWhile- , group -- group :: Monad m => ByteStream m r -> Stream (ByteStream m) m r+ , group , groupBy- , span -- span :: Monad m => (Word8 -> Bool) -> ByteStream m r -> ByteStream m (ByteStream m r)- , splitAt -- splitAt :: Monad m => GHC.Int.Int64 -> ByteStream m r -> ByteStream m (ByteStream m r)- , splitWith -- splitWith :: Monad m => (Word8 -> Bool) -> ByteStream m r -> Stream (ByteStream m) m r- , take -- take :: Monad m => GHC.Int.Int64 -> ByteStream m r -> ByteStream m ()- , takeWhile -- takeWhile :: (Word8 -> Bool) -> ByteStream m r -> ByteStream m ()+ , span+ , splitAt+ , splitWith+ , take+ , takeWhile -- ** Breaking into many substrings- , split -- split :: Monad m => Word8 -> ByteStream m r -> Stream (ByteStream m) m r+ , split -- ** Special folds- , concat -- concat :: Monad m => Stream (ByteStream m) m r -> ByteStream m r+ , concat+ , denull -- * Builders- , toStreamingByteStringWith , toStreamingByteString++ , toStreamingByteStringWith+ , toBuilder , concatBuilders -- * Building ByteStreams -- ** Infinite ByteStreams- , repeat -- repeat :: Word8 -> ByteStream m r- , iterate -- iterate :: (Word8 -> Word8) -> Word8 -> ByteStream m r- , cycle -- cycle :: Monad m => ByteStream m r -> ByteStream m s+ , repeat+ , iterate+ , cycle -- ** Unfolding ByteStreams- , unfoldM -- unfoldr :: (a -> m (Maybe (Word8, a))) -> m a -> ByteStream m ()- , unfoldr -- unfold :: (a -> Either r (Word8, a)) -> a -> ByteStream m r+ , unfoldM+ , unfoldr , reread -- * Folds, including support for `Control.Foldl`- , foldr -- foldr :: Monad m => (Word8 -> a -> a) -> a -> ByteStream m () -> m a- , fold -- fold :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteStream m () -> m b- , fold_ -- fold' :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteStream m r -> m (b, r)+ , foldr+ , fold+ , fold_ , head , head_ , last@@ -139,29 +142,27 @@ -- * I\/O with 'ByteStream's -- ** Standard input and output- , getContents -- getContents :: ByteStream IO ()- , stdin -- stdin :: ByteStream IO ()- , stdout -- stdout :: ByteStream IO r -> IO r- , interact -- interact :: (ByteStream IO () -> ByteStream IO r) -> IO r+ , getContents+ , stdin+ , stdout+ , interact -- ** Files- , readFile -- readFile :: FilePath -> ByteStream IO ()- , writeFile -- writeFile :: FilePath -> ByteStream IO r -> IO r- , appendFile -- appendFile :: FilePath -> ByteStream IO r -> IO r+ , readFile+ , writeFile+ , appendFile -- ** I\/O with Handles- , fromHandle -- fromHandle :: Handle -> ByteStream IO ()- , toHandle -- toHandle :: Handle -> ByteStream IO r -> IO r- , hGet -- hGet :: Handle -> Int -> ByteStream IO ()- , hGetContents -- hGetContents :: Handle -> ByteStream IO ()- , hGetContentsN -- hGetContentsN :: Int -> Handle -> ByteStream IO ()- , hGetN -- hGetN :: Int -> Handle -> Int -> ByteStream IO ()- , hGetNonBlocking -- hGetNonBlocking :: Handle -> Int -> ByteStream IO ()- , hGetNonBlockingN -- hGetNonBlockingN :: Int -> Handle -> Int -> ByteStream IO ()- , hPut -- hPut :: Handle -> ByteStream IO r -> IO r- -- , hPutNonBlocking -- hPutNonBlocking :: Handle -> ByteStream IO r -> ByteStream IO r- -- * Etc.- , zipWithStream -- zipWithStream :: Monad m => (forall x. a -> ByteStream m x -> ByteStream m x) -> [a] -> Stream (ByteStream m) m r -> Stream (ByteStream m) m r+ , fromHandle+ , toHandle+ , hGet+ , hGetContents+ , hGetContentsN+ , hGetN+ , hGetNonBlocking+ , hGetNonBlockingN+ , hPut+ -- , hPutNonBlocking -- * Simple chunkwise operations , unconsChunk@@ -174,6 +175,12 @@ , chunkMap , chunkMapM , chunkMapM_++ -- * Etc.+ , dematerialize+ , materialize+ , distribute+ , zipWithStream ) where import Prelude hiding@@ -286,8 +293,12 @@ -- Note that this is an /expensive/ operation that forces the whole monadic -- ByteString into memory and then copies all the data. If possible, try to -- avoid converting back and forth between streaming and strict bytestrings.-toStrict_ :: Monad m => ByteStream m () -> m B.ByteString+toStrict_ :: Monad m => ByteStream m r -> m B.ByteString+#if MIN_VERSION_streaming (0,2,2) toStrict_ = fmap B.concat . SP.toList_ . toChunks+#else+toStrict_ = fmap B.concat . SP.toList_ . void . toChunks+#endif {-# INLINE toStrict_ #-} -- | /O(n)/ Convert a monadic byte stream into a single strict 'ByteString',@@ -410,15 +421,44 @@ -- | Remove empty ByteStrings from a stream of bytestrings. denull :: Monad m => Stream (ByteStream m) m r -> Stream (ByteStream m) m r-denull = hoist (run . maps effects) . separate . mapped nulls-{-# INLINE denull #-}+{-# INLINABLE denull #-}+denull = loop . Right+ where+ -- Scan each substream, dropping empty chunks along the way. As soon as a+ -- non-empty chunk is found, just apply the loop to the next substream in+ -- the terminal value via fmap. If Empty comes up before that happens,+ -- continue the current stream instead with its denulled tail.+ --+ -- This implementation is tail recursive:+ -- * Recursion via 'loop . Left' continues scanning an inner ByteStream.+ -- * Recursion via 'loop . Right' moves to the next substream.+ --+ -- The old version below was shorter, but noticeably slower, especially+ -- when empty substreams are frequent:+ --+ -- denull = hoist (run . maps effects) . separate . mapped nulls+ --+ loop = \ case+ Left mbs -> case mbs of+ Chunk c cs | B.length c > 0 -> Step $ Chunk c $ fmap (loop . Right) cs+ | otherwise -> loop $ Left cs+ Go m -> Effect $ loop . Left <$> m+ Empty r -> loop $ Right r+ Right strm -> case strm of+ Step mbs -> case mbs of+ Chunk c cs | B.length c > 0 -> Step $ Chunk c $ fmap (loop . Right) cs+ | otherwise -> loop $ Left cs+ Go m -> Effect $ loop . Left <$> m+ Empty r -> loop $ Right r+ Effect m -> Effect $ fmap (loop . Right) m+ r@(Return _) -> r {-| /O1/ Distinguish empty from non-empty lines, while maintaining streaming; the empty ByteStrings are on the right >>> nulls :: ByteStream m r -> m (Sum (ByteStream m) (ByteStream m) r) - There are many ways to remove null bytestrings from a+ There are many (generally slower) ways to remove null bytestrings from a @Stream (ByteStream m) m r@ (besides using @denull@). If we pass next to >>> mapped nulls bs :: Stream (Sum (ByteStream m) (ByteStream m)) m r@@ -521,49 +561,40 @@ head (Go m) = m >>= head {-# INLINABLE head #-} --- | /O(1)/ Extract the head and tail of a 'ByteStream', or 'Nothing' if it is--- empty.-uncons :: Monad m => ByteStream m r -> m (Maybe (Word8, ByteStream m r))-uncons (Empty _) = return Nothing-uncons (Chunk c cs)- = return $ Just (B.unsafeHead c- , if B.length c == 1- then cs- else Chunk (B.unsafeTail c) cs )-uncons (Go m) = m >>= uncons-{-# INLINABLE uncons #-}- -- | /O(1)/ Extract the head and tail of a 'ByteStream', or its return value if -- it is empty. This is the \'natural\' uncons for an effectful byte stream.+uncons :: Monad m => ByteStream m r -> m (Either r (Word8, ByteStream m r))+uncons (Chunk c@(B.length -> len) cs)+ | len > 0 = let !h = B.unsafeHead c+ !t = if len > 1 then Chunk (B.unsafeTail c) cs else cs+ in return $ Right (h, t)+ | otherwise = uncons cs+uncons (Go m) = m >>= uncons+uncons (Empty r) = return (Left r)+{-# INLINABLE uncons #-}++-- | The same as `uncons`, will be removed in the next version. nextByte :: Monad m => ByteStream m r -> m (Either r (Word8, ByteStream m r))-nextByte (Empty r) = return (Left r)-nextByte (Chunk c cs)- = if B.null c- then nextByte cs- else return $ Right (B.unsafeHead c- , if B.length c == 1- then cs- else Chunk (B.unsafeTail c) cs )-nextByte (Go m) = m >>= nextByte+nextByte = uncons {-# INLINABLE nextByte #-}+{-# DEPRECATED nextByte "Use uncons instead." #-} -- | Like `uncons`, but yields the entire first `B.ByteString` chunk that the--- stream is holding onto. If there wasn't one, it tries to fetch it.-unconsChunk :: Monad m => ByteStream m r -> m (Maybe (B.ByteString, ByteStream m r))-unconsChunk (Empty _) = return Nothing-unconsChunk (Chunk c cs) = return (Just (c,cs))-unconsChunk (Go m) = m >>= unconsChunk+-- stream is holding onto. If there wasn't one, it tries to fetch it. Yields+-- the final @r@ return value when the 'ByteStream' is empty.+unconsChunk :: Monad m => ByteStream m r -> m (Either r (B.ByteString, ByteStream m r))+unconsChunk (Chunk c cs)+ | B.null c = unconsChunk cs+ | otherwise = return (Right (c,cs))+unconsChunk (Go m) = m >>= unconsChunk+unconsChunk (Empty r) = return (Left r) {-# INLINABLE unconsChunk #-} --- | Similar to `unconsChunk`, but yields the final @r@ return value when there--- is no subsequent chunk.+-- | The same as `unconsChunk`, will be removed in the next version. nextChunk :: Monad m => ByteStream m r -> m (Either r (B.ByteString, ByteStream m r))-nextChunk (Empty r) = return (Left r)-nextChunk (Go m) = m >>= nextChunk-nextChunk (Chunk c cs)- | B.null c = nextChunk cs- | otherwise = return (Right (c,cs))+nextChunk = unconsChunk {-# INLINABLE nextChunk #-}+{-# DEPRECATED nextChunk "Use unconsChunk instead." #-} -- | /O(n\/c)/ Extract the last element of a 'ByteStream', which must be finite -- and non-empty.@@ -622,49 +653,49 @@ intersperse :: Monad m => Word8 -> ByteStream m r -> ByteStream m r intersperse _ (Empty r) = Empty r intersperse w (Go m) = Go (fmap (intersperse w) m)-intersperse w (Chunk c cs) = Chunk (B.intersperse w c)- (dematerialize cs Empty (Chunk . intersperse') Go)+intersperse w (Chunk c cs) | B.null c = intersperse w cs+ | otherwise =+ Chunk (B.intersperse w c)+ (dematerialize cs Empty (Chunk . intersperse') Go) where intersperse' :: P.ByteString -> P.ByteString- intersperse' (B.PS fp o l) =- B.unsafeCreate (2*l) $ \p' -> withForeignPtr fp $ \p -> do- poke p' w- B.c_intersperse (p' `plusPtr` 1) (p `plusPtr` o) (fromIntegral l) w-+ intersperse' (B.PS fp o l)+ | l > 0 = B.unsafeCreate (2*l) $ \p' -> withForeignPtr fp $ \p -> do+ poke p' w+ B.c_intersperse (p' `plusPtr` 1) (p `plusPtr` o) (fromIntegral l) w+ | otherwise = B.empty {-# INLINABLE intersperse #-} -- | 'foldr', applied to a binary operator, a starting value (typically the -- right-identity of the operator), and a ByteStream, reduces the ByteStream -- using the binary operator, from right to left. ----- > foldr cons = id--- foldr :: Monad m => (Word8 -> a -> a) -> a -> ByteStream m () -> m a foldr k = foldrChunks (flip (B.foldr k)) {-# INLINE foldr #-} --- | 'fold', applied to a binary operator, a starting value (typically the+-- | 'fold_', applied to a binary operator, a starting value (typically the -- left-identity of the operator), and a ByteStream, reduces the ByteStream -- using the binary operator, from left to right. We use the style of the foldl--- libarary for left folds-fold :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteStream m () -> m b-fold step0 begin finish p0 = loop p0 begin+-- library for left folds+fold_ :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteStream m () -> m b+fold_ step0 begin finish p0 = loop p0 begin where loop p !x = case p of Chunk bs bss -> loop bss $! B.foldl' step0 x bs Go m -> m >>= \p' -> loop p' x Empty _ -> return (finish x)-{-# INLINABLE fold #-}+{-# INLINABLE fold_ #-} --- | 'fold_' keeps the return value of the left-folded bytestring. Useful for+-- | 'fold' keeps the return value of the left-folded bytestring. Useful for -- simultaneous folds over a segmented bytestream.-fold_ :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteStream m r -> m (Of b r)-fold_ step0 begin finish p0 = loop p0 begin+fold :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteStream m r -> m (Of b r)+fold step0 begin finish p0 = loop p0 begin where loop p !x = case p of Chunk bs bss -> loop bss $! B.foldl' step0 x bs Go m -> m >>= \p' -> loop p' x Empty r -> return (finish x :> r)-{-# INLINABLE fold_ #-}+{-# INLINABLE fold #-} -- --------------------------------------------------------------------- -- Special folds@@ -786,10 +817,9 @@ >>> Q.putStrLn $ Q.drop 6 "Wisconsin" sin >>> Q.putStrLn $ Q.drop 16 "Wisconsin"-->>>+<BLANKLINE> -}-drop :: Monad m => Int64 -> ByteStream m r -> ByteStream m r+drop :: Monad m => Int64 -> ByteStream m r -> ByteStream m r drop i p | i <= 0 = p drop i cs0 = drop' i cs0 where drop' 0 cs = cs@@ -975,20 +1005,13 @@ -- 'ByteStream's and concatenates the list after interspersing the first -- argument between each element of the list. intercalate :: Monad m => ByteStream m () -> Stream (ByteStream m) m r -> ByteStream m r-intercalate _ (Return r) = Empty r-intercalate s (Effect m) = Go $ fmap (intercalate s) m-intercalate s (Step bs0) = do -- this isn't quite right- ls <- bs0- s- intercalate s ls- -- where- -- loop (Return r) = Empty r -- concat . (L.intersperse s)- -- loop (Effect m) = Go $ fmap loop m- -- loop (Step bs) = do- -- ls <- bs- -- case ls of- -- Return r -> Empty r -- no '\n' before end, in this case.- -- x -> s >> loop x+intercalate s = loop+ where+ loop (Return r) = Empty r+ loop (Effect m) = Go $ fmap loop m+ loop (Step bs) = bs >>= \case+ Return r -> Empty r -- not between final substream and stream end+ x -> s >> loop x {-# INLINABLE intercalate #-} -- | Returns the number of times its argument appears in the `ByteStream`.@@ -1250,8 +1273,8 @@ -- | Zip a list and a stream-of-byte-streams together. zipWithStream- :: (Monad m)- => (forall x . a -> ByteStream m x -> ByteStream m x)+ :: Monad m+ => (forall x . a -> ByteStream m x -> ByteStream m x) -> [a] -> Stream (ByteStream m) m r -> Stream (ByteStream m) m r@@ -1271,8 +1294,7 @@ -- 哈斯克尔 98 -- -- <https://gist.github.com/michaelt/6ea89ca95a77b0ef91f3 This benchmark> shows--- its indistinguishable performance is indistinguishable from--- @toLazyByteString@+-- its performance is indistinguishable from @toLazyByteString@ toStreamingByteString :: MonadIO m => Builder -> ByteStream m () toStreamingByteString = toStreamingByteStringWith (safeStrategy BI.smallChunkSize BI.defaultChunkSize)
lib/Streaming/ByteString/Char8.hs view
@@ -4,6 +4,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE ViewPatterns #-} -- | -- Module : Streaming.ByteString.Char8@@ -26,20 +27,20 @@ , ByteString -- * Introducing and eliminating 'ByteStream's- , empty -- empty :: ByteStream m ()- , pack -- pack :: Monad m => String -> ByteStream m ()+ , empty+ , pack , unpack , string , unlines , unwords- , singleton -- singleton :: Monad m => Char -> ByteStream m ()- , fromChunks -- fromChunks :: Monad m => Stream (Of ByteString) m r -> ByteStream m r- , fromLazy -- fromLazy :: Monad m => ByteString -> ByteStream m ()- , fromStrict -- fromStrict :: ByteString -> ByteStream m ()- , toChunks -- toChunks :: Monad m => ByteStream m r -> Stream (Of ByteString) m r- , toLazy -- toLazy :: Monad m => ByteStream m () -> m ByteString+ , singleton+ , fromChunks+ , fromLazy+ , fromStrict+ , toChunks+ , toLazy , toLazy_- , toStrict -- toStrict :: Monad m => ByteStream m () -> m ByteString+ , toStrict , toStrict_ , effects , copy@@ -47,72 +48,74 @@ , mwrap -- * Transforming ByteStreams- , map -- map :: Monad m => (Char -> Char) -> ByteStream m r -> ByteStream m r- , intercalate -- intercalate :: Monad m => ByteStream m () -> Stream (ByteStream m) m r -> ByteStream m r- , intersperse -- intersperse :: Monad m => Char -> ByteStream m r -> ByteStream m r+ , map+ , intercalate+ , intersperse -- * Basic interface- , cons -- cons :: Monad m => Char -> ByteStream m r -> ByteStream m r- , cons' -- cons' :: Char -> ByteStream m r -> ByteStream m r+ , cons+ , cons' , snoc- , append -- append :: Monad m => ByteStream m r -> ByteStream m s -> ByteStream m s- , filter -- filter :: (Char -> Bool) -> ByteStream m r -> ByteStream m r- , head -- head :: Monad m => ByteStream m r -> m Char- , head_ -- head' :: Monad m => ByteStream m r -> m (Of Char r)- , last -- last :: Monad m => ByteStream m r -> m Char- , last_ -- last' :: Monad m => ByteStream m r -> m (Of Char r)- , null -- null :: Monad m => ByteStream m r -> m Bool+ , append+ , filter+ , head+ , head_+ , last+ , last_+ , null , null_+ , nulls , testNull- , nulls -- null' :: Monad m => ByteStream m r -> m (Of Bool r)- , uncons -- uncons :: Monad m => ByteStream m r -> m (Either r (Char, ByteStream m r))+ , uncons , nextChar , skipSomeWS -- * Substrings -- ** Breaking strings- , break -- break :: Monad m => (Char -> Bool) -> ByteStream m r -> ByteStream m (ByteStream m r)- , drop -- drop :: Monad m => GHC.Int.Int64 -> ByteStream m r -> ByteStream m r+ , break+ , drop , dropWhile- , group -- group :: Monad m => ByteStream m r -> Stream (ByteStream m) m r+ , group , groupBy- , span -- span :: Monad m => (Char -> Bool) -> ByteStream m r -> ByteStream m (ByteStream m r)- , splitAt -- splitAt :: Monad m => GHC.Int.Int64 -> ByteStream m r -> ByteStream m (ByteStream m r)- , splitWith -- splitWith :: Monad m => (Char -> Bool) -> ByteStream m r -> Stream (ByteStream m) m r- , take -- take :: Monad m => GHC.Int.Int64 -> ByteStream m r -> ByteStream m ()- , takeWhile -- takeWhile :: (Char -> Bool) -> ByteStream m r -> ByteStream m ()+ , span+ , splitAt+ , splitWith+ , take+ , takeWhile -- ** Breaking into many substrings- , split -- split :: Monad m => Char -> ByteStream m r -> Stream (ByteStream m) m r+ , split , lines- , words , lineSplit- , denull+ , words -- ** Special folds- , concat -- concat :: Monad m => Stream (ByteStream m) m r -> ByteStream m r+ , concat+ , denull -- * Builders , toStreamingByteString+ , toStreamingByteStringWith+ , toBuilder , concatBuilders -- * Building ByteStreams -- ** Infinite ByteStreams- , repeat -- repeat :: Char -> ByteStream m ()- , iterate -- iterate :: (Char -> Char) -> Char -> ByteStream m ()- , cycle -- cycle :: Monad m => ByteStream m r -> ByteStream m s+ , repeat+ , iterate+ , cycle -- ** Unfolding ByteStreams- , unfoldr -- unfoldr :: (a -> Maybe (Char, a)) -> a -> ByteStream m ()- , unfoldM -- unfold :: (a -> Either r (Char, a)) -> a -> ByteStream m r+ , unfoldr+ , unfoldM , reread -- * Folds, including support for `Control.Foldl`--- , foldr -- foldr :: Monad m => (Char -> a -> a) -> a -> ByteStream m () -> m a- , fold -- fold :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteStream m () -> m b- , fold_ -- fold' :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteStream m r -> m (b, r)+ -- , foldr+ , fold+ , fold_ , length , length_ , count@@ -121,29 +124,29 @@ -- * I\/O with 'ByteStream's -- ** Standard input and output- , getContents -- getContents :: ByteStream IO ()- , stdin -- stdin :: ByteStream IO ()- , stdout -- stdout :: ByteStream IO r -> IO r- , interact -- interact :: (ByteStream IO () -> ByteStream IO r) -> IO r+ , getContents+ , stdin+ , stdout+ , interact , putStr , putStrLn -- ** Files- , readFile -- readFile :: FilePath -> ByteStream IO ()- , writeFile -- writeFile :: FilePath -> ByteStream IO r -> IO r- , appendFile -- appendFile :: FilePath -> ByteStream IO r -> IO r+ , readFile+ , writeFile+ , appendFile -- ** I\/O with Handles- , fromHandle -- fromHandle :: Handle -> ByteStream IO ()- , toHandle -- toHandle :: Handle -> ByteStream IO r -> IO r- , hGet -- hGet :: Handle -> Int -> ByteStream IO ()- , hGetContents -- hGetContents :: Handle -> ByteStream IO ()- , hGetContentsN -- hGetContentsN :: Int -> Handle -> ByteStream IO ()- , hGetN -- hGetN :: Int -> Handle -> Int -> ByteStream IO ()- , hGetNonBlocking -- hGetNonBlocking :: Handle -> Int -> ByteStream IO ()- , hGetNonBlockingN -- hGetNonBlockingN :: Int -> Handle -> Int -> ByteStream IO ()- , hPut -- hPut :: Handle -> ByteStream IO r -> IO r--- , hPutNonBlocking -- hPutNonBlocking :: Handle -> ByteStream IO r -> ByteStream IO r+ , fromHandle+ , toHandle+ , hGet+ , hGetContents+ , hGetContentsN+ , hGetN+ , hGetNonBlocking+ , hGetNonBlockingN+ , hPut+ -- , hPutNonBlocking -- * Simple chunkwise operations , unconsChunk@@ -158,10 +161,10 @@ , chunkMapM_ -- * Etc.--- , zipWithStream -- zipWithStream :: Monad m => (forall x. a -> ByteStream m x -> ByteStream m x) -> [a] -> Stream (ByteStream m) m r -> Stream (ByteStream m) m r- , distribute -- distribute :: ByteStream (t m) a -> t (ByteStream m) a- , materialize , dematerialize+ , materialize+ , distribute+ , zipWithStream ) where import Prelude hiding@@ -194,8 +197,9 @@ length_, nextChunk, null, null_, nulls, readFile, splitAt, stdin, stdout, take, testNull, toBuilder, toChunks, toHandle, toLazy, toLazy_, toStreamingByteString, toStreamingByteStringWith, toStrict, toStrict_,- unconsChunk, writeFile)+ unconsChunk, writeFile, zipWithStream) +import Data.Bits ((.&.)) import Data.Word (Word8) import Foreign.ForeignPtr (withForeignPtr) import Foreign.Ptr@@ -203,7 +207,7 @@ import qualified System.IO as IO -- | Given a stream of bytes, produce a vanilla `Stream` of characters.-unpack :: Monad m => ByteStream m r -> Stream (Of Char) m r+unpack :: Monad m => ByteStream m r -> Stream (Of Char) m r unpack bs = case bs of Empty r -> Return r Go m -> Effect (fmap unpack m)@@ -303,18 +307,24 @@ groupBy rel = Q.groupBy (\w w' -> rel (w2c w) (w2c w')) {-# INLINE groupBy #-} --- | /O(1)/ Extract the head and tail of a ByteStream, returning Nothing--- if it is empty.+-- | /O(1)/ Extract the head and tail of a 'ByteStream', or its return value if+-- it is empty. This is the \'natural\' uncons for an effectful byte stream. uncons :: Monad m => ByteStream m r -> m (Either r (Char, ByteStream m r))+uncons (Chunk c@(B.length -> len) cs)+ | len > 0 = let !h = w2c $ B.unsafeHead c+ !t = if len > 1 then Chunk (B.unsafeTail c) cs else cs+ in return $ Right (h, t)+ | otherwise = uncons cs+uncons (Go m) = m >>= uncons uncons (Empty r) = return (Left r)-uncons (Chunk c cs)- = return $ Right (w2c (B.unsafeHead c)- , if B.length c == 1- then cs- else Chunk (B.unsafeTail c) cs )-uncons (Go m) = m >>= uncons {-# INLINABLE uncons #-} +-- | The same as `uncons`, will be removed in the next version.+nextChar :: Monad m => ByteStream m r -> m (Either r (Char, ByteStream m r))+nextChar = uncons+{-# INLINABLE nextChar #-}+{-# DEPRECATED nextChar "Use uncons instead." #-}+ -- --------------------------------------------------------------------- -- Transformations @@ -514,17 +524,13 @@ -- the \"lines\" had embedded newlines. -- -- * @'unlines' . 'lines'@ will replace @\\r\\n@ with @\\n@.-unlines :: Monad m => Stream (ByteStream m) m r -> ByteStream m r+unlines :: Monad m => Stream (ByteStream m) m r -> ByteStream m r unlines = loop where loop str = case str of Return r -> Empty r Step bstr -> do st <- bstr- let bs = unlines st- case bs of- Chunk "" (Empty r) -> Empty r- Chunk "\n" (Empty _) -> bs- _ -> cons' '\n' bs+ cons' '\n' $ unlines st Effect m -> Go (fmap unlines m) {-# INLINABLE unlines #-} @@ -536,7 +542,7 @@ -- can write @mapped toStrict . words@ or the like, if strict bytestrings are -- needed. words :: Monad m => ByteStream m r -> Stream (ByteStream m) m r-words = filtered . Q.splitWith B.isSpaceWord8+words = filtered . Q.splitWith w8IsSpace where filtered stream = case stream of Return r -> Return r@@ -663,15 +669,6 @@ count c = Q.count (c2w c) {-# INLINE count #-} --- | /O(1)/ Extract the head and tail of a 'ByteStream', or its return value if--- it is empty. This is the \'natural\' uncons for an effectful byte stream.-nextChar :: Monad m => ByteStream m r -> m (Either r (Char, ByteStream m r))-nextChar b = do- e <- Q.nextByte b- case e of- Left r -> return $! Left r- Right (w,bs) -> return $! Right (w2c w, bs)- -- | Print a stream of bytes to STDOUT. putStr :: MonadIO m => ByteStream m r -> m r putStr = hPut IO.stdout@@ -702,10 +699,11 @@ -- the conversion from Word8 to Word is free. let w :: Word !w = fromIntegral w8- in w - 0x21 > 0x7e -- not [x21..0x9f]- && ( w == 0x20 -- SP- || w - 0x09 < 5 -- HT, NL, VT, FF, CR- || w == 0xa0 ) -- NBSP+ in w .&. 0x50 == 0 -- Quick non-wsp filter+ && w - 0x21 > 0x7e -- 2nd non-wsp filter+ && ( w == 0x20 -- SP+ || w - 0x09 < 5 -- HT, NL, VT, FF, CR+ || w == 0xa0 ) -- NBSP {-# INLINE w8IsSpace #-} -- | Try to position the stream at the next non-whitespace input, by@@ -730,14 +728,14 @@ go _ r = r -- | Try to read an 'Int' value from the 'ByteString', returning--- @m (Compose -- (Just val :> str))@ on success, where @val@ is the--- value read and @str@ is the rest of the input stream. If the stream--- of digits decodes to a value larger than can be represented by an--- 'Int', the returned value will be @m (Compose (Nothing :> str))@,--- where the content of @str@ is the same as the original stream, but--- some of the monadic effects may already have taken place, so the--- original stream MUST NOT be used. To read the remaining data, you--- MUST use the returned @str@.+-- @m (Compose (Just val :> str))@ on success, where @val@ is the value+-- read and @str@ is the rest of the input stream. If the stream of+-- digits decodes to a value larger than can be represented by an 'Int',+-- the returned value will be @m (Compose (Nothing :> str))@, where the+-- content of @str@ is the same as the original stream, but some of the+-- monadic effects may already have taken place, so the original stream+-- MUST NOT be used. To read the remaining data, you MUST use the+-- returned @str@. -- -- This function will not read an /unreasonably/ long stream of leading -- zero digits when trying to decode a number. When reading the first@@ -795,9 +793,13 @@ Chunk c cs | !l <- B.length c , l > 0 -> case accumWord acc c of- (0, !_, !_)+ (0, !_, !inrange)+ | inrange -- no more digits found -> result nbytes acc str+ | otherwise+ -- Overlow on first digit of chunk+ -> overflow nbytes acc str (!n, !a, !inrange) | False <- inrange -- result out of 'Int' range
lib/Streaming/ByteString/Internal.hs view
@@ -19,16 +19,16 @@ module Streaming.ByteString.Internal ( ByteStream(..) , ByteString- , consChunk -- :: ByteString -> ByteStream m r -> ByteStream m r- , chunkOverhead -- :: Int- , defaultChunkSize -- :: Int- , materialize -- :: (forall x. (r -> x) -> (ByteString -> x -> x) -> (m x -> x) -> x) -> ByteStream m r- , dematerialize -- :: Monad m => ByteStream m r -> forall x. (r -> x) -> (ByteString -> x -> x) -> (m x -> x) -> x- , foldrChunks -- :: Monad m => (ByteString -> a -> a) -> a -> ByteStream m r -> m a- , foldlChunks -- :: Monad m => (a -> ByteString -> a) -> a -> ByteStream m r -> m a+ , consChunk+ , chunkOverhead+ , defaultChunkSize+ , materialize+ , dematerialize+ , foldrChunks+ , foldlChunks - , foldrChunksM -- :: Monad m => (ByteString -> m a -> m a) -> m a -> ByteStream m r -> m a- , foldlChunksM -- :: Monad m => (ByteString -> m a -> m a) -> m a -> ByteStream m r -> m a+ , foldrChunksM+ , foldlChunksM , chunkFold , chunkFoldM , chunkMap@@ -38,10 +38,10 @@ , unfoldrChunks , packChars- , smallChunkSize -- :: Int- , unpackBytes -- :: Monad m => ByteStream m r -> Stream (Of Word8) m r , packBytes- , chunk -- :: ByteString -> ByteStream m ()+ , unpackBytes+ , chunk+ , smallChunkSize , mwrap , unfoldrNE , reread@@ -81,13 +81,13 @@ import Foreign.ForeignPtr (withForeignPtr) import Foreign.Ptr import Foreign.Storable-import GHC.Exts (SpecConstrAnnotation(..))+import GHC.Types (SPEC(..)) import Data.Functor.Identity import Data.Word import GHC.Base (realWorld#) import GHC.IO (IO(IO))-import System.IO.Unsafe+import System.IO.Unsafe (unsafePerformIO) import Control.Monad.Base import Control.Monad.Catch (MonadCatch(..))@@ -215,9 +215,6 @@ Chunk bs rest -> Chunk bs (loop rest) {-# INLINABLE bracketByteString #-} -data SPEC = SPEC | SPEC2-{-# ANN type SPEC ForceSpecConstr #-}- -- -- ------------------------------------------------------------------------ -- -- | Smart constructor for 'Chunk'.@@ -234,7 +231,7 @@ {- | Reconceive an effect that results in an effectful bytestring as an effectful bytestring.- Compare Streaming.mwrap. The closes equivalent of+ Compare Streaming.mwrap. The closest equivalent of >>> Streaming.wrap :: f (Stream f m r) -> Stream f m r @@ -316,14 +313,17 @@ -- assert (l' <= l) $ return (B.PS fp 0 l', res) -- {-# INLINABLE packBytes' #-} +-- | Convert a `Stream` of pure `Word8` into a chunked 'ByteStream'. packBytes :: Monad m => Stream (Of Word8) m r -> ByteStream m r packBytes cs0 = do+ -- XXX: Why 32? It seems like a rather small chunk size, wouldn't+ -- smallChunkSize make a better choice? (bytes :> rest) <- lift $ SP.toList $ SP.splitAt 32 cs0 case bytes of [] -> case rest of Return r -> Empty r Step as -> packBytes (Step as) -- these two pattern matches- Effect m -> Go $ fmap packBytes m -- should be evaded.+ Effect m -> Go $ fmap packBytes m -- should be avoided. _ -> Chunk (B.packBytes bytes) (packBytes rest) {-# INLINABLE packBytes #-} @@ -331,7 +331,21 @@ -- -- /Note:/ Each `Char` value is truncated to 8 bits. packChars :: Monad m => Stream (Of Char) m r -> ByteStream m r-packChars = packBytes . SP.map B.c2w+packChars str = do+ -- XXX: Why 32? It seems like a rather small chunk size, wouldn't+ -- smallChunkSize make a better choice?+ --+ -- We avoid the cost of converting the stream of Chars to a stream+ -- of Word8 (passed to packBytes), and instead pass the original+ -- `Char` arrays to 'B.packChars', which will be more efficient,+ -- the conversion there will be essentially free.+ (chars :> rest) <- lift $ SP.toList $ SP.splitAt 32 str+ case chars of+ [] -> case rest of+ Return r -> Empty r+ Step as -> packChars (Step as) -- these two pattern matches+ Effect m -> Go $ fmap packChars m -- should be avoided.+ _ -> Chunk (B.packChars chars) (packChars rest) {-# INLINABLE packChars #-} -- | The reverse of `packChars`. Given a stream of bytes, produce a `Stream`@@ -340,8 +354,8 @@ unpackBytes bss = dematerialize bss Return unpackAppendBytesLazy Effect where unpackAppendBytesLazy :: B.ByteString -> Stream (Of Word8) m r -> Stream (Of Word8) m r- unpackAppendBytesLazy (B.PS fp off len) xs- | len <= 100 = unpackAppendBytesStrict (B.PS fp off len) xs+ unpackAppendBytesLazy b@(B.PS fp off len) xs+ | len <= 100 = unpackAppendBytesStrict b xs | otherwise = unpackAppendBytesStrict (B.PS fp off 100) remainder where remainder = unpackAppendBytesLazy (B.PS fp (off+100) (len-100)) xs@@ -351,11 +365,11 @@ B.accursedUnutterablePerformIO $ withForeignPtr fp $ \base -> loop (base `plusPtr` (off-1)) (base `plusPtr` (off-1+len)) xs where- loop !sentinal !p acc- | p == sentinal = return acc+ loop !sentinel !p acc+ | p == sentinel = return acc | otherwise = do x <- peek p- loop sentinal (p `plusPtr` (-1)) (Step (x :> acc))+ loop sentinel (p `plusPtr` (-1)) (Step (x :> acc)) {-# INLINABLE unpackBytes #-} -- | Copied from Data.ByteString.Unsafe for compatibility with older bytestring.@@ -478,16 +492,18 @@ Right (bs,s') -> return $ Chunk bs (loop s') {-# INLINABLE unfoldrChunks #-} --- | Stream chunks from something that contains @IO (Maybe ByteString)@ until it+-- | Stream chunks from something that contains @m (Maybe ByteString)@ until it -- returns 'Nothing'. 'reread' is of particular use rendering @io-streams@ input -- streams as byte streams in the present sense. ----- > Q.reread Streams.read :: InputStream B.ByteString -> Q.ByteString IO ()--- > Q.reread (liftIO . Streams.read) :: MonadIO m => InputStream B.ByteString -> Q.ByteString m ()+-- > import qualified Data.ByteString as B+-- > import qualified System.IO.Streams as S+-- > Q.reread S.read :: S.InputStream B.ByteString -> Q.ByteStream IO ()+-- > Q.reread (liftIO . S.read) :: MonadIO m => S.InputStream B.ByteString -> Q.ByteStream m () -- -- The other direction here is ----- > Streams.unfoldM Q.unconsChunk :: Q.ByteString IO r -> IO (InputStream B.ByteString)+-- > S.unfoldM Q.unconsChunk :: Q.ByteString IO r -> IO (S.InputStream B.ByteString) reread :: Monad m => (s -> m (Maybe B.ByteString)) -> s -> ByteStream m () reread step s = loop where loop = Go $ do@@ -525,8 +541,9 @@ world! >>> :! cat hello1.txt hello-+<BLANKLINE> world+<BLANKLINE> As with the parallel operations in @Streaming.Prelude@, we have
streaming-bytestring.cabal view
@@ -1,6 +1,6 @@ cabal-version: >=1.10 name: streaming-bytestring-version: 0.1.7+version: 0.2.0 synopsis: Fast, effectful byte streams. description: This library enables fast and safe streaming of byte data, in either @Word8@ or@@ -71,6 +71,7 @@ , bytestring , deepseq , exceptions+ , ghc-prim >=0.4 && <0.8 , mmorph >=1.0 && <1.2 , mtl >=2.1 && <2.3 , resourcet
tests/Test.hs view
@@ -314,6 +314,19 @@ $ addEffect cnt $ QI.Empty 16 check cnt res Nothing (smin10 :> 16)+ -- maxBound with cross-chunk overflow+ IOR.writeIORef cnt 4+ res <- readIntSkip+ $ QI.Chunk " \t\n\v\f\r\xa0"+ $ addEffect cnt+ $ QI.Chunk (B.take 4 smax)+ $ addEffect cnt+ $ QI.Chunk (B.drop 4 smax)+ $ addEffect cnt+ $ QI.Chunk "00"+ $ addEffect cnt+ $ QI.Empty 17+ check cnt res Nothing (smax `B.append` "00" :> 17) where -- Count down to zero from initial value readIntSkip = Q8.readInt . Q8.skipSomeWS