diff --git a/Data/ByteString/Streaming.hs b/Data/ByteString/Streaming.hs --- a/Data/ByteString/Streaming.hs +++ b/Data/ByteString/Streaming.hs @@ -99,6 +99,7 @@ , break -- break :: Monad m => (Word8 -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , drop -- drop :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m r , group -- group :: Monad m => ByteString m r -> Stream (ByteString m) m r + , groupBy , span -- span :: Monad m => (Word8 -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , splitAt -- splitAt :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m (ByteString m r) , splitWith -- splitWith :: Monad m => (Word8 -> Bool) -> ByteString m r -> Stream (ByteString m) m r @@ -1158,7 +1159,7 @@ -- --- | The 'group' function take`5s a ByteString and returns a list of +-- | The 'group' function takes a ByteString and returns a list of -- ByteStrings such that the concatenation of the result is equal to the -- argument. Moreover, each sublist in the result contains only equal -- elements. For example, @@ -1191,6 +1192,33 @@ (S.unsafeTake n c : acc) (Empty (go (Chunk (S.unsafeDrop n c) cs))) +{-#INLINABLE group #-} + +-- | The 'groupBy' function is a generalized version of 'group'. +groupBy :: Monad m => (Word8 -> Word8 -> Bool) -> ByteString m r -> Stream (ByteString m) m r +groupBy rel = go + where + go (Empty r) = Return r + go (Go m) = Effect $ liftM go m + go (Chunk c cs) + | S.length c == 1 = Step $ to [c] (S.unsafeHead c) cs + | otherwise = Step $ to [S.unsafeTake 1 c] (S.unsafeHead c) + (Chunk (S.unsafeTail c) cs) + + to acc !_ (Empty r) = revNonEmptyChunks + acc + (Empty (Return r)) + to acc !w (Chunk c cs) = + case findIndexOrEnd (not . rel w) c of + 0 -> revNonEmptyChunks + acc + (Empty (go (Chunk c cs))) + n | n == S.length c -> to (S.unsafeTake n c : acc) w cs + | otherwise -> revNonEmptyChunks + (S.unsafeTake n c : acc) + (Empty (go (Chunk (S.unsafeDrop n c) cs))) +{-#INLINABLE groupBy #-} + -- -- | The 'groupBy' function is the non-overloaded version of 'group'. -- -- -- groupBy :: (Word8 -> Word8 -> Bool) -> ByteString -> [ByteString] @@ -1414,7 +1442,7 @@ -- | hGetNonBlockingN is similar to 'hGetContentsN', except that it will never block -- waiting for data to become available, instead it returns only whatever data -- is available. Chunks are read on demand, in @k@-sized chunks. --- + hGetNonBlockingN :: MonadIO m => Int -> Handle -> Int -> ByteString m () hGetNonBlockingN k h n | n > 0 = readChunks n where diff --git a/Data/ByteString/Streaming/Char8.hs b/Data/ByteString/Streaming/Char8.hs --- a/Data/ByteString/Streaming/Char8.hs +++ b/Data/ByteString/Streaming/Char8.hs @@ -65,6 +65,7 @@ , break -- break :: Monad m => (Char -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , drop -- drop :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m r , group -- group :: Monad m => ByteString m r -> Stream (ByteString m) m r + , groupBy , span -- span :: Monad m => (Char -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , splitAt -- splitAt :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m (ByteString m r) , splitWith -- splitWith :: Monad m => (Char -> Bool) -> ByteString m r -> Stream (ByteString m) m r @@ -188,6 +189,7 @@ import Foreign.Storable import Data.Functor.Compose import Data.Functor.Sum +import qualified Data.List as L unpack :: Monad m => ByteString m r -> Stream (Of Char) m r unpack bs = case bs of @@ -274,6 +276,10 @@ last = liftM (\(m:>r) -> fmap (w2c) m :> r) . R.last {-# INLINE last #-} +groupBy :: Monad m => (Char -> Char -> Bool) -> ByteString m r -> Stream (ByteString m) m r +groupBy rel = R.groupBy (\w w' -> rel (w2c w) (w2c w')) +{-#INLINE groupBy #-} + -- | /O(1)/ Extract the head and tail of a ByteString, returning Nothing -- if it is empty. uncons :: Monad m => ByteString m r -> m (Either r (Char, ByteString m r)) @@ -481,21 +487,37 @@ -} lines :: Monad m => ByteString m r -> Stream (ByteString m) m r -lines = R.split 10 -{-#INLINE lines #-} +-- lines = loop +-- where +-- loop !x = case x of +-- Empty r -> Return r +-- Go m -> Effect $ liftM loop m +-- Chunk c0 cs0 -> comb [] (B.split 10 c0) cs0 +lines (Empty r) = Return r +lines (Go m) = Effect $ liftM lines m +lines (Chunk c0 cs0) = comb [] (B.split 10 c0) cs0 where + comb !acc [] (Empty r) = Step (revChunks acc (Return r)) + comb acc [] (Chunk c cs) = comb acc (B.split 10 c) cs + comb acc (s:[]) (Empty r) = Step (revChunks (s:acc) (Return r)) + comb acc (s:[]) (Chunk c cs) = comb (s:acc) (B.split 10 c) cs + comb acc b (Go m) = Effect (liftM (comb acc b) m) + comb acc (s:ss) cs = Step (revChunks (s:acc) (comb [] ss cs)) + revChunks cs r = L.foldl' (flip Chunk) (Empty r) cs +{-#INLINABLE lines #-} -- | The 'unlines' function restores line breaks between layers unlines :: Monad m => Stream (ByteString m) m r -> ByteString m r -unlines str = case str of - Return r -> Empty r - Step bstr -> do - st <- bstr - let bs = unlines st - case bs of - Chunk "" (Empty r) -> Empty r - Chunk "\n" (Empty r) -> bs - _ -> cons' '\n' bs - Effect m -> Go (liftM unlines m) +unlines = loop where + loop str = case str of + Return r -> Empty r + Step bstr -> do + st <- bstr + let bs = unlines st + case bs of + Chunk "" (Empty r) -> Empty r + Chunk "\n" (Empty r) -> bs + _ -> cons' '\n' bs + Effect m -> Go (liftM unlines m) {-#INLINABLE unlines #-} -- | 'words' breaks a byte stream up into a succession of byte streams diff --git a/streaming-bytestring.cabal b/streaming-bytestring.cabal --- a/streaming-bytestring.cabal +++ b/streaming-bytestring.cabal @@ -1,41 +1,14 @@ name: streaming-bytestring -version: 0.1.4.0 +version: 0.1.4.2 synopsis: effectful byte steams, or: bytestring io done right. description: This is an implementation of effectful, memory-constrained bytestrings (byte streams) and functions for streaming bytestring manipulation, adequate for non-lazy-io. . - Interoperation with @pipes@ uses this isomorphism: - . - > Streaming.unfoldrChunks Pipes.next :: Monad m => Producer ByteString m r -> ByteString m r - > Pipes.unfoldr Streaming.nextChunk :: Monad m => ByteString m r -> Producer ByteString m r - . - Interoperation with @io-streams@ is thus: - . - > IOStreams.unfoldM Streaming.unconsChunk :: ByteString IO () -> IO (InputStream ByteString) - > Streaming.reread IOStreams.read :: InputStream ByteString -> ByteString IO () - . - and similarly for other rational streaming io libraries. - . - Problems and questions about the library can be put as issues on - the github page, or mailed to the - . - . - A tutorial module is in the works; - , - for the moment, - is a sequence of simplified implementations of familiar shell utilities. - The same programs are implemented at the end of the excellent - . - It is generally much simpler; in some case simpler than what - you would write with lazy bytestrings. - - is a simple GET request that returns a byte stream. - . - The implementation is idiot-simple; it follows the + The implementation follows the details of @Data.ByteString.Lazy@ and @Data.ByteString.Lazy.Char8@ - as far as is possible, replacing the lazy bytestring type: + in unrelenting detail, replacing the lazy bytestring type: . > data ByteString = Empty | Chunk Strict.ByteString ByteString . @@ -43,7 +16,7 @@ . > data ByteString m r = Empty r | Chunk Strict.ByteString (ByteString m r) | Go (m (ByteString m r)) . - (Constructors are necessarily hidden in internal modules in both cases.) + (Constructors are necessarily hidden in internal modules in both the @Lazy@ and the @Streaming@.) . That's it. As a lazy bytestring is implemented internally by a sort of list of strict bytestring chunks, a streaming bytestring is @@ -64,7 +37,7 @@ The default I/O chunk size is 32k, which should be good in most circumstances.\" . ... which is very much the idea of this library: the default chunk size for - 'hGetContents' and the like follows @Data.ByteString.Lazy@ and operations + 'hGetContents' and the like follows @Data.ByteString.Lazy@; operations like @lines@ and @append@ and so on are tailored not to increase chunk size. . The present library is thus nothing but /lazy bytestring done right/. @@ -117,7 +90,7 @@ These concepts belong to the ABCs of streaming; @lines@ is just a textbook example, and it is of course handled correctly in @Data.ByteString.Lazy@. - But the concepts are catastrophically mishandled in the streaming io libraries + But the concepts are /catastrophically mishandled/ in /all/ streaming io libraries other than pipes. Already the @enumerator@ and @iteratee@ libraries were completely defeated by @lines@: see e.g. the @enumerator@ implementation of @@ -125,14 +98,13 @@ This will concatenate strict text forever, if that's what is coming in. The rot spreads from there. It is just a fact that in all of the general streaming io - frameworks other than pipes, - it becomes torture to express elementary distinctions - that are transparently - and immediately contained in any idea of streaming whatsoever. + frameworks other than pipes,it becomes torture to express elementary distinctions + that are transparently and immediately contained in any + idea of streaming whatsoever. . - Though we barely alter signatures in @Data.ByteString.Lazy@ - more than is required by the types, - the point of view that emerges is very much that of + Though, as was said above, we barely alter signatures in @Data.ByteString.Lazy@ + more than is required by the types, the point of view that emerges + is very much that of @pipes-bytestring@ and @pipes-group@. In particular we have these correspondences: . @@ -149,13 +121,33 @@ where the @Stream@ type expresses the sequencing of @ByteString m _@ layers with the usual \'free monad\' sequencing. . - If you are unfamiliar with this - way of structuring material you might take a look at the tutorial for - - and the examples in the documentation for the streaming library. See also - - of the shell-like examples mentioned above. Or, again, put a question on - the issues page or to the pipes list. + Interoperation with @pipes-bytestring@ uses this isomorphism: + . + > Streaming.ByteString.unfoldrChunks Pipes.next :: Monad m => Producer ByteString m r -> ByteString m r + > Pipes.unfoldr Streaming.ByteString.nextChunk :: Monad m => ByteString m r -> Producer ByteString m r + . + Interoperation with @io-streams@ is thus: + . + > IOStreams.unfoldM Streaming.ByteString.unconsChunk :: ByteString IO () -> IO (InputStream ByteString) + > Streaming.ByteString.reread IOStreams.read :: InputStream ByteString -> ByteString IO () + . + and similarly for other rational streaming io libraries. + . + Problems and questions about the library can be put as issues on + the github page, or mailed to the + . + . + A tutorial module is in the works; + , + for the moment, + is a sequence of simplified implementations of familiar shell utilities. + The same programs are implemented at the end of the excellent + . + It is generally much simpler; in some case simpler than what + you would write with lazy bytestrings. + + is a simple GET request that returns a byte stream. + . license: BSD3