streaming-bytestring 0.1.4.0 → 0.1.4.2
raw patch · 3 files changed
+103/−61 lines, 3 files
Files
- Data/ByteString/Streaming.hs +30/−2
- Data/ByteString/Streaming/Char8.hs +34/−12
- streaming-bytestring.cabal +39/−47
Data/ByteString/Streaming.hs view
@@ -99,6 +99,7 @@ , break -- break :: Monad m => (Word8 -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , drop -- drop :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m r , group -- group :: Monad m => ByteString m r -> Stream (ByteString m) m r + , groupBy , span -- span :: Monad m => (Word8 -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , splitAt -- splitAt :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m (ByteString m r) , splitWith -- splitWith :: Monad m => (Word8 -> Bool) -> ByteString m r -> Stream (ByteString m) m r @@ -1158,7 +1159,7 @@ ----- | The 'group' function take`5s a ByteString and returns a list of+-- | The 'group' function takes a ByteString and returns a list of -- ByteStrings such that the concatenation of the result is equal to the -- argument. Moreover, each sublist in the result contains only equal -- elements. For example,@@ -1191,6 +1192,33 @@ (S.unsafeTake n c : acc) (Empty (go (Chunk (S.unsafeDrop n c) cs))) +{-#INLINABLE group #-}++-- | The 'groupBy' function is a generalized version of 'group'.+groupBy :: Monad m => (Word8 -> Word8 -> Bool) -> ByteString m r -> Stream (ByteString m) m r+groupBy rel = go+ where+ go (Empty r) = Return r+ go (Go m) = Effect $ liftM go m+ go (Chunk c cs)+ | S.length c == 1 = Step $ to [c] (S.unsafeHead c) cs+ | otherwise = Step $ to [S.unsafeTake 1 c] (S.unsafeHead c)+ (Chunk (S.unsafeTail c) cs)++ to acc !_ (Empty r) = revNonEmptyChunks + acc + (Empty (Return r))+ to acc !w (Chunk c cs) =+ case findIndexOrEnd (not . rel w) c of+ 0 -> revNonEmptyChunks + acc + (Empty (go (Chunk c cs)))+ n | n == S.length c -> to (S.unsafeTake n c : acc) w cs+ | otherwise -> revNonEmptyChunks + (S.unsafeTake n c : acc)+ (Empty (go (Chunk (S.unsafeDrop n c) cs)))+{-#INLINABLE groupBy #-}+ -- -- | The 'groupBy' function is the non-overloaded version of 'group'. -- -- -- groupBy :: (Word8 -> Word8 -> Bool) -> ByteString -> [ByteString]@@ -1414,7 +1442,7 @@ -- | hGetNonBlockingN is similar to 'hGetContentsN', except that it will never block -- waiting for data to become available, instead it returns only whatever data -- is available. Chunks are read on demand, in @k@-sized chunks.---+ hGetNonBlockingN :: MonadIO m => Int -> Handle -> Int -> ByteString m () hGetNonBlockingN k h n | n > 0 = readChunks n where
Data/ByteString/Streaming/Char8.hs view
@@ -65,6 +65,7 @@ , break -- break :: Monad m => (Char -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , drop -- drop :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m r , group -- group :: Monad m => ByteString m r -> Stream (ByteString m) m r + , groupBy , span -- span :: Monad m => (Char -> Bool) -> ByteString m r -> ByteString m (ByteString m r) , splitAt -- splitAt :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m (ByteString m r) , splitWith -- splitWith :: Monad m => (Char -> Bool) -> ByteString m r -> Stream (ByteString m) m r @@ -188,6 +189,7 @@ import Foreign.Storable import Data.Functor.Compose import Data.Functor.Sum+import qualified Data.List as L unpack :: Monad m => ByteString m r -> Stream (Of Char) m r unpack bs = case bs of @@ -274,6 +276,10 @@ last = liftM (\(m:>r) -> fmap (w2c) m :> r) . R.last {-# INLINE last #-} +groupBy :: Monad m => (Char -> Char -> Bool) -> ByteString m r -> Stream (ByteString m) m r+groupBy rel = R.groupBy (\w w' -> rel (w2c w) (w2c w'))+{-#INLINE groupBy #-}+ -- | /O(1)/ Extract the head and tail of a ByteString, returning Nothing -- if it is empty. uncons :: Monad m => ByteString m r -> m (Either r (Char, ByteString m r))@@ -481,21 +487,37 @@ -} lines :: Monad m => ByteString m r -> Stream (ByteString m) m r-lines = R.split 10-{-#INLINE lines #-}+-- lines = loop+-- where+-- loop !x = case x of+-- Empty r -> Return r+-- Go m -> Effect $ liftM loop m+-- Chunk c0 cs0 -> comb [] (B.split 10 c0) cs0+lines (Empty r) = Return r+lines (Go m) = Effect $ liftM lines m+lines (Chunk c0 cs0) = comb [] (B.split 10 c0) cs0 where+ comb !acc [] (Empty r) = Step (revChunks acc (Return r))+ comb acc [] (Chunk c cs) = comb acc (B.split 10 c) cs+ comb acc (s:[]) (Empty r) = Step (revChunks (s:acc) (Return r))+ comb acc (s:[]) (Chunk c cs) = comb (s:acc) (B.split 10 c) cs+ comb acc b (Go m) = Effect (liftM (comb acc b) m)+ comb acc (s:ss) cs = Step (revChunks (s:acc) (comb [] ss cs))+ revChunks cs r = L.foldl' (flip Chunk) (Empty r) cs+{-#INLINABLE lines #-} -- | The 'unlines' function restores line breaks between layers unlines :: Monad m => Stream (ByteString m) m r -> ByteString m r-unlines str = case str of- Return r -> Empty r- Step bstr -> do - st <- bstr - let bs = unlines st- case bs of - Chunk "" (Empty r) -> Empty r- Chunk "\n" (Empty r) -> bs - _ -> cons' '\n' bs- Effect m -> Go (liftM unlines m)+unlines = loop where+ loop str = case str of+ Return r -> Empty r+ Step bstr -> do + st <- bstr + let bs = unlines st+ case bs of + Chunk "" (Empty r) -> Empty r+ Chunk "\n" (Empty r) -> bs + _ -> cons' '\n' bs+ Effect m -> Go (liftM unlines m) {-#INLINABLE unlines #-} -- | 'words' breaks a byte stream up into a succession of byte streams
streaming-bytestring.cabal view
@@ -1,41 +1,14 @@ name: streaming-bytestring-version: 0.1.4.0+version: 0.1.4.2 synopsis: effectful byte steams, or: bytestring io done right. description: This is an implementation of effectful, memory-constrained bytestrings (byte streams) and functions for streaming bytestring manipulation, adequate for non-lazy-io. .- Interoperation with @pipes@ uses this isomorphism:- . - > Streaming.unfoldrChunks Pipes.next :: Monad m => Producer ByteString m r -> ByteString m r- > Pipes.unfoldr Streaming.nextChunk :: Monad m => ByteString m r -> Producer ByteString m r- .- Interoperation with @io-streams@ is thus:- .- > IOStreams.unfoldM Streaming.unconsChunk :: ByteString IO () -> IO (InputStream ByteString)- > Streaming.reread IOStreams.read :: InputStream ByteString -> ByteString IO ()- .- and similarly for other rational streaming io libraries. - .- Problems and questions about the library can be put as issues on - the github page, or mailed to the - <https://groups.google.com/forum/#!forum/haskell-pipes pipes list>.- .- A tutorial module is in the works; - <https://gist.github.com/michaelt/6c6843e6dd8030e95d58 here>,- for the moment, - is a sequence of simplified implementations of familiar shell utilities. - The same programs are implemented at the end of the excellent- <http://hackage.haskell.org/package/io-streams-1.3.2.0/docs/System-IO-Streams-Tutorial.html io-streams tutorial>.- It is generally much simpler; in some case simpler than what- you would write with lazy bytestrings. - <https://gist.github.com/michaelt/2dcea1ba32562c091357 Here>- is a simple GET request that returns a byte stream.- .- The implementation is idiot-simple; it follows the+ The implementation follows the details of @Data.ByteString.Lazy@ and @Data.ByteString.Lazy.Char8@- as far as is possible, replacing the lazy bytestring type:+ in unrelenting detail, replacing the lazy bytestring type: . > data ByteString = Empty | Chunk Strict.ByteString ByteString .@@ -43,7 +16,7 @@ . > data ByteString m r = Empty r | Chunk Strict.ByteString (ByteString m r) | Go (m (ByteString m r)) .- (Constructors are necessarily hidden in internal modules in both cases.) + (Constructors are necessarily hidden in internal modules in both the @Lazy@ and the @Streaming@.) . That's it. As a lazy bytestring is implemented internally by a sort of list of strict bytestring chunks, a streaming bytestring is @@ -64,7 +37,7 @@ The default I/O chunk size is 32k, which should be good in most circumstances.\" . ... which is very much the idea of this library: the default chunk size for- 'hGetContents' and the like follows @Data.ByteString.Lazy@ and operations+ 'hGetContents' and the like follows @Data.ByteString.Lazy@; operations like @lines@ and @append@ and so on are tailored not to increase chunk size. . The present library is thus nothing but /lazy bytestring done right/. @@ -117,7 +90,7 @@ These concepts belong to the ABCs of streaming; @lines@ is just a textbook example, and it is of course handled correctly in @Data.ByteString.Lazy@.- But the concepts are catastrophically mishandled in the streaming io libraries + But the concepts are /catastrophically mishandled/ in /all/ streaming io libraries other than pipes. Already the @enumerator@ and @iteratee@ libraries were completely defeated by @lines@: see e.g. the @enumerator@ implementation of @@ -125,14 +98,13 @@ This will concatenate strict text forever, if that's what is coming in. The rot spreads from there. It is just a fact that in all of the general streaming io - frameworks other than pipes, - it becomes torture to express elementary distinctions - that are transparently- and immediately contained in any idea of streaming whatsoever. + frameworks other than pipes,it becomes torture to express elementary distinctions + that are transparently and immediately contained in any + idea of streaming whatsoever. .- Though we barely alter signatures in @Data.ByteString.Lazy@ - more than is required by the types, - the point of view that emerges is very much that of+ Though, as was said above, we barely alter signatures in @Data.ByteString.Lazy@ + more than is required by the types, the point of view that emerges + is very much that of @pipes-bytestring@ and @pipes-group@. In particular we have these correspondences: .@@ -149,13 +121,33 @@ where the @Stream@ type expresses the sequencing of @ByteString m _@ layers with the usual \'free monad\' sequencing. .- If you are unfamiliar with this- way of structuring material you might take a look at the tutorial for - <http://hackage.haskell.org/package/pipes-group-1.0.2/docs/Pipes-Group-Tutorial.html pipes-group>- and the examples in the documentation for the streaming library. See also- <https://gist.github.com/michaelt/6c6843e6dd8030e95d58 simple implementations> - of the shell-like examples mentioned above. Or, again, put a question on- the issues page or to the pipes list.+ Interoperation with @pipes-bytestring@ uses this isomorphism:+ . + > Streaming.ByteString.unfoldrChunks Pipes.next :: Monad m => Producer ByteString m r -> ByteString m r+ > Pipes.unfoldr Streaming.ByteString.nextChunk :: Monad m => ByteString m r -> Producer ByteString m r+ .+ Interoperation with @io-streams@ is thus:+ .+ > IOStreams.unfoldM Streaming.ByteString.unconsChunk :: ByteString IO () -> IO (InputStream ByteString)+ > Streaming.ByteString.reread IOStreams.read :: InputStream ByteString -> ByteString IO ()+ .+ and similarly for other rational streaming io libraries. + .+ Problems and questions about the library can be put as issues on + the github page, or mailed to the + <https://groups.google.com/forum/#!forum/haskell-pipes pipes list>.+ .+ A tutorial module is in the works; + <https://gist.github.com/michaelt/6c6843e6dd8030e95d58 here>,+ for the moment, + is a sequence of simplified implementations of familiar shell utilities. + The same programs are implemented at the end of the excellent+ <http://hackage.haskell.org/package/io-streams-1.3.2.0/docs/System-IO-Streams-Tutorial.html io-streams tutorial>.+ It is generally much simpler; in some case simpler than what+ you would write with lazy bytestrings. + <https://gist.github.com/michaelt/2dcea1ba32562c091357 Here>+ is a simple GET request that returns a byte stream.+ . license: BSD3