streaming-bytestring 0.1.1.0 → 0.1.2.0
raw patch · 4 files changed
+327/−281 lines, 4 filesdep ~streaming
Dependency ranges changed: streaming
Files
- Data/ByteString/Streaming.hs +238/−181
- Data/ByteString/Streaming/Char8.hs +71/−89
- Data/ByteString/Streaming/Internal.hs +16/−9
- streaming-bytestring.cabal +2/−2
Data/ByteString/Streaming.hs view
@@ -57,14 +57,15 @@ , unpack -- unpack :: Monad m => ByteString m r -> Stream (Of Word8) m r , fromLazy -- fromLazy :: Monad m => ByteString -> ByteString m () , toLazy -- toLazy :: Monad m => ByteString m () -> m ByteString- , toLazy' -- toLazy' :: Monad m => ByteString m () -> m (Of ByteString r) + , toLazy_ -- toLazy' :: Monad m => ByteString m () -> m (Of ByteString r) , fromChunks -- fromChunks :: Monad m => Stream (Of ByteString) m r -> ByteString m r , toChunks -- toChunks :: Monad m => ByteString m r -> Stream (Of ByteString) m r , fromStrict -- fromStrict :: ByteString -> ByteString m () , toStrict -- toStrict :: Monad m => ByteString m () -> m ByteString - , toStrict' -- toStrict' :: Monad m => ByteString m r -> m (Of ByteString r) - , drain- , wrap+ , toStrict_ -- toStrict' :: Monad m => ByteString m r -> m (Of ByteString r) + , effects+ , drained+ , mwrap -- * Transforming ByteStrings , map -- map :: Monad m => (Word8 -> Word8) -> ByteString m r -> ByteString m r @@ -79,7 +80,7 @@ , filter -- filter :: (Word8 -> Bool) -> ByteString m r -> ByteString m r , uncons -- uncons :: Monad m => ByteString m r -> m (Either r (Word8, ByteString m r)) , nextByte -- nextByte :: Monad m => ByteString m r -> m (Either r (Word8, ByteString m r))-+ , denull -- * Direct chunk handling , unconsChunk@@ -100,7 +101,6 @@ , splitWith -- splitWith :: Monad m => (Word8 -> Bool) -> ByteString m r -> Stream (ByteString m) m r , take -- take :: Monad m => GHC.Int.Int64 -> ByteString m r -> ByteString m () , takeWhile -- takeWhile :: (Word8 -> Bool) -> ByteString m r -> ByteString m () - , denull -- ** Breaking into many substrings , split -- split :: Monad m => Word8 -> ByteString m r -> Stream (ByteString m) m r @@ -130,17 +130,18 @@ -- * Folds, including support for `Control.Foldl` , foldr -- foldr :: Monad m => (Word8 -> a -> a) -> a -> ByteString m () -> m a , fold -- fold :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteString m () -> m b - , fold' -- fold' :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteString m r -> m (b, r) + , fold_ -- fold' :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteString m r -> m (b, r) , head- , head'+ , head_ , last- , last'+ , last_ , length- , length'+ , length_ , null- , null'+ , nulls+ , null_ , count- , count'+ , count_ -- * I\/O with 'ByteString's -- ** Standard input and output@@ -187,7 +188,7 @@ import Data.ByteString.Builder.Internal hiding (hPut, defaultChunkSize, empty, append) import Data.ByteString.Streaming.Internal -import Streaming hiding (concats, unfold, distribute, wrap)+import Streaming hiding (concats, unfold, distribute, mwrap) import Streaming.Internal (Stream (..)) import qualified Streaming.Prelude as SP @@ -205,6 +206,7 @@ import Foreign.Storable import Foreign.Ptr import Data.Functor.Compose+import Data.Functor.Sum -- | /O(n)/ Concatenate a stream of byte streams. concat :: Monad m => Stream (ByteString m) m r -> ByteString m r concat x = destroy x join Go Empty @@ -221,37 +223,56 @@ (join . hoist (Go . liftM Empty)) {-# INLINE distribute #-} +{-| Perform the effects contained in an effectful bytestring, ignoring the bytes. -drain :: Monad m => ByteString m r -> m r-drain bs = case bs of +-}+effects :: Monad m => ByteString m r -> m r+effects bs = case bs of Empty r -> return r- Go m -> m >>= drain- Chunk _ rest -> drain rest-{-# INLINABLE drain #-}+ Go m -> m >>= effects+ Chunk _ rest -> effects rest+{-# INLINABLE effects #-}+++{-| Perform the effects contained in the second in an effectful pair of bytestrings, + ignoring the bytes. It would typically be used at the type++> ByteString m (ByteString m r) -> ByteString m r++-}++drained :: (Monad m, MonadTrans t, Monad (t m)) => t m (ByteString m r) -> t m r+drained t = t >>= lift . effects -- ----------------------------------------------------------------------------- -- Introducing and eliminating 'ByteString's --- | /O(1)/ The empty 'ByteString' -- i.e. return ()+{-| /O(1)/ The empty 'ByteString' -- i.e. @return ()@ Note that @ByteString m w@ is+ generally a monoid for monoidal values of @w@, like @()@+-} empty :: ByteString m () empty = Empty () {-# INLINE empty #-} --- | /O(1)/ Yield a 'Word8' as a minimal 'ByteString'+{-| /O(1)/ Yield a 'Word8' as a minimal 'ByteString'+-} singleton :: Monad m => Word8 -> ByteString m () singleton w = Chunk (S.singleton w) (Empty ()) {-# INLINE singleton #-} --- | /O(n)/ Convert a monadic stream of individual 'Word8's into a packed byte stream.+{-| /O(n)/ Convert a monadic stream of individual 'Word8's into a packed byte stream.+-} pack :: Monad m => Stream (Of Word8) m r -> ByteString m r pack = packBytes {-#INLINE pack #-} --- | /O(n)/ Converts a packed byte stream into a stream of individual bytes.+{-| /O(n)/ Converts a packed byte stream into a stream of individual bytes.+-} unpack :: Monad m => ByteString m r -> Stream (Of Word8) m r unpack = unpackBytes --- | /O(c)/ Convert a monadic stream of individual strict 'ByteString' --- chunks into a byte stream.+{-| /O(c)/ Convert a monadic stream of individual strict 'ByteString' + chunks into a byte stream.+-} fromChunks :: Monad m => Stream (Of P.ByteString) m r -> ByteString m r fromChunks cs = destroy cs (\(bs :> rest) -> Chunk bs rest)@@ -259,8 +280,9 @@ return {-#INLINE fromChunks#-} --- | /O(c)/ Convert a byte stream into a stream of individual strict bytestrings.--- This of course exposes the internal chunk structure.+{-| /O(c)/ Convert a byte stream into a stream of individual strict bytestrings.+ This of course exposes the internal chunk structure.+-} toChunks :: Monad m => ByteString m r -> Stream (Of P.ByteString) m r toChunks bs = dematerialize bs@@ -269,38 +291,40 @@ Delay {-#INLINE toChunks#-} --- |/O(1)/ yield a strict 'ByteString' chunk. +{-| /O(1)/ yield a strict 'ByteString' chunk. +-} fromStrict :: P.ByteString -> ByteString m () fromStrict bs | S.null bs = Empty () | otherwise = Chunk bs (Empty ()) {-# INLINE fromStrict #-} --- |/O(n)/ Convert a byte stream into a single strict 'ByteString'.------ Note that this is an /expensive/ operation that forces the whole monadic--- ByteString into memory and then copies all the data. If possible, try to--- avoid converting back and forth between streaming and strict bytestrings.+{-| /O(n)/ Convert a byte stream into a single strict 'ByteString'. -toStrict :: Monad m => ByteString m () -> m (S.ByteString)-toStrict = liftM S.concat . SP.toListM . toChunks-{-# INLINE toStrict #-}+ Note that this is an /expensive/ operation that forces the whole monadic+ ByteString into memory and then copies all the data. If possible, try to+ avoid converting back and forth between streaming and strict bytestrings.+-}+toStrict_ :: Monad m => ByteString m () -> m (S.ByteString)+toStrict_ = liftM S.concat . SP.toList_ . toChunks+{-# INLINE toStrict_ #-} {-| /O(n)/ Convert a monadic byte stream into a single strict 'ByteString', retaining the return value of the original pair. This operation is for use with 'mapsM'. -> mapsM R.toStrict' :: Monad m => Stream (ByteString m) m r -> Stream (Of ByteString) m r +> mapsM R.toStrict :: Monad m => Stream (ByteString m) m r -> Stream (Of ByteString) m r - It is subject to all the objections one makes to 'toStrict'. + It is subject to all the objections one makes to Data.ByteString.Lazy 'toStrict'; + all of these are devastating. -}-toStrict' :: Monad m => ByteString m r -> m (Of S.ByteString r)-toStrict' bs = do - (bss :> r) <- SP.toListM' (toChunks bs)+toStrict :: Monad m => ByteString m r -> m (Of S.ByteString r)+toStrict bs = do + (bss :> r) <- SP.toList (toChunks bs) return $ (S.concat bss :> r)-{-# INLINE toStrict' #-}+{-# INLINE toStrict #-} -{- |/O(c)/ Transmute a lazy bytestring to its representation+{- |/O(c)/ Transmute a pseudo-pure lazy bytestring to its representation as a monadic stream of chunks. >>> Q.putStrLn $ Q.fromLazy "hi"@@ -315,46 +339,56 @@ fromLazy = BI.foldrChunks Chunk (Empty ()) {-# INLINE fromLazy #-} -{- |/O(n)/ Convert a monadic byte stream into a single lazy 'ByteString'- with the same internal chunk structure.-->>> Q.toLazy "hello"-"hello"+{-| /O(n)/ Convert an effectful byte stream into a single lazy 'ByteString'+ with the same internal chunk structure. See @toLazy@ which preserve+ connectedness by keeping the return value of the effectful bytestring. -}-toLazy :: Monad m => ByteString m () -> m BI.ByteString-toLazy bs = dematerialize bs- (\() -> return (BI.Empty))+toLazy_ :: Monad m => ByteString m r -> m BI.ByteString+toLazy_ bs = dematerialize bs+ (\_ -> return (BI.Empty)) (\b mx -> liftM (BI.Chunk b) mx) join-{-#INLINE toLazy #-} +{-#INLINE toLazy_ #-} -{- |/O(n)/ Convert a monadic byte stream into a single lazy 'ByteString'- with the same invisible chunk structure, retaining the original+{-| /O(n)/ Convert an effectful byte stream into a single lazy 'ByteString'+ with the same internal chunk structure, retaining the original return value. ->>> Q.toLazy' "hello"+ This is the canonical way of breaking streaming (@toStrict@ and the+ like are far more demonic). Essentially one is dividing the interleaved+ layers of effects and bytes into one immense layer of effects, + followed by the memory of the succession of bytes. ++ Because one preserves the return value, @toLazy@ is a suitable argument+ for 'Streaming.mapsM'++> S.mapsM Q.toLazy :: Stream (ByteString m) m r -> Stream (Of L.ByteString) m r++>>> Q.toLazy "hello" "hello" :> ()->>> S.toListM $ mapsM Q.toLazy' $ Q.lines $ "one\ntwo\three\nfour\nfive\n"-["one","two\three","four","five",""]+>>> S.toListM $ traverses Q.toLazy $ Q.lines "one\ntwo\nthree\nfour\nfive\n"+["one","two","three","four","five",""] -- [L.ByteString] -}-toLazy' :: Monad m => ByteString m r -> m (Of BI.ByteString r)-toLazy' bs0 = dematerialize bs0+toLazy :: Monad m => ByteString m r -> m (Of BI.ByteString r)+toLazy bs0 = dematerialize bs0 (\r -> return (BI.Empty :> r)) (\b mx -> do (bs :> x) <- mx return $ BI.Chunk b bs :> x ) join-{-#INLINE toLazy' #-} +{-#INLINE toLazy #-} + -- --------------------------------------------------------------------- -- Basic interface ---{-| /O(1)/ Test whether a ByteString is empty. The value is of course in the base monad.+{-| /O(1)/ Test whether an ByteString is empty. The value is of course in + the monad of the effects. >>> Q.null "one\ntwo\three\nfour\nfive\n" False@@ -363,56 +397,100 @@ >>> :t Q.null $ Q.take 0 Q.stdin Q.null $ Q.take 0 Q.stdin :: MonadIO m => m Bool -}-null :: Monad m => ByteString m r -> m Bool-null (Empty _) = return True-null (Go m) = m >>= null-null (Chunk bs rest) = if S.null bs - then null rest +null_ :: Monad m => ByteString m r -> m Bool+null_ (Empty _) = return True+null_ (Go m) = m >>= null_ +null_ (Chunk bs rest) = if S.null bs + then null_ rest else return False-{-# INLINABLE null #-}+{-# INLINABLE null_ #-} +{-| Remove empty ByteStrings from a stream of bytestrings.++-}+denull :: Monad m => Stream (ByteString m) m r -> Stream (ByteString m) m r +denull = hoist (run . maps effects) . separate . mapsM nulls+{-#INLINE denull #-}+ {- | /O(1)/ Test whether a ByteString is empty, collecting its return value; -- to reach the return value, this operation must check the whole length of the string. ->>> Q.null' "one\ntwo\three\nfour\nfive\n"+>>> Q.null "one\ntwo\three\nfour\nfive\n" False :> ()-[*Main]->>> Q.null' ""+>>> Q.null "" True :> ()->>> S.print $ mapsM R.null' $ Q.lines "yours,\nMeredith"+>>> S.print $ mapsM R.null $ Q.lines "yours,\nMeredith" False False -}-null' :: Monad m => ByteString m r -> m (Of Bool r)-null' (Empty r) = return $! True :> r-null' (Go m) = m >>= null'-null' (Chunk bs rest) = if S.null bs - then null' rest +null :: Monad m => ByteString m r -> m (Of Bool r)+null (Empty r) = return $! True :> r+null (Go m) = m >>= null+null (Chunk bs rest) = if S.null bs + then null rest else do - r <- SP.drain (toChunks rest)+ r <- SP.effects (toChunks rest) return (False :> r)-{-# INLINABLE null' #-}+{-# INLINABLE null #-} +{-| /O1/ Distinguish empty from non-empty lines, while maintaining streaming; + the empty ByteStrings are on the right -length :: Monad m => ByteString m r -> m Int-length = liftM (\(n:> _) -> n) . foldlChunks (\n c -> n + fromIntegral (S.length c)) 0 -{-# INLINE length #-}+>>> nulls :: ByteString m r -> m (Sum (ByteString m) (ByteString m) r) -{-| /O(n\/c)/ 'length'' returns the length of a byte stream as an 'Int'+ There are many ways to remove null bytestrings from a + @Stream (ByteString m) m r@ (besides using @denull@). If we pass next to++>>> mapsM nulls bs :: Stream (Sum (ByteString m) (ByteString m)) m r++ then can then apply @Streaming.separate@ to get++>>> separate (mapsM nulls bs) :: Stream (ByteString m) (Stream (ByteString m) m) r++ The inner monad is now made of the empty bytestrings; we act on this + with @hoist@ , considering that ++>>> :t Q.effects . Q.concat+Q.effects . Q.concat+ :: Monad m => Stream (Q.ByteString m) m r -> m r++ we have ++>>> hoist (Q.effects . Q.concat) . separate . mapsM Q.nulls+ :: Monad n => Stream (Q.ByteString n) n b -> Stream (Q.ByteString n) n b++++-}++nulls :: Monad m => ByteString m r -> m (Sum (ByteString m) (ByteString m) r)+nulls (Empty r) = return (InR (return r))+nulls (Go m) = m >>= nulls+nulls (Chunk bs rest) = if S.null bs + then nulls rest + else return (InL (Chunk bs rest))+{-# INLINABLE nulls #-}+++length_ :: Monad m => ByteString m r -> m Int+length_ = liftM (\(n:> _) -> n) . foldlChunks (\n c -> n + fromIntegral (S.length c)) 0 +{-# INLINE length_ #-}++{-| /O(n\/c)/ 'length' returns the length of a byte stream as an 'Int' together with the return value. This makes various maps possible ->>> Q.length' "one\ntwo\three\nfour\nfive\n"+>>> Q.length "one\ntwo\three\nfour\nfive\n" 23 :> ()->>> S.print $ S.take 3 $ mapsM Q.length' $ Q.lines "one\ntwo\three\nfour\nfive\n" +>>> S.print $ S.take 3 $ mapsM Q.length $ Q.lines "one\ntwo\three\nfour\nfive\n" 3 8 4 -}-length' :: Monad m => ByteString m r -> m (Of Int r)-length' cs = foldlChunks (\n c -> n + fromIntegral (S.length c)) 0 cs-{-# INLINE length' #-}+length :: Monad m => ByteString m r -> m (Of Int r)+length cs = foldlChunks (\n c -> n + fromIntegral (S.length c)) 0 cs+{-# INLINE length #-} -- infixr 5 `cons` -- , `cons'` --same as list (:) -- -- nfixl 5 `snoc`@@ -450,22 +528,22 @@ {-# INLINE snoc #-} -- | /O(1)/ Extract the first element of a ByteString, which must be non-empty.-head :: Monad m => ByteString m r -> m Word8-head (Empty _) = error "head"-head (Chunk c _) = return $ S.unsafeHead c-head (Go m) = m >>= head-{-# INLINE head #-}+head_ :: Monad m => ByteString m r -> m Word8+head_ (Empty _) = error "head"+head_ (Chunk c _) = return $ S.unsafeHead c+head_ (Go m) = m >>= head_+{-# INLINE head_ #-} -- | /O(c)/ Extract the first element of a ByteString, which must be non-empty.-head' :: Monad m => ByteString m r -> m (Of (Maybe Word8) r)-head' (Empty r) = return (Nothing :> r)-head' (Chunk c rest) = case S.uncons c of - Nothing -> head' rest+head :: Monad m => ByteString m r -> m (Of (Maybe Word8) r)+head (Empty r) = return (Nothing :> r)+head (Chunk c rest) = case S.uncons c of + Nothing -> head rest Just (w,_) -> do- r <- SP.drain $ toChunks rest+ r <- SP.effects $ toChunks rest return $! (Just w) :> r-head' (Go m) = m >>= head'-{-# INLINE head' #-}+head (Go m) = m >>= head+{-# INLINE head #-} -- | /O(1)/ Extract the head and tail of a ByteString, or Nothing -- if it is empty@@ -511,28 +589,37 @@ -- | /O(n\/c)/ Extract the last element of a ByteString, which must be finite -- and non-empty.-last :: Monad m => ByteString m r -> m Word8-last (Empty _) = error "Data.ByteString.Streaming.last: empty string"-last (Go m) = m >>= last-last (Chunk c0 cs0) = go c0 cs0+last_ :: Monad m => ByteString m r -> m Word8+last_ (Empty _) = error "Data.ByteString.Streaming.last: empty string"+last_ (Go m) = m >>= last_+last_ (Chunk c0 cs0) = go c0 cs0 where go c (Empty _) = if S.null c then error "Data.ByteString.Streaming.last: empty string" else return $ unsafeLast c go _ (Chunk c cs) = go c cs go x (Go m) = m >>= go x-{-# INLINABLE last #-}+{-# INLINABLE last_ #-} -last' :: Monad m => ByteString m r -> m (Of (Maybe Word8) r)-last' (Empty r) = return (Nothing :> r)-last' (Go m) = m >>= last'-last' (Chunk c0 cs0) = go c0 cs0++last :: Monad m => ByteString m r -> m (Of (Maybe Word8) r)+last (Empty r) = return (Nothing :> r)+last (Go m) = m >>= last+last (Chunk c0 cs0) = go c0 cs0 where go c (Empty r) = return $ (Just (unsafeLast c) :> r) go _ (Chunk c cs) = go c cs go x (Go m) = m >>= go x -{-# INLINABLE last' #-}+{-# INLINABLE last #-} ++isPrefixOf :: Monad m => S.ByteString -> ByteString m r -> m (Sum (ByteString m) (ByteString m) r)+isPrefixOf bytes bs = do+ let len = S.length bytes+ (bytes' :> rest) <- toStrict $ splitAt (fromIntegral len) bs+ if bytes' == bytes + then return $ InR $ chunk bytes' >> rest+ else return $ InL $ chunk bytes' >> rest -- -- | /O(n\/c)/ Return all the elements of a 'ByteString' except the last one. -- init :: ByteString -> ByteString -- init Empty = errorEmptyStream "init"@@ -625,14 +712,14 @@ -- | 'fold\'' keeps the return value of the left-folded bytestring. Useful for -- simultaneous folds over a segmented bytestream -fold' :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteString m r -> m (Of b r)-fold' step0 begin done p0 = loop p0 begin+fold_ :: Monad m => (x -> Word8 -> x) -> x -> (x -> b) -> ByteString m r -> m (Of b r)+fold_ step0 begin done p0 = loop p0 begin where loop p !x = case p of Chunk bs bss -> loop bss $! S.foldl' step0 x bs Go m -> m >>= \p' -> loop p' x Empty r -> return (done x :> r)-{-# INLINABLE fold' #-}+{-# INLINABLE fold_ #-} -- @@ -659,7 +746,7 @@ -- --------------------------------------------------------------------- -- Special folds --- | /O(n)/ Concatenate a list of ByteStrings.+-- /O(n)/ Concatenate a list of ByteStrings. -- concat :: (Monad m) => [ByteString m ()] -> ByteString m () -- concat css0 = to css0 -- where@@ -763,7 +850,7 @@ > iterate f x == [x, f x, f (f x), ...] >>> R.stdout $ R.take 50 $ R.iterate succ 39-()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXY>>> +()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXY >>> Q.putStrLn $ Q.take 50 $ Q.iterate succ '\'' ()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXY @@ -776,7 +863,7 @@ element. >>> R.stdout $ R.take 50 $ R.repeat 60-<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<>>> +<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< >>> Q.putStrLn $ Q.take 50 $ Q.repeat 'z' zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz -}@@ -860,7 +947,7 @@ True >>> rest <- Q.putStrLn $ Q.splitAt 8 $ "Is there a God?" >> return True Is there->>> Q.drain rest+>>> Q.effects rest True -}@@ -1140,14 +1227,14 @@ -- -- > count = length . elemIndices ---count :: Monad m => Word8 -> ByteString m r -> m Int-count w = liftM (\(n :> _) -> n) . foldlChunks (\n c -> n + fromIntegral (S.count w c)) 0 -{-# INLINE count #-}+count_ :: Monad m => Word8 -> ByteString m r -> m Int+count_ w = liftM (\(n :> _) -> n) . foldlChunks (\n c -> n + fromIntegral (S.count w c)) 0 +{-# INLINE count_ #-} -- But more efficiently than using length on the intermediate list.-count' :: Monad m => Word8 -> ByteString m r -> m (Of Int r)-count' w cs = foldlChunks (\n c -> n + fromIntegral (S.count w c)) 0 cs-{-# INLINE count' #-}+count :: Monad m => Word8 -> ByteString m r -> m (Of Int r)+count w cs = foldlChunks (\n c -> n + fromIntegral (S.count w c)) 0 cs+{-# INLINE count #-} -- -- | The 'findIndex' function takes a predicate and a 'ByteString' and -- -- returns the index of the first element in the ByteString@@ -1277,17 +1364,17 @@ -- If so, close when done. -- --- | Read entire handle contents /lazily/ into a 'ByteString'. Chunks--- are read on demand, in at most @k@-sized chunks. It does not block--- waiting for a whole @k@-sized chunk, so if less than @k@ bytes are--- available then they will be returned immediately as a smaller chunk.------ The handle is closed on EOF.------ Note: the 'Handle' should be placed in binary mode with--- 'System.IO.hSetBinaryMode' for 'hGetContentsN' to--- work correctly.---+{- | Read entire handle contents /lazily/ into a 'ByteString'. Chunks+ are read on demand, in at most @k@-sized chunks. It does not block+ waiting for a whole @k@-sized chunk, so if less than @k@ bytes are+ available then they will be returned immediately as a smaller chunk.++ The handle is closed on EOF.++ Note: the 'Handle' should be placed in binary mode with+ 'System.IO.hSetBinaryMode' for 'hGetContentsN' to+ work correctly.+-} hGetContentsN :: MonadIO m => Int -> Handle -> ByteString m () hGetContentsN k h = loop -- TODO close on exceptions where@@ -1341,15 +1428,15 @@ msg = fn ++ ": illegal ByteString size " ++ showsPrec 9 sz [] {-# INLINABLE illegalBufferSize #-} --- | Read entire handle contents /lazily/ into a 'ByteString'. Chunks--- are read on demand, using the default chunk size.------ Once EOF is encountered, the Handle is closed.------ Note: the 'Handle' should be placed in binary mode with--- 'System.IO.hSetBinaryMode' for 'hGetContents' to--- work correctly.+{-| Read entire handle contents /lazily/ into a 'ByteString'. Chunks+ are read on demand, using the default chunk size. + Once EOF is encountered, the Handle is closed.++ Note: the 'Handle' should be placed in binary mode with+ 'System.IO.hSetBinaryMode' for 'hGetContents' to+ work correctly.+-} hGetContents :: MonadIO m => Handle -> ByteString m () hGetContents = hGetContentsN defaultChunkSize {-#INLINE hGetContents #-}@@ -1429,7 +1516,7 @@ stdout = hPut IO.stdout {-#INLINE stdout#-} --- | Similar to 'hPut' except that it will never block. Instead it returns+-- -- | Similar to 'hPut' except that it will never block. Instead it returns -- any tail that did not get written. This tail may be 'empty' in the case that -- the whole string was written, or the whole original string if nothing was -- written. Partial writes are also possible.@@ -1462,17 +1549,17 @@ -- putStrLn :: ByteString -> IO () -- putStrLn ps = hPut stdout ps >> hPut stdout (singleton 0x0a) ----- {-# DEPRECATED putStrLn--- "Use Data.ByteString.Lazy.Char8.putStrLn instead. (Functions that rely on ASCII encodings belong in Data.ByteString.Lazy.Char8)"--- #-}------ -- | The interact function takes a function of type @ByteString -> ByteString@--- -- as its argument. The entire input from the standard input device is passed--- -- to th is function as its argument, and the resulting string is output on the--- -- standard output device.--- --+++{- | The interact function takes a function of type @ByteString -> ByteString@+ as its argument. The entire input from the standard input device is passed+ to this function as its argument, and the resulting string is output on the+ standard output device.++> interact morph = stdout (morph stdin)+-} interact :: (ByteString IO () -> ByteString IO r) -> IO r-interact transformer = stdout (transformer stdin)+interact f = stdout (f stdin) {-# INLINE interact #-} -- -- ---------------------------------------------------------------------@@ -1534,35 +1621,6 @@ {-#INLINABLE zipWithStream #-} -{- Remove empty bytestrings from a stream of connected bytestrings,- as with Prelude @filter (not . null)@ This does not block streaming.-->>> let humpty = "all the\n\nking\'s horses"->>> Q.putStrLn humpty-all the--king's horses->>> Q.putStrLn $ Q.unlines $ Q.denull $ Q.lines humpty-all the-king's horses -->>> putStrLn $ unlines $ filter (not.null) $ lines humpty-all the-king's horses---}-denull :: Monad m => Stream (ByteString m) m r -> Stream (ByteString m) m r-denull = loop where- loop stream = do- e <- lift $ inspect stream- case e of- Left r -> Return r- Right bsstream -> do- e <- lift $ nextChunk bsstream- case e of - Left stream -> loop stream- Right (bs, qbs) -> Step (chunk bs >> fmap loop qbs)- {- Take a builder constructed otherwise and convert it to a genuine streaming bytestring. @@ -1580,7 +1638,6 @@ toStreamingByteString = toStreamingByteStringWith (safeStrategy BI.smallChunkSize BI.defaultChunkSize) {-#INLINE toStreamingByteString #-}-{-#SPECIALIZE toStreamingByteString :: Builder -> ByteString IO () #-} {-| Take a builder and convert it to a genuine streaming bytestring, using a specific allocation strategy.
Data/ByteString/Streaming/Char8.hs view
@@ -15,19 +15,18 @@ , string , unlines , unwords- , unlinesIndividual- , unwordsIndividual , singleton -- singleton :: Monad m => Char -> ByteString m () , fromChunks -- fromChunks :: Monad m => Stream (Of ByteString) m r -> ByteString m r , fromLazy -- fromLazy :: Monad m => ByteString -> ByteString m () , fromStrict -- fromStrict :: ByteString -> ByteString m () , toChunks -- toChunks :: Monad m => ByteString m r -> Stream (Of ByteString) m r , toLazy -- toLazy :: Monad m => ByteString m () -> m ByteString - , toLazy'+ , toLazy_ , toStrict -- toStrict :: Monad m => ByteString m () -> m ByteString - , toStrict'- , drain- , wrap+ , toStrict_+ , effects+ , drained+ , mwrap @@ -43,11 +42,11 @@ , append -- append :: Monad m => ByteString m r -> ByteString m s -> ByteString m s , filter -- filter :: (Char -> Bool) -> ByteString m r -> ByteString m r , head -- head :: Monad m => ByteString m r -> m Char- , head' -- head' :: Monad m => ByteString m r -> m (Of Char r)+ , head_ -- head' :: Monad m => ByteString m r -> m (Of Char r) , last -- last :: Monad m => ByteString m r -> m Char- , last' -- last' :: Monad m => ByteString m r -> m (Of Char r)+ , last_ -- last' :: Monad m => ByteString m r -> m (Of Char r) , null -- null :: Monad m => ByteString m r -> m Bool - , null' -- null' :: Monad m => ByteString m r -> m (Of Bool r)+ , nulls -- null' :: Monad m => ByteString m r -> m (Of Bool r) , uncons -- uncons :: Monad m => ByteString m r -> m (Either r (Char, ByteString m r)) , nextChar @@ -75,8 +74,6 @@ , split -- split :: Monad m => Char -> ByteString m r -> Stream (ByteString m) m r , lines , words- , linesIndividual- , wordsIndividual , denull -- ** Special folds @@ -103,11 +100,13 @@ -- * Folds, including support for `Control.Foldl` -- , foldr -- foldr :: Monad m => (Char -> a -> a) -> a -> ByteString m () -> m a , fold -- fold :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteString m () -> m b - , fold' -- fold' :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteString m r -> m (b, r) + , fold_ -- fold' :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteString m r -> m (b, r) , length- , length'+ , length_ , count- , count'+ , count_+ , null_+ , readInt -- * I\/O with 'ByteString's -- ** Standard input and output@@ -155,7 +154,7 @@ import qualified Data.ByteString.Unsafe as B import qualified Data.ByteString.Char8 as Char8 -import Streaming hiding (concats, unfold, distribute, wrap)+import Streaming hiding (concats, unfold, distribute, mwrap) import Streaming.Internal (Stream (..)) import qualified Streaming.Prelude as S import qualified Streaming as S@@ -164,11 +163,11 @@ import Data.ByteString.Streaming.Internal import Data.ByteString.Streaming- (fromLazy, toLazy, toLazy', nextChunk, unconsChunk, - fromChunks, toChunks, fromStrict, toStrict, toStrict', - concat, distribute, drain, toStreamingByteStringWith,+ (fromLazy, toLazy, toLazy_, nextChunk, unconsChunk, + fromChunks, toChunks, fromStrict, toStrict, toStrict_, + concat, distribute, effects, drained, mwrap, toStreamingByteStringWith, toStreamingByteString, toBuilder, concatBuilders,- empty, null, null', length, length', append, cycle, + empty, null, nulls, null_, length, length_, append, cycle, take, drop, splitAt, intercalate, group, denull, appendFile, stdout, stdin, fromHandle, toHandle, hGetContents, hGetContentsN, hGet, hGetN, hPut, @@ -182,11 +181,12 @@ import qualified System.IO as IO import System.IO.Unsafe import Control.Exception (bracket)-+import Data.Char (isDigit) import Foreign.ForeignPtr (withForeignPtr) import Foreign.Ptr import Foreign.Storable import Data.Functor.Compose+import Data.Functor.Sum unpack :: Monad m => ByteString m r -> Stream (Of Char) m r unpack bs = case bs of @@ -216,7 +216,7 @@ -- | /O(n)/ Convert a stream of separate characters into a packed byte stream. pack :: Monad m => Stream (Of Char) m r -> ByteString m r pack = fromChunks - . mapsM (liftM (\(str :> r) -> Char8.pack str :> r) . S.toListM') + . mapsM (liftM (\(str :> r) -> Char8.pack str :> r) . S.toList) . chunksOf 32 {-# INLINABLE pack #-} @@ -254,24 +254,24 @@ {-# INLINE snoc #-} -- | /O(1)/ Extract the first element of a ByteString, which must be non-empty.-head :: Monad m => ByteString m r -> m Char-head = liftM (w2c) . R.head-{-# INLINE head #-}+head_ :: Monad m => ByteString m r -> m Char+head_ = liftM (w2c) . R.head_+{-# INLINE head_ #-} -- | /O(1)/ Extract the first element of a ByteString, which may be non-empty-head' :: Monad m => ByteString m r -> m (Of (Maybe Char) r)-head' = liftM (\(m:>r) -> fmap w2c m :> r) . R.head'-{-# INLINE head' #-}+head :: Monad m => ByteString m r -> m (Of (Maybe Char) r)+head = liftM (\(m:>r) -> fmap w2c m :> r) . R.head+{-# INLINE head #-} -- | /O(n\/c)/ Extract the last element of a ByteString, which must be finite -- and non-empty.-last :: Monad m => ByteString m r -> m Char-last = liftM (w2c) . R.last-{-# INLINE last #-}+last_ :: Monad m => ByteString m r -> m Char+last_ = liftM (w2c) . R.last_+{-# INLINE last_ #-} -last' :: Monad m => ByteString m r -> m (Of (Maybe Char) r)-last' = liftM (\(m:>r) -> fmap (w2c) m :> r) . R.last'-{-# INLINE last' #-}+last :: Monad m => ByteString m r -> m (Of (Maybe Char) r)+last = liftM (\(m:>r) -> fmap (w2c) m :> r) . R.last+{-# INLINE last #-} -- | /O(1)/ Extract the head and tail of a ByteString, returning Nothing -- if it is empty.@@ -316,24 +316,24 @@ -- -- -- --------------------------------------------------------------------- -- -- Reducing 'ByteString's-fold :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteString m () -> m b-fold step begin done p0 = loop p0 begin+fold_ :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteString m () -> m b+fold_ step begin done p0 = loop p0 begin where loop p !x = case p of Chunk bs bss -> loop bss $! Char8.foldl' step x bs Go m -> m >>= \p' -> loop p' x Empty _ -> return (done x)-{-# INLINABLE fold #-}+{-# INLINABLE fold_ #-} -fold' :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteString m r -> m (Of b r)-fold' step begin done p0 = loop p0 begin+fold :: Monad m => (x -> Char -> x) -> x -> (x -> b) -> ByteString m r -> m (Of b r)+fold step begin done p0 = loop p0 begin where loop p !x = case p of Chunk bs bss -> loop bss $! Char8.foldl' step x bs Go m -> m >>= \p' -> loop p' x Empty r -> return (done x :> r)-{-# INLINABLE fold' #-}+{-# INLINABLE fold #-} -- --------------------------------------------------------------------- -- Unfolds and replicates @@ -474,10 +474,8 @@ {- | 'lines' turns a ByteString into a connected stream of ByteStrings at divide at newline characters. The resulting strings do not contain newlines. This is the genuinely streaming 'lines' which only breaks chunks, and- thus never increases the use of memory. It is crucial to distinguish its- type from that of 'linesIndividual'+ thus never increases the use of memory. -> linesIndividual :: Monad m => ByteString m r -> Stream (Of B.ByteString) m r > lines :: Monad m => ByteString m r -> Stream (ByteString m) m r -} @@ -499,22 +497,13 @@ Delay m -> Go (liftM unlines m) {-#INLINABLE unlines #-} -{-| 'linesIndividual' breaks streaming by concatening the chunks between line breaks--> linesIndividual = mapsM toStrict' . lines--}-linesIndividual :: Monad m => ByteString m r -> Stream (Of B.ByteString) m r-linesIndividual = mapsM R.toStrict' . lines---- | -unlinesIndividual :: Monad m => Stream (Of B.ByteString) m r -> ByteString m r -unlinesIndividual bss = R.concat $ for bss (\bs -> elevate $ R.chunk bs >> singleton '\n')- -- | 'words' breaks a byte stream up into a succession of byte streams -- corresponding to words, breaking Chars representing white space. This is --- the genuinely streaming 'words' to be distinguished from--- 'wordsIndividual', which will attempt to concatenate even infinitely--- long words like @cycle "y"@ in memory.+-- the genuinely streaming 'words'. A function that returns individual+-- strict bytestrings would concatenate even infinitely+-- long words like @cycle "y"@ in memory. It is best for the user who+-- has reflected on her materials to write `mapsM toStrict . words` or the like,+-- if strict bytestrings are needed. words :: Monad m => ByteString m r -> Stream (ByteString m) m r words = filtered . R.splitWith B.isSpaceWord8 where @@ -535,50 +524,22 @@ unwords = intercalate (singleton ' ') {-# INLINE unwords #-} -{- | 'wordsIndividual' breaks a bytestream into a sequence of individual- @Data.ByteString.ByteString@s, delimited by Chars representing white space. - It involves concatenation, of course, and is thus potentially unsafe.- Distinguish the types -> wordsIndividual :: Monad m => ByteString m r -> Stream (Of B.ByteString) m r-> words :: Monad m => ByteString m r -> Stream (ByteString m) m r - The latter, genuinely streaming, 'words' can only break up chunks- hidden in the stream that is given; the former potentially concatenates -> wordsIndividual = mapsM toStrict' . words---}-wordsIndividual :: Monad m => ByteString m r -> Stream (Of B.ByteString) m r-wordsIndividual = mapsM R.toStrict' . words---{- | 'unwordsIndividual' returns to a genuine bytestream by interspersing- white space between a sequence of individual Data.ByteString.ByteString - Distinguish the types--> unwordsIndividual :: Monad m => Stream (Of B.ByteString) m r -> ByteString m r -> unwords :: Monad m => Stream (ByteString m) m r -> ByteString m r---}-unwordsIndividual :: Monad m => Stream (Of B.ByteString) m r -> ByteString m r -unwordsIndividual bss = R.concat $ for bss (\bs -> elevate $ R.chunk bs >> singleton ' ')--- string :: String -> ByteString m () string = chunk . B.pack . Prelude.map B.c2w {-# INLINE string #-} -count :: Monad m => Char -> ByteString m r -> m Int+count_ :: Monad m => Char -> ByteString m r -> m Int+count_ c = R.count_ (c2w c)+{-# INLINE count_ #-}++count :: Monad m => Char -> ByteString m r -> m (Of Int r) count c = R.count (c2w c) {-# INLINE count #-} -count' :: Monad m => Char -> ByteString m r -> m (Of Int r)-count' c = R.count' (c2w c)-{-# INLINE count' #-}- nextChar :: Monad m => ByteString m r -> m (Either r (Char, ByteString m r)) nextChar b = do e <- R.nextByte b@@ -602,3 +563,24 @@ -- , null' -- , count -- , count'++{-| This will read positive or negative Ints that require 18 or fewer characters.+-}+readInt :: Monad m => ByteString m r -> m (Compose (Of (Maybe Int)) (ByteString m) r)+readInt = go . toStrict . splitAt 18 where+ go m = do + (bs :> rest) <- m+ case Char8.readInt bs of+ Nothing -> return (Compose (Nothing :> (chunk bs >> rest)))+ Just (n,more) -> if B.null more + then do + e <- uncons rest+ return $ case e of+ Left r -> Compose (Just n :> return r)+ Right (c,rest') -> if isDigit c + then Compose (Nothing :> (chunk bs >> cons' c rest'))+ else Compose (Just n :> (chunk more >> cons' c rest'))+ else return (Compose (Just n :> (chunk more >> rest)))+{-#INLINABLE readInt #-}++ -- uncons :: Monad m => ByteString m r -> m (Either r (Char, ByteString m r))
Data/ByteString/Streaming/Internal.hs view
@@ -21,7 +21,7 @@ , unpackBytes -- :: Monad m => ByteString m r -> Stream Word8_ m r , packBytes , chunk -- :: ByteString -> ByteString m ()- , wrap + , mwrap , unfoldrNE , reread , inlinePerformIO@@ -45,7 +45,7 @@ import qualified Data.ByteString.Internal as S import Streaming (Of(..))-import Streaming.Internal hiding (concats, wrap, step)+import Streaming.Internal hiding (concats, mwrap, step) import qualified Streaming.Prelude as SP import Foreign.ForeignPtr (withForeignPtr)@@ -155,10 +155,17 @@ {-# INLINE chunk #-} --- | Smart constructor for 'Go'.-wrap :: m (ByteString m r) -> ByteString m r-wrap = Go-{-# INLINE wrap #-}+{- | Reconceive an effect that results in an effectful bytestring as an effectful bytestring. + Compare Streaming.mwrap. The closes equivalent of + +>>> Streaming.wrap :: f (Stream f m r) -> Stream f m r++ is here @consChunk@. @mwrap@ is the smart constructor for the internal @Go@ constructor.+-}+mwrap :: m (ByteString m r) -> ByteString m r+mwrap = Go+{-# INLINE mwrap #-}+ -- | Construct a succession of chunks from its Church encoding (compare @GHC.Exts.build@) materialize :: (forall x . (r -> x) -> (S.ByteString -> x -> x) -> (m x -> x) -> x) -> ByteString m r@@ -170,12 +177,12 @@ dematerialize :: Monad m => ByteString m r -> (forall x . (r -> x) -> (S.ByteString -> x -> x) -> (m x -> x) -> x)-dematerialize x0 nil cons wrap = loop SPEC x0+dematerialize x0 nil cons mwrap = loop SPEC x0 where loop !_ x = case x of Empty r -> nil r Chunk b bs -> cons b (loop SPEC bs )- Go ms -> wrap (liftM (loop SPEC) ms)+ Go ms -> mwrap (liftM (loop SPEC) ms) {-# INLINABLE dematerialize #-} ------------------------------------------------------------------------ @@ -230,7 +237,7 @@ packBytes :: Monad m => Stream (Of Word8) m r -> ByteString m r packBytes cs0 = do - (bytes :> rest) <- lift $ SP.toListM' $ SP.splitAt 32 cs0+ (bytes :> rest) <- lift $ SP.toList $ SP.splitAt 32 cs0 case bytes of [] -> case rest of Return r -> Empty r
streaming-bytestring.cabal view
@@ -1,5 +1,5 @@ name: streaming-bytestring-version: 0.1.1.0+version: 0.1.2.0 synopsis: effectful byte steams, or: bytestring io done right. description: This is an implementation of effectful, memory-constrained @@ -189,7 +189,7 @@ , mtl >=2.1 && <2.3 , mmorph >=1.0 && <1.2 , transformers >=0.3 && <0.5- , streaming > 0.1.0.20 && < 0.1.1.2+ , streaming > 0.1.1.1 && < 0.1.2.2 if impl(ghc < 7.8) build-depends: bytestring < 0.10.4.0