streaming-bytestring 0.1.0.1 → 0.1.0.2
raw patch · 6 files changed
+148/−306 lines, 6 filesdep −attoparsecdep −foldldep −http-clientdep ~basedep ~deepseqdep ~mmorph
Dependencies removed: attoparsec, foldl, http-client, http-client-tls, syb
Dependency ranges changed: base, deepseq, mmorph, mtl, streaming, transformers
Files
- Data/Attoparsec/ByteString/Streaming.hs +0/−112
- Data/ByteString/Streaming.hs +118/−29
- Data/ByteString/Streaming/Char8.hs +1/−1
- Data/ByteString/Streaming/HTTP.hs +0/−133
- Data/ByteString/Streaming/Internal.hs +15/−4
- streaming-bytestring.cabal +14/−27
− Data/Attoparsec/ByteString/Streaming.hs
@@ -1,112 +0,0 @@-{-# LANGUAGE CPP #-}-{-# LANGUAGE Trustworthy #-} -- Imports internal modules---- |--- Module : Data.Attoparsec.ByteString.Streaming--- Copyright : Bryan O'Sullivan 2007-2015--- License : BSD3------ Maintainer : bos@serpentine.com--- Stability : experimental--- Portability : unknown------ Simple, efficient combinator parsing that can consume lazy--- 'ByteString' strings, loosely based on the Parsec library.------ This is essentially the same code as in the 'Data.Attoparsec'--- module, only with a 'parse' function that can consume a lazy--- 'ByteString' incrementally, and a 'Result' type that does not allow--- more input to be fed in. Think of this as suitable for use with a--- lazily read file, e.g. via 'L.readFile' or 'L.hGetContents'.------ /Note:/ The various parser functions and combinators such as--- 'string' still expect /strict/ 'B.ByteString' parameters, and--- return strict 'B.ByteString' results. Behind the scenes, strict--- 'B.ByteString' values are still used internally to store parser--- input and manipulate it efficiently.--module Data.Attoparsec.ByteString.Streaming- (- parse- , parsed- , atto- , atto_- , module Data.Attoparsec.ByteString-- )- where--import qualified Data.ByteString as B-import Control.Monad.Trans.State.Strict-import Control.Monad.Trans.Except-import Control.Monad.Trans--import qualified Data.Attoparsec.ByteString as A-import qualified Data.Attoparsec.Internal.Types as T-import Data.Attoparsec.ByteString- hiding (IResult(..), Result, eitherResult, maybeResult,- parse, parseWith, parseTest)- -import Streaming hiding (concats, unfold)-import Streaming.Internal (Stream (..))-import Data.ByteString.Streaming-import Data.ByteString.Streaming.Internal----- | The result of a parse.--parse :: Monad m - => A.Parser a - -> ByteString m x - -> m (Either a ([String], String), ByteString m x)-parse p s = case s of- Chunk x xs -> go (A.parse p x) xs- Empty r -> go (A.parse p B.empty) (Empty r)- Go m -> m >>= parse p- where- go (T.Fail x stk msg) ys = return $ (Right (stk, msg), Chunk x ys)- go (T.Done x r) ys = return $ (Left r, Chunk x ys)- go (T.Partial k) (Chunk y ys) = go (k y) ys- go (T.Partial k) (Go m) = m >>= go (T.Partial k)- go (T.Partial k) empty = go (k B.empty) empty----- | Run a parser and return its result.-atto :: Monad m => A.Parser a -> StateT (ByteString m x) m (Either a ([String], String))-atto p = StateT (parse p)--atto_ :: Monad m => A.Parser a -> ExceptT ([String], String) (StateT (ByteString m x) m) a-atto_ p = ExceptT $ StateT loop where- loop s = case s of- Chunk x xs -> go (A.parse p x) xs- Empty r -> go (A.parse p B.empty) (Empty r)- Go m -> m >>= loop-- go (T.Fail x stk msg) ys = return $ (Left (stk, msg), Chunk x ys)- go (T.Done x r) ys = return $ (Right r, Chunk x ys)- go (T.Partial k) (Chunk y ys) = go (k y) ys- go (T.Partial k) (Go m) = m >>= go (T.Partial k)- go (T.Partial k) blank = go (k B.empty) blank---parsed- :: Monad m- => A.Parser a -- ^ Attoparsec parser- -> ByteString m r -- ^ Raw input- -> Stream (Of a) m (Either (([String],String), ByteString m r) r)-parsed parser = go- where- go p0 = do- x <- lift (nextChunk p0)- case x of- Left r -> Return (Right r)- Right (bs,p1) -> step (chunk bs >>) (A.parse parser bs) p1- step diffP res p0 = case res of- A.Fail _ c m -> Return (Left ((c,m), diffP p0))- A.Done bs b -> Step (b :> go (chunk bs >> p0))- A.Partial k -> do- x <- lift (nextChunk p0)- case x of- Left e -> step diffP (k mempty) (return e)- Right (a,p1) -> step (diffP . (chunk a >>)) (k a) p1-{-# INLINABLE parsed #-}
Data/ByteString/Streaming.hs view
@@ -182,7 +182,7 @@ import Streaming.Internal (Stream (..)) import qualified Streaming.Prelude as SP -import Control.Monad (liftM)+import Control.Monad (liftM, forever) import Data.Word (Word8) import Data.Int (Int64)@@ -208,7 +208,7 @@ distribute ls = dematerialize ls return (\bs x -> join $ lift $ Chunk bs (Empty x) )- (join . hoist (Go . fmap Empty))+ (join . hoist (Go . liftM Empty)) {-# INLINE distribute #-} @@ -290,14 +290,28 @@ return $ (S.concat bss :> r) {-# INLINE toStrict' #-} --- |/O(c)/ Transmute a lazy bytestring to its representation--- as a monadic stream of chunks.+{- |/O(c)/ Transmute a lazy bytestring to its representation+ as a monadic stream of chunks.++>>> Q.putStrLn $ Q.fromLazy "hi"+hi+>>> Q.fromLazy "hi"+Chunk "hi" (Empty (())) -- note: a 'show' instance works in the identity monad+>>> Q.fromLazy $ BL.fromChunks ["here", "are", "some", "chunks"]+Chunk "here" (Chunk "are" (Chunk "some" (Chunk "chunks" (Empty (())))))++-} fromLazy :: Monad m => BI.ByteString -> ByteString m () fromLazy = BI.foldrChunks Chunk (Empty ()) {-# INLINE fromLazy #-} --- |/O(n)/ Convert a monadic byte stream into a single lazy 'ByteString'--- with the same internal chunk structure.+{- |/O(n)/ Convert a monadic byte stream into a single lazy 'ByteString'+ with the same internal chunk structure.++>>> Q.toLazy "hello"+"hello"++-} toLazy :: Monad m => ByteString m () -> m BI.ByteString toLazy bs = dematerialize bs (\() -> return (BI.Empty))@@ -305,9 +319,16 @@ join {-#INLINE toLazy #-} --- |/O(n)/ Convert a monadic byte stream into a single lazy 'ByteString'--- with the same invisible chunk structure, retaining the original--- return value. +{- |/O(n)/ Convert a monadic byte stream into a single lazy 'ByteString'+ with the same invisible chunk structure, retaining the original+ return value. ++>>> Q.toLazy' "hello"+"hello" :> ()+>>> S.toListM $ mapsM Q.toLazy' $ Q.lines $ "one\ntwo\three\nfour\nfive\n"+["one","two\three","four","five",""]++-} toLazy' :: Monad m => ByteString m r -> m (Of BI.ByteString r) toLazy' bs0 = dematerialize bs0 (\r -> return (BI.Empty :> r))@@ -323,7 +344,15 @@ -- --------------------------------------------------------------------- -- Basic interface ----- | /O(1)/ Test whether a ByteString is empty.+{-| /O(1)/ Test whether a ByteString is empty. The value is of course in the base monad.++>>> Q.null "one\ntwo\three\nfour\nfive\n"+False+>>> Q.null $ Q.take 0 Q.stdin+True+>>> :t Q.null $ Q.take 0 Q.stdin+Q.null $ Q.take 0 Q.stdin :: MonadIO m => m Bool+-} null :: Monad m => ByteString m r -> m Bool null (Empty _) = return True null (Go m) = m >>= null@@ -334,8 +363,13 @@ {- | /O(1)/ Test whether a ByteString is empty, collecting its return value;--- this operation must check the whole length of the string.+-- to reach the return value, this operation must check the whole length of the string. +>>> Q.null' "one\ntwo\three\nfour\nfive\n"+False :> ()+[*Main]+>>> Q.null' ""+True :> () >>> S.print $ mapsM R.null' $ Q.lines "yours,\nMeredith" False False@@ -356,7 +390,16 @@ length = liftM (\(n:> _) -> n) . foldlChunks (\n c -> n + fromIntegral (S.length c)) 0 {-# INLINE length #-} --- | /O(n\/c)/ 'length' returns the length of a byte stream as an 'Int64'+{-| /O(n\/c)/ 'length'' returns the length of a byte stream as an 'Int'+ together with the return value. This makes various maps possible++>>> Q.length' "one\ntwo\three\nfour\nfive\n"+23 :> ()+>>> S.print $ S.take 3 $ mapsM Q.length' $ Q.lines "one\ntwo\three\nfour\nfive\n" +3+8+4+-} length' :: Monad m => ByteString m r -> m (Of Int r) length' cs = foldlChunks (\n c -> n + fromIntegral (S.length c)) 0 cs {-# INLINE length' #-}@@ -703,18 +746,29 @@ -- --------------------------------------------------------------------- -- Unfolds and replicates --- | @'iterate' f x@ returns an infinite ByteString of repeated applications+{-| @'iterate' f x@ returns an infinite ByteString of repeated applications -- of @f@ to @x@: --- > iterate f x == [x, f x, f (f x), ...]+> iterate f x == [x, f x, f (f x), ...] +>>> R.stdout $ R.take 50 $ R.iterate succ 39+()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXY>>> +>>> Q.putStrLn $ Q.take 50 $ Q.iterate succ '\''+()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXY++-} iterate :: (Word8 -> Word8) -> Word8 -> ByteString m r iterate f = unfoldr (\x -> case f x of !x' -> Right (x', x')) {-# INLINABLE iterate #-} --- | @'repeat' x@ is an infinite ByteString, with @x@ the value of every--- element.---+{- | @'repeat' x@ is an infinite ByteString, with @x@ the value of every+ element.++>>> R.stdout $ R.take 50 $ R.repeat 60+<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<>>> +>>> Q.putStrLn $ Q.take 50 $ Q.repeat 'z'+zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz+-} repeat :: Word8 -> ByteString m r repeat w = cs where cs = Chunk (S.replicate BI.smallChunkSize w) cs {-# INLINABLE repeat #-}@@ -735,13 +789,19 @@ -- nChunks 0 = Empty -- nChunks m = Chunk c (nChunks (m-1)) --- | 'cycle' ties a finite ByteString into a circular one, or equivalently,--- the infinite repetition of the original ByteString.---+{- | 'cycle' ties a finite ByteString into a circular one, or equivalently,+ the infinite repetition of the original ByteString. For an empty bytestring+ (like @return 17@) it of course makes an unproductive loop + +>>> Q.putStrLn $ Q.take 7 $ Q.cycle "y\n"+y+y+y+y+-} cycle :: Monad m => ByteString m r -> ByteString m s-cycle (Empty _) = error "cycle" -- errorEmptyStream "cycle"-cycle cs = cs >> cycle cs -- ' where cs' = foldrChunks Chunk cs' cs-{-# INLINABLE cycle #-}+cycle = forever+{-# INLINE cycle #-} -- | /O(n)/ The 'unfoldr' function is analogous to the Stream \'unfoldr\'. -- 'unfoldr' builds a ByteString from a seed value. The function takes@@ -776,8 +836,23 @@ -- --------------------------------------------------------------------- -- Substrings --- | /O(n\/c)/ 'take' @n@, applied to a ByteString @xs@, returns the prefix--- of @xs@ of length @n@, or @xs@ itself if @n > 'length' xs@.+{-| /O(n\/c)/ 'take' @n@, applied to a ByteString @xs@, returns the prefix+ of @xs@ of length @n@, or @xs@ itself if @n > 'length' xs@.++ Note that in the streaming context this drops the final return value;+ 'splitAt' preserves this information, and is sometimes to be preferred.++>>> Q.putStrLn $ Q.take 8 $ "Is there a God?" >> return True+Is there+>>> Q.putStrLn $ "Is there a God?" >> return True+Is there a God?+True+>>> rest <- Q.putStrLn $ Q.splitAt 8 $ "Is there a God?" >> return True+Is there+>>> Q.drain rest+True++-} take :: Monad m => Int64 -> ByteString m r -> ByteString m () take i _ | i <= 0 = Empty () take i cs0 = take' i cs0@@ -790,8 +865,15 @@ take' n (Go m) = Go (liftM (take' n) m) {-# INLINABLE take #-} --- | /O(n\/c)/ 'drop' @n xs@ returns the suffix of @xs@ after the first @n@--- elements, or @[]@ if @n > 'length' xs@.+{-| /O(n\/c)/ 'drop' @n xs@ returns the suffix of @xs@ after the first @n@+ elements, or @[]@ if @n > 'length' xs@.++>>> Q.putStrLn $ Q.drop 6 "Wisconsin"+sin+>>> Q.putStrLn $ Q.drop 16 "Wisconsin"++>>>+-} drop :: Monad m => Int64 -> ByteString m r -> ByteString m r drop i p | i <= 0 = p drop i cs0 = drop' i cs0@@ -805,7 +887,14 @@ {-# INLINABLE drop #-} --- | /O(n\/c)/ 'splitAt' @n xs@ is equivalent to @('take' n xs, 'drop' n xs)@.+{-| /O(n\/c)/ 'splitAt' @n xs@ is equivalent to @('take' n xs, 'drop' n xs)@.++>>> rest <- Q.putStrLn $ Q.splitAt 3 "therapist is a danger to good hyphenation, as Knuth notes"+the+>>> Q.putStrLn $ Q.splitAt 19 rest+rapist is a danger ++-} splitAt :: Monad m => Int64 -> ByteString m r -> ByteString m (ByteString m r) splitAt i cs0 | i <= 0 = Empty cs0 splitAt i cs0 = splitAt' i cs0@@ -1408,7 +1497,7 @@ -- of the string if no element is found, rather than Nothing. findIndexOrEnd :: (Word8 -> Bool) -> P.ByteString -> Int findIndexOrEnd k (S.PS x s l) =- S.accursedUnutterablePerformIO $+ inlinePerformIO $ withForeignPtr x $ \f -> go (f `plusPtr` s) 0 where go !ptr !n | n >= l = return l
Data/ByteString/Streaming/Char8.hs view
@@ -195,7 +195,7 @@ unpackAppendCharsStrict :: B.ByteString -> Stream (Of Char) m r -> Stream (Of Char) m r unpackAppendCharsStrict (B.PS fp off len) xs =- B.accursedUnutterablePerformIO $ withForeignPtr fp $ \base -> do+ inlinePerformIO $ withForeignPtr fp $ \base -> do loop (base `plusPtr` (off-1)) (base `plusPtr` (off-1+len)) xs where loop !sentinal !p acc
− Data/ByteString/Streaming/HTTP.hs
@@ -1,133 +0,0 @@--- | This module, including the documentation, replicates `pipes-http` as--- closely as will type-check.--- --- Here is an example GET request that streams the response body to standard--- output:------ > import qualified Data.ByteString.Streaming as S--- > import Data.ByteString.Streaming.HTTP--- >--- > main = do--- > req <- parseUrl "https://www.example.com"--- > m <- newManager tlsManagerSettings --- > withHTTP req m $ \resp -> S.stdout (responseBody resp) --- > ------ Here is an example POST request that also streams the request body from--- standard input:------ > {-#LANGUAGE OverloadedStrings #-}--- > import qualified Data.ByteString.Streaming as S--- > import Data.ByteString.Streaming.HTTP--- > --- > main = do--- > req <- parseUrl "https://www.example.com"--- > let req' = req--- > { method = "POST"--- > , requestBody = stream S.stdin--- > }--- > m <- newManager tlsManagerSettings--- > withHTTP req' m $ \resp -> S.stdout (responseBody resp)------ For non-streaming request bodies, study the 'RequestBody' type, which also--- accepts strict \/ lazy bytestrings or builders.---module Data.ByteString.Streaming.HTTP (- -- * http-client- -- $httpclient- module Network.HTTP.Client- , module Network.HTTP.Client.TLS-- -- * Streaming Interface- , withHTTP- , streamN- , stream-- ) where--import Control.Monad (unless)-import qualified Data.ByteString as B-import Data.Int (Int64)-import Data.IORef (newIORef, readIORef, writeIORef)-import Network.HTTP.Client-import Network.HTTP.Client.TLS-import Data.ByteString.Streaming-import Data.ByteString.Streaming.Internal-import Control.Monad.Trans--{- $httpclient- This module is a thin @streaming-bytestring@ wrapper around the @http-client@ and- @http-client-tls@ libraries.-- Read the documentation in the "Network.HTTP.Client" module of the- @http-client@ library to learn about how to:-- * manage connections using connection pooling,-- * use more advanced request\/response features,-- * handle exceptions, and:-- * manage cookies.-- @http-client-tls@ provides support for TLS connections (i.e. HTTPS).--}---- | Send an HTTP 'Request' and wait for an HTTP 'Response'-withHTTP- :: Request- -- ^- -> Manager- -- ^- -> (Response (ByteString IO ()) -> IO a)- -- ^ Handler for response- -> IO a-withHTTP r m k = withResponse r m k'- where- k' resp = do- let p = (from . brRead . responseBody) resp- k (resp { responseBody = p})-{-# INLINABLE withHTTP #-}---- | Create a 'RequestBody' from a content length and an effectful 'ByteString'-streamN :: Int64 -> ByteString IO () -> RequestBody-streamN n p = RequestBodyStream n (to p)-{-# INLINABLE streamN #-}--{-| Create a 'RequestBody' from an effectful 'ByteString'-- 'stream' is more flexible than 'streamN', but requires the server to support- chunked transfer encoding.--}-stream :: ByteString IO () -> RequestBody-stream p = RequestBodyStreamChunked (to p)-{-# INLINABLE stream #-}--to :: ByteString IO () -> (IO B.ByteString -> IO ()) -> IO ()-to p0 k = do- ioref <- newIORef p0- let readAction :: IO B.ByteString- readAction = do- p <- readIORef ioref- case p of- Empty () -> do- writeIORef ioref (return ())- return B.empty- Go m -> do - p' <- m- writeIORef ioref p'- readAction- Chunk bs p' -> do- writeIORef ioref p'- return bs- k readAction --from :: IO B.ByteString -> ByteString IO ()-from io = go- where- go = do- bs <- lift io- unless (B.null bs) $ do- chunk bs- go
Data/ByteString/Streaming/Internal.hs view
@@ -1,5 +1,6 @@ {-# LANGUAGE CPP, BangPatterns #-} {-#LANGUAGE RankNTypes, GADTs #-}+{-# LANGUAGE UnliftedFFITypes, MagicHash, UnboxedTuples #-} module Data.ByteString.Streaming.Internal ( ByteString (..) , consChunk -- :: S.ByteString -> ByteString m r -> ByteString m r@@ -23,6 +24,7 @@ , wrap , unfoldrNE , reread+ , inlinePerformIO ) where import Prelude hiding@@ -34,7 +36,9 @@ import qualified Prelude import Control.Monad.Trans import Control.Monad+import Control.Applicative import Control.Monad.Morph+import Data.Monoid (Monoid(..)) import qualified Data.ByteString as S -- S for strict (hmm...) import qualified Data.ByteString.Internal as S@@ -48,9 +52,12 @@ import Foreign.Storable import GHC.Exts ( SpecConstrAnnotation(..) ) import Data.String+ import Data.Functor.Identity import Data.Word import System.IO.Unsafe+import GHC.Base (realWorld#,unsafeChr)+import GHC.IO (IO(IO)) -- | A space-efficient representation of a succession of 'Word8' vectors, supporting many -- efficient operations.@@ -107,7 +114,7 @@ hoist phi bs = case bs of Empty r -> Empty r Chunk bs' rest -> Chunk bs' (hoist phi rest)- Go m -> Go (phi (fmap (hoist phi) m))+ Go m -> Go (phi (liftM (hoist phi) m)) {-#INLINABLE hoist #-} instance (r ~ ()) => IsString (ByteString m r) where@@ -122,9 +129,9 @@ instance (Monoid r, Monad m) => Monoid (ByteString m r) where mempty = Empty mempty- {-#INLINE mempty#-}+ {-# INLINE mempty #-} mappend = liftM2 mappend- {-#INLINE mappend#-}+ {-# INLINE mappend #-} -- data Word8_ r = Word8_ {-#UNPACK#-} !Word8 r -- This might be preferable to (Of Word8 r), but the present approach is simpler.@@ -252,14 +259,18 @@ unpackAppendBytesStrict :: S.ByteString -> Stream (Of Word8) m r -> Stream (Of Word8) m r unpackAppendBytesStrict (S.PS fp off len) xs =- S.accursedUnutterablePerformIO $ withForeignPtr fp $ \base -> do+ inlinePerformIO $ withForeignPtr fp $ \base -> do loop (base `plusPtr` (off-1)) (base `plusPtr` (off-1+len)) xs where+ accursedUnutterablePerformIO (IO m) = case m realWorld# of (# _, r #) -> r loop !sentinal !p acc | p == sentinal = return acc | otherwise = do x <- peek p loop sentinal (p `plusPtr` (-1)) (Step (x :> acc)) {-# INLINABLE unpackBytes #-}++inlinePerformIO :: IO a -> a+inlinePerformIO (IO m) = case m realWorld# of (# _, r #) -> r -- | Consume the chunks of an effectful ByteString with a natural right fold. foldrChunks :: Monad m => (S.ByteString -> a -> a) -> a -> ByteString m r -> m a
streaming-bytestring.cabal view
@@ -1,9 +1,9 @@ name: streaming-bytestring-version: 0.1.0.1-synopsis: Lazy bytestring done right-description: This is an implementation of effectful memory-constrained - bytestrings, or byte streams, and streaming bytestring manipulation, - adequate for non-lazy-io. +version: 0.1.0.2+synopsis: effectful bytestrings, or: lazy bytestring done right+description: This is an implementation of effectful, memory-constrained + bytestrings (byte streams) and functions for streaming + bytestring manipulation, adequate for non-lazy-io. . Interoperation with @pipes@ uses this isomorphism: . @@ -131,13 +131,7 @@ and the examples in the documentation for the streaming library. See also <https://gist.github.com/michaelt/6c6843e6dd8030e95d58 simple implementations> of the shell-like examples mentioned above.- .- The modules for applying attoparsec parsers and making http clients should- properly be in separate packages, but are included for the moment- to simplify experimentation. They simply replicate the corresponding pipes- modules. Those modules don't involve pipes, but only the - @Producer ByteString m r@ type, which pipes globally reserves - to mean exactly what is here called @ByteString m r@.+ license: BSD3 license-file: LICENSE@@ -153,25 +147,18 @@ exposed-modules: Data.ByteString.Streaming , Data.ByteString.Streaming.Char8 , Data.ByteString.Streaming.Internal- -- , Data.ByteString.Streaming.Aeson- , Data.ByteString.Streaming.HTTP- , Data.Attoparsec.ByteString.Streaming+ -- other-modules: other-extensions: CPP, BangPatterns, ForeignFunctionInterface, DeriveDataTypeable, Unsafe- build-depends: base >=4.8 && <4.9+ build-depends: base >=4.7 && <4.9 , bytestring >=0.10 && <0.11- , deepseq >=1.4 && <1.5- , syb >=0.5 && <0.6- , mtl >=2.2 && <2.3- , mmorph >=1.0 && <1.1- , attoparsec- , transformers- , foldl- -- , aeson- , streaming- , http-client - , http-client-tls+ , deepseq + , mtl >=2.1 && <2.3+ , mmorph >=1.0 && <1.2+ , transformers >=0.3 && <0.5+ , streaming > 0.1.0.8 && < 0.1.1+ -- hs-source-dirs: default-language: Haskell2010 -- ghc-options: -Wall