pipes-bytestring 1.0.3 → 2.1.7
raw patch · 4 files changed
Files
- LICENSE +1/−1
- pipes-bytestring.cabal +11/−10
- src/Pipes/ByteString.hs +553/−314
- src/Pipes/ByteString/Parse.hs +0/−142
LICENSE view
@@ -1,4 +1,4 @@-Copyright (c) 2012, 2013 Gabriel Gonzalez+Copyright (c) 2012-2014 Gabriel Gonzalez All rights reserved. Redistribution and use in source and binary forms, with or without modification,
pipes-bytestring.cabal view
@@ -1,10 +1,10 @@ Name: pipes-bytestring-Version: 1.0.3-Cabal-Version: >=1.8.0.2+Version: 2.1.7+Cabal-Version: >=1.10 Build-Type: Simple License: BSD3 License-File: LICENSE-Copyright: 2012, 2013 Gabriel Gonzalez+Copyright: 2012-2014 Gabriel Gonzalez Author: Gabriel Gonzalez Maintainer: Gabriel439@gmail.com Bug-Reports: https://github.com/Gabriel439/Haskell-Pipes-ByteString-Library/issues@@ -19,11 +19,12 @@ Hs-Source-Dirs: src Build-Depends: base >= 4 && < 5 ,- bytestring >= 0.9.2.1 && < 0.11,- pipes >= 4.0 && < 4.1 ,- pipes-parse >= 2.0.0 && < 2.1 ,- transformers >= 0.2.0.0 && < 0.4- Exposed-Modules:- Pipes.ByteString,- Pipes.ByteString.Parse+ bytestring >= 0.9.2.1 && < 0.12,+ pipes >= 4.0 && < 4.4 ,+ pipes-group >= 1.0.0 && < 1.1 ,+ pipes-parse >= 3.0.0 && < 3.1 ,+ stringsearch >= 0.3.0 && < 0.4 ,+ transformers >= 0.2.0.0 && < 0.6+ Exposed-Modules: Pipes.ByteString GHC-Options: -O2 -Wall+ Default-Language: Haskell2010
src/Pipes/ByteString.hs view
@@ -1,9 +1,4 @@-{-# LANGUAGE RankNTypes, CPP #-}---- The rewrite rules require the Trustworthy annotation-#if __GLASGOW_HASKELL__ >= 702-{-# LANGUAGE Trustworthy #-}-#endif+{-# LANGUAGE RankNTypes, Trustworthy #-} {-| This module provides @pipes@ utilities for \"byte streams\", which are streams of strict 'ByteString's chunks. Use byte streams to interact@@ -22,11 +17,11 @@ > runEffect $ P.fromHandle hIn >-> P.toHandle hOut You can stream to and from 'stdin' and 'stdout' using the predefined 'stdin'- and 'stdout' proxies, like in the following \"echo\" program:+ and 'stdout' pipes, like in the following \"echo\" program: > main = runEffect $ P.stdin >-> P.stdout - You can also translate pure lazy 'BL.ByteString's to and from proxies:+ You can also translate pure lazy 'BL.ByteString's to and from pipes: > import qualified Data.ByteString.Lazy.Char8 as BL >@@ -37,13 +32,12 @@ example, to stream only the first three lines of 'stdin' to 'stdout' you would write: +> import Lens.Family (over) > import Pipes > import qualified Pipes.ByteString as PB-> import qualified Pipes.Parse as PP+> import Pipes.Group (takes) >-> main = runEffect $ takeLines 3 PB.stdin >-> PB.stdout-> where-> takeLines n = PB.unlines . PP.takeFree n . PB.lines+> main = runEffect $ over PB.lines (takes 3) PB.stdin >-> PB.stdout The above program will never bring more than one chunk (~ 32 KB) into memory, no matter how long the lines are.@@ -61,8 +55,9 @@ , stdin , fromHandle , hGetSome+ , hGetNonBlocking , hGet- , pack+ , hGetRange -- * Servers , hGetSomeN@@ -76,19 +71,17 @@ , map , concatMap , take- , drop , takeWhile- , dropWhile , filter , elemIndices , findIndices , scan- , unpack -- * Folds , toLazy , toLazyM- , fold+ , toLazyM'+ , foldBytes , head , last , null@@ -105,72 +98,83 @@ , findIndex , count - -- * Splitters+ -- * Parsing+ -- $parse+ , nextByte+ , drawByte+ , unDrawByte+ , peekByte+ , isEndOfBytes++ -- * Parsing Lenses , splitAt- , chunksOf , span , break- , splitWith- , split+ , breakOn , groupBy , group- , lines- , words+ , word+ , line - -- * Transformations+ -- * Transforming Byte Streams+ , drop+ , dropWhile , intersperse+ , pack+ , unpack+ , chunksOf' - -- * Joiners- , intercalate+ -- * FreeT Transformations+ , chunksOf+ , splitsWith+ , splits+ , splitOn+ , groupsBy+ , groups+ , lines , unlines+ , words , unwords - -- * Low-level Parsers- -- $parse- , nextByte- , drawByte- , unDrawByte- , peekByte- , isEndOfBytes- , takeWhile'- -- * Re-exports -- $reexports , module Data.ByteString , module Data.Word+ , module Pipes.Group , module Pipes.Parse ) where +import Control.Applicative ((<*)) import Control.Exception (throwIO, try)-import Control.Monad (liftM)-import Control.Monad.Trans.State.Strict (StateT)+import Control.Monad (liftM, join)+import Control.Monad.Trans.State.Strict (modify) import qualified Data.ByteString as BS import Data.ByteString (ByteString) import Data.ByteString.Internal (isSpaceWord8) import qualified Data.ByteString.Lazy as BL+import qualified Data.ByteString.Search import Data.ByteString.Lazy.Internal (foldrChunks, defaultChunkSize)-import Data.ByteString.Unsafe (unsafeTake, unsafeDrop)+import Data.ByteString.Unsafe (unsafeTake) import Data.Char (ord)+import Data.Monoid (mempty, (<>))+import Data.Functor.Constant (Constant(Constant, getConstant)) import Data.Functor.Identity (Identity) import qualified Data.List as List import Data.Word (Word8) import Foreign.C.Error (Errno(Errno), ePIPE) import qualified GHC.IO.Exception as G import Pipes-import qualified Pipes.ByteString.Parse as PBP-import Pipes.ByteString.Parse (- nextByte, drawByte, unDrawByte, peekByte, isEndOfBytes ) import Pipes.Core (respond, Server')-import Pipes.Lift (evalStateP)+import qualified Pipes.Group as PG+import Pipes.Group (concats, intercalates, FreeT) import qualified Pipes.Parse as PP-import Pipes.Parse (input, concat, FreeT, isEndOfInput)+import Pipes.Parse (Parser) import qualified Pipes.Prelude as P import qualified System.IO as IO import Prelude hiding ( all , any , break- , concat , concatMap , drop , dropWhile@@ -195,7 +199,7 @@ ) -- | Convert a lazy 'BL.ByteString' into a 'Producer' of strict 'ByteString's-fromLazy :: (Monad m) => BL.ByteString -> Producer' ByteString m ()+fromLazy :: Monad m => BL.ByteString -> Producer' ByteString m () fromLazy bs = foldrChunks (\e a -> yield e >> a) (return ()) bs {-# INLINABLE fromLazy #-} @@ -229,6 +233,23 @@ {-| Convert a handle into a byte stream using a fixed chunk size + Similar to 'hGet' except that it will never block waiting for data+ to become available.+-}+hGetNonBlocking :: MonadIO m => Int -> IO.Handle -> Producer' ByteString m ()+hGetNonBlocking size h = go where+ go = do+ eof <- liftIO (IO.hIsEOF h)+ if eof+ then return ()+ else do+ bs <- liftIO (BS.hGetNonBlocking h size)+ yield bs+ go+{-# INLINABLE hGetNonBlocking #-}++{-| Convert a handle into a byte stream using a fixed chunk size+ 'hGet' waits until exactly the requested number of bytes are available for each chunk. -}@@ -244,23 +265,26 @@ go {-# INLINABLE hGet #-} --- | Convert a 'Word8' producer into a byte stream using a default chunk size-pack :: Monad m => Producer Word8 m () -> Producer ByteString m ()-pack p = evalStateP p go where- go = do- eof <- lift isEndOfInput- if eof- then return ()- else do- bytes <- lift $ P.toListM (P.take defaultChunkSize <-< input)- yield $ BS.pack bytes- go-{-# INLINABLE pack #-}+{-| Like 'hGet' but with an extra parameter specifying an initial handle offset+-}+hGetRange+ :: MonadIO m+ => Int -- ^ Offset+ -> Int -- ^ Size+ -> IO.Handle+ -> Producer' ByteString m ()+hGetRange offset size h = do+ liftIO $ IO.hSeek h IO.AbsoluteSeek (fromIntegral offset)+ hGet size h+{-# INLINABLE hGetRange #-} +(^.) :: a -> ((b -> Constant b b) -> (a -> Constant b a)) -> b+a ^. lens = getConstant (lens Constant a)+ {-| Like 'hGetSome', except you can vary the maximum chunk size for each request -} hGetSomeN :: MonadIO m => IO.Handle -> Int -> Server' Int ByteString m ()-hGetSomeN h = go+hGetSomeN h size = go size where go size = do bs <- liftIO (BS.hGetSome h size)@@ -273,7 +297,7 @@ -- | Like 'hGet', except you can vary the chunk size for each request hGetN :: MonadIO m => IO.Handle -> Int -> Server' Int ByteString m ()-hGetN h = go+hGetN h size = go size where go size = do bs <- liftIO (BS.hGet h size)@@ -287,9 +311,6 @@ {-| Stream bytes to 'stdout' Unlike 'toHandle', 'stdout' gracefully terminates on a broken output pipe.-- Note: For best performance, use @(for source (liftIO . putStr))@ instead of- @(source >-> stdout)@. -} stdout :: MonadIO m => Consumer' ByteString m () stdout = go@@ -308,30 +329,28 @@ {-| Convert a byte stream into a 'Handle' - Note: For best performance, use @(for source (liftIO . hPutStr handle))@- instead of @(source >-> toHandle handle)@.+> p >-> toHandle handle = for p (liftIO . hPutStr handle) -} toHandle :: MonadIO m => IO.Handle -> Consumer' ByteString m r toHandle h = for cat (liftIO . BS.hPut h)-{-# INLINABLE toHandle #-}+{-# INLINABLE [1] toHandle #-} {-# RULES "p >-> toHandle h" forall p h . p >-> toHandle h = for p (\bs -> liftIO (BS.hPut h bs)) #-} -- | Apply a transformation to each 'Word8' in the stream-map :: (Monad m) => (Word8 -> Word8) -> Pipe ByteString ByteString m r+map :: Monad m => (Word8 -> Word8) -> Pipe ByteString ByteString m r map f = P.map (BS.map f)-{-# INLINABLE map #-}+{-# INLINE map #-} -- | Map a function over the byte stream and concatenate the results-concatMap- :: (Monad m) => (Word8 -> ByteString) -> Pipe ByteString ByteString m r+concatMap :: Monad m => (Word8 -> ByteString) -> Pipe ByteString ByteString m r concatMap f = P.map (BS.concatMap f) {-# INLINABLE concatMap #-} -- | @(take n)@ only allows @n@ bytes to pass-take :: (Monad m, Integral a) => a -> Pipe ByteString ByteString m ()+take :: (Monad m, Integral n) => n -> Pipe ByteString ByteString m () take n0 = go n0 where go n | n <= 0 = return ()@@ -345,23 +364,8 @@ go (n - len) {-# INLINABLE take #-} --- | @(drop n)@ drops the first @n@ bytes-drop :: (Monad m, Integral a) => a -> Pipe ByteString ByteString m r-drop n0 = go n0 where- go n- | n <= 0 = cat- | otherwise = do- bs <- await- let len = fromIntegral (BS.length bs)- if (len >= n)- then do- yield (unsafeDrop (fromIntegral n) bs)- cat- else go (n - len)-{-# INLINABLE drop #-}- -- | Take bytes until they fail the predicate-takeWhile :: (Monad m) => (Word8 -> Bool) -> Pipe ByteString ByteString m ()+takeWhile :: Monad m => (Word8 -> Bool) -> Pipe ByteString ByteString m () takeWhile predicate = go where go = do@@ -374,20 +378,8 @@ else yield prefix {-# INLINABLE takeWhile #-} --- | Drop bytes until they fail the predicate-dropWhile :: (Monad m) => (Word8 -> Bool) -> Pipe ByteString ByteString m r-dropWhile predicate = go where- go = do- bs <- await- case BS.findIndex (not . predicate) bs of- Nothing -> go- Just i -> do- yield (unsafeDrop i bs)- cat-{-# INLINABLE dropWhile #-}- -- | Only allows 'Word8's to pass if they satisfy the predicate-filter :: (Monad m) => (Word8 -> Bool) -> Pipe ByteString ByteString m r+filter :: Monad m => (Word8 -> Bool) -> Pipe ByteString ByteString m r filter predicate = P.map (BS.filter predicate) {-# INLINABLE filter #-} @@ -402,34 +394,26 @@ where go n = do bs <- await- each $ List.map (\i -> n + fromIntegral i) (BS.findIndices predicate bs)+ each $ List.map (\i -> n + fromIntegral i) (BS.findIndices predicate bs) go $! n + fromIntegral (BS.length bs) {-# INLINABLE findIndices #-} -- | Strict left scan over the bytes scan- :: (Monad m)+ :: Monad m => (Word8 -> Word8 -> Word8) -> Word8 -> Pipe ByteString ByteString m r-scan step begin = go begin+scan step begin = do+ yield (BS.singleton begin)+ go begin where go w8 = do bs <- await let bs' = BS.scanl step w8 bs w8' = BS.last bs'- yield bs'+ yield (BS.tail bs') go w8' {-# INLINABLE scan #-} --- | Unpack the bytes-unpack :: Monad m => Pipe ByteString Word8 m ()-unpack = for cat (mapM_ yield . BS.unpack)-{-# INLINABLE unpack #-}--{-# RULES- "p >-> unpack" forall p .- p >-> unpack = for p (\bs -> mapM_ yield (BS.unpack bs))- #-}- {-| Fold a pure 'Producer' of strict 'ByteString's into a lazy 'BL.ByteString' -}@@ -444,19 +428,35 @@ simple testing purposes. Idiomatic @pipes@ style consumes the chunks immediately as they are generated instead of loading them all into memory. -}-toLazyM :: (Monad m) => Producer ByteString m () -> m BL.ByteString+toLazyM :: Monad m => Producer ByteString m () -> m BL.ByteString toLazyM = liftM BL.fromChunks . P.toListM {-# INLINABLE toLazyM #-} --- | Reduce the stream of bytes using a strict left fold-fold+{-| Fold an effectful 'Producer' of strict 'ByteString's into a lazy+ 'BL.ByteString' alongside the return value++ Note: 'toLazyM'' is not an idiomatic use of @pipes@, but I provide it for+ simple testing purposes. Idiomatic @pipes@ style consumes the chunks+ immediately as they are generated instead of loading them all into memory.+-}+toLazyM' :: Monad m => Producer ByteString m a -> m (BL.ByteString, a)+toLazyM' p = do (chunks, a) <- P.toListM' p+ return (BL.fromChunks chunks, a)+{-# INLINABLE toLazyM' #-}++{-| Reduce the stream of bytes using a strict left fold++ Note: It's more efficient to use folds from @Control.Foldl.ByteString@ in+ conjunction with @Pipes.Prelude.'Pipes.Prelude.fold'@ when possible+-}+foldBytes :: Monad m => (x -> Word8 -> x) -> x -> (x -> r) -> Producer ByteString m () -> m r-fold step begin done = P.fold (\x bs -> BS.foldl' step x bs) begin done-{-# INLINABLE fold #-}+foldBytes step begin done = P.fold (\x bs -> BS.foldl' step x bs) begin done+{-# INLINABLE foldBytes #-} -- | Retrieve the first 'Word8'-head :: (Monad m) => Producer ByteString m () -> m (Maybe Word8)+head :: Monad m => Producer ByteString m () -> m (Maybe Word8) head = go where go p = do@@ -467,7 +467,7 @@ {-# INLINABLE head #-} -- | Retrieve the last 'Word8'-last :: (Monad m) => Producer ByteString m () -> m (Maybe Word8)+last :: Monad m => Producer ByteString m () -> m (Maybe Word8) last = go Nothing where go r p = do@@ -481,7 +481,7 @@ {-# INLINABLE last #-} -- | Determine if the stream is empty-null :: (Monad m) => Producer ByteString m () -> m Bool+null :: Monad m => Producer ByteString m () -> m Bool null = P.all BS.null {-# INLINABLE null #-} @@ -491,17 +491,17 @@ {-# INLINABLE length #-} -- | Fold that returns whether 'M.Any' received 'Word8's satisfy the predicate-any :: (Monad m) => (Word8 -> Bool) -> Producer ByteString m () -> m Bool+any :: Monad m => (Word8 -> Bool) -> Producer ByteString m () -> m Bool any predicate = P.any (BS.any predicate) {-# INLINABLE any #-} -- | Fold that returns whether 'M.All' received 'Word8's satisfy the predicate-all :: (Monad m) => (Word8 -> Bool) -> Producer ByteString m () -> m Bool+all :: Monad m => (Word8 -> Bool) -> Producer ByteString m () -> m Bool all predicate = P.all (BS.all predicate) {-# INLINABLE all #-} -- | Return the maximum 'Word8' within a byte stream-maximum :: (Monad m) => Producer ByteString m () -> m (Maybe Word8)+maximum :: Monad m => Producer ByteString m () -> m (Maybe Word8) maximum = P.fold step Nothing id where step mw8 bs =@@ -513,7 +513,7 @@ {-# INLINABLE maximum #-} -- | Return the minimum 'Word8' within a byte stream-minimum :: (Monad m) => Producer ByteString m () -> m (Maybe Word8)+minimum :: Monad m => Producer ByteString m () -> m (Maybe Word8) minimum = P.fold step Nothing id where step mw8 bs =@@ -525,29 +525,29 @@ {-# INLINABLE minimum #-} -- | Determine whether any element in the byte stream matches the given 'Word8'-elem :: (Monad m) => Word8 -> Producer ByteString m () -> m Bool+elem :: Monad m => Word8 -> Producer ByteString m () -> m Bool elem w8 = P.any (BS.elem w8) {-# INLINABLE elem #-} {-| Determine whether all elements in the byte stream do not match the given 'Word8' -}-notElem :: (Monad m) => Word8 -> Producer ByteString m () -> m Bool+notElem :: Monad m => Word8 -> Producer ByteString m () -> m Bool notElem w8 = P.all (BS.notElem w8) {-# INLINABLE notElem #-} -- | Find the first element in the stream that matches the predicate find- :: (Monad m)+ :: Monad m => (Word8 -> Bool) -> Producer ByteString m () -> m (Maybe Word8) find predicate p = head (p >-> filter predicate) {-# INLINABLE find #-} -- | Index into a byte stream index- :: (Monad m, Integral a)- => a-> Producer ByteString m () -> m (Maybe Word8)-index n p = head (p >-> drop n)+ :: (Monad m, Integral n)+ => n -> Producer ByteString m () -> m (Maybe Word8)+index n p = head (drop n p) {-# INLINABLE index #-} -- | Find the index of an element that matches the given 'Word8'@@ -568,55 +568,124 @@ count w8 p = P.fold (+) 0 id (p >-> P.map (fromIntegral . BS.count w8)) {-# INLINABLE count #-} --- | Splits a 'Producer' after the given number of bytes-splitAt- :: (Monad m, Integral n)- => n- -> Producer ByteString m r- -> Producer' ByteString m (Producer ByteString m r)-splitAt = go+{-| Consume the first byte from a byte stream++ 'next' either fails with a 'Left' if the 'Producer' has no more bytes or+ succeeds with a 'Right' providing the next byte and the remainder of the+ 'Producer'.+-}+nextByte+ :: Monad m+ => Producer ByteString m r+ -> m (Either r (Word8, Producer ByteString m r))+nextByte = go where- go 0 p = return p- go n p = do- x <- lift (next p)+ go p = do+ x <- next p case x of- Left r -> return (return r)- Right (bs, p') -> do- let len = fromIntegral (BS.length bs)- if (len <= n)- then do- yield bs- go (n - len) p'- else do- let (prefix, suffix) = BS.splitAt (fromIntegral n) bs- yield prefix- return (yield suffix >> p')-{-# INLINABLE splitAt #-}+ Left r -> return (Left r)+ Right (bs, p') -> case (BS.uncons bs) of+ Nothing -> go p'+ Just (w8, bs') -> return (Right (w8, yield bs' >> p'))+{-# INLINABLE nextByte #-} --- | Split a byte stream into 'FreeT'-delimited byte streams of fixed size-chunksOf+{-| Draw one 'Word8' from the underlying 'Producer', returning 'Nothing' if the+ 'Producer' is empty+-}+drawByte :: Monad m => Parser ByteString m (Maybe Word8)+drawByte = do+ x <- PP.draw+ case x of+ Nothing -> return Nothing+ Just bs -> case (BS.uncons bs) of+ Nothing -> drawByte+ Just (w8, bs') -> do+ PP.unDraw bs'+ return (Just w8)+{-# INLINABLE drawByte #-}++-- | Push back a 'Word8' onto the underlying 'Producer'+unDrawByte :: Monad m => Word8 -> Parser ByteString m ()+unDrawByte w8 = modify (yield (BS.singleton w8) >>)+{-# INLINABLE unDrawByte #-}++{-| 'peekByte' checks the first 'Word8' in the stream, but uses 'unDrawByte' to+ push the 'Word8' back++> peekByte = do+> x <- drawByte+> case x of+> Nothing -> return ()+> Just w8 -> unDrawByte w8+> return x+-}+peekByte :: Monad m => Parser ByteString m (Maybe Word8)+peekByte = do+ x <- drawByte+ case x of+ Nothing -> return ()+ Just w8 -> unDrawByte w8+ return x+{-# INLINABLE peekByte #-}++{-| Check if the underlying 'Producer' has no more bytes++ Note that this will skip over empty 'ByteString' chunks, unlike+ 'Pipes.Parse.isEndOfInput' from @pipes-parse@.++> isEndOfBytes = liftM isNothing peekByte+-}+isEndOfBytes :: Monad m => Parser ByteString m Bool+isEndOfBytes = do+ x <- peekByte+ return (case x of+ Nothing -> True+ Just _ -> False )+{-# INLINABLE isEndOfBytes #-}++type Lens' a b = forall f . Functor f => (b -> f b) -> (a -> f a)++-- | Improper lens that splits a 'Producer' after the given number of bytes+splitAt :: (Monad m, Integral n)- => n -> Producer ByteString m r -> FreeT (Producer ByteString m) m r-chunksOf n = go+ => n+ -> Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x))+splitAt n0 k p0 = fmap join (k (go n0 p0)) where- go p = PP.FreeT $ do- x <- next p- return $ case x of- Left r -> PP.Pure r- Right (bs, p') -> PP.Free $ do- p'' <- splitAt n (yield bs >> p')- return (go p'')-{-# INLINABLE chunksOf #-}+ -- go :: (Monad m, Integral n)+ -- => n+ -- -> Producer ByteString m r+ -- -> Producer' ByteString m (Producer ByteString m r)+ go n p =+ if (n <= 0)+ then return p+ else do+ x <- lift (next p)+ case x of+ Left r -> return (return r)+ Right (bs, p') -> do+ let len = fromIntegral (BS.length bs)+ if (len <= n)+ then do+ yield bs+ go (n - len) p'+ else do+ let (prefix, suffix) =+ BS.splitAt (fromIntegral n) bs+ yield prefix+ return (yield suffix >> p')+{-# INLINABLE splitAt #-} -{-| Split a byte stream in two, where the first byte stream is the longest- consecutive group of bytes that satisfy the predicate+{-| Improper lens that splits after the longest consecutive group of bytes that+ satisfy the given predicate -} span- :: (Monad m)+ :: Monad m => (Word8 -> Bool)- -> Producer ByteString m r- -> Producer' ByteString m (Producer ByteString m r)-span predicate = go+ -> Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x))+span predicate k p0 = fmap join (k (go p0)) where go p = do x <- lift (next p)@@ -633,129 +702,147 @@ return (yield suffix >> p') {-# INLINABLE span #-} -{-| Split a byte stream in two, where the first byte stream is the longest- consecutive group of bytes that don't satisfy the predicate+{-| Improper lens that splits after the longest consecutive group of bytes that+ fail the given predicate -} break- :: (Monad m)+ :: Monad m => (Word8 -> Bool)- -> Producer ByteString m r- -> Producer ByteString m (Producer ByteString m r)+ -> Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x)) break predicate = span (not . predicate) {-# INLINABLE break #-} -{-| Split a byte stream into sub-streams delimited by bytes that satisfy the- predicate+{-| Improper lens that splits at the first occurrence of the pattern. -}-splitWith- :: (Monad m)- => (Word8 -> Bool)- -> Producer ByteString m r- -> PP.FreeT (Producer ByteString m) m r-splitWith predicate p0 = PP.FreeT (go0 p0)+breakOn+ :: Monad m+ => ByteString+ -> Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x))+breakOn needle k p0 =+ fmap join (k (go mempty p0)) where- go0 p = do- x <- next p- case x of- Left r -> return (PP.Pure r)- Right (bs, p') ->- if (BS.null bs)- then go0 p'- else go1 (yield bs >> p')- go1 p = return $ PP.Free $ do- p' <- span (not . predicate) p- return $ PP.FreeT $ do- x <- nextByte p'- case x of- Left r -> return (PP.Pure r)- Right (_, p'') -> go1 p''-{-# INLINABLE splitWith #-}+ len0 = BS.length needle --- | Split a byte stream using the given 'Word8' as the delimiter-split :: (Monad m)- => Word8- -> Producer ByteString m r- -> FreeT (Producer ByteString m) m r-split w8 = splitWith (w8 ==)-{-# INLINABLE split #-}+ go leftovers p =+ if BS.length leftovers < len0+ then do+ x <- lift (next p)+ case x of+ Left r -> do+ yield leftovers+ return (return r)+ Right (bytes, p') -> do+ go (leftovers <> bytes) p'+ else do+ let (prefix, suffix) = Data.ByteString.Search.breakOn needle leftovers+ if BS.null suffix+ then do+ let len = BS.length leftovers+ let (output, leftovers') =+ BS.splitAt (len + 1 - len0) leftovers+ yield output+ go leftovers' p+ else do+ yield prefix+ return (yield suffix >> p)+{-# INLINABLE breakOn #-} -{-| Group a byte stream into 'FreeT'-delimited byte streams using the supplied- equality predicate+{-| Improper lens that splits after the first group of matching bytes, as+ defined by the given equality predicate -} groupBy- :: (Monad m)+ :: Monad m => (Word8 -> Word8 -> Bool)- -> Producer ByteString m r- -> FreeT (Producer ByteString m) m r-groupBy equal p0 = PP.FreeT (go p0)+ -> Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x))+groupBy equals k p0 = fmap join (k (_groupBy p0)) where- go p = do- x <- next p+ -- _groupBy+ -- :: Monad m+ -- => Producer ByteString m r+ -- -> Producer ByteString m (Producer ByteString m r)+ _groupBy p = do+ x <- lift (next p) case x of- Left r -> return (PP.Pure r)+ Left r -> return (return r) Right (bs, p') -> case (BS.uncons bs) of- Nothing -> go p'- Just (w8, _) -> do- return $ PP.Free $ do- p'' <- span (equal w8) (yield bs >> p')- return $ PP.FreeT (go p'')+ Nothing -> _groupBy p'+ Just (w8, _) -> (yield bs >> p')^.span (equals w8) {-# INLINABLE groupBy #-} --- | Group a byte stream into 'FreeT'-delimited byte streams of identical bytes+-- | Like 'groupBy', where the equality predicate is ('==') group- :: (Monad m) => Producer ByteString m r -> FreeT (Producer ByteString m) m r+ :: Monad m+ => Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x)) group = groupBy (==) {-# INLINABLE group #-} -{-| Split a byte stream into 'FreeT'-delimited lines+{-| Improper lens that splits a 'Producer' after the first word + Unlike 'words', this does not drop leading whitespace+ Note: This function is purely for demonstration purposes since it assumes a particular encoding. You should prefer the 'Data.Text.Text' equivalent of- this function from the upcoming @pipes-text@ library.+ this function from the @pipes-text@ library. -}-lines- :: (Monad m) => Producer ByteString m r -> FreeT (Producer ByteString m) m r-lines p0 = PP.FreeT (go0 p0)+word+ :: Monad m+ => Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x))+word k p0 = fmap join (k (to p0)) where- go0 p = do- x <- next p- case x of- Left r -> return (PP.Pure r)- Right (bs, p') ->- if (BS.null bs)- then go0 p'- else return $ PP.Free $ go1 (yield bs >> p')- go1 p = do- p' <- break (fromIntegral (ord '\n') ==) p- return $ PP.FreeT $ do- x <- nextByte p'- case x of- Left r -> return (PP.Pure r)- Right (_, p'') -> go0 p''-{-# INLINABLE lines #-}+ -- to+ -- :: Monad m+ -- => Producer ByteString m r+ -- -> Producer ByteString m (Producer ByteString m r)+ to p = do+ p' <- p^.span isSpaceWord8+ p'^.break isSpaceWord8+{-# INLINABLE word #-} -{-| Split a byte stream into 'FreeT'-delimited words+nl :: Word8+nl = fromIntegral (ord '\n') +{-| Improper lens that splits a 'Producer' after the first line++ Unlike 'lines', this does not consume the newline marker, which is stored+ within the inner 'Producer'+ Note: This function is purely for demonstration purposes since it assumes a particular encoding. You should prefer the 'Data.Text.Text' equivalent of- this function from the upcoming @pipes-text@ library.+ this function from the @pipes-text@ library. -}-words- :: (Monad m) => Producer ByteString m r -> FreeT (Producer ByteString m) m r-words = go- where- go p = PP.FreeT $ do- x <- next (p >-> dropWhile isSpaceWord8)- return $ case x of- Left r -> PP.Pure r- Right (bs, p') -> PP.Free $ do- p'' <- break isSpaceWord8 (yield bs >> p')- return (go p'')-{-# INLINABLE words #-}+line+ :: Monad m+ => Lens' (Producer ByteString m x)+ (Producer ByteString m (Producer ByteString m x))+line = break (== nl)+{-# INLINABLE line #-} +-- | @(drop n)@ drops the first @n@ bytes+drop+ :: (Monad m, Integral n)+ => n -> Producer ByteString m r -> Producer ByteString m r+drop n p = do+ p' <- lift $ runEffect (for (p ^. splitAt n) discard)+ p'+{-# INLINABLE drop #-}++-- | Drop bytes until they fail the predicate+dropWhile+ :: Monad m+ => (Word8 -> Bool) -> Producer ByteString m r -> Producer ByteString m r+dropWhile predicate p = do+ p' <- lift $ runEffect (for (p ^. span predicate) discard)+ p'+{-# INLINABLE dropWhile #-}+ -- | Intersperse a 'Word8' in between the bytes of the byte stream intersperse- :: (Monad m) => Word8 -> Producer ByteString m r -> Producer ByteString m r+ :: Monad m => Word8 -> Producer ByteString m r -> Producer ByteString m r intersperse w8 = go0 where go0 p = do@@ -775,62 +862,226 @@ go1 p' {-# INLINABLE intersperse #-} -{-| 'intercalate' concatenates the 'FreeT'-delimited byte streams after- interspersing a byte stream in between them+-- | Improper lens from unpacked 'Word8's to packaged 'ByteString's+pack :: Monad m => Lens' (Producer Word8 m x) (Producer ByteString m x)+pack k p = fmap _unpack (k (_pack p))+{-# INLINABLE pack #-}++-- | Improper lens from packed 'ByteString's to unpacked 'Word8's+unpack :: Monad m => Lens' (Producer ByteString m x) (Producer Word8 m x)+unpack k p = fmap _pack (k (_unpack p))+{-# INLINABLE unpack #-}++_pack :: Monad m => Producer Word8 m x -> Producer ByteString m x+_pack p = PG.folds step id done (p^.PG.chunksOf defaultChunkSize)+ where+ step diffAs w8 = diffAs . (w8:)++ done diffAs = BS.pack (diffAs [])+{-# INLINABLE _pack #-}++_unpack :: Monad m => Producer ByteString m x -> Producer Word8 m x+_unpack p = for p (each . BS.unpack)+{-# INLINABLE _unpack #-}++{-| Group byte stream chunks into chunks of fixed length++ Note: This is the /only/ function in this API that concatenates+ 'ByteString' chunks, which requires allocating new `ByteString`s -}-intercalate- :: (Monad m)- => Producer ByteString m ()- -> FreeT (Producer ByteString m) m r- -> Producer ByteString m r-intercalate p0 = go0+chunksOf'+ :: (Monad m, Integral n)+ => n -> Producer ByteString m r -> Producer ByteString m r+chunksOf' n p =+ PG.folds+ (\diffBs bs -> diffBs . (bs:))+ id+ (\diffBs -> BS.concat (diffBs []))+ (p ^. chunksOf n)+{-# INLINABLE chunksOf' #-}++-- | Split a byte stream into 'FreeT'-delimited byte streams of fixed size+chunksOf+ :: (Monad m, Integral n)+ => n -> Lens' (Producer ByteString m x) (FreeT (Producer ByteString m) m x)+chunksOf n k p0 = fmap concats (k (go p0)) where- go0 f = do- x <- lift (PP.runFreeT f)- case x of- PP.Pure r -> return r- PP.Free p -> do- f' <- p- go1 f'- go1 f = do- x <- lift (PP.runFreeT f)+ go p = PG.FreeT $ do+ x <- next p+ return $ case x of+ Left r -> PG.Pure r+ Right (bs, p') -> PG.Free $ do+ p'' <- (yield bs >> p')^.splitAt n+ return (go p'')+{-# INLINABLE chunksOf #-}++{-| Split a byte stream into groups separated by bytes that satisfy the+ predicate+-}+splitsWith+ :: Monad m+ => (Word8 -> Bool)+ -> Producer ByteString m x -> FreeT (Producer ByteString m) m x+splitsWith predicate p0 = PG.FreeT (go0 p0)+ where+ go0 p = do+ x <- next p case x of- PP.Pure r -> return r- PP.Free p -> do- p0- f' <- p- go1 f'-{-# INLINABLE intercalate #-}+ Left r -> return (PG.Pure r)+ Right (bs, p') ->+ if (BS.null bs)+ then go0 p'+ else go1 (yield bs >> p')+ go1 p = return $ PG.Free $ do+ p' <- p^.break predicate+ return $ PG.FreeT $ do+ x <- nextByte p'+ case x of+ Left r -> return (PG.Pure r)+ Right (_, p'') -> go1 p''+{-# INLINABLE splitsWith #-} -{-| Join 'FreeT'-delimited lines into a byte stream+-- | Split a byte stream into groups separated by the given byte+splits+ :: Monad m+ => Word8+ -> Lens' (Producer ByteString m x) (FreeT (Producer ByteString m) m x)+splits w8 k p =+ fmap (PG.intercalates (yield (BS.singleton w8))) (k (splitsWith (w8 ==) p))+{-# INLINABLE splits #-} +-- | Split a byte stream into groups separated by the given `ByteString`+splitOn+ :: Monad m+ => ByteString+ -> Lens' (Producer ByteString m x) (FreeT (Producer ByteString m) m x)+splitOn needle k p0 =+ fmap+ (PG.intercalates (yield needle))+ (k (go p0))+ where+ len0 = BS.length needle+ go p = PG.FreeT $ do+ x <- next p+ return $ case x of+ Left r -> PG.Pure r+ Right (bs, p') -> PG.Free $ do+ p'' <- (yield bs >> p')^.(breakOn needle)+ return (go (drop len0 p''))+{-# INLINABLE splitOn #-}++{-| Isomorphism between a byte stream and groups of identical bytes using the+ supplied equality predicate+-}+groupsBy+ :: Monad m+ => (Word8 -> Word8 -> Bool)+ -> Lens' (Producer ByteString m x) (FreeT (Producer ByteString m) m x)+groupsBy equals k p0 = fmap concats (k (_groupsBy p0))+ where+ -- _groupsBy+ -- :: Monad m+ -- => (Word8 -> Word8 -> Bool)+ -- -> Producer ByteString m x+ -- -> FreeT (Producer ByteString m) m x+ _groupsBy p0' = PG.FreeT (go p0')+ where+ go p = do+ x <- next p+ case x of+ Left r -> return (PG.Pure r)+ Right (bs, p') -> case (BS.uncons bs) of+ Nothing -> go p'+ Just (w8, _) -> do+ return $ PG.Free $ do+ p'' <- (yield bs >> p')^.span (equals w8)+ return $ PG.FreeT (go p'')+{-# INLINABLE groupsBy #-}++-- | Like 'groupsBy', where the equality predicate is ('==')+groups+ :: Monad m+ => Lens' (Producer ByteString m x) (FreeT (Producer ByteString m) m x)+groups = groupsBy (==)+{-# INLINABLE groups #-}++{-| Improper lens between a bytestream and its lines+ Note: This function is purely for demonstration purposes since it assumes a particular encoding. You should prefer the 'Data.Text.Text' equivalent of- this function from the upcoming @pipes-text@ library.+ this function from the @pipes-text@ library. -}+lines+ :: Monad m+ => Lens' (Producer ByteString m x) (FreeT (Producer ByteString m) m x)+lines k p = fmap _unlines (k (_lines p))+{-# INLINABLE lines #-}++{-| Improper lens between lines and a bytestream++ Note: This function is purely for demonstration purposes since it assumes a+ particular encoding. You should prefer the 'Data.Text.Text' equivalent of+ this function from the @pipes-text@ library.+-} unlines- :: (Monad m) => FreeT (Producer ByteString m) m r -> Producer ByteString m r-unlines = go+ :: Monad m+ => Lens' (FreeT (Producer ByteString m) m x) (Producer ByteString m x)+unlines k p = fmap _lines (k (_unlines p))+{-# INLINABLE unlines #-}++_lines+ :: Monad m => Producer ByteString m x -> FreeT (Producer ByteString m) m x+_lines p0 = PG.FreeT (go0 p0) where- go f = do- x <- lift (PP.runFreeT f)+ go0 p = do+ x <- next p case x of- PP.Pure r -> return r- PP.Free p -> do- f' <- p- yield $ BS.singleton $ fromIntegral (ord '\n')- go f'-{-# INLINABLE unlines #-}+ Left r -> return (PG.Pure r)+ Right (bs, p') ->+ if (BS.null bs)+ then go0 p'+ else return $ PG.Free $ go1 (yield bs >> p')+ go1 p = do+ p' <- p^.line+ return $ PG.FreeT $ do+ x <- nextByte p'+ case x of+ Left r -> return (PG.Pure r)+ Right (_, p'') -> go0 p''+{-# INLINABLE _lines #-} -{-| Join 'FreeT'-delimited words into a byte stream+_unlines+ :: Monad m => FreeT (Producer ByteString m) m x -> Producer ByteString m x+_unlines = concats . PG.maps addNewline+ where+ addNewline p = p <* yield (BS.singleton nl)+{-# INLINABLE _unlines #-} +{-| Convert a bytestream to delimited words+ Note: This function is purely for demonstration purposes since it assumes a particular encoding. You should prefer the 'Data.Text.Text' equivalent of- this function from the upcoming @pipes-text@ library.+ this function from the @pipes-text@ library. -}+words :: Monad m => Producer ByteString m x -> FreeT (Producer ByteString m) m x+words p = PG.FreeT $ do+ x <- next (dropWhile isSpaceWord8 p)+ return $ case x of+ Left r -> PG.Pure r+ Right (bs, p') -> PG.Free $ do+ p'' <- (yield bs >> p')^.break isSpaceWord8+ return (words p'')+{-# INLINABLE words #-}++{-| Convert delimited words back to a byte stream++ Note: This function is purely for demonstration purposes since it assumes a+ particular encoding. You should prefer the 'Data.Text.Text' equivalent of+ this function from the @pipes-text@ library.+-} unwords- :: (Monad m) => FreeT (Producer ByteString m) m r -> Producer ByteString m r-unwords = intercalate (yield $ BS.singleton $ fromIntegral $ ord ' ')+ :: Monad m => FreeT (Producer ByteString m) m x -> Producer ByteString m x+unwords = PG.intercalates (yield $ BS.singleton $ fromIntegral $ ord ' ') {-# INLINABLE unwords #-} {- $parse@@ -838,25 +1089,13 @@ in @pipes-parse@. -} -{-| Take bytes until they fail the predicate-- Unlike 'takeWhile', this 'PP.unDraw's unused bytes--}-takeWhile'- :: (Monad m)- => (Word8 -> Bool)- -> Pipe ByteString ByteString (StateT (Producer ByteString m r) m) ()-takeWhile' = PBP.takeWhile-{-# INLINABLE takeWhile' #-}-{-# DEPRECATED takeWhile' "Use Pipes.ByteString.Parse.takeWhile instead" #-}- {- $reexports- "Pipes.ByteString.Parse" re-exports 'nextByte', 'drawByte', 'unDrawByte',- 'peekByte', and 'isEndOfBytes'.- @Data.ByteString@ re-exports the 'ByteString' type. @Data.Word@ re-exports the 'Word8' type. - @Pipes.Parse@ re-exports 'input', 'concat', and 'FreeT' (the type).+ @Pipes.Parse@ re-exports 'Parser'.++ @Pipes.Group@ re-exports 'concats', 'intercalates', and 'FreeT'+ (the type). -}
− src/Pipes/ByteString/Parse.hs
@@ -1,142 +0,0 @@--- | Parsing utilities for bytestrings, in the style of @pipes-parse@--module Pipes.ByteString.Parse (- -- * Parsers- nextByte- , drawByte- , unDrawByte- , peekByte- , isEndOfBytes- , take- , takeWhile- ) where--import Control.Monad.Trans.State.Strict (StateT, modify)-import qualified Data.ByteString as BS-import Data.ByteString (ByteString)-import Data.ByteString.Unsafe (unsafeTake, unsafeDrop)-import Data.Word (Word8)-import Pipes-import qualified Pipes.Parse as PP--import Prelude hiding (take, takeWhile)--{-| Consume the first byte from a byte stream-- 'next' either fails with a 'Left' if the 'Producer' has no more bytes or- succeeds with a 'Right' providing the next byte and the remainder of the- 'Producer'.--}-nextByte- :: (Monad m)- => Producer ByteString m r- -> m (Either r (Word8, Producer ByteString m r))-nextByte = go- where- go p = do- x <- next p- case x of- Left r -> return (Left r)- Right (bs, p') -> case (BS.uncons bs) of- Nothing -> go p'- Just (w8, bs') -> return (Right (w8, yield bs' >> p'))-{-# INLINABLE nextByte #-}--{-| Draw one 'Word8' from the underlying 'Producer', returning 'Left' if the- 'Producer' is empty--}-drawByte :: (Monad m) => StateT (Producer ByteString m r) m (Either r Word8)-drawByte = do- x <- PP.draw- case x of- Left r -> return (Left r)- Right bs -> case (BS.uncons bs) of- Nothing -> drawByte- Just (w8, bs') -> do- PP.unDraw bs'- return (Right w8)-{-# INLINABLE drawByte #-}---- | Push back a 'Word8' onto the underlying 'Producer'-unDrawByte :: (Monad m) => Word8 -> StateT (Producer ByteString m r) m ()-unDrawByte w8 = modify (yield (BS.singleton w8) >>)-{-# INLINABLE unDrawByte #-}--{-| 'peekByte' checks the first 'Word8' in the stream, but uses 'unDrawByte' to- push the 'Word8' back--> peekByte = do-> x <- drawByte-> case x of-> Left _ -> return ()-> Right w8 -> unDrawByte w8-> return x--}-peekByte :: (Monad m) => StateT (Producer ByteString m r) m (Either r Word8)-peekByte = do- x <- drawByte- case x of- Left _ -> return ()- Right w8 -> unDrawByte w8- return x-{-# INLINABLE peekByte #-}--{-| Check if the underlying 'Producer' has no more bytes-- Note that this will skip over empty 'ByteString' chunks, unlike- 'PP.isEndOfInput' from @pipes-parse@.--> isEndOfBytes = liftM isLeft peekByte--}-isEndOfBytes :: (Monad m) => StateT (Producer ByteString m r) m Bool-isEndOfBytes = do- x <- peekByte- return (case x of- Left _ -> True- Right _ -> False )-{-# INLINABLE isEndOfBytes #-}--{-| @(take n)@ only allows @n@ bytes to pass-- Unlike @Pipes.ByteString.'Pipes.ByteString.take'@, this 'PP.unDraw's unused- bytes--}-take :: (Monad m, Integral a) => a -> Pipe ByteString ByteString (StateT (Producer ByteString m r) m) ()-take n0 = go n0 where- go n- | n <= 0 = return ()- | otherwise = do- bs <- await- let len = fromIntegral (BS.length bs)- if (len > n)- then do- let n' = fromIntegral n- lift . PP.unDraw $ unsafeDrop n' bs- yield $ unsafeTake n' bs- else do- yield bs- go (n - len)-{-# INLINABLE take #-}--{-| Take bytes until they fail the predicate-- Unlike @Pipes.ByteString.'Pipes.ByteString.takeWhile'@, this 'PP.unDraw's- unused bytes--}-takeWhile- :: (Monad m)- => (Word8 -> Bool)- -> Pipe ByteString ByteString (StateT (Producer ByteString m r) m) ()-takeWhile predicate = go- where- go = do- bs <- await- let (prefix, suffix) = BS.span predicate bs- if (BS.null suffix)- then do- yield bs- go- else do- lift $ PP.unDraw suffix- yield prefix-{-# INLINABLE takeWhile #-}