pipes-text 0.0.0.16 → 0.0.0.17
raw patch · 4 files changed
+65/−41 lines, 4 files
Files
- Pipes/Text.hs +11/−8
- Pipes/Text/Encoding.hs +51/−29
- Pipes/Text/IO.hs +1/−2
- pipes-text.cabal +2/−2
Pipes/Text.hs view
@@ -1,4 +1,4 @@-{-# LANGUAGE RankNTypes, TypeFamilies, BangPatterns, Trustworthy #-}+{-# LANGUAGE RankNTypes, TypeFamilies, BangPatterns#-} {-| The module @Pipes.Text@ closely follows @Pipes.ByteString@ from the @pipes-bytestring@ package. A draft tutorial can be found in@@ -81,13 +81,17 @@ import Control.Applicative ((<*)) import Control.Monad (liftM, join)-import Control.Monad.Trans.State.Strict (StateT(..), modify)+import Data.Functor.Constant (Constant(..))+import Data.Functor.Identity (Identity)+import Control.Monad.Trans.State.Strict (modify)+ import qualified Data.Text as T import Data.Text (Text) import qualified Data.Text.Lazy as TL import Data.ByteString (ByteString)-import Data.Functor.Constant (Constant(Constant, getConstant))-import Data.Functor.Identity (Identity)+import Data.Char (isSpace)+import Foreign.Storable (sizeOf)+import Data.Bits (shiftL) import Pipes import Pipes.Group (folds, maps, concats, intercalates, FreeT(..), FreeF(..))@@ -95,10 +99,9 @@ import qualified Pipes.Parse as PP import Pipes.Parse (Parser) import qualified Pipes.Prelude as P-import Data.Char (isSpace)-import Data.Word (Word8)-import Foreign.Storable (sizeOf)-import Data.Bits (shiftL)+++ import Prelude hiding ( all, any,
Pipes/Text/Encoding.hs view
@@ -53,7 +53,6 @@ import Data.Functor.Constant (Constant(..)) import Data.Char (ord) import Data.ByteString as B -import Data.ByteString (ByteString) import Data.ByteString.Char8 as B8 import Data.Text (Text) import qualified Data.Text as T @@ -61,49 +60,70 @@ import qualified Data.Streaming.Text as Stream import Data.Streaming.Text (DecodeResult(..)) import Control.Monad (join, liftM)-import Data.Word (Word8) import Pipes {- $usage- Given + Encoding is of course simple. Given > text :: Producer Text IO () - we can encode it with @Data.Text.Encoding@ and ordinary pipe operations:+ we can encode it with @Data.Text.Encoding.encodeUtf8@ +> TE.encodeUtf8 :: Text -> ByteString++ and ordinary pipe operations:+ > text >-> P.map TE.encodeUtf8 :: Producer.ByteString IO () - or, using this module, with+ or, equivalently +> for text (yield . TE.encodeUtf8)++ But, using this module, we might use++> encodeUtf8 :: Text -> Producer ByteString m ()++ to write+ > for text encodeUtf8 :: Producer.ByteString IO () - Given + All of the above come to the same. -> bytes :: Producer ByteString Text IO () + Given++> bytes :: Producer ByteString IO ()+ we can apply a decoding function from this module: > decodeUtf8 bytes :: Producer Text IO (Producer ByteString IO ()) - The Text producer ends wherever decoding first fails. Thus we can re-encode+ The Text producer ends wherever decoding first fails. The un-decoded+ material is returned. If we are confident it is of no interest, we can+ write: ++> void $ decodeUtf8 bytes :: Producer Text IO ()++ Thus we can re-encode as uft8 as much of our byte stream as is decodeUtf16BE decodable, with, e.g. > for (decodeUtf16BE bytes) encodeUtf8 :: Producer ByteString IO (Producer ByteString IO ()) The bytestring producer that is returned begins with where utf16BE decoding- failed; it it didn't fail the producer is empty. + failed; if it didn't fail the producer is empty. -} {- $lenses- We get a bit more flexibility, though, if we use a lens like @utf8@ or @utf16BE@ - that looks for text in an appropriately encoded byte stream.+ We get a bit more flexibility, particularly in the use of pipes-style "parsers", + if we use a lens like @utf8@ or @utf16BE@ + that focusses on the text in an appropriately encoded byte stream. > type Lens' a b = forall f . Functor f => (b -> f b) -> (a -> f a) - is just an alias for a Prelude type. We abbreviate this further, for our use case, as+ is just an alias for a Prelude type. We abbreviate this further, for our use case, as > type Codec > = forall m r . Monad m => Lens' (Producer ByteString m r) (Producer Text m (Producer ByteString m r))@@ -111,9 +131,11 @@ and call the decoding lenses @utf8@, @utf16BE@ \"codecs\", since they can re-encode what they have decoded. Thus you use any particular codec with the @view@ / @(^.)@ , @zoom@ and @over@ functions from the standard lens libraries;- we presuppose neither <http://hackage.haskell.org/package/lens lens> - nor <http://hackage.haskell.org/package/lens-family lens-family> - since we already have access to the types they require. + <http://hackage.haskell.org/package/lens lens>,+ <http://hackage.haskell.org/package/lens-family lens-family>,+ <http://hackage.haskell.org/package/lens-simple lens-simple>, or one of the+ and <http://hackage.haskell.org/package/microlens microlens> packages will all work+ the same, since we already have access to the types they require. Each decoding lens looks into a byte stream that is supposed to contain text. The particular lenses are named in accordance with the expected @@ -124,8 +146,7 @@ > decode utf8 Byte.stdin :: Producer Text IO (Producer ByteString IO r) > Bytes.stdin ^. utf8 :: Producer Text IO (Producer ByteString IO r) - These simple uses of a codec with @view@ or @(^.)@ or 'decode' can always be replaced by - the specialized decoding functions exported here, e.g. + Of course, we could always do this with the specialized decoding functions, e.g. > decodeUtf8 :: Producer ByteString m r -> Producer Text m (Producer ByteString m r) > decodeUtf8 Byte.stdin :: Producer Text IO (Producer ByteString IO r)@@ -163,7 +184,7 @@ > return (Left bad_bytestream) - @zoom@ converts a Text parser into a ByteString parser:+ @zoom utf8@ converts a Text parser into a ByteString parser: > zoom utf8 drawChar :: Monad m => StateT (Producer ByteString m r) m (Maybe Char) @@ -180,7 +201,7 @@ Though @charPlusByte@ is partly defined with a Text parser 'drawChar'; but it is a ByteString parser; it will return the first valid utf8-encoded - Char in a ByteString, whatever its byte-length, + Char in a ByteString, /whatever its byte-length/, and the first byte following, if both exist. Because we \'draw\' one and \'peek\' at the other, the parser as a whole only advances one Char's length along the bytestring, whatever that length may be.@@ -229,9 +250,9 @@ -} -eof :: Monad m => Lens' (Producer Text m (Producer ByteString m r))- (Producer Text m (Either (Producer ByteString m r) r))-eof k p = fmap fromEither (k (toEither p)) where+eof :: (Monad m, Monad (t m), MonadTrans t) => Lens' (t m (Producer ByteString m r))+ (t m (Either (Producer ByteString m r) r))+eof k p0 = fmap fromEither (k (toEither p0)) where fromEither = liftM (either id return) @@ -270,13 +291,14 @@ decodeStream = loop where loop dec0 p = do x <- lift (next p) - case x of Left r -> return (return r)- Right (chunk, p') -> case dec0 chunk of - DecodeResultSuccess text dec -> do yield text- loop dec p'- DecodeResultFailure text bs -> do yield text - return (do yield bs - p')+ case x of + Left r -> return (return r)+ Right (chunk, p') -> case dec0 chunk of + DecodeResultSuccess text dec -> do yield text+ loop dec p'+ DecodeResultFailure text bs -> do yield text + return (do yield bs + p') {-# INLINABLE decodeStream#-}
Pipes/Text/IO.hs view
@@ -28,8 +28,7 @@ import qualified Data.Text.IO as T import Pipes import qualified Pipes.Safe.Prelude as Safe-import qualified Pipes.Safe as Safe-import Pipes.Safe (MonadSafe(..), Base(..))+import Pipes.Safe (MonadSafe(..)) import Prelude hiding (readFile, writeFile) {- $textio
pipes-text.cabal view
@@ -1,5 +1,5 @@ name: pipes-text-version: 0.0.0.16+version: 0.0.0.17 synopsis: Text pipes. description: * This package will be in a draft, or testing, phase until version 0.0.1. Please report any installation difficulties, or any wisdom about the api, on the github page or the <https://groups.google.com/forum/#!forum/haskell-pipes pipes list> .@@ -34,7 +34,7 @@ library exposed-modules: Pipes.Text, Pipes.Text.Encoding- build-depends: base >= 4 && < 5 ,+ build-depends: base >= 4 && < 5 , bytestring >= 0.9.2.1 && < 0.11, text >= 0.11.2 && < 1.3 , streaming-commons >= 0.1 && < 0.2 ,