pipes-text 0.0.1.0 → 0.0.2.0
raw patch · 4 files changed
+230/−31 lines, 4 files
Files
- Pipes/Prelude/Text.hs +191/−0
- Pipes/Text/IO.hs +22/−22
- Pipes/Text/Tutorial.hs +9/−5
- pipes-text.cabal +8/−4
+ Pipes/Prelude/Text.hs view
@@ -0,0 +1,191 @@+{-#LANGUAGE RankNTypes#-}+++module Pipes.Prelude.Text+ ( + -- * Simple line-based Text IO+ -- $lineio+ + fromHandleLn+ , toHandleLn+ , stdinLn+ , stdoutLn+ , stdoutLn'+ , readFileLn+ , writeFileLn+ ) where++import qualified System.IO as IO+import Control.Exception (throwIO, try)+import Foreign.C.Error (Errno(Errno), ePIPE)+import qualified GHC.IO.Exception as G+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.IO as T+import Pipes+import qualified Pipes.Safe.Prelude as Safe+import Pipes.Safe (MonadSafe(..), runSafeT, runSafeP)+import Prelude hiding (readFile, writeFile)++{- $lineio+ Line-based operations are marked with a final \-@Ln@, like 'stdinLn', 'readFileLn', etc. + They are drop-in 'Text' replacements for the corresponding 'String' operations in + @Pipes.Prelude@ and @Pipes.Safe.Prelude@ - a final \-@Ln@ being added where necessary. + In using them, one is producing and consuming semantically significant individual texts, + understood as lines, just as one would produce or pipe 'Int's or 'Char's or anything else.+ The standard materials from @Pipes@ and @Pipes.Prelude@ and+ @Data.Text@ are all you need to work with them, and+ you can use these operations without using any of the other modules in this package. ++ Thus, to take a trivial case, here we upper-case three lines from standard input and write + them to a file.++>>> import Pipes+>>> import qualified Pipes.Prelude as P+>>> import qualified Pipes.Prelude.Text as Text+>>> import qualified Data.Text as T+>>> Text.runSafeT $ runEffect $ Text.stdinLn >-> P.take 3 >-> P.map T.toUpper >-> Text.writeFileLn "threelines.txt"+one<Enter>+two<Enter>+three<Enter>+>>> :! cat "threelines.txt"+ONE+TWO+THREE++ Here @runSafeT@ from @Pipes.Safe@ just makes sure to close any handles opened in its scope. + Otherwise the point of view is very much that of @Pipes.Prelude@, substituting @Text@ for @String@. + It would still be the same even if+ we did something a bit more sophisticated, like run an ordinary attoparsec 'Text' parser on+ each line, as is frequently desirable. Here we use+ a minimal attoparsec number parser, @scientific@, on separate lines of standard input, + dropping bad parses with @P.concat@:++>>> import Data.Attoparsec.Text (parseOnly, scientific)+>>> P.toListM $ Text.stdinLn >-> P.takeWhile (/= "quit") >-> P.map (parseOnly scientific) >-> P.concat +1<Enter>+2<Enter>+bad<Enter>+3<Enter>+quit<Enter>+[1.0,2.0,3.0]++ The line-based operations are, however, subject to a number of caveats.+ First, where they read from a handle, they will of course happily + accumulate indefinitely long lines. This is likely to be legitimate for input + typed in by a user, and for locally produced files of known characteristics, but+ otherwise not. See the post on+ <http://www.haskellforall.com/2013/09/perfect-streaming-using-pipes-bytestring.html perfect streaming> + to see why @pipes-bytestring@ and this package, outside this module, take a different approach. + Furthermore, the line-based operations, + like those in @Data.Text.IO@, use the system encoding (and @T.hGetLine@, @T.hPutLine@ etc.)+ and thus are slower than the \'official\' route, which would use the very fast + bytestring IO operations from @Pipes.ByteString@ and+ encoding and decoding functions in @Pipes.Text.Encoding@. Finally, the line-based+ operations will generate text exceptions after the fashion of + @Data.Text.Encoding@, rather than returning the undigested bytes in the + style of @Pipes.Text.Encoding@.++-}+++{-| Read separate lines of 'Text' from 'IO.stdin' using 'T.getLine' + This function will accumulate indefinitely long strict 'Text's. See the caveats above.++ Terminates on end of input+-}+stdinLn :: MonadIO m => Producer' T.Text m ()+stdinLn = fromHandleLn IO.stdin+{-# INLINABLE stdinLn #-}+++{-| Write 'Text' lines to 'IO.stdout' using 'putStrLn'++ Unlike 'toHandle', 'stdoutLn' gracefully terminates on a broken output pipe+-}+stdoutLn :: MonadIO m => Consumer' T.Text m ()+stdoutLn = go+ where+ go = do+ str <- await+ x <- liftIO $ try (T.putStrLn str)+ case x of+ Left (G.IOError { G.ioe_type = G.ResourceVanished+ , G.ioe_errno = Just ioe })+ | Errno ioe == ePIPE+ -> return ()+ Left e -> liftIO (throwIO e)+ Right () -> go+{-# INLINABLE stdoutLn #-}++{-| Write lines of 'Text' to 'IO.stdout'.++ This does not handle a broken output pipe, but has a polymorphic return+ value.+-}+stdoutLn' :: MonadIO m => Consumer' T.Text m r+stdoutLn' = for cat (\str -> liftIO (T.putStrLn str))+{-# INLINABLE stdoutLn' #-}++{-# RULES+ "p >-> stdoutLn'" forall p .+ p >-> stdoutLn' = for p (\str -> liftIO (T.putStrLn str))+ #-}++{-| Read separate lines of 'Text' from a 'IO.Handle' using 'T.hGetLine'.+ This operation will accumulate indefinitely large strict texts. See the caveats above.++ Terminates on end of input+-}+fromHandleLn :: MonadIO m => IO.Handle -> Producer' Text m ()+fromHandleLn h = go where+ getLine :: IO (Either G.IOException Text)+ getLine = try (T.hGetLine h)++ go = do txt <- liftIO getLine+ case txt of+ Left e -> return ()+ Right y -> do yield y+ go+{-# INLINABLE fromHandleLn #-}++-- to do: investigate differences from the above: +-- fromHandleLn :: MonadIO m => IO.Handle -> Producer' T.Text m ()+-- fromHandleLn h = go+-- where+-- go = do+-- eof <- liftIO $ IO.hIsEOF h+-- unless eof $ do+-- str <- liftIO $ T.hGetLine h+-- yield str+-- go+-- {-# INLINABLE fromHandleLn #-}+++-- | Write separate lines of 'Text' to a 'IO.Handle' using 'T.hPutStrLn'+toHandleLn :: MonadIO m => IO.Handle -> Consumer' T.Text m r+toHandleLn handle = for cat (\str -> liftIO (T.hPutStrLn handle str))+{-# INLINABLE toHandleLn #-}++{-# RULES+ "p >-> toHandleLn handle" forall p handle .+ p >-> toHandleLn handle = for p (\str -> liftIO (T.hPutStrLn handle str))+ #-}+++{-| Stream separate lines of text from a file. This operation will accumulate+ indefinitely long strict text chunks. See the caveats above.+-}+readFileLn :: MonadSafe m => FilePath -> Producer Text m ()+readFileLn file = Safe.withFile file IO.ReadMode fromHandleLn+{-# INLINE readFileLn #-}++++{-| Write lines to a file, automatically opening and closing the file as+ necessary+-}+writeFileLn :: (MonadSafe m) => FilePath -> Consumer' Text m r+writeFileLn file = Safe.withFile file IO.WriteMode toHandleLn+{-# INLINABLE writeFileLn #-}+
Pipes/Text/IO.hs view
@@ -3,7 +3,8 @@ module Pipes.Text.IO ( - -- * Text IO++ -- * Simple streaming text IO -- $textio -- * Caveats@@ -13,10 +14,17 @@ fromHandle , stdin , readFile+ -- * Consumers , toHandle , stdout , writeFile+ + -- * Re-exports+ , MonadSafe(..)+ , runSafeT+ , runSafeP+ , Safe.withFile ) where import qualified System.IO as IO@@ -28,14 +36,19 @@ import qualified Data.Text.IO as T import Pipes import qualified Pipes.Safe.Prelude as Safe-import Pipes.Safe (MonadSafe(..))+import Pipes.Safe (MonadSafe(..), runSafeT, runSafeP) import Prelude hiding (readFile, writeFile) + {- $textio Where pipes @IO@ replaces lazy @IO@, @Producer Text IO r@ replaces lazy 'Text'. - This module exports some convenient functions for producing and consuming - pipes 'Text' in @IO@, namely, 'readFile', 'writeFile', 'fromHandle', 'toHandle', - 'stdin' and 'stdout'. Some caveats described below. + The official IO of this package and the pipes ecosystem generally would use the+ IO functions in @Pipes.ByteString@ and the encoding and decoding material in + @Pipes.Text.Encoding@.++ The streaming functions exported here, namely, 'readFile', 'writeFile', 'fromHandle', 'toHandle', + 'stdin' and 'stdout' simplify this and use the system encoding on the model of @Data.Text.IO@ + and @Data.Text.Lazy.IO@ Some caveats described below. The main points are as in <https://hackage.haskell.org/package/pipes-bytestring-1.0.0/docs/Pipes-ByteString.html Pipes.ByteString>:@@ -67,6 +80,8 @@ > main = runEffect $ Text.stdin >-> Text.stdout + These programs, unlike the corresponding programs written with the line-based functions,+ will pass along a 1 terabyte line without affecting memory use. -} @@ -85,23 +100,6 @@ * Like the functions in @Data.Text.IO@ , they use Text exceptions, not the standard Pipes protocols. - Something like - -> view utf8 . Bytes.fromHandle :: Handle -> Producer Text IO (Producer ByteString m ()) -- yields a stream of Text, and follows- standard pipes protocols by reverting to (i.e. returning) the underlying byte stream- upon reaching any decoding error. (See especially the pipes-binary package.) -- By contrast, something like --> Text.fromHandle :: Handle -> Producer Text IO () -- supplies a stream of text returning '()', which is convenient for many tasks, - but violates the pipes @pipes-binary@ approach to decoding errors and - throws an exception of the kind characteristic of the @text@ library instead.-- -} {-| Convert a 'IO.Handle' into a text stream using a text size @@ -119,6 +117,7 @@ go {-# INLINABLE fromHandle#-} + -- | Stream text from 'stdin' stdin :: MonadIO m => Producer Text m () stdin = fromHandle IO.stdin@@ -134,6 +133,7 @@ readFile :: MonadSafe m => FilePath -> Producer Text m () readFile file = Safe.withFile file IO.ReadMode fromHandle {-# INLINE readFile #-}+ {-| Stream text to 'stdout'
Pipes/Text/Tutorial.hs view
@@ -43,8 +43,8 @@ {- $intro This package provides @pipes@ utilities for /character streams/, realized as streams of 'Text' chunks. The individual chunks are uniformly /strict/,- and thus the @Text@ type we are using is the one from @Data.Text@, not @Data.Text.Lazy@ - But the type @Producer Text m r@, as we are using it, is a sort of /pipes/ equivalent of + and thus the @Text@ type we are using is always the one from @Data.Text@, not @Data.Text.Lazy@ + The type @Producer Text m r@, as we are using it, is a sort of /pipes/ equivalent of the lazy @Text@ type. -} @@ -60,13 +60,16 @@ {- $pipestextencoding In the @text@ library, @Data.Text.Lazy.Encoding@ - handles inter-operation with @Data.ByteString.Lazy@. Here, @Pipes.Text.Encoding@ + handles inter-operation with @Data.ByteString.Lazy@. Similarly here, @Pipes.Text.Encoding@ provides for interoperation with the \'effectful ByteStrings\' of @Pipes.ByteString@. -} {- $pipestextio Simple /IO/ operations are defined in @Pipes.Text.IO@ - as lazy IO @Text@- operations are in @Data.Text.Lazy.IO@. It is generally + operations are in @Data.Text.Lazy.IO@. There are also some simple line-based operations+ in @Pipes.Prelude.Text@. The latter do not depend on the conception of effectful text+ implemented elsewhere in this package, but just improve on the @stdinLn@ and @writeFile@ of+ @Pipes.Prelude@ and @Pipes.Safe.Prelude@ by replacing 'String' with 'Text' -} @@ -139,7 +142,8 @@ are a distraction. The lens combinators to keep in mind, the ones that make sense for our lenses, are @view@, @over@, and @zoom@. - One need only keep in mind that if @l@ is a @Lens' a b@, then:+ One need only keep in mind that if @l@ is a @Lens' a b@, then the action of the + leading operations, @view@, @over@, and @zoom@ are as follows: -} {- $view
pipes-text.cabal view
@@ -1,16 +1,20 @@ name: pipes-text-version: 0.0.1.0+version: 0.0.2.0 synopsis: Text pipes. description: * This organization of the package follows the rule . * @pipes-text : pipes-bytestring :: text : bytestring@ .- Familiarity with the other three packages should give one an idea what to expect where. The package has three modules, @Pipes.Text@ , @Pipes.Text.Encoding@ and @Pipes.Text.IO@; the division has more or less the significance it has in the @text@ library. + Familiarity with the other three packages should give one an idea what to expect where. The package has three principal modules, @Pipes.Text@ , @Pipes.Text.Encoding@ and @Pipes.Text.IO@; the division has more or less the significance it has in the @text@ library. A fourth module @Pipes.Prelude.Text@ is explained below. . Note that the module @Pipes.Text.IO@ is present as a convenience (as is @Data.Text.IO@). Official pipes IO would use @Pipes.ByteString@ together with the bytestring decoding functions in @Pipes.Text.Encoding@. In particular, the @Pipes.Text.IO@ functions use Text exceptions. .- @Pipes.Text.IO@ uses version 0.11.3 or later of the @text@ library. It thus works with the version of @text@ that came with the 2013 Haskell Platform. To use an older @text@, install with the flag @-fnoio@+ The fourth module @Pipes.Prelude.Text@ exports line-based Text producers and consumers as a drop-in replacement for the String material in @Pipes.Prelude@ and @Pipes.Safe.Prelude@. They can be used as one uses @Pipes.Prelude@ without reference to the rest of this package. See the caveats in the documentation for that module.+ .+ @Pipes.Text.IO@ and @Pipes.Prelude.Text@ use version 0.11.3 or later of the @text@ library. To use a (very) old version of @text@, install with the flag @-fnoio@ ++ homepage: https://github.com/michaelt/text-pipes bug-reports: https://github.com/michaelt/text-pipes/issues license: BSD3@@ -48,5 +52,5 @@ ghc-options: -O2 if !flag(noio)- exposed-modules: Pipes.Text.IO, Pipes.Text.Tutorial+ exposed-modules: Pipes.Text.IO, Pipes.Text.Tutorial, Pipes.Prelude.Text build-depends: text >=0.11.3 && < 1.3