diff --git a/Pipes/Prelude/Text.hs b/Pipes/Prelude/Text.hs new file mode 100644 --- /dev/null +++ b/Pipes/Prelude/Text.hs @@ -0,0 +1,191 @@ +{-#LANGUAGE RankNTypes#-} + + +module Pipes.Prelude.Text + ( + -- * Simple line-based Text IO + -- $lineio + + fromHandleLn + , toHandleLn + , stdinLn + , stdoutLn + , stdoutLn' + , readFileLn + , writeFileLn + ) where + +import qualified System.IO as IO +import Control.Exception (throwIO, try) +import Foreign.C.Error (Errno(Errno), ePIPE) +import qualified GHC.IO.Exception as G +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.IO as T +import Pipes +import qualified Pipes.Safe.Prelude as Safe +import Pipes.Safe (MonadSafe(..), runSafeT, runSafeP) +import Prelude hiding (readFile, writeFile) + +{- $lineio + Line-based operations are marked with a final \-@Ln@, like 'stdinLn', 'readFileLn', etc. + They are drop-in 'Text' replacements for the corresponding 'String' operations in + @Pipes.Prelude@ and @Pipes.Safe.Prelude@ - a final \-@Ln@ being added where necessary. + In using them, one is producing and consuming semantically significant individual texts, + understood as lines, just as one would produce or pipe 'Int's or 'Char's or anything else. + The standard materials from @Pipes@ and @Pipes.Prelude@ and + @Data.Text@ are all you need to work with them, and + you can use these operations without using any of the other modules in this package. + + Thus, to take a trivial case, here we upper-case three lines from standard input and write + them to a file. + +>>> import Pipes +>>> import qualified Pipes.Prelude as P +>>> import qualified Pipes.Prelude.Text as Text +>>> import qualified Data.Text as T +>>> Text.runSafeT $ runEffect $ Text.stdinLn >-> P.take 3 >-> P.map T.toUpper >-> Text.writeFileLn "threelines.txt" +one +two +three +>>> :! cat "threelines.txt" +ONE +TWO +THREE + + Here @runSafeT@ from @Pipes.Safe@ just makes sure to close any handles opened in its scope. + Otherwise the point of view is very much that of @Pipes.Prelude@, substituting @Text@ for @String@. + It would still be the same even if + we did something a bit more sophisticated, like run an ordinary attoparsec 'Text' parser on + each line, as is frequently desirable. Here we use + a minimal attoparsec number parser, @scientific@, on separate lines of standard input, + dropping bad parses with @P.concat@: + +>>> import Data.Attoparsec.Text (parseOnly, scientific) +>>> P.toListM $ Text.stdinLn >-> P.takeWhile (/= "quit") >-> P.map (parseOnly scientific) >-> P.concat +1 +2 +bad +3 +quit +[1.0,2.0,3.0] + + The line-based operations are, however, subject to a number of caveats. + First, where they read from a handle, they will of course happily + accumulate indefinitely long lines. This is likely to be legitimate for input + typed in by a user, and for locally produced files of known characteristics, but + otherwise not. See the post on + + to see why @pipes-bytestring@ and this package, outside this module, take a different approach. + Furthermore, the line-based operations, + like those in @Data.Text.IO@, use the system encoding (and @T.hGetLine@, @T.hPutLine@ etc.) + and thus are slower than the \'official\' route, which would use the very fast + bytestring IO operations from @Pipes.ByteString@ and + encoding and decoding functions in @Pipes.Text.Encoding@. Finally, the line-based + operations will generate text exceptions after the fashion of + @Data.Text.Encoding@, rather than returning the undigested bytes in the + style of @Pipes.Text.Encoding@. + +-} + + +{-| Read separate lines of 'Text' from 'IO.stdin' using 'T.getLine' + This function will accumulate indefinitely long strict 'Text's. See the caveats above. + + Terminates on end of input +-} +stdinLn :: MonadIO m => Producer' T.Text m () +stdinLn = fromHandleLn IO.stdin +{-# INLINABLE stdinLn #-} + + +{-| Write 'Text' lines to 'IO.stdout' using 'putStrLn' + + Unlike 'toHandle', 'stdoutLn' gracefully terminates on a broken output pipe +-} +stdoutLn :: MonadIO m => Consumer' T.Text m () +stdoutLn = go + where + go = do + str <- await + x <- liftIO $ try (T.putStrLn str) + case x of + Left (G.IOError { G.ioe_type = G.ResourceVanished + , G.ioe_errno = Just ioe }) + | Errno ioe == ePIPE + -> return () + Left e -> liftIO (throwIO e) + Right () -> go +{-# INLINABLE stdoutLn #-} + +{-| Write lines of 'Text' to 'IO.stdout'. + + This does not handle a broken output pipe, but has a polymorphic return + value. +-} +stdoutLn' :: MonadIO m => Consumer' T.Text m r +stdoutLn' = for cat (\str -> liftIO (T.putStrLn str)) +{-# INLINABLE stdoutLn' #-} + +{-# RULES + "p >-> stdoutLn'" forall p . + p >-> stdoutLn' = for p (\str -> liftIO (T.putStrLn str)) + #-} + +{-| Read separate lines of 'Text' from a 'IO.Handle' using 'T.hGetLine'. + This operation will accumulate indefinitely large strict texts. See the caveats above. + + Terminates on end of input +-} +fromHandleLn :: MonadIO m => IO.Handle -> Producer' Text m () +fromHandleLn h = go where + getLine :: IO (Either G.IOException Text) + getLine = try (T.hGetLine h) + + go = do txt <- liftIO getLine + case txt of + Left e -> return () + Right y -> do yield y + go +{-# INLINABLE fromHandleLn #-} + +-- to do: investigate differences from the above: +-- fromHandleLn :: MonadIO m => IO.Handle -> Producer' T.Text m () +-- fromHandleLn h = go +-- where +-- go = do +-- eof <- liftIO $ IO.hIsEOF h +-- unless eof $ do +-- str <- liftIO $ T.hGetLine h +-- yield str +-- go +-- {-# INLINABLE fromHandleLn #-} + + +-- | Write separate lines of 'Text' to a 'IO.Handle' using 'T.hPutStrLn' +toHandleLn :: MonadIO m => IO.Handle -> Consumer' T.Text m r +toHandleLn handle = for cat (\str -> liftIO (T.hPutStrLn handle str)) +{-# INLINABLE toHandleLn #-} + +{-# RULES + "p >-> toHandleLn handle" forall p handle . + p >-> toHandleLn handle = for p (\str -> liftIO (T.hPutStrLn handle str)) + #-} + + +{-| Stream separate lines of text from a file. This operation will accumulate + indefinitely long strict text chunks. See the caveats above. +-} +readFileLn :: MonadSafe m => FilePath -> Producer Text m () +readFileLn file = Safe.withFile file IO.ReadMode fromHandleLn +{-# INLINE readFileLn #-} + + + +{-| Write lines to a file, automatically opening and closing the file as + necessary +-} +writeFileLn :: (MonadSafe m) => FilePath -> Consumer' Text m r +writeFileLn file = Safe.withFile file IO.WriteMode toHandleLn +{-# INLINABLE writeFileLn #-} + diff --git a/Pipes/Text/IO.hs b/Pipes/Text/IO.hs --- a/Pipes/Text/IO.hs +++ b/Pipes/Text/IO.hs @@ -3,7 +3,8 @@ module Pipes.Text.IO ( - -- * Text IO + + -- * Simple streaming text IO -- $textio -- * Caveats @@ -13,10 +14,17 @@ fromHandle , stdin , readFile + -- * Consumers , toHandle , stdout , writeFile + + -- * Re-exports + , MonadSafe(..) + , runSafeT + , runSafeP + , Safe.withFile ) where import qualified System.IO as IO @@ -28,14 +36,19 @@ import qualified Data.Text.IO as T import Pipes import qualified Pipes.Safe.Prelude as Safe -import Pipes.Safe (MonadSafe(..)) +import Pipes.Safe (MonadSafe(..), runSafeT, runSafeP) import Prelude hiding (readFile, writeFile) + {- $textio Where pipes @IO@ replaces lazy @IO@, @Producer Text IO r@ replaces lazy 'Text'. - This module exports some convenient functions for producing and consuming - pipes 'Text' in @IO@, namely, 'readFile', 'writeFile', 'fromHandle', 'toHandle', - 'stdin' and 'stdout'. Some caveats described below. + The official IO of this package and the pipes ecosystem generally would use the + IO functions in @Pipes.ByteString@ and the encoding and decoding material in + @Pipes.Text.Encoding@. + + The streaming functions exported here, namely, 'readFile', 'writeFile', 'fromHandle', 'toHandle', + 'stdin' and 'stdout' simplify this and use the system encoding on the model of @Data.Text.IO@ + and @Data.Text.Lazy.IO@ Some caveats described below. The main points are as in : @@ -67,6 +80,8 @@ > main = runEffect $ Text.stdin >-> Text.stdout + These programs, unlike the corresponding programs written with the line-based functions, + will pass along a 1 terabyte line without affecting memory use. -} @@ -85,23 +100,6 @@ * Like the functions in @Data.Text.IO@ , they use Text exceptions, not the standard Pipes protocols. - Something like - -> view utf8 . Bytes.fromHandle :: Handle -> Producer Text IO (Producer ByteString m ()) - - yields a stream of Text, and follows - standard pipes protocols by reverting to (i.e. returning) the underlying byte stream - upon reaching any decoding error. (See especially the pipes-binary package.) - - By contrast, something like - -> Text.fromHandle :: Handle -> Producer Text IO () - - supplies a stream of text returning '()', which is convenient for many tasks, - but violates the pipes @pipes-binary@ approach to decoding errors and - throws an exception of the kind characteristic of the @text@ library instead. - - -} {-| Convert a 'IO.Handle' into a text stream using a text size @@ -119,6 +117,7 @@ go {-# INLINABLE fromHandle#-} + -- | Stream text from 'stdin' stdin :: MonadIO m => Producer Text m () stdin = fromHandle IO.stdin @@ -134,6 +133,7 @@ readFile :: MonadSafe m => FilePath -> Producer Text m () readFile file = Safe.withFile file IO.ReadMode fromHandle {-# INLINE readFile #-} + {-| Stream text to 'stdout' diff --git a/Pipes/Text/Tutorial.hs b/Pipes/Text/Tutorial.hs --- a/Pipes/Text/Tutorial.hs +++ b/Pipes/Text/Tutorial.hs @@ -43,8 +43,8 @@ {- $intro This package provides @pipes@ utilities for /character streams/, realized as streams of 'Text' chunks. The individual chunks are uniformly /strict/, - and thus the @Text@ type we are using is the one from @Data.Text@, not @Data.Text.Lazy@ - But the type @Producer Text m r@, as we are using it, is a sort of /pipes/ equivalent of + and thus the @Text@ type we are using is always the one from @Data.Text@, not @Data.Text.Lazy@ + The type @Producer Text m r@, as we are using it, is a sort of /pipes/ equivalent of the lazy @Text@ type. -} @@ -60,13 +60,16 @@ {- $pipestextencoding In the @text@ library, @Data.Text.Lazy.Encoding@ - handles inter-operation with @Data.ByteString.Lazy@. Here, @Pipes.Text.Encoding@ + handles inter-operation with @Data.ByteString.Lazy@. Similarly here, @Pipes.Text.Encoding@ provides for interoperation with the \'effectful ByteStrings\' of @Pipes.ByteString@. -} {- $pipestextio Simple /IO/ operations are defined in @Pipes.Text.IO@ - as lazy IO @Text@ - operations are in @Data.Text.Lazy.IO@. It is generally + operations are in @Data.Text.Lazy.IO@. There are also some simple line-based operations + in @Pipes.Prelude.Text@. The latter do not depend on the conception of effectful text + implemented elsewhere in this package, but just improve on the @stdinLn@ and @writeFile@ of + @Pipes.Prelude@ and @Pipes.Safe.Prelude@ by replacing 'String' with 'Text' -} @@ -139,7 +142,8 @@ are a distraction. The lens combinators to keep in mind, the ones that make sense for our lenses, are @view@, @over@, and @zoom@. - One need only keep in mind that if @l@ is a @Lens' a b@, then: + One need only keep in mind that if @l@ is a @Lens' a b@, then the action of the + leading operations, @view@, @over@, and @zoom@ are as follows: -} {- $view diff --git a/pipes-text.cabal b/pipes-text.cabal --- a/pipes-text.cabal +++ b/pipes-text.cabal @@ -1,16 +1,20 @@ name: pipes-text -version: 0.0.1.0 +version: 0.0.2.0 synopsis: Text pipes. description: * This organization of the package follows the rule . * @pipes-text : pipes-bytestring :: text : bytestring@ . - Familiarity with the other three packages should give one an idea what to expect where. The package has three modules, @Pipes.Text@ , @Pipes.Text.Encoding@ and @Pipes.Text.IO@; the division has more or less the significance it has in the @text@ library. + Familiarity with the other three packages should give one an idea what to expect where. The package has three principal modules, @Pipes.Text@ , @Pipes.Text.Encoding@ and @Pipes.Text.IO@; the division has more or less the significance it has in the @text@ library. A fourth module @Pipes.Prelude.Text@ is explained below. . Note that the module @Pipes.Text.IO@ is present as a convenience (as is @Data.Text.IO@). Official pipes IO would use @Pipes.ByteString@ together with the bytestring decoding functions in @Pipes.Text.Encoding@. In particular, the @Pipes.Text.IO@ functions use Text exceptions. . - @Pipes.Text.IO@ uses version 0.11.3 or later of the @text@ library. It thus works with the version of @text@ that came with the 2013 Haskell Platform. To use an older @text@, install with the flag @-fnoio@ + The fourth module @Pipes.Prelude.Text@ exports line-based Text producers and consumers as a drop-in replacement for the String material in @Pipes.Prelude@ and @Pipes.Safe.Prelude@. They can be used as one uses @Pipes.Prelude@ without reference to the rest of this package. See the caveats in the documentation for that module. + . + @Pipes.Text.IO@ and @Pipes.Prelude.Text@ use version 0.11.3 or later of the @text@ library. To use a (very) old version of @text@, install with the flag @-fnoio@ + + homepage: https://github.com/michaelt/text-pipes bug-reports: https://github.com/michaelt/text-pipes/issues license: BSD3 @@ -48,5 +52,5 @@ ghc-options: -O2 if !flag(noio) - exposed-modules: Pipes.Text.IO, Pipes.Text.Tutorial + exposed-modules: Pipes.Text.IO, Pipes.Text.Tutorial, Pipes.Prelude.Text build-depends: text >=0.11.3 && < 1.3