diff --git a/Control/Frame.hs b/Control/Frame.hs new file mode 100644 --- /dev/null +++ b/Control/Frame.hs @@ -0,0 +1,477 @@ +{-| + 'Frame's extend 'Pipe's with: + + * The ability to fold input + + * Prompt and deterministic finalization + + 'Frame's differ from 'Pipe's because they form restricted monads rather than + forming ordinary monads. This means you must rebind @do@ notation to use + restricted monads from the @index-core@ package. See the \"Create Frames\" + section for details. For even more details, consult the @index-core@ + package. +-} + +{-# LANGUAGE GADTs, TypeOperators #-} + +module Control.Frame ( + -- * Types + -- $types + C, + O, + M, + FrameF(..), + Frame, + Stack, + -- * Create Frames + -- $create + + -- ** Primitives + -- $primitives + yieldF, + awaitF, + close, + -- ** Pipe-like primitives + -- $pipeprims + yield, + await, + -- * Finalize Frames + -- $finalization + catchD, + catchF, + finallyD, + finallyF, + -- * Compose Frames + -- $compose + (<-<), + (>->), + idF, + FrameC(..), + -- * Run Frames + -- $run + runFrame + ) where + +import Control.Category +import Control.IMonad +import Control.IMonad.Trans +import Control.IMonad.Trans.Free +import Control.Monad.Instances () +import Data.Maybe +import Data.Void +import Prelude hiding ((.), id) + +-- For documentation +import Control.Pipe hiding (await, yield, Await, Yield) + +{- $types + The first step to convert 'Pipe' code to 'Frame' code is to translate the + types. All types of the form \"@Pipe a b m r@\" become + \"@Frame b m (M a) C r@\". For example, given the following type signatures + from the tutorial: + +> printer :: (Show a) => Pipe b Void IO r +> take' :: Int -> Pipe b b IO () +> fromList :: (Monad m) => [b] -> Pipe () b m () + + ... you would replace them with: + +> printer :: (Show a) => Frame Void IO (M a) C r +> take' :: Int -> Frame a IO (M a) C () +> fromList :: (Monad m) => [a] -> Frame a m (M ()) C () +> -- To use the finalization example, change fromList's base monad to 'IO' +> fromList :: [a] -> Frame a IO (M ()) C () +-} + +-- | Index representing an open input end, receiving values of type @a@ +data O a + +-- | Index representing a closed input end +data C + +-- | Index representing an open input end, receiving values of type @Maybe a@ +type M a = O (Maybe a) + +{-| + Base functor for a pipe that can close its input end + + * @b@ - Output type + + * @x@ - Next step + + * @i@ - Current step's index +-} +data FrameF b x i where + Yield :: b -> x i -> FrameF b x i + Await :: (a -> x (O a)) -> FrameF b x (O a) + Close :: x C -> FrameF b x (O a) + +instance IFunctor (FrameF b) where + fmapI f p = case p of + Yield b y -> Yield b (f y) + Await a -> Await (f . a) + Close c -> Close (f c) + +{-| + A 'Frame' is like a 'Pipe' with an indexed input end: + + * @b@ - The type of the 'Frame's output + + * @m@ - The base monad + + * @i@ - The initial index of the input end ('O'pen or 'C'losed) + + * @j@ - The final index of the input end ('O'pen or 'C'losed) + + * @r@ - The return value +-} +type Frame b m i j r = IFreeT (FrameF (m (), b)) (U m) (r := j) i + +-- | A self-contained 'Frame' that is ready to be run +type Stack m r = Frame Void m (M ()) C r + +-- $create +-- The second step to convert 'Pipe' code to 'Frame' code is to change your +-- module header to: +-- +-- > {-# LANGUAGE RebindableSyntax #-} +-- > +-- > import Control.IMonad.Do +-- > import Control.Frame +-- > import Prelude hiding (Monad(..)) +-- +-- "Control.Frame" replaces all 'Pipe' 'await's and 'yield's with their +-- corresponding 'Frame' counterparts. @Control.IMonad.Do@ rebinds @do@ +-- notation to work with restricted monads, which also requires using the +-- @RebindableSyntax@ extension and hiding the 'Monad' class from the @Prelude@. +-- +-- You also must use the restricted monad utility functions, which have the +-- same name as their ordinary monad counterparts except with an \'@R@\' suffix, +-- such as 'foreverR' instead of 'forever'. Finally, you must use 'liftU' +-- instead of 'lift' to invoke operations in the base monad. +-- +-- Finally, every terminating 'Frame' must be 'close'd exactly once before being +-- passed to composition. +-- +-- > printer = foreverR $ do +-- > a <- await +-- > liftU $ print a +-- > +-- > take' n = do +-- > replicateMR_ n $ do +-- > a <- await +-- > yield a +-- > close +-- > liftU $ putStrLn "You shall not pass!" +-- > +-- > fromList xs = do +-- > close +-- > mapMR_ yield xs + +{- $primitives + 'yieldF' guards against downstream termination by yielding the most + up-to-date finalization alongside each value, so that downstream can call + that finalizer if it terminates before requesting another value. + + 'awaitF' intercepts upstream termination by returning a 'Nothing' if + upstream terminates before providing a value. Further attempts to request + input from upstream will terminate the current 'Frame' using the + return value provided from upstream. + + While 'awaitF' is useful for folds, 'yieldF' is less useful for end-users of + this library and the higher-order 'catchF' / 'finallyF' finalization + functions are much more user-friendly. + + Composing two 'Frame's requires that each 'Frame' invokes 'close' exactly + once. Anything else will not type-check. Leave out the 'close' statement + when writing library components and let the person assembling the components + for composition specify where the 'close' goes. + + The earlier you 'close' the upstream 'Frame', the earlier it is finalized. + However, once you 'close' it you may no longer 'await'. +-} + +-- | 'Yield' the most current finalizer for this 'Frame' alongside the value +yieldF :: (Monad m) => m () -> b -> Frame b m i i () +yieldF m x = liftF $ Yield (m, x) (V ()) + +-- | 'Await' a value from upstream, returning 'Nothing' if upstream terminates +awaitF :: (Monad m) => Frame b m (M a) (M a) (Maybe a) +awaitF = liftF $ Await V + +-- | 'Close' the input end, calling the finalizers of every upstream 'Frame' +close :: (Monad m) => Frame b m (M a) C () +close = liftF $ Close (V ()) + +{- $pipeprims + The following 'Pipe'-like primitives are built on top of the 'Frame' + primitives. They behave identically to their 'Pipe' counterparts and can + be used as drop-in replacements for them. +-} + +-- | 'yield' a value upstream alongside an empty finalizer +yield :: (Monad m) => b -> Frame b m i i () +yield = yieldF (return ()) + +-- | 'await' a value from upstream and terminate if upstream terminates +await :: (Monad m) => Frame b m (M a) (M a) a +await = awaitF !>= maybe await returnR + +{- $finalization + The third (and optional) step to convert 'Pipe' code to 'Frame' code is to + register finalizers for your 'Frame'. These finalizers may be arbitrarily + nested: + +> printer = foreverR $ catchF (putStrLn "printer interrupted") $ do +> a <- await +> liftU $ print a +> +> take' n = finallyF (putStrLn "You shall not pass!") $ do +> replicateMR_ n $ do +> a <- catchF (putStrLn "take' interrupted") await +> yield a +> close +> +> fromList xs = catchF (putStrLn "fromList interrupted") $ do +> close +> mapMR_ yield xs + + These convenience functions register block-level finalizers to be called if + another 'Frame' terminates first. The naming conventions are: + + * \"catch\" functions (i.e. 'catchD' / 'catchF') call the finalizer only if + another 'Frame' terminates before the block completes, but will not call + the finalizer if the block terminates normally. + + * \"finally\" functions (i.e. 'finallyD' / 'finallyF') are like \"catch\" + functions except that they also call the finalizer if the block terminates + normally. + + * Functions that end in a \'@D@\' suffix (i.e. 'catchD' / 'finallyD') only + guard against downstream termination. + + * Functions that end in a \'@F@\' suffix (i.e. 'catchF' / 'finallyF') guard + against termination in both directions. You usually want these ones. + + Note that finalization blocks that /begin/ after the 'close' statement may + only use the \'@D@\'-suffixed version as upstream has been closed off. This + is a consequence of a deficiency in Haskell's type system that will take + time to work around. However an \'@F@\'-suffixed block that begins before a + 'close' statement may continue through it normally. So, for code blocks + after a 'close' statement, use 'catchD' \/ 'finallyD', otherwise use + 'catchF' \/ 'finallyF'. In future releases, the \'@D@\'-suffixed versions + will be removed and merged into the \'@F@\'-suffixed versions. +-} + +{-| + @catchD m p@ calls the finalizer @m@ if a downstream 'Frame' terminates + before @p@ finishes. +-} +catchD :: (Monad m) => m () -> Frame b m i j r -> Frame b m i j r +catchD m p = IFreeT $ U $ do + x <- unU $ runIFreeT p + unU $ runIFreeT $ case x of + Return r -> returnI r + Wrap (Close p') -> wrap $ Close (catchD m p') + Wrap (Yield (m', b) p') -> wrap $ Yield (m' >> m, b) (catchD m p') + Wrap (Await f ) -> wrap $ Await $ fmap (catchD m) f + +{-| + @catchF m p@ calls the finalizer @m@ if any 'Frame' terminates before @p@ + finishes. +-} +catchF :: (Monad m) => m () -> Frame b m (M a) j r -> Frame b m (M a) j r +catchF m p = IFreeT $ U $ do + x <- unU $ runIFreeT p + unU $ runIFreeT $ case x of + Return r -> returnI r + Wrap (Close p') -> wrap $ Close $ catchD m p' + Wrap (Yield (m', b) p') -> wrap $ Yield (m' >> m, b) (catchF m p') + Wrap (Await f ) -> wrap $ Await $ \e -> case e of + Nothing -> liftU m !> catchF m (f e) + Just _ -> catchF m (f e) + +{-| + @finallyD m p@ calls the finalizer @m@ if a downstream 'Frame' terminates + before @p@ finishes or if @p@ completes normally. +-} +finallyD :: (Monad m) => m () -> Frame b m i j r -> Frame b m i j r +finallyD m p = IFreeT $ U $ do + x <- unU $ runIFreeT p + unU $ runIFreeT $ case x of + Return r -> liftU m !> returnI r + Wrap (Close p') -> wrap $ Close (finallyD m p') + Wrap (Yield (m', b) p') -> wrap $ Yield (m' >> m, b) (finallyD m p') + Wrap (Await f ) -> wrap $ Await $ fmap (finallyD m) f + +{-| + @finallyF m p@ calls the finalizer @m@ if any 'Frame' terminates before @p@ + finishes or if @p@ completes normally. +-} +finallyF :: (Monad m) => m () -> Frame b m (M a) j r -> Frame b m (M a) j r +finallyF m p = IFreeT $ U $ do + x <- unU $ runIFreeT p + unU $ runIFreeT $ case x of + Return r -> liftU m !> returnI r + Wrap (Close p') -> wrap $ Close $ finallyD m p' + Wrap (Yield (m', b) p') -> wrap $ Yield (m' >> m, b) (finallyF m p') + Wrap (Await f ) -> wrap $ Await $ \e -> case e of + Nothing -> liftU m !> finallyF m (f e) + Just _ -> finallyF m (f e) + +(<~<) :: (Monad m) + => IFreeT (FrameF c) (U m) (r := C) (O b) + -> IFreeT (FrameF b) (U m) (r := C) (O a) + -> IFreeT (FrameF c) (U m) (r := C) (O a) +p1 <~< p2 = IFreeT $ U $ do + x1 <- unU $ runIFreeT p1 + unU $ runIFreeT $ case x1 of + Wrap (Close p1') -> wrap $ Close p1' + Wrap (Yield c p1') -> wrap $ Yield c (p1' <~< p2) + Wrap (Await f1 ) -> IFreeT $ U $ do + x2 <- unU $ runIFreeT p2 + let p1' = IFreeT $ returnI x1 + unU $ runIFreeT $ case x2 of + Wrap (Close p2') -> wrap $ Close $ p1' <~| p2' + Wrap (Yield b p2') -> f1 b <~< p2' + Wrap (Await f2) -> wrap $ Await $ fmap (\p2'-> p1' <~< p2') f2 + +(<~|) :: (Monad m) + => IFreeT (FrameF c) (U m) (r := C) (O b) + -> IFreeT (FrameF b) (U m) (r := C) C + -> IFreeT (FrameF c) (U m) (r := C) C +p1 <~| p2 = IFreeT $ U $ do + x1 <- unU $ runIFreeT p1 + unU $ runIFreeT $ case x1 of + Wrap (Close p1') -> p1' + Wrap (Yield c p1') -> wrap $ Yield c (p1' <~| p2) + Wrap (Await f1 ) -> IFreeT $ U $ do + x2 <- unU $ runIFreeT p2 + let p1' = IFreeT $ returnI x1 + unU $ runIFreeT $ case x2 of + Return r -> returnI r + Wrap (Yield b p2') -> f1 b <~| p2' + +heap :: (Monad m) + => m () + -> IFreeT (FrameF (m (), c)) (U m) (r := C) (M b ) + -> IFreeT (FrameF (m (), c)) (U m) (r := C) (M (m (), b)) +heap m p = IFreeT $ U $ do + x <- unU $ runIFreeT p + unU $ runIFreeT $ case x of + Wrap (Close p') -> wrap $ Close $ liftU m !> p' + Wrap (Yield (m', c) p') -> wrap $ Yield (m >> m', c) (heap m p') + Wrap (Await f ) -> wrap $ Await $ \e -> case e of + Nothing -> heap (return ()) (f Nothing) + Just (m', b) -> heap m' (f $ Just b) + +stack :: (Monad m) + => Bool + -> IFreeT (FrameF b ) (U m) (r := C) (M a) + -> IFreeT (FrameF (Maybe b)) (U m) (r := C) (M a) +stack t p = IFreeT $ U $ do + x <- unU $ runIFreeT p + unU $ runIFreeT $ case x of + Wrap (Close p') -> wrap $ Close $ warn p' + Wrap (Yield b p') -> wrap $ Yield (Just b) (stack t p') + Wrap (Await f ) -> + let p' = wrap $ Await $ \e -> stack (isNothing e) (f e) + in case t of + False -> p' + True -> wrap $ Yield Nothing p' + +warn :: (Monad m) + => IFreeT (FrameF b ) (U m) (r := C) C + -> IFreeT (FrameF (Maybe b)) (U m) (r := C) C +warn p = IFreeT $ U $ do + x <- unU $ runIFreeT p + unU $ runIFreeT $ case x of + Return r -> wrap $ Yield Nothing (returnI r) + Wrap (Yield b p') -> wrap $ Yield (Just b) (warn p') + +{- $compose + The fourth step to convert 'Pipe' code to 'Frame' code is to replace ('<+<') + with ('<-<'): + +> printer <-< take' 3 <-< fromList [1..] + + Like 'Pipe's, Frames form a 'Category' where composition pipes the output + from the upstream 'Frame' to the input of the downstream 'Frame'. + Additionally, composition guarantees the following behaviors: + + * 'Frame's receive exactly one 'Nothing' if an upstream 'Frame' terminates. + + * Registered finalizers get called exactly once if a downstream 'Frame' + terminates. + + * Finalizers are always ordered from upstream to downstream. + + The 'Category' laws cannot be broken, so you don't have to be careful when + using 'Frame's. + + Note that you may only compose 'Frame's that begin open and end closed. +-} + +-- | Corresponds to ('<<<')/('.') from @Control.Category@ +(<-<) :: Monad m + => Frame c m (M b) C r -> Frame b m (M a) C r -> Frame c m (M a) C r +p1 <-< p2 = heap (return ()) p1 <~< stack False p2 + +-- | Corresponds to ('>>>') from @Control.Category@ +(>->) :: Monad m + => Frame b m (M a) C r -> Frame c m (M b) C r -> Frame c m (M a) C r +(>->) = flip (<-<) + +infixr 9 <-< +infixr 9 >-> + +-- | Corresponds to 'id' from @Control.Category@ +idF :: (Monad m) => Frame a m (M a) C r +idF = foreverR $ await !>= yield + +-- | 'Frame's form a 'Category' instance when you rearrange the type variables +newtype FrameC m r a b = FrameC { unFrameC :: Frame b m (M a) C r } + +instance (Monad m) => Category (FrameC m r) where + id = FrameC idF + (FrameC p1) . (FrameC p2) = FrameC (p1 <-< p2) + +{- $run + The fifth step to convert 'Pipe' code to 'Frame' code is to use 'runFrame' + instead of 'runPipe': + +>>> runFrame $ printer <-< take' 3 <-< fromList [1..] +1 +2 +3 +fromList interrupted +You shall not pass! +printer interrupted +>>> runFrame $ printer <-< take' 3 <-< fromList [1] +1 +You shall not pass! +take' interrupted +printer interrupted + +-} + +{-| + Run the 'Frame' monad transformer, converting it back to the base monad. + + 'runFrame' is the 'Frame' equivalent to 'runPipe' and requires a + self-contained 'Stack'. +-} +runFrame :: (Monad m) => Stack m r -> m r +runFrame p = do + x <- unU $ runIFreeT p + case x of + Wrap (Close p') -> runFrame' p' + Wrap (Yield _ p') -> runFrame p' + Wrap (Await f ) -> runFrame (f $ Just ()) + +runFrame' :: (Monad m) => Frame Void m C C r -> m r +runFrame' p = do + x <- unU $ runIFreeT p + case x of + Return (V r) -> return r + Wrap (Yield _ p') -> runFrame' p' diff --git a/Control/Frame/Tutorial.hs b/Control/Frame/Tutorial.hs new file mode 100644 --- /dev/null +++ b/Control/Frame/Tutorial.hs @@ -0,0 +1,487 @@ +{-| + This module provides the tutorial for "Control.Frame". +-} + +module Control.Frame.Tutorial ( + -- * Restricted Monads + -- $restrict1 + + -- $extension + + -- $restrict2 + + -- * Type Signatures + -- $types + + -- * Prompt Finalization + -- $prompt + + -- * Composition + -- $compose + + -- * Finalization + -- $ensure + + -- * Folds + -- $fold + + -- * Strictness + -- $strict + + -- * Robustness + -- $robust + ) where + +-- For documentation +import Control.Category +import Control.Frame +import Control.IMonad +import Control.IMonad.Trans +import Control.Monad.Trans.Class +import Control.Pipe hiding (await, yield, Await, Yield) + +{- $restrict1 + 'Frame's extend 'Pipe's with two new features: + + * Folding input and intercepting upstream termination + + * Guaranteeing prompt and deterministic finalization + + However, these extra features comes with some added complexity: restricted + monads, also known as indexed monads. Restricted monads sound scarier than + they are, so I'll demonstrate that if you are comfortable using monads, then + you'll be comfortable using restricted monads. + + Let's translate the @take'@ function from the 'Pipe's tutorial into a + 'Frame' to see what changes when we use restricted monads: + +-} +-- $extension +-- > {-# LANGUAGE RebindableSyntax #-} +-- > +-- > import Control.Frame +-- > import Control.IMonad.Do +-- > import Control.IMonad.Trans +-- > import Prelude hiding (Monad(..)) +-- > +-- > take' :: Int -> Frame a IO (M a) C () +-- > take' n = do +-- > replicateMR_ n $ do +-- > x <- await +-- > yield x +-- > close +-- > liftU $ putStrLn "You shall not pass!" +{- $restrict2 + This time I included all imports and highlighted the new @RebindableSyntax@ + extension. The new imports belong to the @Control.IMonad@ hierarchy from + the @index-core@ package, which provides the core restricted monad + functionality. + + Yet, you almost wouldn't even know you were using an restricted monad just + by looking at the code. This is because @index-core@ can rebind @do@ + notation to use restricted monads instead of ordinary extensions. Three + things make this possible: + + * The @RebindableSyntax@ extension, which allows libraries to override + @do@ syntax (among other things) + + * The @Control.IMonad.Do@ module which exports the new bindings for @do@ + notation + + * Hiding 'Monad' from the Prelude so that it does not conflict with the + bindings from @index-core@ + + However, you are not obligated to rebind @do@ notation to use 'Frame's. You + can choose to keep ordinary @do@ notation and desugar the restricted monad + by hand. Just import @Control.IMonad@ instead, drop the @RebindableSyntax@ + extension, and don't hide 'Monad'. Then you can desugar @take'@ manually + using the restricted monad operators: + +> import Control.Frame +> import Control.IMonad +> import Control.IMonad.Trans +> +> take' :: Int -> Frame a IO (M a) C () +> take' n = +> (replicateMR_ n $ +> await !>= \x -> +> yield x) !>= \_ -> +> close !>= \_ -> +> liftU $ putStrLn "You shall not pass!" + + However, for this tutorial I will use the @do@ notation, since it's prettier + and easier to use. + + You'll also notice functions that resemble the ones in @Control.Monad@, + except with an \'@R@\' suffix on the end of them, like 'replicateMR_'. + Most functions in @Control.Monad@ have a restricted counterpart provided by + @Control.IMonad.Restrict@ (which is in turn re-exported by + @Control.IMonad@), such as 'whenR', 'foreverR', and 'mapMR'. + + Also, every time you lift an operation from the base monad, you must use + 'liftU' instead of 'lift'. 'Frame's are \"restricted monad transformers\", + and they would normally lift a base restricted monad using 'liftI', but + they can also lift ordinary monads, too, using 'liftU' (mnemonic: \"lift\" + an ordinary monad and \'U\'pgrade it to a restricted monad). +-} + +{- $types + The 'Frame' type constructor also looks a bit different, too: + +> Frame a IO (M a) C () + + Let's dissect that to understand how 'Frame's work: + +> | Output | Base monad | Initial Input | Final Input | Return Value +> Frame a IO (M a) C () + + 'Frame's differ from 'Pipe's in that their input end indexes the beginning + and end of the operation. Our @take'@ function starts off with an open + input end (@M a@), and ends with a closed input end (@C@). + + @take'@ finishes with a closed input end because it called the 'close' + function, which seals off and finalizes upstream. You can see that the + 'close' primitive changes the index just by looking at its type: + +> close :: Monad m => Frame b m (M a) C () + + The 'close' instruction begins with an open input end (@M a@) and finishes + with a closed input end (@C@). If you tried to call 'close' twice, you'd + get a type error: + +> -- wrong! +> do close +> close + + This prevents you from accidentally finalizing upstream twice. + + 'close' is the only primitive that changes the index, and there is no way to + reopen the input once you have closed it. 'close' also forbids you from + 'await'ing input from upstream after you have already closed it. If you + try, you will get a type error + +> -- wrong! +> do close +> await + + This prevents you from requesting input from a finalized pipe. In fact, + once you 'close' your input end, every upstream 'Frame' disappears + completely. You couldn't get input from upstream anyway, even if you + somehow allowed 'await' statements after 'close'. + + You can check out 'await''s type signature to see why it won't type-check + after 'close': + +> await :: Monad m => Frame b m (M a) (M a) a + + 'await' must begin with the input end open (@M a@) and it leaves the input + end open when done (@M a@). However, you can still use a 'yield' anywhere: + +> yield :: Monad m => b -> Frame b m i i () + + 'yield' will work whether or not the input end is open, and it leaves the + input end in the same state once 'yield' is done. +-} + +{- $prompt + Every 'Frame' must close its input end /exactly/ one time before you can + compose it with other 'Frame's. The only exception is if a 'Frame' never + terminates: + +> -- This type-checks because foreverR is polymorphic in the final index +> printer :: (Show b) => Frame Void IO (M b) C r +> printer = foreverR $ do +> a <- await +> liftU $ print a + + However, when a 'Frame' no longer needs input then you should 'close' it as + early as possible. The earlier you 'close' upstream, the more promptly + upstream gets finalized. + + If you write a stand-alone producer from start to finish, you can be sure it + will never need upstream, so you can close it immediately: + +> -- I'm keeping fromList's input end polymorphic for a later example +> fromList :: (M.Monad m) => [b] -> Frame b m (M a) C () +> fromList xs = do +> close +> mapMR_ yield xs + + However, if @fromList@ were a library function, you would remove the 'close' + statement as you cannot guarantee that your user won't want to 'await' after + @fromList@. Or, the user might want to call @fromList@ twice within the + same 'Frame', and having two close statements would lead to a type error. + Therefore, a good rule of thumb when writing library code for 'Frame's is to + always let the user decide when to 'close' the 'Frame' unless you are + writing a stand-alone 'Frame'. + + So for right now, I will leave the 'close' in @fromList@ for simplicity and + treat it as a stand-alone 'Frame'. Also, it will come in handy for a later + example. +-} + +{- $compose + Composition works just like 'Pipe's, except you use the ('<-<') composition + operator instead of ('<+<'): + +> stack :: Stack IO () +> stack = printer <-< take' 3 <-< fromList [1..] + + The 'Frame' equivalent to 'Pipeline' is a 'Stack' (mnemonic: call stack; + also the name 'Frame' refers to a call stack frame): + +> type Stack m r = Frame Void m (M ()) C r + + Similarly, you use 'runFrame' instead of 'runPipe' to convert the 'Frame' + back to the base monad: + +>>> runFrame stack +1 +2 +3 +You shall not pass! + + However, let's carefully inspect the type of composition: + +> (<-<) :: Monad m +> => Frame c m (M b) C r +> -> Frame b m (M a) C r +> -> Frame c m (M a) C r + + Each argument 'Frame' must begin in an open state and end in a closed state. + This means that each 'Frame' in a 'Stack' must call 'close' exactly once + before it may be used. 'runFrame' has the exact same restriction: + +> runFrame :: Monad m => Stack m r -> m r +> runFrame ~ Monad m => Frame Void m (M ()) C r -> m r + + Composition specifically requires the user to define when to finalize + upstream and does not assume this occurs at the end of the 'Frame'. This + doesn't pose a problem for stand-alone 'Frame's, since they will know when + they no longer need input, but smaller library components designed to be + assembled into larger 'Frame's should let the user decide at the very last + moment where to 'close' the 'Pipe'. There is no way to know ahead of time + where the 'close' should be until the complete 'Frame' has been assembled. +-} + +{- $ensure + With 'Frame's in hand, we can now write a safe @read'@ function: + +> readFile' :: Handle -> Frame Text IO C C () +> readFile' h = do +> eof <- liftU $ hIsEOF h +> whenR (not eof) $ do +> s <- liftU $ hGetLine h +> yield s +> readFile' h +> +> read' :: FilePath -> Frame Text IO C C () +> read' file = do +> liftU $ putStrLn "Opening file..." +> h <- liftU $ openFile file ReadMode +> -- The following requires "import qualified Control.Monad as M" +> finallyD (putStrLn "Closing file ..." M.>> hClose h) $ readFile' h + + The 'finallyD' function registers a block-level finalizer that executes if a + downstream 'Pipe' terminates or if the block completes normally. The more + general 'finallyF' function will call the finalizer if /any/ 'Frame' + terminates. + + Usually you would always want to use 'finallyF', but because of some type + limitations you can only use 'finallyD' after a 'Frame' is closed. A future + release of this library will fix this and merge 'finallyD' into 'finallyF'. + So that means that for everything beginning before a 'close' statement, use + 'finallyF', otherwise use 'finallyD'. + + Similarly, you can use the 'catchF' / 'catchD' counterparts to the + \"finally\" functions. The \"catch\" functions run the finalizer only if + another 'Frame' terminates before the block is done, but not if the block + terminates normally. + + We don't 'close' the @read'@ function because it's not a stand-alone + 'Frame'. We want to be able to concatenate multiple @read'@s together + within the same 'Frame', like so: + +> files = do +> close +> read' "file1.txt" +> read' "file2.txt" + + So let's assume those two files have the following contents: + + \"@file1.txt@\" + +> Line 1 +> Line 2 +> Line 3 + + \"@file2.txt@\" + +> A +> B +> C + + We can now check to see if our @files@ producer works: + +>>> runFrame $ printer <-< files +Opening file... +"Line1" +"Line2" +"Line3" +Closing file ... +Opening file... +"A" +"B" +"C" +Closing file ... + + More importantly, files are never opened if they aren't demanded and they + are always properly finalized if the consumer terminates early: + +>>> runFrame $ printer <-< take' 2 <-< files +Opening file... +"Line1" +"Line2" +Closing file ... +You shall not pass! + + So we get lazy, deterministic, and prompt resource management. Nice! + +-} + +{- $fold + 'Frame's can actually do more than just manage finalization! Using + 'Frame's, we can now correctly implement folds like @toList@ in a way that + is truly compositional: + +> toList :: (M.Monad m) => Frame b m (M a) (M a) [a] +> toList = do +> a' <- awaitF +> case a' of +> Nothing -> return [] +> Just a -> do +> as <- toList +> return (a:as) + + We used one new function this time: 'awaitF'. This is like 'await' except + that it returns a 'Nothing' if upstream terminates before 'yield'ing back a + value. This allows you to intercept upstream termination and do some + cleanup, and in our case we use it to end the fold. + + You only receive a 'Nothing' once when you use 'awaitF'. Any attempt to + request more input after you receive the first 'Nothing' will terminate the + current 'Frame' using the upstream return value. In fact, 'await' is built + on top of 'awaitF': + +> await = do +> a' <- awaitF +> case a' of +> Nothing -> await +> Just a -> return a + + If it gets a 'Nothing', it just ignores it and 'await's again, choosing to + not do any cleanup. + + Now let's make sure our @toList@ function works. I didn't make @toList@ a + stand-alone 'Frame', so we will have to include a 'close' statement to + complete it before composing it: + +> p1 = do +> xs <- toList +> close +> return (Just xs) +> +> p2 xs = do +> fromList xs +> return Nothing -- Remember: they need the same return type + +>>> runFrame $ p1 <-< p2 [1..10] +Just [1,2,3,4,5,6,7,8,9,10] +-} + +{- $strict + Lazy resource management has one important disadvantage: we can't free the + resource until downstream no longer needs input. Many libraries duplicate + their code to provide Lazy and Strict versions, allowing the user to decide + if they want: + + * Lazy input, which conserves memory, but holds onto the resource until + downstream is done processing it + + * Strict input, which loads everything into memory, but can then immediately + dispose of the resource before the input is processed + + What if there were a way to seamlessly switch between those semantics or + even choose something in between? Well, it turns out we can! + + First, we can combine @fromList@ and @toList@ into something even cooler: + +> strict :: (M.Monad m) => Frame a m (M a) C () +> strict = do +> xs <- toList +> fromList xs + + As the name suggests, the @strict@ function is strict in its input. + @strict@ loads the entire input into memory, finalizes upstream, then + proceeds to hand the input off to downstream. We can prove this just by + using it: + +>>> runFrame $ printer <-< strict <-< files +> Opening file... +> Closing file ... +> Opening file... +> Closing file ... +> "Line1" +> "Line2" +> "Line3" +> "A" +> "B" +> "C" + + Both files were disposed of immediately, at the expense of using more + memory. + + But what if we want something in between strictness and laziness? Maybe + something like this: + +>>> runFrame $ printer <-< strict <-< take' 2 <-< files +Opening file... +Closing file ... +You shall not pass! +"Line1" +"Line2" + + Now we have the best of both worlds. We can pick and choose how much of + our source to strictly load into memory. In the above example, we specified + that we wanted to be strict only in the first two lines of our input, and as + a result the third line of \"@file1.txt@\" is never read and \"@file2.txt@\" + is never even opened! + + Now we have a way to seamlessly slide anywhere on the spectrum between + laziness and strictness, and it's all implemented entirely within Haskell + in a way that is elegant and intuitive without the use of artificial and + clumsy 'seq' annotations. +-} + +{- $robust + The 'Frame' implementation exposes all internals, yet this does not + compromise safety or invariants in any way. The library's implementation is + \"correct-by-construction\", meaning that you can extend it with your own + features if you so choose, and you never have to worry about accidentally + breaking any laws, such as the associativity of composition. + + This has the following important practical benefits for finalization and + folds: + + * Finalizers never get duplicated or dropped + + * Folds can be performed anywhere within the 'Stack', not just at the most + downstream 'Frame', as the @strict@ example illustrates. + + * You can reason about each 'Frame's finalization behavior completely + independently of other 'Frame's. + + Composition elegantly handles every single corner case. This directly + follows from strictly enforcing the 'Category' laws, because categories have + no corners! +-} diff --git a/Control/IMonad/Trans/Free.hs b/Control/IMonad/Trans/Free.hs new file mode 100644 --- /dev/null +++ b/Control/IMonad/Trans/Free.hs @@ -0,0 +1,56 @@ +-- | This module is the indexed version of "Control.Monad.Trans.Free" + +{-# LANGUAGE KindSignatures, TypeOperators #-} + +module Control.IMonad.Trans.Free ( + -- * Free monad transformers + -- $freet + IFreeF(..), + IFreeT(..), + wrap, + liftF + ) where + +import Control.Category.Index +import Control.IMonad +import Control.IMonad.Trans + +{- $freet + Indexed free monad transformers lift the constructor signatures to + the category of indexed Haskell functions: (':->'). + +> Return :: r :-> IFreeF f r x +> Wrap :: f x :-> IFreeF f r x +> +> IFreeT :: m (IFreeF f r (IFreeT f m r)) :-> IFreeT f m r +-} + +-- | Indexed equivalent to @FreeF@ +data IFreeF f r (x :: * -> *) i = Return (r i) | Wrap (f x i) + +-- | Indexed equivalent to @FreeT@ +data IFreeT f m r i = IFreeT { runIFreeT :: m (IFreeF f r (IFreeT f m r)) i } + +instance (IFunctor f, IMonad m) => IFunctor (IFreeT f m) where + fmapI f x = x ?>= returnI . f + +instance (IFunctor f, IMonad m) => IMonad (IFreeT f m) where + returnI = IFreeT . returnI . Return + bindI f m = IFreeT $ + runIFreeT m ?>= \x -> + runIFreeT $ case x of + Return r -> f r + Wrap w -> wrap $ fmapI (bindI f) w + +instance (IFunctor f) => IMonadTrans (IFreeT f) where + liftI = IFreeT . fmapI Return + +-- | Indexed equivalent to @wrap@ +wrap :: (IMonad m) => f (IFreeT f m r) :-> IFreeT f m r +wrap = IFreeT . returnI . Wrap + +-- | Indexed equivalent to @liftF@ +liftF :: (IFunctor f, IMonad m) => f r :-> IFreeT f m r +liftF x = wrap $ fmapI returnI x + +-- FIXME: Add IIdentity so that IFree can be defined in terms of IFreeT diff --git a/Control/Monad/Trans/Free.hs b/Control/Monad/Trans/Free.hs --- a/Control/Monad/Trans/Free.hs +++ b/Control/Monad/Trans/Free.hs @@ -1,19 +1,29 @@ -{-| Every functor @f@ gives rise to a corresponding free monad: @Free f@. - - A free monad over a functor resembles a \"list\" of that functor: +{-| + People commonly misconstrue 'Free' as defining a monad transformer with + 'liftF' behaving like 'lift', however that approach violates the monad + transformer laws. Another common mistake is to include the base monad as a + term in the functor, which also gives rise to an incorrect monad + transformer. - * 'pure' behaves like @[]@ by not using the functor at all + To solve this, this module provides 'FreeT', which properly generalizes the + free monad to a free monad transformer which is correct by construction. - * 'wrap' behaves like @(:)@ by prepending another layer of the functor + The 'FreeT' type commonly arises in coroutine and iteratee libraries that + wish to provide a monad transformer that correctly obeys the monad + transformer laws. -} + module Control.Monad.Trans.Free ( - -- * The Free monad + -- * Free monad transformer + -- $freet FreeF(..), - Free(..), - wrap, - runFree, - -- * The FreeT monad transformer FreeT(..), + wrap, + liftF, + -- * Free monad + -- $free + Free, + runFree ) where import Control.Applicative @@ -21,57 +31,84 @@ import Control.Monad.Trans.Class import Data.Functor.Identity -data FreeF f r x = Pure r | Wrap (f x) - -{-| - The 'Free' type is isomorphic to: +{- $freet + This differs substantially from the non-monad-transformer version because + of the requirement to nest the constructors within the base monad. -> data Free f r = Pure r | Wrap (f (Free f r)) + To deconstruct a free monad transformer, use 'runFreeT' to unwrap it and + bind the result in the base monad. You can then pattern match against the + bound value to obtain the next constructor: - ... except that if you want to pattern match against those constructors, you - must first use 'runFree' to unwrap the value first. +> do x <- runFreeT f +> case x of +> Return r -> ... +> Wrap w -> ... - Similarly, you don't use the raw constructors to build a value of type - 'Free'. You instead use the smart constructors 'pure' (from - @Control.Applicative@) and 'wrap'. + Because of this, you cannot create free monad transformers using the raw + constructors from 'FreeF'. Instead you use the smart constructors 'return' + (from @Control.Monad@) and 'wrap'. -} -type Free f = FreeT f Identity -wrap :: (Monad m) => f (FreeT f m r) -> FreeT f m r -wrap = FreeT . return . Wrap - -runFree :: Free f r -> FreeF f r (Free f r) -runFree = runIdentity . runFreeT +-- | The signature for 'Free' +data FreeF f r x = Return r | Wrap (f x) {-| - A free monad transformer alternates nesting the base functor @f@ and the - base monad @m@. + A free monad transformer alternates nesting the base monad @m@ and the base + functor @f@. - * @f@ - The functor that generates the free monad + * @f@ - The functor that generates the free monad transformer * @m@ - The base monad * @r@ - The type of the return value - - This type commonly arises in coroutine/iteratee libraries under various - names. -} data FreeT f m r = FreeT { runFreeT :: m (FreeF f r (FreeT f m r)) } -instance (Functor f, Monad m) => Monad (FreeT f m) where - return = FreeT . return . Pure - m >>= f = FreeT $ do - x <- runFreeT m - runFreeT $ case x of - Pure r -> f r - Wrap a -> wrap $ fmap (>>= f) a - instance (Functor f, Monad m) => Functor (FreeT f m) where fmap = liftM instance (Functor f, Monad m) => Applicative (FreeT f m) where - pure = return + pure = return (<*>) = ap +instance (Functor f, Monad m) => Monad (FreeT f m) where + return = FreeT . return . Return + m >>= f = FreeT $ do + x <- runFreeT m + runFreeT $ case x of + Return r -> f r + Wrap w -> wrap $ fmap (>>= f) w + instance MonadTrans (FreeT f) where - lift = FreeT . liftM Pure + lift = FreeT . liftM Return + +-- | Smart constructor for 'Wrap' +wrap :: (Monad m) => f (FreeT f m r) -> FreeT f m r +wrap = FreeT . return . Wrap + +-- | Equivalent to @liftF@ from "Control.Monad.Free" +liftF :: (Functor f, Monad m) => f r -> FreeT f m r +liftF x = wrap $ fmap return x + +{- $free + The 'Free' type is isomorphic to the following simple implementation: + +> data Free f r = Return r | Wrap (f (Free f r)) + + ... except that if you want to pattern match against those constructors, you + must first use 'runFree' to unwrap the value first. + +> case (runFreeT f) of +> Return r -> ... +> Wrap w -> ... + + Similarly, you use the smart constructors 'return' and 'wrap' to build a + value of type 'Free'. +-} + +-- | 'FreeT' reduces to 'Free' when specialized to the 'Identity' monad. +type Free f = FreeT f Identity + +-- | Observation function that exposes the next 'FreeF' constructor +runFree :: Free f r -> FreeF f r (Free f r) +runFree = runIdentity . runFreeT diff --git a/Control/Pipe.hs b/Control/Pipe.hs --- a/Control/Pipe.hs +++ b/Control/Pipe.hs @@ -1,764 +1,273 @@ -module Control.Pipe ( - -- * Types - -- $type - - -- * Composition - -- $compose - - -- * Modularity - -- $modular - - -- * Vertical Concatenation - -- $vertical - - -- * Return Values - -- $return - - -- * Termination - -- $terminate - - -- * Resource Management - -- $resource - - -- * Frames - -- $frame - - -- * Frame Composition - -- $framecompose - - -- * Frame vs. Ensure - -- $frameensure - - -- * Folds - -- $fold +{-| + 'Pipe' is a monad transformer that enriches the base monad with the ability + to 'await' or 'yield' data to and from other 'Pipe's. +-} - -- * Strictness - -- $strict +module Control.Pipe ( + -- * Introduction + -- $summary - module Control.Pipe.Common, - module Control.Pipe.Final + -- * Types + -- $types + PipeF(..), + Pipe, + Producer, + Consumer, + Pipeline, + -- * Create Pipes + -- $create + await, + yield, + pipe, + -- * Compose Pipes + -- $category + (<+<), + (>+>), + idP, + PipeC(..), + -- * Run Pipes + -- $runpipe + runPipe ) where +import Control.Applicative import Control.Category -import Control.Monad.Trans.Class -import Control.Pipe.Common -import Control.Pipe.Final -import Data.Void - -{- $type - This library represents streaming computations using a single data type: - 'Pipe'. - - 'Pipe' is a monad transformer that extends the base monad with the ability - to 'await' input from or 'yield' output to other pipes. Pipes resemble - enumeratees in other libraries because they receive an input stream and - transform it into a new output stream. - - I'll introduce our first 'Pipe', which is a verbose version of the Prelude's - 'take' function: - -> take' :: Int -> Pipe a a IO () -> take' n = do -> replicateM_ n $ do -> x <- await -> yield x -> lift $ putStrLn "You shall not pass!" - - This pipe forwards the first @n@ values it receives undisturbed, then it - outputs a cute message. - - Let's dissect the above pipe's type to learn a bit about how pipes work: - -> | Input Type | Output Type | Base monad | Return value -> Pipe a a IO () - - So @take'@ 'await's input values of type @a@ from upstream pipes and - 'yield's output values of type @a@ to downstream pipes. @take'@ uses 'IO' - as its base monad because it invokes the 'putStrLn' function. If we were to - remove the call to 'putStrLn', the compiler would infer the following type - instead, which is polymorphic in the base monad: - -> take' :: (Monad m) => Int -> Pipe a a m () - - Now let's create a function that converts a list into a pipe by 'yield'ing - each element of the list: - -> fromList :: (Monad m) => [b] -> Pipe a b m () -> fromList = mapM_ yield - - Note that @fromList xs@ is polymorphic in its input. This is because it - does not 'await' any input. If we wanted, we could type-restrict it to: - -> fromList :: (Monad m) => [b] -> Pipe () b m () - - There is no type that forbids a pipe from 'await'ing, but you can guarantee - that if it does 'await', the request is trivially satisfiable by supplying - it with @()@. - - A pipe that doesn't 'await' (any useful input) can serve as the first stage - in a 'Pipeline'. I provide a type synonym for this common case: - -> type Producer b m r = Pipe () b m r - - 'Producer's resemble enumerators in other libraries because they function as - data sources. - - You can then use the 'Producer' type synonym to rewrite the type signature - for @fromList@ as: - -> fromList :: (Monad m) => [b] -> Producer b m () - - Now let's create a pipe that prints every value delivered to it: - -> printer :: (Show b) => Pipe b c IO r -> printer = forever $ do -> x <- await -> lift $ print x - - Here, @printer@ is polymorphic in its output. We could type-restrict it to - guarantee it will never 'yield' by setting the output to 'Void', from - @Data.Void@: - -> printer :: (Show a) => Pipe b Void IO r - - A pipe that never yields can be the final stage in a 'Pipeline'. Again, - I provide a type synonym for this common case: - -> type Consumer b m r = Pipe b Void m r - - So we could instead write @printer@'s type as: +import Control.Monad (forever) +import Control.Monad.Trans.Class (lift) +import Control.Monad.Trans.Free +import Data.Void (Void) +import Prelude hiding ((.), id) -> printer :: (Show b) => Consumer b IO r +{- $summary + I completely expose the 'Pipe' data type and internals in order to encourage + people to write their own 'Pipe' functions. This does not compromise the + correctness or safety of the library at all and you can feel free to use the + constructors directly without violating any laws or invariants. - 'Consumer's resemble iteratees in other libraries because they function as - data sinks. + I promote using the 'Monad' and 'Category' instances to build and compose + pipes, but this does not mean that they are the only option. In fact, any + combinator provided by other iteratee libraries can be recreated for pipes, + too. However, this core library does not provide many of the functions + found in other libraries in order to encourage people to find principled and + theoretically grounded solutions rather than devise ad-hoc solutions + characteristic of other iteratee implementations. -} -{- $compose - What distinguishes pipes from every other iteratee implementation is that - they form a true 'Category'. Because of this, you can literally compose - pipes into 'Pipeline's using ordinary composition: - -> newtype Lazy m r a b = Lazy { unLazy :: Pipe a b m r } -> instance Category (Lazy m r) where ... - - For example, you can compose the above pipes with: - -> pipeline :: Pipe () Void IO () -> pipeline = unLazy $ Lazy printer . Lazy (take' 3) . Lazy (fromList [1..]) - - The compiler deduces that the final pipe must be blocked at both ends, - meaning it will never 'await' useful input and it will never 'yield' any - output. This represents a self-contained 'Pipeline' and I provide a type - synonym for this common case: - -> type Pipeline m r = Pipe () Void m r - - Also, I provide '<+<' as a convenience operator for composing pipes without - the burden of wrapping and unwrapping newtypes: - -> p1 <+< p2 = unLazy $ Lazy p1 . Lazy p2 - - So you can rewrite @pipeline@ as: - -> pipeline :: Pipeline IO () -> pipeline = printer <+< take' 3 <+< fromList [1..] - - Like many other monad transformers, you convert the 'Pipe' monad back to the - base monad using some sort of \"@run...@\" function. In this case, it's the - 'runPipe' function: - -> runPipe :: (Monad m) => Pipeline m r -> m r - - 'runPipe' only works on self-contained 'Pipeline's, but you don't need to - worry about explicitly type-restricting any of your pipes. Self-contained - pipelines will automatically have polymorphic input and output ends and they - will type-check when you provide them to 'runPipe'. - - Let's try using 'runPipe': - ->>> runPipe pipeline -1 -2 -3 -You shall not pass! - - Fascinating! Our pipe terminates even though @printer@ never terminates - and @fromList@ never terminates when given an infinite list. To illustrate - why our pipe terminates, let's outline the pipe flow control rules for - composition: - - * Pipes are lazy, so execution begins at the most downstream pipe - (@printer@ in our example). - - * Upstream pipes only run if input is requested from them and they only run - as long as necessary to 'yield' back a value. - - * If a pipe terminates, it terminates every other pipe composed with it. - - Another way to think of this is like a stack where each pipe is a frame on - that stack: - - * If a pipe 'await's input, it blocks and pushes the next pipe upstream onto - the stack until that pipe 'yield's back a value. - - * If a pipe 'yield's output, it pops itself off the stack and restores - control to the original downstream pipe that was 'await'ing its input. - This binds its result to the return value of the pending 'await' command. - - All of these flow control rules uniquely follow from the 'Category' laws. - - It might surprise you that termination brings down the entire pipeline until - you realize that: - - * Downstream pipes depending on the terminated pipe cannot proceed - - * Upstream pipes won't be further evaluated because the terminated pipe will - not request any further input from them - - So in our previous example, the 'Pipeline' terminated because @take' 3@ - terminated and brought down the entire 'Pipeline' with it. - - Actually, these flow control rules will mislead you into thinking that - composed pipes behave as a collection of sub-pipes with some sort of message passing architecture between them, but nothing could be further from the - truth! When you compose pipes, they automatically fuse into a single pipe - that corresponds to how you would have written the control flow by hand. - - For example, if you compose @printer@ and @fromList@: - -> printer <+< fromList [1..] - - The result is indistinguishable from: - -> lift (mapM_ print [1..]) +{- $types + The 'Pipe' type is strongly inspired by Mario Blazevic's @Coroutine@ type in + his concurrency article from Issue 19 of The Monad Reader and is formulated + in the exact same way. - ... which is what we would have written by hand if we had not used pipes at - all! All 'runPipe' does is just remove the 'lift'! + His @Coroutine@ type is actually a free monad transformer (i.e. 'FreeT') + and his @InOrOut@ functor corresponds to 'PipeF'. -} -{- $modular - Given a loop like: - -> loop :: IO r -> loop = forever $ do -> x <- dataSource -> y <- processData x -> dataSink y - - We could decompose it into three separate parts: - -> stage1 :: Producer a IO r -> stage1 = forever $ do -> x <- dataSource -> yield x -> -> stage2 :: Pipe a b IO r -> stage2 = forever $ do -> x <- await -> y <- processData x -> yield y -> -> -> stage3 :: Consumer b IO r -> stage3 = forever $ do -> y <- await -> dataSink -> -> stage3 <+< stage2 <+< stage1 == lift loop +-- | The base functor for the 'Pipe' type +data PipeF a b x = Await (a -> x) | Yield (b, x) - In other words, pipes let you decompose loops into modular components, which - promotes loose coupling and allows you to freely mix and match those - components. +-- I could use the "DerivingFunctor" extension, but I want to remain portable +instance Functor (PipeF a b) where + fmap f (Await a) = Await $ fmap f a + fmap f (Yield y) = Yield $ fmap f y - To demonstrate this, let's define a new data source that indefinitely - prompts the user for integers: +{-| + The base type for pipes -> prompt :: Producer Int IO a -> prompt = forever $ do -> lift $ putStrLn "Enter a number: " -> n <- read <$> lift getLine -> yield n + * @a@ - The type of input received from upstream pipes - Now we can use it as a drop-in replacement for @fromList@: + * @b@ - The type of output delivered to downstream pipes ->>> runPipe $ printer <+< take' 3 <+< prompt -Enter a number: -1 -1 -Enter a number: -2 -2 -Enter a number: -3 -3 -You shall not pass! + * @m@ - The base monad + * @r@ - The type of the return value -} - -{- $vertical - You can easily \"vertically\" concatenate pipes, 'Producer's, and - 'Consumer's, all using simple monad sequencing: ('>>'). For example, here - is how you concatenate 'Producer's: - ->>> runPipe $ printer <+< (fromList [1..3] >> fromList [10..12]) -1 -2 -3 -10 -11 -12 +type Pipe a b = FreeT (PipeF a b) - Here's how you would concatenate 'Consumer's: +-- | A pipe that produces values +type Producer b = Pipe () b ->>> let print' n = printer <+< take' n :: (Show a) => Int -> Consumer a IO () ->>> runPipe $ (print' 3 >> print' 4) <+< fromList [1..] -1 -2 -3 -You shall not pass! -4 -5 -6 -7 -You shall not pass! +-- | A pipe that consumes values +type Consumer b = Pipe b Void - ... but the above example is gratuitous because we could have just - concatenated the intermediate @take'@ pipe: +-- | A self-contained pipeline that is ready to be run +type Pipeline = Pipe () Void ->>> runPipe $ printer <+< (take' 3 >> take' 4) <+< fromList [1..] -1 -2 -3 -You shall not pass! -4 -5 -6 -7 -You shall not pass! +{- $create + 'yield' and 'await' are the only two primitives you need to create pipes. + Since @Pipe a b m@ is a monad, you can assemble 'yield' and 'await' + statements using ordinary @do@ notation. Since @Pipe a b@ is also a monad + transformer, you can use 'lift' to invoke the base monad. For example, you + could write a pipe stage that requests permission before forwarding any + output: +> check :: (Show a) => Pipe a a IO r +> check = forever $ do +> x <- await +> lift $ putStrLn $ "Can '" ++ (show x) ++ "' pass?" +> ok <- read <$> lift getLine +> when ok (yield x) -} -{- $return - Pipe composition imposes an important requirement: You can only compose - pipes that have the same return type. For example, I could write the - following function: - -> deliver :: (Monad m) => Int -> Consumer a m [a] -> deliver n = replicateM n await - - ... and I might try to compose it with @fromList@: - ->>> runPipe $ deliver 3 <+< fromList [1..10] -- wrong! - - ... but this wouldn't type-check, because @fromList@ has a return type of - @()@ and @deliver@ has a return type of @[Int]@. Composition requires that - every pipe has a return value ready in case it terminates first. - - Fortunately, we don't have to rewrite the @fromList@ function because we can - just add a return value using vertical concatenation: - ->>> runPipe $ deliver 3 <+< (fromList [1..10] >> return []) -[1,2,3] - - ... although a more idiomatic Haskell version would be: - ->>> runPipe $ (Just <$> deliver 3) <+< (fromList [1..10] *> pure Nothing) -Just [1,2,3] - - This forces you to cover all code paths by thinking about what return value - you would provide if something were to go wrong. For example, let's say I - were to make a mistake and request more input than @fromList@ can deliver: - ->>> runPipe $ (Just <$> deliver 99) <+< (fromList [1..10] *> pure Nothing) -Nothing +{-| + Wait for input from upstream. - The type system saved me by forcing me to cover all corner cases and handle - every way my program could terminate. + 'await' blocks until input is available from upstream. -} - -{- $terminate - - Now what if you wanted to write a pipe that only reads from its input end - (i.e. a 'Consumer') and returns a list of every value delivered to it when - its input pipe terminates? - -> toList :: (Monad m) => Consumer a m [a] -> toList = ??? - - You can't write such a pipe because if its input terminates then it brings - down @toList@ with it! This is correct because @toList@ as defined is not - compositional (yet!). - - To see why, let's say you somehow got @toList@ to work and the following - imaginary code sample worked: - ->>> runPipe $ toList <+< (fromList [1..5] >> return []) -[1,2,3,4,5] - - @toList@ is defined to return its value when the pipe immediately upstream - (@fromList@ in this case) terminates. This behavior immediately leads to a - problem. What if I were to insert an \"identity\" pipe between @toList@ and - @fromList@: - -> identity = forever $ await >>= yield -> -- This is how id is actually implemented! - - This pipe forwards every valued untouched, so we would expect it to not have - any affect if we were to insert it in the middle: - ->>> runPipe $ toList <+< identity <+< (fromList [1..5] >> return []) -??? -- Oops! Something other than [1,2,3,4,5], perhaps even non-termination - - The answer couldn't be @[1,2,3,4,5]@ because @toList@ would monitor - @identity@ instead of @fromList@ and since @identity@ never terminates - @toList@ never terminates. This is what I mean when I say that @toList@'s - specified behavior is non-compositional. It only works if it is coupled - directly to the desired pipe and breaks when you introduce intermediate - stages. - - This was not an intentional design choice, but rather a direct consequence - of enforcing the 'Category' laws when I was implementing 'Pipe''s 'Category' - instance. Satisfying the 'Category' laws forces code to be compositional. - - Note that a terminated pipe only brings down pipes composed with it. To - illustrate this, let's use the following example: - -> p = do a <+< b -> c +await :: (Monad m) => Pipe a b m a +await = wrap $ Await return - @a@, @b@, and @c@ are pipes, and @c@ shares the same input and output as - the composite pipe @a <+< b@, otherwise we cannot combine them within the - same monad. In the above example, either @a@ or @b@ could terminate and - bring down the other one since they are composed, but @c@ is guaranteed to - continue after @a <+< b@ terminates because it is not composed with them. - Conceptually, we can think of this as @c@ automatically taking over the - pipe's channeling responsibilities when @a <+< b@ can no longer continue. - There is no need to \"restart\" the input or output manually as in some - other iteratee libraries. +{-| + Deliver output downstream. - The @pipes@ library, unlike other iteratee libraries, grounds its vertical - and horizontal concatenation in category theory by deriving horizontal - concatenation ('.') from its 'Category' instance and vertical concatenation - ('>>') from its 'Monad' instance. This makes it easier to reason about - pipes because you can leverage your intuition about 'Category's and 'Monad's - to understand their behavior. The only 'Pipe'-specific primitives are - 'await' and 'yield'. + 'yield' restores control back upstream and binds the result to 'await'. -} - -{- $resource - Here's another problem with 'Pipe' composition: resource finalization. - Let's say we have the file \"test.txt\" with the following contents: - -> Line 1 -> Line 2 -> Line 3 - - .. and we wish to lazily read one line at a time from it: - -> readFile' :: Handle -> Producer Text IO () -> readFile' h = do -> eof <- lift $ hIsEOF h -> when (not eof) $ do -> s <- lift $ hGetLine h -> yield s -> readFile' h - - We could then try to be slick and write a lazy version that only reads as - many lines as we request: - -> read' :: FilePath -> Producer Text IO () -> read' = do -> lift $ putStrLn "Opening file ..." -> h <- lift $ openFile file ReadMode -> readFile' h -> lift $ putStrLn "Closing file ..." -> lift $ hClose h - - Now compose! - ->>> runPipe $ printer <+< read' "test.xt" -Opening file ... -"Line 1" -"Line 2" -"Line 3" -Closing file ... - - So far, so good. Equally important, the file is never opened if we replace - @printer@ with a pipe that never demands input: - ->>> runPipe $ (lift $ putStrLn "I don't need input") <+< read' "test.txt" -I don't need input - - There is still one problem, though. What if we wrote: +yield :: (Monad m) => b -> Pipe a b m () +yield b = wrap $ Yield (b, return ()) ->>> runPipe $ printer <+< take' 2 <+< read' "test.txt" -Opening file ... -"Line 1" -"Line 2" -You shall not pass! +{-| + Convert a pure function into a pipe - Oh no! While it was lazy and only read two lines from the file, it was also - too lazy to properly close our file! @take' 2@ terminated before @read'@, - preventing @read'@ from properly closing \"test.txt\". This is why 'Pipe' - composition fails to guarantee deterministic finalization. +> pipe = forever $ do +> x <- await +> yield (f x) -} - -{- $frame - So how could we implement finalization, then? The answer is to build a - higher-order type on top of 'Pipe' and define a new composition that permits - prompt, deterministic finalization. - - To do this, we import "Control.Pipe.Final", which exports the 'Frame' type, - analogous to the 'Pipe' type, except more powerful. To demonstrate it in - action, let's rewrite our @take'@ function to be a 'Frame' instead. - -> take' :: Int -> Frame a a IO () -> take' n -> | n < 1 = Frame $ close $ lift $ putStrLn "You shall not pass!" -> | otherwise = Frame $ do -> replicateM_ (n - 1) $ do -> x <- awaitF -> yieldF x -> x <- awaitF -> close $ do -> lift $ putStrLn "You shall not pass!" -> yieldF x - - The type signature looks the same, except 'Pipe' has been replaced with - 'Frame'. Also, now we have 'awaitF' instead of 'await' and 'yieldF' instead - of 'yield'. However, you'll notice two new things: 'close' and 'Frame'. - - 'close' signals when we no longer need input from upstream. If you try to - request input other than @()@ after the 'close', you will get a type error. - Whenever you 'close' a frame, composition finalizes every upstream frame and - removes them from the pipeline. The type error reflects the fact that if - you 'awaitF' past that point there is no longer anything upstream to request - input from. - - 'Frame' is a newtype constructor that I use to give clearer type errors and - abstract away the underlying implementation. The reason is that if you were - to expand out the full type that 'Frame' wraps you would get: - -> Frame a b m r ~ Pipe (Maybe a) (m (), b) m (Pipe (Maybe ()) (m (), b) m r) -> -- Yuck! - - Really, the only reason the type is that complicated is because I avoid - using language extensions to implement 'Frame's, otherwise it would look - more like: - -> Pipe (Maybe a) (m (), b) m r - - ... which isn't so bad. In fact, it's not hard to understand what that - type is doing. The 'Maybe' is used to supply a 'Nothing' to 'await's when - upstream terminates before 'yield'ing a value. The @m ()@ is the most - recent finalizer which is yielded alongside every value so that downstream - pipes can finalize you if they terminate before requesting another value. - The finalization machinery uses these tricks behind the scene to guarantee - that your finalizers get called. I provide a type synonym for this: +pipe :: (Monad m) => (a -> b) -> Pipe a b m r +pipe f = forever $ await >>= yield . f -> type Ensure a b m r = Pipe (Maybe a) (m (), b) m r +{- $category + 'Pipe's form a 'Category', meaning that you can compose 'Pipe's and also + define an identity 'Pipe'. - In other words, an 'Ensure'd pipe can intercept upstream termination and - register finalizers for downstream to call in the event of premature - termination. A good way to think about the distinction between 'Ensure' - and 'Frame' is that 'Ensure' is the 'Monad' and 'Frame' is the 'Category', - unlike 'Pipe', which is both at the same time. + 'Pipe' composition binds the output of the upstream 'Pipe' to the input of + the downstream 'Pipe'. Like Haskell functions, 'Pipe's are lazy, meaning + that upstream 'Pipe's are only evaluated as far as necessary to generate + enough input for downstream 'Pipe's. If any 'Pipe' terminates, it also + terminates every 'Pipe' composed with it. - Using this type synonym, we can rewrite the type that 'Frame' wraps: + If you want to define a proper 'Category' instance you have to wrap the + 'Pipe' type using the newtype 'PipeC' in order to rearrange the type + variables. -> Frame a b m r ~ Ensure a b m (Ensure () b m r) + This means that if you want to compose pipes using ('.') from the 'Category' + type class, you end up with a newtype mess: - The first half of the type is the part of the pipe before you call 'close', - the second half of the type is the part of the pipe after you call 'close'. - Notice how the second half has a blocked input end. +> unPipeC (PipeC p1 . PipeC p2) - However, I haven't yet shown you how to register finalizers. That's easy, - though, since you just use 'catchP' or 'finallyP', which are identical to - their exception-handling counterparts, except they catch 'Frame' - terminations in either direction. Let's rewrite our @read'@ function using - finalizers: + You can avoid this by using convenient operators that do this newtype + wrapping and unwrapping for you: -> readFile' :: Handle -> Ensure () Text IO () -> readFile' h = do -> eof <- lift $ hIsEOF h -> when (not eof) $ do -> s <- lift $ hGetLine h -> yieldF s -> readFile' h +> p1 <+< p2 = unPipeC $ PipeC p1 . PipeC p2 > -> read' :: FilePath -> Frame () Text IO () -> read' = Frame $ close $ do -> lift $ putStrLn "Opening file ..." -> h <- lift $ openFile file ReadMode -> finallyP (putStrLn "Closing file ..." >> hClose h) -> (readFile' h) - - Notice how @read'@ closes its input end immediately because it never - requires input. Also, the 'finallyP' ensures that the finalizer is called - both if @read'@ terminates normally or is interrupted by another 'Frame' - terminating first. - - Now, all we need to do is rewrite @printer@ to be a 'Frame': - -> printer :: (Show b) => Frame b Void IO r -> printer = Frame $ forever $ do -> x <- awaitF -> lift $ print x - - This time we don't even need a 'close' statement because @printer@ never - stops needing input. Any non-terminating 'Frame' with a polymorphic return - type can skip calling 'close' altogether, and it will type-check. --} - -{- $framecompose - - Just like with 'Pipe's, we can compose 'Frame's, except now we use ('<-<'): - -> stack :: Frame Void () IO () -> stack = printer <-< take' 1 <-< read' "test.txt" +> idP = unPipeC id - I call a complete set of 'Frame's a 'Stack', to reflect the fact that - 'Frame' composition uses the exact same tricks stack-based programming uses - to guarantee deterministic finalization. When a 'Frame' terminates it - finalizes upstream 'Frame's as if they were a heap and it propagates an - exceptional value ('Nothing' in this case) for downstream 'Frame's to - intercept. I provide a type synonym to reflect this: + The 'Category' instance obeys the 'Category' laws. In other words: -> type Stack m r = Frame Void () IO r + * Composition is truly associative. The result of composition produces the + exact same composite 'Pipe' regardless of how you group composition, so it + is perfectly safe to omit the parentheses altogether: - So we can rewrite the type of @stack@ to be: +> (p1 <+< p2) <+< p3 = p1 <+< (p2 <+< p3) = p1 <+< p2 <+< p3 -> stack :: Stack IO () + * 'idP' is a true identity pipe. Composing a pipe with 'idP' returns the + exact same original pipe: - To run a 'Stack', we use 'runFrame', which is the 'Frame'-based analog to - 'runPipe': +> p <+< idP = p +> idP <+< p = p ->>> runFrame stack -Opening file ... -"Line 1" -Closing file ... -"Line 2" -You shall not pass! + The 'Category' laws are \"correct by construction\", meaning that you cannot + break them despite the library's internals being fully exposed. The above + equalities are true using the strongest denotational semantics possible in + Haskell, namely that both sides of the equals sign correspond to the exact + same value in Haskell, constructor-for-constructor, value-for-value. You + cannot create a function that can distinguish the results. - Not only did it correctly finalize the file this time, but it did so as - promptly as possible! I programmed @take'@ so that it knew it would not - need @read'@ any longer before it 'yield'ed the second value, so it - finalized the file before 'yield'ing the second value for @printer@. - @take'@ did this without knowing anything about the 'Frame' upstream of it. - One of the big advantages of 'Frame's is that you can reason about the - finalization behavior of each 'Frame' in complete isolation from other - 'Frame's, allowing you to completely decouple their finalization - behavior. + Actually, all other class instances in this library provide the same strong + guarantees for their corresponding laws. I only emphasize the guarantee for + the 'Category' instance because it is one of the most distinguishing + features of this library, making it far more extensible than other + implementations. -} -{- $frameensure - Unfortunately, in the absence of extensions I have to split the 'Monad' and - 'Category' into two separate types. 'Ensure' is the 'Monad', 'Frame' is the - 'Category'. - - However, you can achieve the best of both worlds by programming all your - pipes in the 'Ensure' monad, and then only adding 'close' at the last minute when you are building your 'Stack'. For example, if we wanted to read from - multiple files, it would be much better to just remove the 'close' function - from the @read'@ implementation, so it operates in the 'Ensure' monad: +-- | 'Pipe's form a 'Category' instance when you rearrange the type variables +newtype PipeC m r a b = PipeC { unPipeC :: Pipe a b m r} -> read' :: FilePath -> Ensure () Text IO () +instance (Monad m) => Category (PipeC m r) where + id = PipeC idP + PipeC p1 . PipeC p2 = PipeC $ p1 <+< p2 - Then use 'close' only after we've already concatenated our files: +-- | Corresponds to ('<<<')/('.') from @Control.Category@ +(<+<) :: (Monad m) => Pipe b c m r -> Pipe a b m r -> Pipe a c m r +p1 <+< p2 = FreeT $ do + x1 <- runFreeT p1 + let p1' = FreeT $ return x1 + runFreeT $ case x1 of + Return r -> return r + Wrap (Yield y ) -> wrap $ Yield $ fmap (<+< p2) y + Wrap (Await f1) -> FreeT $ do + x2 <- runFreeT p2 + runFreeT $ case x2 of + Return r -> return r + Wrap (Yield (x, p)) -> f1 x <+< p + Wrap (Await f2 ) -> wrap $ Await $ fmap (p1' <+<) f2 -> files :: Frame () Text IO () -> files = close $ do -> read' "test.txt" -> read' "dictionary.txt" -> read' "poem.txt" +-- | Corresponds to ('>>>') from @Control.Category@ +(>+>) :: (Monad m) => Pipe a b m r -> Pipe b c m r -> Pipe a c m r +(>+>) = flip (<+<) - This is a more idiomatic 'Frame' programming style that lets you take - advantage of both the 'Monad' and 'Category' instances. +{- These associativities might help performance since pipe evaluation is + downstream-biased. I set them to the same priority as (.). -} +infixr 9 <+< +infixl 9 >+> - The beauty of compositional finalization is we can decompose complicated - problems into smaller ones. Imagine that we have a resource that needs a - fine-grained finalization behavior like in our @take'@ function which does - a cute little optimization to finalize early. We can always decompose our - frame into one that does the straight-forward thing (like @read'@) and then - just compose it with @take'@ to get the cute optimization for free. In this - way we've decomposed the problem into two separate problems: generating the - resource and doing the cute optimization. --} +-- | Corresponds to 'id' from @Control.Category@ +idP :: (Monad m) => Pipe a a m r +idP = pipe id -{- $fold - 'Frame's can actually do much more than manage finalization! Using - 'Frame's, we can now correctly implement folds like @toList@ in a way that - is truly compositional: +{- $runpipe + Note that you can also unwrap a 'Pipe' a single step at a time using + 'runFreeT' (since 'Pipe' is just a type synonym for a free monad + transformer). This will take you to the next /external/ 'await' or 'yield' + statement. -> toList :: (Monad m) => Frame a Void m [a] -> toList = Frame go where -> go = do -> x <- await -> case x of -> Nothing -> close $ pure [] -> Just a -> fmap (fmap (a:)) go -> -- the extra fmap is an unfortunate extra detail + This means that a closed 'Pipeline' will unwrap to a single step, in which + case you would have been better served by 'runPipe'. This directly follows + from the 'Category' laws, which guarantee that you cannot resolve a + composite pipe into its component pipes. When you compose two pipes, the + internal await and yield statements fuse and completely disappear. - This time I used an ordinary 'await', instead of 'awaitF', so I could access - the underlying 'Maybe' values that these 'Frame's are passing around. - Similarly, you could use 'yield' instead of 'yieldF' if you wanted to - manually manage the finalizers passed downstream at each 'yield' statement - instead of using the 'catchP' or 'finallyP' convenience functions. Using - these advanced features does not break any of the 'Category' laws. I could - expose every single internal of the library and you would not be able to - break the 'Category' laws because the 'Frame's generated are still - indistinguishable at the value level and fuse into the hand-written - implementation. The compositionality of 'Frame's is just as strong as the - compositionality of 'Pipe's. + 'runFreeT' is ideal for more advanced users who wish to write their own + 'Pipe' functions while waiting for me to find more elegant solutions. +-} +{-| + Run the 'Pipe' monad transformer, converting it back into the base monad. - Now let's use our @toList@ function: + 'runPipe' imposes two conditions: ->>> runFrame $ (Just <$> toList) <-< (Nothing <$ fromList [1..3]) -Just [1,2,3] + * The pipe's input, if any, is trivially satisfiable (i.e. @()@) - I still had to provide a return value for @fromList@ ('Nothing' in this - case), because when @fromList@ terminates, it cannot guarantee that its - return value will come from itself or from @toList@. When @toList@ receives - a 'Nothing' from upstream, it can choose to terminate and over-ride the - return value from upstream or 'await' again and defer to the upstream return - value (@fromList@ in this case). It doesn't even have to immediately - decide. It could just 'yield' more values downstream and forget it had even - received a 'Nothing' and if downstream terminates then composition will - still ensure that everything \"just works\" the way you would expect and no - finalizers are missed or duplicated. + * The pipe does not 'yield' any output - Composition handles every single corner case of finalization. This directly - follows from enforcing the 'Category' laws, because categories have no - corners! --} + The latter restriction makes 'runPipe' less polymorphic than it could be, + and I settled on the restriction for three reasons: -{- $strict - We can go a step further and modify @toList@ into something even cooler: + * It prevents against accidental data loss. -> strict :: (Monad m) => Frame a a m () -> strict = Frame $ do -> xs <- go -> close $ mapM_ yieldF xs -> where -> go = do -> x <- await -> case x of -> Nothing -> pure [] -> Just a -> fmap (a:) go + * It prevents wastefully draining a scarce resource by gratuitously + demanding values from it. - As the name suggests, @strict@ is strict in its input. We can use @strict@ - to load the entire resource into memory immediately, allowing us to finalize - it early. Let's use this to create a strict version of our file reader: + * It encourages an idiomatic pipe programming style where input is consumed + in a structured way using a 'Consumer'. ->>> runFrame $ printer <-< take' 2 <-< strict <-< read' "test.txt" -Opening file ... -Closing file ... -"Line 1" -"Line 2" -You shall not pass! + If you believe that discarding output is the appropriate behavior, you can + specify this by explicitly feeding your output to a pipe that gratuitously + discards it: - Now we have a way to seamlessly switch from laziness to strictness all - implemented entirely within Haskell without the use of artificial 'seq' - annotations. +> runPipe $ forever await <+< p -} - - +runPipe :: (Monad m) => Pipeline m r -> m r +runPipe p = do + e <- runFreeT p + case e of + Return r -> return r + Wrap (Await f) -> runPipe $ f () + Wrap (Yield y) -> runPipe $ snd y diff --git a/Control/Pipe/Common.hs b/Control/Pipe/Common.hs deleted file mode 100644 --- a/Control/Pipe/Common.hs +++ /dev/null @@ -1,283 +0,0 @@ -{-# LANGUAGE Rank2Types #-} - -module Control.Pipe.Common ( - -- * Introduction - -- $summary - - -- * Types - -- $types - PipeF(..), - Pipe, - Producer, - Consumer, - Pipeline, - -- * Create Pipes - -- $create - await, - yield, - pipe, - -- * Compose Pipes - -- $newtype - Lazy(..), - -- $convenience - (<+<), - (>+>), - idP, - -- $category - -- * Run Pipes - -- $runpipe - runPipe - ) where - -import Control.Applicative -import Control.Category -import Control.Monad (forever) -import Control.Monad.Trans.Class (lift) -import Control.Monad.Trans.Free -import Data.Void (Void) -import Prelude hiding ((.), id) - -{- $summary - I completely expose the 'Pipe' data type and internals in order to encourage - people to write their own 'Pipe' functions. This does not compromise the - correctness or safety of the library at all and you can feel free to use the - constructors directly without violating any laws or invariants. - - I promote using the 'Monad' and 'Category' instances to build and compose - pipes, but this does not mean that they are the only option. In fact, any - combinator provided by other iteratee libraries can be recreated for pipes, - too. However, this core library does not provide many of the functions - found in other libraries in order to encourage people to find principled and - theoretically grounded solutions rather than devise ad-hoc solutions - characteristic of other iteratee implementations. --} - -{- $types - The 'Pipe' type is strongly inspired by Mario Blazevic's @Coroutine@ type in - his concurrency article from Issue 19 of The Monad Reader and is formulated - in the exact same way. - - His @Coroutine@ type is actually a free monad transformer (i.e. 'FreeT') - and his @InOrOut@ functor corresponds to 'PipeF'. --} -data PipeF a b x = Await (a -> x) | Yield (b, x) - --- I could use the "DerivingFunctor" extension, but I want to remain portable -instance Functor (PipeF a b) where - fmap f (Await a) = Await $ fmap f a - fmap f (Yield y) = Yield $ fmap f y - -{-| - The base type for pipes - - * @a@ - The type of input received from upstream pipes - - * @b@ - The type of output delivered to downstream pipes - - * @m@ - The base monad - - * @r@ - The type of the return value --} -type Pipe a b = FreeT (PipeF a b) - --- | A pipe that produces values -type Producer b = Pipe () b - --- | A pipe that consumes values -type Consumer b = Pipe b Void - --- | A self-contained pipeline that is ready to be run -type Pipeline = Pipe () Void - -{- $create - 'yield' and 'await' are the only two primitives you need to create pipes. - Since 'Pipe a b m' is a monad, you can assemble 'yield' and 'await' - statements using ordinary @do@ notation. Since 'Pipe a b' is also a monad - transformer, you can use 'lift' to invoke the base monad. For example, you - could write a pipe stage that requests permission before forwarding any - output: - -> check :: (Show a) => Pipe a a IO r -> check = forever $ do -> x <- await -> lift $ putStrLn $ "Can '" ++ (show x) ++ "' pass?" -> ok <- read <$> lift getLine -> when ok (yield x) --} - -{-| - Wait for input from upstream. - - 'await' blocks until input is available from upstream. --} -await :: (Monad m) => Pipe a b m a -await = wrap $ Await return - -{-| - Deliver output downstream. - - 'yield' restores control back upstream and binds the result to 'await'. --} -yield :: (Monad m) => b -> Pipe a b m () -yield b = wrap $ Yield (b, return ()) - -{-| - Convert a pure function into a pipe - -> pipe = forever $ do -> x <- await -> yield (f x) --} -pipe :: (Monad m) => (a -> b) -> Pipe a b m r -pipe f = forever $ await >>= yield . f - -{- $newtype - Pipes form a 'Category', but if you want to define a proper 'Category' - instance you have to wrap the 'Pipe' type using a newtype in order to - rearrange the type variables: --} -newtype Lazy m r a b = Lazy { unLazy :: Pipe a b m r} - -instance (Monad m) => Category (Lazy m r) where - id = Lazy idP - Lazy p1 . Lazy p2 = Lazy $ p1 <+< p2 - -{- $convenience - This means that if you want to compose pipes using ('.') from the 'Category' - type class, you end up with a newtype mess: @unLazy (Lazy p1 . Lazy p2)@. - - You can avoid this by using convenient operators that do this newtype - wrapping and unwrapping for you: - -> p1 <+< p2 = unLazy $ Lazy p1 . Lazy p2 -> -> idP = unLazy id --} - --- | Corresponds to ('<<<')/('.') from @Control.Category@ -(<+<) :: (Monad m) => Pipe b c m r -> Pipe a b m r -> Pipe a c m r -p1 <+< p2 = FreeT $ do - x1 <- runFreeT p1 - let p1' = FreeT $ return x1 - runFreeT $ case x1 of - Pure r -> pure r - Wrap (Yield y) -> wrap $ Yield $ fmap (<+< p2) y - Wrap (Await f1) -> FreeT $ do - x2 <- runFreeT p2 - runFreeT $ case x2 of - Pure r -> pure r - Wrap (Yield (x, p)) -> f1 x <+< p - Wrap (Await f2 ) -> wrap $ Await $ fmap (p1' <+<) f2 - --- | Corresponds to ('>>>') from @Control.Category@ -(>+>) :: (Monad m) => Pipe a b m r -> Pipe b c m r -> Pipe a c m r -(>+>) = flip (<+<) - -{- These associativities might help performance since pipe evaluation is - downstream-biased. I set them to the same priority as (.). -} -infixr 9 <+< -infixl 9 >+> - --- | Corresponds to 'id' from @Control.Category@ -idP :: (Monad m) => Pipe a a m r -idP = pipe id - -{- $category - You can compose two pipes using @p1 <+< p2@, which binds the output of @p2@ - to the input of @p1@. For example: - -> (await >>= lift . print) <+< yield 0 -> = lift (print 0) - - 'idP' is the identity pipe which forwards all output untouched: - -> idP = forever $ do -> x <- await -> yield x - - Pipes are lazy, meaning that control begins at the downstream pipe and - control only transfers upstream when the downstream pipe 'await's input from - upstream. If a pipe never 'await's input, then pipes upstream of it will - never run. - - Upstream pipes relinquish control back downstream whenever they 'yield' an - output value. This binds the 'yield'ed value to the return value of the - downstream 'await'. The upstream pipe does not regain control unless the - downstream pipe requests input again. - - When a pipe terminates, it also terminates any pipes composed with it. - - The 'Category' instance obeys the 'Category' laws. In other words: - - * Composition is truly associative. The result of composition produces the - exact same composite 'Pipe' regardless of how you group composition: - -> (p1 <+< p2) <+< p3 = p1 <+< (p2 <+< p3) - - * 'idP' is a true identity pipe. Composing a pipe with 'idP' returns the - exact same original pipe: - -> p <+< idP = p -> idP <+< p = p - - The 'Category' laws are \"correct by construction\", meaning that you cannot - break them despite the library's internals being fully exposed. The above - equalities are true using the strongest denotational semantics possible in - Haskell, namely that both sides of the equals sign correspond to the exact - same value in Haskell, constructor-for-constructor, value-for-value. You - cannot create a function that can distinguish the results. - - Actually, all other class instances for 'Pipe's provide the same strong - guarantees for their corresponding laws. I only emphasize the guarantee for - the 'Category' instance because it is one of the most distinguishing - features of this library. --} - -{- $runpipe - Note that you can also unwrap a 'Pipe' a single step at a time using - 'runFreeT' (since 'Pipe' is just a type synonym for a free monad - transformer). This will take you to the next /external/ 'await' or 'yield' - statement. - - This means that a closed 'Pipeline' will unwrap to a single step, in which - case you would have been better served by 'runPipe'. This directly follows - from the 'Category' laws, which guarantee that you cannot resolve a - composite pipe into its component pipes. When you compose two pipes, the - internal await and yield statements fuse and completely disappear. - - 'runFreeT' is ideal for more advanced users who wish to write their own - 'Pipe' functions while waiting for me to find more elegant solutions. --} -{-| - Run the 'Pipe' monad transformer, converting it back into the base monad. - - 'runPipe' imposes two conditions: - - * The pipe's input, if any, is trivially satisfiable (i.e. @()@) - - * The pipe does not 'yield' any output - - The latter restriction makes 'runPipe' less polymorphic than it could be, - and I settled on the restriction for three reasons: - - * It prevents against accidental data loss. - - * It prevents wastefully draining a scarce resource by gratuitously - demanding values from it. - - * It encourages an idiomatic pipe programming style where input is consumed - in a structured way using a 'Consumer'. - - If you believe that discarding output is the appropriate behavior, you can - specify this by explicitly feeding your output to a pipe that gratuitously - discards it: - -> runPipe $ forever await <+< p --} -runPipe :: (Monad m) => Pipeline m r -> m r -runPipe p = do - e <- runFreeT p - case e of - Pure r -> return r - Wrap (Await f) -> runPipe $ f () - Wrap (Yield y) -> runPipe $ snd y diff --git a/Control/Pipe/Final.hs b/Control/Pipe/Final.hs deleted file mode 100644 --- a/Control/Pipe/Final.hs +++ /dev/null @@ -1,424 +0,0 @@ -module Control.Pipe.Final ( - -- * Introduction - -- $intro - - -- * Types - Prompt, - Ensure, - Frame(..), - Stack, - -- * Create Frames - -- $create - yieldF, - awaitF, - -- * Prompt Finalization - -- $prompt - close, - bindClosed, - reopen, - -- * Ensure Finalization - -- $ensure - catchP, - finallyP, - -- * Compose Frames - -- $compose - (<-<), - (>->), - idF, - FrameC(..), - -- * Run Frames - -- $run - runFrame - ) where - -import Control.Applicative -import Control.Category -import Control.Monad -import Control.Monad.Trans.Class -import Control.Monad.Trans.Free -import Control.Pipe.Common -import Data.Void -import Prelude hiding ((.), id) - -{- $intro - A 'Frame' is a higher-order type built on top of 'Pipe'. It enables a - richer composition with the ability to finalize resources in a manner that - is: - - * Prompt: You can close resources when you no longer need input from them - - * Deterministic: Composition finalizes every 'Frame' when one terminates - - 'Frame's differ from 'Pipe's in that they do not form monads, but instead - form parametrized monads. Unfortunately, parametrized monads are not - mainstream in Haskell and require a ton of extensions along with a modified - Prelude in order to recover @do@ notation, so this first release of the - 'Frame' implementation essentially \"in-lines\" the parametrized monad by - splitting it into two monads. Future releases will split off a version that - takes advantage of parametrized monads for a much simpler underlying type - and a significantly cleaner implementation. - - Ordinary users should start at the section \"Create Frames\", but if you - encounter weird type errors and want to understand them, then consult the - \"Types\" section. --} - -{-| - An illustrative type synonym that demonstrates how 'Prompt' finalization - works - - This type simulates a parametrized monad by breaking it up into two monads - where the first monad returns the second one. The first monad permits any - pipe code and the second monad only permits pipe code that doesn't need - input. - - For example if @p = Pipe@, the first monad becomes an ordinary 'Pipe' and - the second monad becomes a 'Producer': - -> Prompt Pipe a b m r = Pipe a b m (Pipe () b m r) - - The pipe does not require input by the time it reaches the second block, - meaning that the finalization machinery can safely finalize upstream - resources the moment. The earlier you use 'close' the input end, - the more promptly you release upstream resources. - - The finalization machinery also finalizes downstream pipes when the - second monad terminates. I use this trick to ensure a strict ordering of - finalizers from upstream to downstream. - - I don't actually use the 'Prompt' type synonym, since that would require - newtyping everything, but I will reference it in documentation to clarify - type signatures. --} -type Prompt p a b m r = p a b m (p () b m r) - -{-| - A pipe type that 'Ensure's deterministic finalization - - The finalization machinery uses the input and output ends in different ways - to finalize the pipe when another pipe terminates first. - - If an upstream pipe terminates first, the current pipe will receive a - 'Nothing' once. This allows it to finalize itself and if it terminates then - its return value takes precedence over upstream's return value. However, if - it 'await's again, it defers to upstream's return value and never regains - control. You do not need to \"rethrow\" the 'Nothing' (nor can you): - composition takes care of this for you. - - On the output end, the pipe must supply its most up-to-date finalizer - alongside every value it 'yield's downstream. This finalizer is guaranteed - to be called if downstream terminates first. You do not need to relay - upstream finalizers alongside the pipe's own finalizer (nor can you): - composition takes care of this for you. - - The combination of these two tricks allows a bidirectional guarantee of - deterministic finalization that satisfies the 'Category' laws. --} -type Ensure a b m r = Pipe (Maybe a) (m (), b) m r - -{-| - A pipe type that combines 'Prompt' and 'Ensure' to enable both prompt and - deterministic finalization. - - The name connotes a stack frame, since finalized pipes can be thought of as - forming the 'Category' of stack frames, where upstream finalization is - equivalent to finalizing the heap, and downstream finalization is equivalent - to throwing an exception up the stack. - - The type is equivalent to: - -> type Frame a b m r = Prompt Ensure a b m r --} -newtype Frame a b m r = Frame { unFrame :: Ensure a b m (Ensure () b m r) } - -instance (Monad m) => Functor (Frame a b m) where - fmap f (Frame p) = Frame $ fmap (fmap f) p - --- | A 'Stack' is a 'Frame' that doesn't need input and doesn't generate output -type Stack = Frame () Void - -{- $create - The first step to convert 'Pipe' code to 'Frame' code is to replace all - 'yield's with 'yieldF's and all 'await's with 'awaitF's. - -> contrived = do --> contrived = do -> x1 <- await --> x1 <- awaitF -> yield x1 --> yieldF x1 -> x2 <- await --> x2 <- awaitF -> yield x2 --> yieldF x2 --} - --- | Like 'yield', but also yields an empty finalizer alongside the value -yieldF :: (Monad m) => b -> Ensure a b m () -yieldF x = yield (unit, x) - --- | Like 'await', but ignores all 'Nothing's and just awaits again -awaitF :: (Monad m) => Ensure a b m a -awaitF = await >>= maybe awaitF return - -{- $prompt - The second step to convert 'Pipe' code to 'Frame' code is to mark the point - where your 'Pipe' no longer 'await's by wrapping it in the 'close' function - and then wrapping the 'Pipe' in a 'Frame' newtype: - -> contrived :: (Monad m) => Frame a a m () -> contrived = Frame $ do -> x1 <- awaitF -> yieldF x1 -> x2 <- awaitF -> close $ yieldF x2 - - If a non-terminating pipe demands input indefinitely, there is no need to - 'close' it. It will type-check if the return value is polymorphic as a - result of non-termination. --} - -{-| - Use this to mark when a 'Frame' no longer requires input. The earlier the - better! --} -close :: (Monad m) => Ensure () b m r -> Ensure a b m (Ensure () b m r) -close = pure - -{-| - Use this to bind to the 'close'd half of the 'Frame' if you want to continue - where it left off but you still don't require input. - - This function would not be necessary if 'Prompt' were implemented as a - parametrized monad, so if it seems ugly, that's because it is. --} -bindClosed :: (Monad m) => - Frame a b m r1 -> (r1 -> Ensure () b m r2) -> Frame a b m r2 -bindClosed (Frame p) f = Frame $ fmap (>>= f) p - -{-| - Use this to 'reopen' a 'Frame' if you change your mind and decide you want - to continue to 'await' input after all. - - This postpones finalization of upstream until you 'close' the input end - again. --} -reopen :: (Monad m) => Frame a b m r -> Ensure a b m r -reopen (Frame p) = join $ fmap (<+< (forever $ yield $ Just ())) p - -{- $ensure - The third (optional) step to convert 'Pipe' code to 'Frame' code is to use - 'catchP' or 'finallyP' to register finalizers for blocks of code. - -> contrived :: Frame a a IO () -> contrived = Frame $ do -> catchP (putStrLn "Stage 1 interrupted") $ do -> x1 <- awaitF -> catchP (putStrLn "Stage 1(b) interrupted") $ yieldF x1 -> catchP (putStrLn "Stage 2 interrupted") $ do -> x2 <- awaitF -> close $ yieldF x2 --} - -{-| - @catchP m p@ registers @m@ to be called only if another composed - pipe terminates before @p@ is done. --} -catchP :: (Monad m) => m () -> Ensure a b m r -> Ensure a b m r -catchP m p = FreeT $ do - x <- runFreeT p - runFreeT $ case x of - Pure r -> pure r - Wrap (Yield ((m', b), p')) -> wrap $ Yield ((m' >> m, b), catchP m p') - Wrap (Await f) -> wrap $ Await $ \e -> case e of - Nothing -> lift m >> catchP m (f e) - Just _ -> catchP m (f e) -{- catchP is equivalent to: - -awaitF' m = await >>= maybe (lift m >> awaitF' m) return - -yieldF' m x = yield (m, x) - -catchP m p = reopen $ - (forever $ awaitF >>= yieldF' m) - <-< Frame (fmap close p) - <-< (forever $ awaitF' m >>= yieldF) -} - -{-| - 'finallyP' is like 'catchP' except that it also calls the finalizer if @p@ - completes normally. --} -finallyP :: (Monad m) => m () -> Ensure a b m r -> Ensure a b m r -finallyP m p = do - r <- catchP m p - lift m - return r - -(<~<) :: (Monad m) - => Pipe b c m (Pipe x c m r) - -> Pipe a b m (Pipe x b m r) - -> Pipe a c m (Pipe x c m r) -p1 <~< p2 = FreeT $ do - x1 <- runFreeT p1 - runFreeT $ case x1 of - Pure p1' -> pure p1' - Wrap (Yield y) -> wrap $ Yield $ fmap (<~< p2) y - Wrap (Await f1) -> FreeT $ do - let p1 = FreeT $ return x1 - x2 <- runFreeT p2 - runFreeT $ case x2 of - Pure p2' -> pure $ p1 <~| p2' - Wrap (Yield (b2, p2')) -> f1 b2 <~< p2' - Wrap (Await f2 ) -> wrap $ Await $ fmap (p1 <~<) f2 - -(<~|) :: (Monad m) - => Pipe b c m (Pipe x c m r) - -> Pipe x b m r - -> Pipe x c m r -p1 <~| p2 = FreeT $ do - x1 <- runFreeT p1 - runFreeT $ case x1 of - Pure p1' -> p1' - Wrap (Yield y) -> wrap $ Yield $ fmap (<~| p2) y - Wrap (Await f) -> FreeT $ do - let p1 = FreeT $ return x1 - x2 <- runFreeT p2 - runFreeT $ case x2 of - Pure r -> pure r - Wrap (Yield (b2, p2')) -> f b2 <~| p2' - Wrap (Await f2 ) -> wrap $ Await $ fmap (p1 <~|) f2 - -unit :: (Monad m) => m () -unit = return () - -mult :: (Monad m) - => m () - -> Pipe (Maybe b ) (m (), c) m (Pipe x (m (), c) m r) - -> Pipe (Maybe (m (), b)) (m (), c) m (Pipe x (m (), c) m r) -mult m p = FreeT $ do - x <- runFreeT p - runFreeT $ case x of - Pure p' -> pure $ lift m >> p' - Wrap (Yield ((m', c), p')) -> wrap $ Yield ((m >> m', c), mult m p') - Wrap (Await f) -> wrap $ Await $ \e -> case e of - Nothing -> mult unit (f Nothing) - Just (m', b) -> mult m' (f $ Just b ) - -comult :: (Monad m) - => Pipe (Maybe a) b m (Pipe x b m r) - -> Pipe (Maybe a) (Maybe b) m (Pipe x (Maybe b) m r) -comult p = FreeT $ do - x <- runFreeT p - runFreeT $ case x of - Pure p' -> pure $ warn p' - Wrap (Yield (b, p')) -> wrap $ Yield (Just b, comult p') - Wrap (Await f) -> wrap $ Await $ \e -> case e of - Nothing -> schedule $ comult (f e) - Just _ -> comult (f e) - -warn :: (Monad m) - => Pipe x b m r - -> Pipe x (Maybe b) m r -warn p = do - r <- pipe Just <+< p - yield Nothing - return r - -schedule :: (Monad m) - => Pipe (Maybe a) (Maybe b) m (Pipe x (Maybe b) m r) - -> Pipe (Maybe a) (Maybe b) m (Pipe x (Maybe b) m r) -schedule p = FreeT $ do - x <- runFreeT p - runFreeT $ case x of - Pure p' -> pure p' - Wrap (Await f) -> wrap $ Yield (Nothing, wrap $ Await f) - Wrap (Yield y) -> wrap $ Yield $ fmap schedule y - -{- $compose - The fourth step to convert 'Pipe' code to 'Frame' code is to use ('<-<') to - compose 'Frame's instead of ('<+<'). - -> printer :: Frame a Void IO r -> fromList :: (Monad m) => [a] -> Frame () a m () -> -> p :: Stack IO () -> p = printer <-< contrived <-< fromList [1..] - - Similarly, 'idF' replaces 'idP'. - - When a 'Frame' terminates, the 'FrameC' category strictly orders the - finalizers from upstream to downstream. Specifically: - - * When any 'Frame' 'close's its input end, it finalizes all 'Frame's - upstream of it. These finalizers are ordered from upstream to downstream. - - * A 'Frame' is responsible for finalizing its own resources under ordinary - operation (either manually, or using 'finallyP'). - - * When a 'Frame' terminates, everything downstream of it is finalized. - These finalizers are ordered from upstream to downstream. - - The 'Category' instance for 'FrameC' provides the same strong guarantees as - the 'Lazy' category. This confers many practical advantages: - - * Finalizers are never duplicated or dropped in corner cases. - - * The grouping of composition will never affect the ordering or behavior of - finalizers. - - * Finalization does not grow more complex the more 'Frame's you add in your - 'Stack'. - - * You can reason about the finalization behavior of each 'Frame' - independently of other 'Frame's it is composed with. --} - --- | Corresponds to 'id' from @Control.Category@ -idF :: (Monad m) => Frame a a m r -idF = Frame $ forever $ awaitF >>= yieldF - --- | Corresponds to ('<<<')/('.') from @Control.Category@ -(<-<) :: (Monad m) => Frame b c m r -> Frame a b m r -> Frame a c m r -(Frame p1) <-< (Frame p2) = Frame $ mult unit p1 <~< comult p2 - --- | Corresponds to ('>>>') from @Control.Category@ -(>->) :: (Monad m) => Frame a b m r -> Frame b c m r -> Frame a c m r -(>->) = flip (<-<) - -newtype FrameC m r a b = FrameC { unFrameC :: Frame a b m r } - -instance (Monad m) => Category (FrameC m r) where - (FrameC p1) . (FrameC p2) = FrameC $ p1 <-< p2 - id = FrameC idF - -{- $run - The final step to convert 'Pipe' code to 'Frame' code is to replace - 'runPipe' with 'runFrame'. - -> printer :: Frame a Void IO r -> take :: (Monad m) => Int -> Frame a a m () -> fromList :: (Monad m) => [a] -> Frame () a m () - ->>> runFrame $ printer <-< contrived <-< fromList [1..] -1 -2 - ->>> runFrame $ printer <-< contrived <-< fromList [1] -1 -Stage 2 interrupted - ->>> runFrame $ printer <-< take 1 <-< contrived <-< fromList [1..] -Stage 1(b) interrupted -Stage 1 interrupted -1 - -For the last example, remember that 'take' is written to 'close' its input end -before yielding its final value, which is why the finalizers run before -@printer@ receives the 1. - --} - --- | Convert a 'Frame' back to the base monad. -runFrame :: (Monad m) => Stack m r -> m r -runFrame p = go (reopen p) where - go p = do - x <- runFreeT p - case x of - Pure r -> return r - Wrap (Await f) -> go $ f (Just ()) - Wrap (Yield y) -> go $ snd y diff --git a/Control/Pipe/Tutorial.hs b/Control/Pipe/Tutorial.hs new file mode 100644 --- /dev/null +++ b/Control/Pipe/Tutorial.hs @@ -0,0 +1,503 @@ +{-| + This module provides the tutorial for "Control.Pipe". +-} + +module Control.Pipe.Tutorial ( + -- * Types + -- $type + + -- * Composition + -- $compose + + -- * Modularity + -- $modular + + -- * Vertical Concatenation + -- $vertical + + -- * Return Values + -- $return + + -- * Termination + -- $terminate + + -- * Resource Management + -- $resource + + -- *Frames + -- $frames + ) where + +-- For documentation +import Control.Category +import Control.Frame hiding (await, yield) +import Control.Monad.Trans.Class +import Control.Pipe +import Data.Void + +{- $type + This library represents streaming computations using a single data type: + 'Pipe'. + + 'Pipe' is a monad transformer that extends the base monad with the ability + to 'await' input from or 'yield' output to other 'Pipe's. 'Pipe's resemble + enumeratees in other libraries because they receive an input stream and + transform it into a new output stream. + + I'll introduce our first 'Pipe', which is a verbose version of the Prelude's + 'take' function: + +> take' :: Int -> Pipe a a IO () +> take' n = do +> replicateM_ n $ do +> x <- await +> yield x +> lift $ putStrLn "You shall not pass!" + + This 'Pipe' forwards the first @n@ values it receives undisturbed, then it + outputs a cute message. + + Let's dissect the above 'Pipe''s type to learn a bit about how 'Pipe's work: + +> | Input Type | Output Type | Base monad | Return value +> Pipe a a IO () + + So @take'@ 'await's input values of type \'@a@\' from upstream 'Pipe's and + 'yield's output values of type \'@a@\' to downstream 'Pipe's. @take'@ uses + 'IO' as its base monad because it invokes the 'putStrLn' function. If we + were to remove the call to 'putStrLn', the compiler would infer the + following type instead, which is polymorphic in the base monad: + +> take' :: (Monad m) => Int -> Pipe a a m () + + Now let's create a function that converts a list into a 'Pipe' by 'yield'ing + each element of the list: + +> fromList :: (Monad m) => [b] -> Pipe a b m () +> fromList = mapM_ yield + + Note that @fromList xs@ is polymorphic in its input. This is because it + does not 'await' any input. If we wanted, we could type-restrict it to: + +> fromList :: (Monad m) => [b] -> Pipe () b m () + + There is no type that forbids a 'Pipe' from 'await'ing, but you can + guarantee that if it does 'await', the request is trivially satisfiable by + supplying it with @()@. + + A 'Pipe' that doesn't 'await' (any useful input) can serve as the first + stage in a 'Pipeline'. I provide a type synonym for this common case: + +> type Producer b m r = Pipe () b m r + + 'Producer's resemble enumerators in other libraries because they function as + data sources. + + You can then use the 'Producer' type synonym to rewrite the type signature + for @fromList@ as: + +> fromList :: (Monad m) => [b] -> Producer b m () + + Now let's create a 'Pipe' that prints every value delivered to it: + +> printer :: (Show b) => Pipe b c IO r +> printer = forever $ do +> x <- await +> lift $ print x + + Here, @printer@ is polymorphic in its output. We could type-restrict it to + guarantee it will never 'yield' by setting the output to 'Void', from + @Data.Void@: + +> printer :: (Show b) => Pipe b Void IO r + + A 'Pipe' that never 'yield's can be the final stage in a 'Pipeline'. Again, + I provide a type synonym for this common case: + +> type Consumer b m r = Pipe b Void m r + + So we could instead write @printer@'s type as: + +> printer :: (Show b) => Consumer b IO r + + 'Consumer's resemble iteratees in other libraries because they function as + data sinks. +-} + +{- $compose + What distinguishes 'Pipe's from every other iteratee implementation is that + they form a true 'Category'. Because of this, you can literally compose + 'Pipe's into 'Pipeline's using ordinary composition: + +> newtype PipeC m r a b = PipeC { unPipeC :: Pipe a b m r } +> instance Category (PipeC m r) where ... + + For example, you can compose the above 'Pipe's with: + +> pipeline :: Pipe () Void IO () +> pipeline = unPipeC $ PipeC printer . PipeC (take' 3) . PipeC (fromList [1..]) + + The compiler deduces that the final 'Pipe' must be blocked at both ends, + meaning it will never 'await' useful input and it will never 'yield' any + output. This represents a self-contained 'Pipeline' and I provide a type + synonym for this common case: + +> type Pipeline m r = Pipe () Void m r + + Also, I provide '<+<' as a convenience operator for composing 'Pipe's + without the burden of wrapping and unwrapping newtypes: + +> p1 <+< p2 == unPipeC $ PipeC p1 . PipeC p2 + + So you can rewrite @pipeline@ as: + +> pipeline :: Pipeline IO () +> pipeline = printer <+< take' 3 <+< fromList [1..] + + Like many other monad transformers, you convert the 'Pipe' monad back to the + base monad using some sort of \"@run...@\" function. In this case, it's the + 'runPipe' function: + +> runPipe :: (Monad m) => Pipeline m r -> m r + + 'runPipe' only works on self-contained 'Pipeline's, but you don't need to + worry about explicitly type-restricting any of your 'Pipe's. Self-contained + 'Pipeline's will automatically have polymorphic input and output ends and + they will type-check when you provide them to 'runPipe'. + + Let's try using 'runPipe': + +>>> runPipe pipeline +1 +2 +3 +You shall not pass! + + Fascinating! Our 'Pipe' terminates even though @printer@ never terminates + and @fromList@ never terminates when given an infinite list. To illustrate + why our 'Pipe' terminates, let's outline the 'Pipe' flow control rules for + composition: + + * 'Pipe's are lazy, so execution begins at the most downstream 'Pipe' + (@printer@ in our example). + + * Upstream 'Pipe's only run if input is requested from them and they only + run as long as necessary to 'yield' back a value. + + * If a 'Pipe' terminates, it terminates every other 'Pipe' composed with it. + + Another way to think of this is like a stack where each 'Pipe' is a frame on + that stack: + + * If a 'Pipe' 'await's input, it blocks and pushes the next 'Pipe' upstream + onto the stack until that 'Pipe' 'yield's back a value. + + * If a 'Pipe' 'yield's output, it pops itself off the stack and restores + control to the original downstream 'Pipe' that was 'await'ing its input. + This binds its result to the return value of the pending 'await' command. + + All of these flow control rules uniquely follow from the 'Category' laws. + + It might surprise you that termination brings down the entire 'Pipeline' + until you realize that: + + * Downstream 'Pipe's depending on the terminated 'Pipe' cannot proceed + + * Upstream 'Pipe's won't be further evaluated because the terminated 'Pipe' + will not request any further input from them + + So in our previous example, the 'Pipeline' terminated because \"@take' 3@\" + terminated and brought down the entire 'Pipeline' with it. + + Actually, these flow control rules will mislead you into thinking that + composed 'Pipe's behave as a collection of sub-'Pipe's with some sort of + message passing architecture between them, but nothing could be further from + the truth! When you compose 'Pipe's, they automatically fuse into a single + 'Pipe' that corresponds to how you would have written the control flow by + hand. + + For example, if you compose @printer@ and @fromList@: + +> printer <+< fromList [1..] + + The result is indistinguishable from: + +> lift (mapM_ print [1..]) + + ... which is what we would have written by hand if we had not used 'Pipe's + at all! All 'runPipe' does is just remove the 'lift'! +-} + +{- $modular + Given a loop like: + +> loop :: IO r +> loop = forever $ do +> x <- dataSource +> y <- processData x +> dataSink y + + We could decompose it into three separate parts: + +> stage1 :: Producer a IO r +> stage1 = forever $ do +> x <- dataSource +> yield x +> +> stage2 :: Pipe a b IO r +> stage2 = forever $ do +> x <- await +> y <- processData x +> yield y +> +> +> stage3 :: Consumer b IO r +> stage3 = forever $ do +> y <- await +> dataSink y +> +> stage3 <+< stage2 <+< stage1 = lift loop + + In other words, 'Pipe's let you decompose loops into modular components, + which promotes loose coupling and allows you to freely mix and match those + components. + + To demonstrate this, let's define a new data source that indefinitely + prompts the user for integers: + +> prompt :: Producer Int IO a +> prompt = forever $ do +> lift $ putStrLn "Enter a number: " +> n <- read <$> lift getLine +> yield n + + Now we can use it as a drop-in replacement for @fromList@: + +>>> runPipe $ printer <+< take' 3 <+< prompt +Enter a number: +1 +1 +Enter a number: +2 +2 +Enter a number: +3 +3 +You shall not pass! + +-} + +{- $vertical + You can easily \"vertically\" concatenate 'Pipe's, 'Producer's, and + 'Consumer's, all using simple monad sequencing: ('>>'). For example, here + is how you concatenate 'Producer's: + +>>> runPipe $ printer <+< (fromList [1..3] >> fromList [10..12]) +1 +2 +3 +10 +11 +12 + + Here's how you would concatenate 'Consumer's: + +>>> let print' n = printer <+< take' n :: (Show a) => Int -> Consumer a IO () +>>> runPipe $ (print' 3 >> print' 4) <+< fromList [1..] +1 +2 +3 +You shall not pass! +4 +5 +6 +7 +You shall not pass! + + ... but the above example is gratuitous because we could have just + concatenated the intermediate @take'@ 'Pipe': + +>>> runPipe $ printer <+< (take' 3 >> take' 4) <+< fromList [1..] +1 +2 +3 +You shall not pass! +4 +5 +6 +7 +You shall not pass! + +-} + +{- $return + 'Pipe' composition imposes an important requirement: You can only compose + 'Pipe's that have the same return type. For example, I could write the + following function: + +> deliver :: (Monad m) => Int -> Consumer a m [a] +> deliver n = replicateM n await + + ... and I might try to compose it with @fromList@: + +>>> runPipe $ deliver 3 <+< fromList [1..10] -- wrong! + + ... but this wouldn't type-check, because @fromList@ has a return type of + @()@ and @deliver@ has a return type of @[Int]@. Composition requires that + every 'Pipe' has a return value ready in case it terminates first. + + Fortunately, we don't have to rewrite the @fromList@ function because we can + just add a return value using vertical concatenation: + +>>> runPipe $ deliver 3 <+< (fromList [1..10] >> return []) +[1,2,3] + + ... although a more idiomatic Haskell version would be: + +>>> runPipe $ (Just <$> deliver 3) <+< (fromList [1..10] *> pure Nothing) +Just [1,2,3] + + This forces you to cover all code paths by thinking about what return value + you would provide if something were to go wrong. For example, let's say I + were to make a mistake and request more input than @fromList@ can deliver: + +>>> runPipe $ (Just <$> deliver 99) <+< (fromList [1..10] *> pure Nothing) +Nothing + + The type system saved me by forcing me to cover all corner cases and handle + every way my program could terminate. +-} + +{- $terminate + + Now what if you wanted to write a 'Pipe' that only reads from its input end + (i.e. a 'Consumer') and returns a list of every value delivered to it when + its input 'Pipe' terminates? + +> toList :: (Monad m) => Consumer a m [a] +> toList = ??? + + You can't write such a 'Pipe' because if its input terminates then it brings + down @toList@ with it! This is correct because @toList@ as defined is not + compositional (yet!). + + To see why, let's say you somehow got @toList@ to work and the following + imaginary code sample worked: + +>>> runPipe $ toList <+< (fromList [1..5] >> return []) +[1,2,3,4,5] + + @toList@ is defined to return its value when the 'Pipe' immediately upstream + (@fromList@ in this case) terminates. This behavior immediately leads to a + problem. What if I were to insert an \"identity\" 'Pipe' between @toList@ + and @fromList@: + +> identity = forever $ await >>= yield +> -- This is how id is actually implemented! + + This 'Pipe' forwards every valued untouched, so we would expect it to not + have any affect if we were to insert it in the middle: + +>>> runPipe $ toList <+< identity <+< (fromList [1..5] >> return []) +??? -- Oops! Something other than [1,2,3,4,5], perhaps even non-termination + + The answer couldn't be @[1,2,3,4,5]@ because @toList@ would monitor + @identity@ instead of @fromList@ and since @identity@ never terminates + @toList@ never terminates. This is what I mean when I say that @toList@'s + specified behavior is non-compositional. It only works if it is coupled + directly to the desired 'Pipe' and breaks when you introduce intermediate + stages. + + This was not an intentional design choice, but rather a direct consequence + of enforcing the 'Category' laws when I was implementing 'Pipe''s 'Category' + instance. Satisfying the 'Category' laws forces code to be compositional. + + Note that a terminated 'Pipe' only brings down 'Pipe's composed with it. To + illustrate this, let's use the following example: + +> p = do a <+< b +> c + + @a@, @b@, and @c@ are 'Pipe's, and @c@ shares the same input and output as + the composite 'Pipe' @a <+< b@, otherwise we cannot combine them within the + same monad. In the above example, either @a@ or @b@ could terminate and + bring down the other one since they are composed, but @c@ is guaranteed to + continue after @a <+< b@ terminates because it is not composed with them. + Conceptually, we can think of this as @c@ automatically taking over the + 'Pipe''s channeling responsibilities when @a <+< b@ can no longer continue. + There is no need to \"restart\" the input or output manually as in some + other iteratee libraries. + + The @pipes@ library, unlike other iteratee libraries, grounds its vertical + and horizontal concatenation in category theory by deriving horizontal + concatenation ('.') from its 'Category' instance and vertical concatenation + ('>>') from its 'Monad' instance. This makes it easier to reason about + 'Pipe's because you can leverage your intuition about 'Category's and + 'Monad's to understand their behavior. The only 'Pipe'-specific primitives + are 'await' and 'yield'. +-} + +{- $resource + Here's another problem with 'Pipe' composition: resource finalization. + Let's say we have the file \"@test.txt@\" with the following contents: + +> Line 1 +> Line 2 +> Line 3 + + .. and we wish to lazily read one line at a time from it: + +> readFile' :: Handle -> Producer Text IO () +> readFile' h = do +> eof <- lift $ hIsEOF h +> when (not eof) $ do +> s <- lift $ hGetLine h +> yield s +> readFile' h + + We could then try to be slick and write a lazy version that only reads as + many lines as we request: + +> read' :: FilePath -> Producer Text IO () +> read' file = do +> lift $ putStrLn "Opening file ..." +> h <- lift $ openFile file ReadMode +> readFile' h +> lift $ putStrLn "Closing file ..." +> lift $ hClose h + + Now compose! + +>>> runPipe $ printer <+< read' "test.xt" +Opening file ... +"Line 1" +"Line 2" +"Line 3" +Closing file ... + + So far, so good. Equally important, the file is never opened if we replace + @printer@ with a 'Pipe' that never demands input: + +>>> runPipe $ (lift $ putStrLn "I don't need input") <+< read' "test.txt" +I don't need input + + There is still one problem, though. What if we wrote: + +>>> runPipe $ printer <+< take' 2 <+< read' "test.txt" +Opening file ... +"Line 1" +"Line 2" +You shall not pass! + + Oh no! While it was lazy and only read two lines from the file, it was also + too lazy to properly close our file! \"@take' 2@\" terminated before + @read'@, preventing @read'@ from properly closing \"test.txt\". This is why + 'Pipe' composition fails to guarantee deterministic finalization. +-} + +{- $frames + So with 'Pipe's, we can neither write folds, nor can we finalize resources + deterministically. Fortunately, there is a solution: 'Frame's. Check out + "Control.Frame.Tutorial" for an introduction to a type that enriches 'Pipe's + with the ability to fold and finalize resources correctly. +-} diff --git a/pipes.cabal b/pipes.cabal --- a/pipes.cabal +++ b/pipes.cabal @@ -1,5 +1,5 @@ Name: pipes -Version: 2.0.0 +Version: 2.1.0 Cabal-Version: >=1.10.1 Build-Type: Simple License: BSD3 @@ -39,20 +39,22 @@ Vertical Concatenation always works the way you expect, picking up where the previous 'Pipe' left off. . - Check out "Control.Pipe" for a copious tutorial and "Control.Pipe.Common" for - the actual implementation. + Check out "Control.Pipe.Tutorial" for a copious introductory tutorial and + "Control.Pipe" for the actual implementation. Category: Control, Enumerator -Tested-With: GHC ==7.0.3 +Tested-With: GHC ==7.4.1 Source-Repository head Type: git Location: https://github.com/Gabriel439/Haskell-Pipes-Library Library - Build-Depends: base >= 4 && < 5, transformers, void + Build-Depends: base >= 4 && < 5, transformers, void, index-core Exposed-Modules: + Control.Frame, + Control.Frame.Tutorial, + Control.IMonad.Trans.Free, + Control.Monad.Trans.Free, Control.Pipe, - Control.Pipe.Common, - Control.Pipe.Final, - Control.Monad.Trans.Free + Control.Pipe.Tutorial GHC-Options: -O2 Default-Language: Haskell2010