diff --git a/Benchmark/System/Process.hs b/Benchmark/System/Process.hs --- a/Benchmark/System/Process.hs +++ b/Benchmark/System/Process.hs @@ -23,7 +23,6 @@ -- Internal imports import qualified Streamly.Internal.FileSystem.Handle as FH -import qualified Streamly.Internal.System.Process as Proc -- XXX replace with streamly versions once they are fixed {-# INLINE rights #-} diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.3.1 (Dec 2023) + +* Allow streamly-0.10.0 and streamly-core-0.2.0 +* Fix a bug in quote escaping in the Command module +* Add APIs in System.Process and System.Command module with ability to set + process attributes. + ## 0.3.0 (Apr 2023) * Added a `Streamly.System.Command` module diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ Command.toBytes [str|echo "hello world"|] -- Stream IO Word8 & Command.pipeBytes [str|tr [a-z] [A-Z]|] -- Stream IO Word8 & Stream.fold Stdio.write -- IO () - :} - HELLO WORLD +:} +HELLO WORLD ``` ## Shell commands as streaming functions @@ -45,8 +45,8 @@ >>> :{ Command.toBytes [str|sh "-c" "echo 'hello world' | tr [a-z] [A-Z]"|] -- Stream IO Word8 & Stream.fold Stdio.write -- IO () - :} - HELLO WORLD +:} +HELLO WORLD ``` ## Running Commands Concurrently @@ -61,7 +61,7 @@ Command.toBytes [str|grep -H "pattern" #{file}|] -- Stream IO Word8 & Stream.handle (\(_ :: Command.ProcessFailure) -> Stream.nil) -- Stream IO Word8 & Stream.foldMany (Fold.takeEndBy (== 10) Array.write) -- Stream IO (Array Word8) - :} +:} >>> :{ pgrep = diff --git a/src/DocTestCommand.hs b/src/DocTestCommand.hs new file mode 100644 --- /dev/null +++ b/src/DocTestCommand.hs @@ -0,0 +1,20 @@ +{- $setup + +>>> :set -XFlexibleContexts +>>> :set -XQuasiQuotes +>>> import Data.Char (toUpper) +>>> import Data.Function ((&)) +>>> import Streamly.Unicode.String (str) +>>> import qualified Streamly.Console.Stdio as Stdio +>>> import qualified Streamly.Data.Array as Array +>>> import qualified Streamly.Data.Fold as Fold +>>> import qualified Streamly.Data.Stream.Prelude as Stream +>>> import qualified Streamly.System.Command as Command +>>> import qualified Streamly.Unicode.Stream as Unicode + +For APIs that have not been released yet. + +>>> import qualified Streamly.Internal.Console.Stdio as Stdio (putBytes, putChars, putChunks) +>>> import qualified Streamly.Internal.FileSystem.Dir as Dir (readFiles) +>>> import qualified Streamly.Internal.System.Process as Process +-} diff --git a/src/DocTestProcess.hs b/src/DocTestProcess.hs new file mode 100644 --- /dev/null +++ b/src/DocTestProcess.hs @@ -0,0 +1,20 @@ +{- $setup + +>>> :set -XFlexibleContexts +>>> :set -XScopedTypeVariables +>>> import Data.Char (toUpper) +>>> import Data.Function ((&)) +>>> import qualified Streamly.Console.Stdio as Stdio +>>> import qualified Streamly.Data.Array as Array +>>> import qualified Streamly.Data.Fold as Fold +>>> import qualified Streamly.Data.Stream.Prelude as Stream +>>> import qualified Streamly.System.Process as Process +>>> import qualified Streamly.Unicode.Stream as Unicode + +For APIs that have not been released yet. + +>>> import qualified Streamly.Internal.Console.Stdio as Stdio (putChars, putChunks) +>>> import qualified Streamly.Internal.FileSystem.Dir as Dir (readFiles) +>>> import qualified Streamly.Internal.System.Process as Process +>>> import qualified Streamly.Internal.Unicode.Stream as Unicode (lines) +-} diff --git a/src/Streamly/Internal/System/Command.hs b/src/Streamly/Internal/System/Command.hs --- a/src/Streamly/Internal/System/Command.hs +++ b/src/Streamly/Internal/System/Command.hs @@ -15,6 +15,7 @@ -- * Generation toBytes , toChunks + , toChunksWith , toChars , toLines @@ -27,8 +28,15 @@ , pipeBytes , pipeChars , pipeChunks + , pipeChunksWith + -- * Standalone Processes + , standalone + , foreground + , daemon + -- * Helpers + , quotedWord , runWith , streamWith , pipeWith @@ -42,6 +50,9 @@ import Streamly.Data.Fold (Fold) import Streamly.Data.Parser (Parser) import Streamly.Data.Stream.Prelude (MonadAsync, Stream) +import Streamly.Internal.System.Process (Config) +import System.Exit (ExitCode(..)) +import System.Process (ProcessHandle) import qualified Streamly.Data.Fold as Fold import qualified Streamly.Data.Parser as Parser @@ -50,23 +61,18 @@ -- Keep it synced with the released module --- $setup --- >>> :set -XFlexibleContexts --- >>> :set -XQuasiQuotes --- >>> import Data.Char (toUpper) --- >>> import Data.Function ((&)) --- >>> import Streamly.Unicode.String (str) --- >>> import qualified Streamly.Data.Array as Array --- >>> import qualified Streamly.Console.Stdio as Stdio --- >>> import qualified Streamly.Data.Fold as Fold --- >>> import qualified Streamly.Data.Stream.Prelude as Stream --- >>> import qualified Streamly.System.Command as Command --- >>> import qualified Streamly.Unicode.Stream as Unicode --- --- >>> import qualified Streamly.Internal.System.Process as Process --- >>> import qualified Streamly.Internal.Console.Stdio as Stdio --- >>> import qualified Streamly.Internal.FileSystem.Dir as Dir +#include "DocTestCommand.hs" +-- | Posix compliant quote escaping: +-- +-- $ echo 'hello\\"world' +-- hello\\"world +-- +-- $ echo "hello\"\\w\'orld" +-- hello"\w\'orld +-- +-- $ echo 'hello\' +-- hello\ {-# INLINE quotedWord #-} quotedWord :: MonadCatch m => Parser Char m String quotedWord = @@ -75,7 +81,15 @@ '"' -> Just x '\'' -> Just x _ -> Nothing - trEsc q x = if q == x then Just x else Nothing + -- Inside ", + -- \\ is translated to \ + -- \" is translated to " + trEsc '"' x = + case x of + '\\' -> Just '\\' + '"' -> Just '"' + _ -> Nothing + trEsc _ _ = Nothing in Parser.wordWithQuotes False trEsc '\\' toRQuote isSpace Fold.toList -- | A modifier for stream generation APIs in "Streamly.System.Process" to @@ -153,6 +167,15 @@ y:ys -> return $ f y ys input _ -> error "streamWith: empty command" +-- | Like 'pipeChunks' but use the specified configuration to run the process. +{-# INLINE pipeChunksWith #-} +pipeChunksWith :: + (MonadCatch m, MonadAsync m) + => (Config -> Config) -- ^ Config modifier + -> String -- ^ Command + -> Stream m (Array Word8) -- ^ Input stream + -> Stream m (Array Word8) -- ^ Output stream +pipeChunksWith modifier = pipeWith (Process.pipeChunksWith modifier) -- | @pipeChunks command input@ runs the executable with arguments specified by -- @command@ and supplying @input@ stream as its standard input. Returns the @@ -165,7 +188,7 @@ -- If the input stream throws an exception or if the output stream is garbage -- collected before it could finish then the process is terminated with SIGTERM. -- --- If the process terminates with a non-zero exit code then a 'ProcessFailure' +-- If the process terminates with a non-zero exit code then a 'Process.ProcessFailure' -- exception is raised. -- -- The following code is equivalent to the shell command @echo "hello world" | @@ -220,10 +243,9 @@ -- Generation ------------------------------------------------------------------------------- --- | --- -- >>> toBytes = streamWith Process.toBytes --- + +-- | -- >>> toBytes "echo hello world" & Stdio.putBytes --hello world -- >>> toBytes "echo hello\\ world" & Stdio.putBytes @@ -238,10 +260,18 @@ toBytes :: (MonadAsync m, MonadCatch m) => String -> Stream m Word8 toBytes = streamWith Process.toBytes --- | --- +-- | Like 'toChunks' but use the specified configuration to run the process. +{-# INLINE toChunksWith #-} +toChunksWith :: + (MonadCatch m, MonadAsync m) + => (Config -> Config) -- ^ Config modifier + -> String -- ^ Command + -> Stream m (Array Word8) -- ^ Output stream +toChunksWith modifier = streamWith (Process.toChunksWith modifier) + -- >>> toChunks = streamWith Process.toChunks --- + +-- | -- >>> toChunks "echo hello world" & Stdio.putChunks --hello world -- @@ -250,9 +280,9 @@ toChunks :: (MonadAsync m, MonadCatch m) => String -> Stream m (Array Word8) toChunks = streamWith Process.toChunks --- | -- >>> toChars = streamWith Process.toChars --- + +-- | -- >>> toChars "echo hello world" & Stdio.putChars --hello world -- @@ -261,9 +291,9 @@ toChars :: (MonadAsync m, MonadCatch m) => String -> Stream m Char toChars = streamWith Process.toChars --- | -- >>> toLines f = streamWith (Process.toLines f) --- + +-- | -- >>> toLines Fold.toList "echo -e hello\\\\nworld" & Stream.fold Fold.toList -- ["hello","world"] -- @@ -276,9 +306,9 @@ -> Stream m a -- ^ Output Stream toLines f = streamWith (Process.toLines f) --- | -- >>> toString = runWith Process.toString --- + +-- | -- >>> toString "echo hello world" --"hello world\n" -- @@ -290,9 +320,9 @@ -> m String toString = runWith Process.toString --- | -- >>> toStdout = runWith Process.toStdout --- + +-- | -- >>> toStdout "echo hello world" -- hello world -- @@ -304,9 +334,9 @@ -> m () toStdout = runWith Process.toStdout --- | -- >>> toNull = runWith Process.toNull --- + +-- | -- >>> toNull "echo hello world" -- -- /Pre-release/ @@ -316,3 +346,62 @@ => String -- ^ Command -> m () toNull = runWith Process.toNull + +------------------------------------------------------------------------------- +-- Processes not interacting with the parent process +------------------------------------------------------------------------------- + +-- | Launch a standlone process i.e. the process does not have a way to attach +-- the IO streams with other processes. The IO streams stdin, stdout, stderr +-- can either be inherited from the parent or closed. +-- +-- This API is more powerful than 'interactive' and 'daemon' and can be used to +-- implement both of these. However, it should be used carefully e.g. if you +-- inherit the IO streams and parent is not waiting for the child process to +-- finish then both parent and child may use the IO streams resulting in +-- garbled IO if both are reading/writing simultaneously. +-- +-- If the parent chooses to wait for the process an 'ExitCode' is returned +-- otherwise a 'ProcessHandle' is returned which can be used to terminate the +-- process, send signals to it or wait for it to finish. +{-# INLINE standalone #-} +standalone :: + Bool -- ^ Wait for process to finish? + -> (Bool, Bool, Bool) -- ^ close (stdin, stdout, stderr) + -> (Config -> Config) + -> String -- ^ Command + -> IO (Either ExitCode ProcessHandle) +standalone wait streams modCfg = + runWith (Process.standalone wait streams modCfg) + +-- | Launch a process interfacing with the user. User interrupts are sent to +-- the launched process and ignored by the parent process. The launched process +-- inherits stdin, stdout, and stderr from the parent, so that the user can +-- interact with the process. The parent waits for the child process to exit, +-- an 'ExitCode' is returned when the process finishes. +-- +-- This is the same as the common @system@ function found in other libraries +-- used to execute commands. +-- +-- On Windows you can pass @setSession NewConsole@ to create a new console. +-- +{-# INLINE foreground #-} +foreground :: + (Config -> Config) + -> String -- ^ Command + -> IO ExitCode +foreground modCfg = runWith (Process.foreground modCfg) + +-- | Launch a daemon process. Closes stdin, stdout and stderr, creates a new +-- session, detached from the terminal, the parent does not wait for the +-- process to finish. +-- +-- The 'ProcessHandle' returned can be used to terminate the daemon or send +-- signals to it. +-- +{-# INLINE daemon #-} +daemon :: + (Config -> Config) + -> String -- ^ Command + -> IO ProcessHandle +daemon modCfg = runWith (Process.daemon modCfg) diff --git a/src/Streamly/Internal/System/Process.hs b/src/Streamly/Internal/System/Process.hs --- a/src/Streamly/Internal/System/Process.hs +++ b/src/Streamly/Internal/System/Process.hs @@ -60,17 +60,18 @@ -} , closeFiles , newProcessGroup + , Session (..) , setSession -- * Posix Only Options -- | These options have no effect on Windows. - , parentIgnoresInterrupt + , interruptChildOnly , setUserId , setGroupId -- * Windows Only Options -- | These options have no effect on Posix. - , waitForChildTree + , waitForDescendants -- * Internal , inheritStdin @@ -111,11 +112,14 @@ , pipeChunksEitherWith -- * Standalone Processes - , standalone - , interactive + , foreground , daemon + , standalone -- * Deprecated + , parentIgnoresInterrupt + , waitForChildTree + , interactive , processBytes , processChunks ) @@ -170,17 +174,7 @@ import qualified Streamly.Unicode.Stream as Unicode import qualified Streamly.Internal.Unicode.Stream as Unicode (lines) --- $setup --- >>> :set -XFlexibleContexts --- >>> import Data.Char (toUpper) --- >>> import Data.Function ((&)) --- >>> import qualified Streamly.Internal.Console.Stdio as Stdio --- >>> import qualified Streamly.Data.Fold as Fold --- >>> import qualified Streamly.Data.Stream.Prelude as Stream --- >>> import qualified Streamly.Internal.System.Process as Process --- >>> import qualified Streamly.Unicode.Stream as Unicode --- >>> import qualified Streamly.Internal.Data.Stream as Stream --- >>> import qualified Streamly.Internal.Unicode.Stream as Unicode +#include "DocTestProcess.hs" ------------------------------------------------------------------------------- -- Config @@ -271,7 +265,7 @@ -- working directory is inherited from the parent process. -- -- Default is 'Nothing' - inherited from the parent process. -setCwd :: Maybe (FilePath) -> Config -> Config +setCwd :: Maybe FilePath -> Config -> Config setCwd path (Config cfg) = Config $ cfg { cwd = path } -- | Set the environment variables for the new process. When 'Nothing', the @@ -336,7 +330,9 @@ -- | If 'True' the new process starts a new process group, becomes a process -- group leader, its pid becoming the process group id. -- --- See the POSIX @setpgid@ man page. +-- See the POSIX +-- +-- man page. -- -- Default is 'False', the new process belongs to the parent's process group. newProcessGroup :: Bool -> Config -> Config @@ -346,7 +342,9 @@ -- parent process. This is the default. -- -- 'NewSession' makes the new process start with a new session without a --- controlling terminal. On POSIX, @setsid@ is used to create a new process +-- controlling terminal. On POSIX, +-- +-- is used to create a new process -- group and session, the pid of the new process is the session id and process -- group id as well. On Windows @DETACHED_PROCESS@ flag is used to detach the -- process from inherited console session. @@ -356,10 +354,12 @@ -- nothing. -- -- For Windows see +-- -- * https://learn.microsoft.com/en-us/windows/win32/procthread/process-creation-flags -- * https://learn.microsoft.com/en-us/windows/console/creation-of-a-console . -- --- For POSIX see, @setsid@ man page. +-- For POSIX see, +-- man page. data Session = InheritSession -- ^ Inherit the parent session | NewSession -- ^ Detach process from the current session @@ -376,11 +376,15 @@ NewSession -> cfg { new_session = True} NewConsole -> cfg {create_new_console = True} --- | Use the POSIX @setuid@ call to set the user id of the new process before +-- | Use the POSIX +-- +-- call to set the user id of the new process before -- executing the command. The parent process must have sufficient privileges to -- set the user id. -- --- POSIX only. See the POSIX @setuid@ man page. +-- POSIX only. See the POSIX +-- +-- man page. -- -- Default is 'Nothing' - inherit from the parent. setUserId :: Maybe Word32 -> Config -> Config @@ -392,11 +396,15 @@ Config $ cfg { child_user = CUid <$> x } #endif --- | Use the POSIX @setgid@ call to set the group id of the new process before +-- | Use the POSIX +-- +-- call to set the group id of the new process before -- executing the command. The parent process must have sufficient privileges to -- set the group id. -- --- POSIX only. See the POSIX @setgid@ man page. +-- POSIX only. See the POSIX +-- +-- man page. -- -- Default is 'Nothing' - inherit from the parent. setGroupId :: Maybe Word32 -> Config -> Config @@ -424,15 +432,23 @@ -- until the child exits. -- -- POSIX only. Default is 'False'. +interruptChildOnly :: Bool -> Config -> Config +interruptChildOnly x (Config cfg) = Config $ cfg { delegate_ctlc = x } + +{-# DEPRECATED parentIgnoresInterrupt "Use interruptChildOnly instead." #-} parentIgnoresInterrupt :: Bool -> Config -> Config -parentIgnoresInterrupt x (Config cfg) = Config $ cfg { delegate_ctlc = x } +parentIgnoresInterrupt = interruptChildOnly --- | On Windows, the parent waits for the entire tree of process i.e. including --- processes that are spawned by the child process. +-- | On Windows, the parent waits for the entire descendant tree of process +-- i.e. including processes that are spawned by the child process. -- -- Default is 'True'. +waitForDescendants :: Bool -> Config -> Config +waitForDescendants x (Config cfg) = Config $ cfg { use_process_jobs = x } + +{-# DEPRECATED waitForChildTree "Use waitForDescendants instead." #-} waitForChildTree :: Bool -> Config -> Config -waitForChildTree x (Config cfg) = Config $ cfg { use_process_jobs = x } +waitForChildTree = waitForDescendants pipeStdErr :: Config -> Config pipeStdErr (Config cfg) = Config $ cfg { std_err = CreatePipe } @@ -595,6 +611,8 @@ alloc = createProc' modCfg path args +-- | Like 'pipeChunksEither' but use the specified configuration to run the +-- process. {-# INLINE pipeChunksEitherWith #-} pipeChunksEitherWith :: (MonadCatch m, MonadAsync m) @@ -614,6 +632,8 @@ `parallel` fmap Right (toChunksClose stdoutH) run _ = error "pipeChunksEitherWith: Not reachable" +-- | Like 'pipeChunks' but also includes stderr as 'Left' stream in the +-- 'Either' output. {-# INLINE pipeChunksEither #-} pipeChunksEither :: (MonadCatch m, MonadAsync m) @@ -657,6 +677,7 @@ rightRdr = fmap Right Array.reader in Stream.unfoldMany (Unfold.either leftRdr rightRdr) output +-- | Like 'pipeChunks' but use the specified configuration to run the process. {-# INLINE pipeChunksWith #-} pipeChunksWith :: (MonadCatch m, MonadAsync m) @@ -790,6 +811,8 @@ -- Generation ------------------------------------------------------------------------------- +-- | Like 'toChunksEither' but use the specified configuration to run the +-- process. {-# INLINE toChunksEitherWith #-} toChunksEitherWith :: (MonadCatch m, MonadAsync m) @@ -807,6 +830,7 @@ `parallel` fmap Right (toChunksClose stdoutH) run _ = error "toChunksEitherWith: Not reachable" +-- | Like 'toChunks' but use the specified configuration to run the process. {-# INLINE toChunksWith #-} toChunksWith :: (MonadCatch m, MonadAsync m) @@ -873,7 +897,7 @@ let output = toChunks path args in Stream.unfoldMany Array.reader output --- | Like 'toBytes' but generates a stream of @Array Word8@ instead of a stream +-- | Like 'toBytesEither' but generates a stream of @Array Word8@ instead of a stream -- of @Word8@. -- -- >>> :{ @@ -886,7 +910,7 @@ -- -- >>> toChunksEither = toChunksEitherWith id -- --- Prefer 'toChunksEither over 'toBytesEither when performance matters. +-- Prefer 'toChunksEither' over 'toBytesEither' when performance matters. -- -- /Pre-release/ {-# INLINE toChunksEither #-} @@ -980,9 +1004,24 @@ toNull path args = toChunks path args & Stream.fold Fold.drain ------------------------------------------------------------------------------- --- Process not interacting with the parent process +-- Processes not interacting with the parent process ------------------------------------------------------------------------------- +-- XXX Make the return type ExitCode/ProcessHandle depend on the wait argument? + +-- | Launch a standalone process i.e. the process does not have a way to attach +-- the IO streams with other processes. The IO streams stdin, stdout, stderr +-- can either be inherited from the parent or closed. +-- +-- This API is more powerful than 'interactive' and 'daemon' and can be used to +-- implement both of these. However, it should be used carefully e.g. if you +-- inherit the IO streams and parent is not waiting for the child process to +-- finish then both parent and child may use the IO streams resulting in +-- garbled IO if both are reading/writing simultaneously. +-- +-- If the parent chooses to wait for the process an 'ExitCode' is returned +-- otherwise a 'ProcessHandle' is returned which can be used to terminate the +-- process, send signals to it or wait for it to finish. {-# INLINE standalone #-} standalone :: Bool -- ^ Wait for process to finish? @@ -998,7 +1037,7 @@ postCreate _ _ _ procHandle = if wait - then fmap Left $ waitForProcess procHandle + then Left <$> waitForProcess procHandle else return $ Right procHandle cfg = @@ -1008,52 +1047,62 @@ s_err = if close_stderr then NoStream else Inherit in c {std_in = s_in, std_out = s_out, std_err = s_err} --- | Inherits stdin, stdout, and stderr from the parent, so that the user can --- interact with the process, user interrupts are handled by the child process, --- the parent waits for the child process to exit. +-- | Launch a process interfacing with the user. User interrupts are sent to +-- the launched process and ignored by the parent process. The launched process +-- inherits stdin, stdout, and stderr from the parent, so that the user can +-- interact with the process. The parent waits for the child process to exit, +-- an 'ExitCode' is returned when the process finishes. -- --- This is same as the common @system@ function found in other libraries used --- to execute commands. +-- This is the same as the common @system@ function found in other libraries +-- used to execute commands. -- -- On Windows you can pass @setSession NewConsole@ to create a new console. -- +{-# INLINE foreground #-} +foreground :: + (Config -> Config) + -> FilePath -- ^ Executable name or path + -> [String] -- ^ Arguments + -> IO ExitCode +foreground modCfg path args = + let r = + standalone + True + (False, False, False) + (parentIgnoresInterrupt True . modCfg) + path args + in fmap (either id undefined) r + +{-# DEPRECATED interactive "Use foreground instead." #-} {-# INLINE interactive #-} interactive :: (Config -> Config) -> FilePath -- ^ Executable name or path -> [String] -- ^ Arguments -> IO ExitCode -interactive modCfg path args = - withCreateProcess cfg (\_ _ _ p -> waitForProcess p) - - where - - -- let child handle SIGINT/QUIT - modCfg1 = (parentIgnoresInterrupt True . modCfg) - - cfg = - let Config c = modCfg1 $ mkConfig path args - in c {std_in = Inherit, std_out = Inherit, std_err = Inherit} +interactive = foreground -- XXX ProcessHandle can be used to terminate the process. re-export -- terminateProcess? --- | Closes stdin, stdout and stderr, creates a new session, detached from the --- terminal, the parent does not wait for the process to finish. +-- | Launch a daemon process. Closes stdin, stdout and stderr, creates a new +-- session, detached from the terminal, the parent does not wait for the +-- process to finish. -- +-- The 'ProcessHandle' returned can be used to terminate the daemon or send +-- signals to it. +-- {-# INLINE daemon #-} daemon :: (Config -> Config) -> FilePath -- ^ Executable name or path -> [String] -- ^ Arguments -> IO ProcessHandle -daemon modCfg path args = withCreateProcess cfg (\_ _ _ p -> return p) - - where - - -- Detach terminal - modCfg1 = (setSession NewSession . modCfg) - - cfg = - let Config c = modCfg1 $ mkConfig path args - in c {std_in = NoStream, std_out = NoStream, std_err = NoStream} +daemon modCfg path args = + let r = + standalone + False + (True, True, True) + (setSession NewSession . modCfg) + path args + in fmap (either undefined id) r diff --git a/src/Streamly/System/Command.hs b/src/Streamly/System/Command.hs --- a/src/Streamly/System/Command.hs +++ b/src/Streamly/System/Command.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} -- | -- Module : Streamly.System.Command -- Copyright : (c) 2023 Composewell Technologies @@ -6,97 +7,18 @@ -- Stability : experimental -- Portability : GHC -- --- Use command strings to execute OS processes. These processes can be used --- just like native Haskell functions - to generate, transform or consume --- streams. It provides a powerful way to write high-level Haskell scripts to --- perform tasks similar to shell scripts without requiring the shell. --- Moreover, the Haskell scripts provide C-like performance. --- --- This module is a wrapper over the "Streamly.System.Process" module. --- --- See also: "Streamly.Internal.System.Command". --- -module Streamly.System.Command - ( - -- * Setup - -- | To execute the code examples provided in this module in ghci, please - -- run the following commands first. - -- - -- $setup - - -- * Overview - -- $overview - - -- * Types - ProcessFailure (..) - - -- * Generation - , toBytes - , toChunks - , toChars - , toLines - - -- * Effects - , toString - , toStdout - , toNull - - -- * Transformation - , pipeBytes - , pipeChars - , pipeChunks - - -- -- * Helpers - -- , runWith - -- , streamWith - -- , pipeWith - ) -where - -import Streamly.Internal.System.Command -import Streamly.Internal.System.Process (ProcessFailure (..)) - --- Keep it synced with the Internal module - --- $setup --- >>> :set -XFlexibleContexts --- >>> :set -XQuasiQuotes --- >>> import Data.Char (toUpper) --- >>> import Data.Function ((&)) --- >>> import Streamly.Unicode.String (str) --- >>> import qualified Streamly.Data.Array as Array --- >>> import qualified Streamly.Console.Stdio as Stdio --- >>> import qualified Streamly.Data.Fold as Fold --- >>> import qualified Streamly.Data.Stream.Prelude as Stream --- >>> import qualified Streamly.System.Command as Command --- >>> import qualified Streamly.Unicode.Stream as Unicode --- --- >>> import qualified Streamly.Internal.System.Process as Process --- >>> import qualified Streamly.Internal.Console.Stdio as Stdio --- >>> import qualified Streamly.Internal.FileSystem.Dir as Dir - --- Note: Commands are not executed using shell --- --- You can use this module to execute system commands and compose them with --- Haskell. It does not use the system shell to execute commands, they are --- executed as independent processes. This provides a convenient and powerful --- way to replace shell scripting with Haskell. Instead of composing commands --- using shell you can use Haskell Streamly streaming APIs to compose them with --- better efficiency and type safety. --- --- Normally, you should not need the system shell but if you want to use shell --- scripts in your program then you can take a look at the @streamly-shell@ --- package which provides convenient wrapper over "Streamly.System.Process" to --- execute shell scripts, commands. - --- $overview --- --- Please see the "Streamly.System.Process" for basics. +-- This module provides a way to invoke external executables and use them +-- seamlessly in a Haskell program, in a streaming fashion. This enables you to +-- write high-level Haskell scripts to perform tasks similar to shell scripts +-- without requiring the shell. Moreover, Haskell scripts provide C-like +-- performance. -- --- "Streamly.System.Process" module requires specifying the command executable --- name and its arguments separately (e.g. "ls" "-al") whereas using this --- module we can specify the executable and its arguments more conveniently as --- a single command string e.g. we can execute "ls -al". +-- Please see the "Streamly.System.Process" for basics. This module is a +-- wrapper over that module. "Streamly.System.Process" requires +-- specifying a command executable name and its arguments separately (e.g. +-- "ls" "-al") whereas using this module we can specify the executable and its +-- arguments more conveniently as a single command string e.g. we can execute +-- "ls -al". -- -- A command string is parsed in the same way as a posix shell would parse it. -- A command string consists of whitespace separated tokens with the first @@ -130,7 +52,9 @@ -- -- = Shell commands as functions -- --- If you want to execute the same command using the shell: +-- We recommend using streamly to compose commands natively in Haskell rather +-- than using the shell as shown in the previous example. However, if for some +-- reason you want to execute commands using the shell: -- -- >>> :{ -- Command.toBytes [str|sh "-c" "echo 'hello world' | tr [a-z] [A-Z]"|] @@ -140,7 +64,8 @@ -- -- = Running Commands Concurrently -- --- Running @grep@ concurrently on many files: +-- This example shows the power of composing in Haskell rather than using the +-- shell. Running @grep@ concurrently on many files: -- -- >>> :{ -- grep file = @@ -156,3 +81,103 @@ -- & Stream.fold Stdio.writeChunks -- :} -- +-- = Experimental APIs +-- +-- See "Streamly.Internal.System.Command" for unreleased APIs. +-- + +module Streamly.System.Command + ( + -- * Setup + -- | To execute the code examples provided in this module in ghci, please + -- run the following commands first. + -- + -- $setup + + -- * Exceptions + Process.ProcessFailure (..) + + -- * Process Configuration + -- | Use the config modifiers to modify the default config. + , Process.Config + + -- ** Common Modifiers + -- | These options apply to both POSIX and Windows. + , Process.setCwd + , Process.setEnv + , Process.closeFiles + , Process.newProcessGroup + , Process.Session (..) + , Process.setSession + + -- ** Posix Only Modifiers + -- | These options have no effect on Windows. + , Process.interruptChildOnly + , Process.setUserId + , Process.setGroupId + + -- ** Windows Only Modifiers + -- | These options have no effect on Posix. + , Process.waitForDescendants + + -- * Generation + , toBytes + , toChunks + , toChunksWith + , toChars + , toLines + + -- * Effects + , toString + , toStdout + , toNull + + -- * Transformation + , pipeChunks + , pipeChunksWith + , pipeBytes + , pipeChars + + -- -- * Including Stderr Stream + -- | Like other "Generation" routines but along with stdout, stderr is also + -- included in the output stream. stdout is converted to 'Right' values in + -- the output stream and stderr is converted to 'Left' values. + -- , toBytesEither + -- , toChunksEither + -- , toChunksEitherWith + -- , pipeBytesEither + -- , pipeChunksEither + -- , pipeChunksEitherWith + + -- * Non-streaming Processes + -- | These processes do not attach the IO streams with other processes. + , foreground + , daemon + , standalone + + -- -- * Helpers + -- , runWith + -- , streamWith + -- , pipeWith + ) +where + +import Streamly.Internal.System.Command +import qualified Streamly.Internal.System.Process as Process -- (ProcessFailure (..)) + +-- Keep it synced with the Internal module + +#include "DocTestCommand.hs" +-- Note: Commands are not executed using shell +-- +-- You can use this module to execute system commands and compose them with +-- Haskell. It does not use the system shell to execute commands, they are +-- executed as independent processes. This provides a convenient and powerful +-- way to replace shell scripting with Haskell. Instead of composing commands +-- using shell you can use Haskell Streamly streaming APIs to compose them with +-- better efficiency and type safety. +-- +-- Normally, you should not need the system shell but if you want to use shell +-- scripts in your program then you can take a look at the @streamly-shell@ +-- package which provides convenient wrapper over "Streamly.System.Process" to +-- execute shell scripts, commands. diff --git a/src/Streamly/System/Process.hs b/src/Streamly/System/Process.hs --- a/src/Streamly/System/Process.hs +++ b/src/Streamly/System/Process.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} -- | -- Module : Streamly.System.Process -- Copyright : (c) 2020 Composewell Technologies @@ -6,98 +7,20 @@ -- Stability : experimental -- Portability : GHC -- --- Use OS processes just like native Haskell functions - to generate, transform --- or consume streams. --- --- See "Streamly.System.Command" module for a higher level wrapper over this --- module. --- --- See also: "Streamly.Internal.System.Process" for unreleased functions. --- -module Streamly.System.Process - ( - -- * Setup - -- | To execute the code examples provided in this module in ghci, please - -- run the following commands first. - -- - -- $setup - - -- * Overview - -- $overview - - -- * Exceptions - -- | Since we are composing using Streamly's streaming pipeline there is - -- nothing special about exception handling, it works the same as in - -- Streamly. Like the @pipefail@ option in shells, exceptions are - -- propagated if any of the stages fail. - ProcessFailure (..) - - -- * Process Configuration - , Config - - {- - -- ** Common Config Options - -- | These options apply to both POSIX and Windows. - , setCwd - , setEnv - , closeFiles - , newProcessGroup - , setSession - - -- * Posix Only Options - -- | These options have no effect on Windows. - , parentIgnoresInterrupt - , setUserId - , setGroupId - - -- * Windows Only Options - -- | These options have no effect on Posix. - , waitForChildTree - -} - - -- * Generation - , toChunks - , toBytes - - -- * Transformation - , pipeChunks - , pipeBytes - - -- * Deprecated - , processChunks - , processBytes - ) -where - -import Streamly.Internal.System.Process - --- $setup --- >>> :set -XFlexibleContexts --- >>> :set -XScopedTypeVariables --- >>> import Data.Char (toUpper) --- >>> import Data.Function ((&)) --- >>> import qualified Streamly.Console.Stdio as Stdio --- >>> import qualified Streamly.Data.Array as Array --- >>> import qualified Streamly.Data.Fold as Fold --- >>> import qualified Streamly.Data.Stream.Prelude as Stream --- >>> import qualified Streamly.System.Process as Process --- >>> import qualified Streamly.Unicode.Stream as Unicode --- --- >>> import qualified Streamly.Internal.FileSystem.Dir as Dir --- >>> import qualified Streamly.Internal.Data.Stream as Stream - --- $overview --- -- This module provides functions to run operating system processes as stream --- source, sink or transformation functions. Thus OS processes can be used in --- the same way as Haskell functions and all the streaming combinators in --- streamly can be used to combine them. This allows you to seamlessly --- integrate external programs into your Haskell program. +-- producers, consumers or stream transformation functions. Thus OS processes +-- can be used in the same way as Haskell functions and all the streaming +-- combinators in streamly can be used to combine them. This allows you to +-- seamlessly integrate external binary executables into your Haskell program. -- --- We recommend using Haskell functions with Streamly threads for performing --- tasks whenever possible. This approach offers a simpler programming model --- compared to system processes, which also have a larger performance overhead. +-- However, we recommend native Haskell functions with Streamly threads over +-- using system processes whenever possible. This approach offers a simpler +-- programming model compared to system processes, which also have a larger +-- performance overhead. -- +-- Prefer "Streamly.System.Command" module as a higher level wrapper over this +-- module. +-- -- = Executables as functions -- -- Processes can be composed in a streaming pipeline just like a Posix shell @@ -150,3 +73,90 @@ -- & Stream.parConcatMap id grep -- & Stream.fold Stdio.writeChunks -- :} +-- +-- = Experimental APIs +-- +-- See "Streamly.Internal.System.Process" for unreleased functions. +-- + +module Streamly.System.Process + ( + -- * Setup + -- | To execute the code examples provided in this module in ghci, please + -- run the following commands first. + -- + -- $setup + + -- * Exceptions + -- | Since we are composing using Streamly's streaming pipeline there is + -- nothing special about exception handling, it works the same as in + -- Streamly. Like the @pipefail@ option in shells, exceptions are + -- propagated if any of the stages fail. + ProcessFailure (..) + + -- * Process Configuration + -- | Use the config modifiers to modify the default config. + , Config + + -- ** Common Modifiers + -- | These options apply to both POSIX and Windows. + , setCwd + , setEnv + , closeFiles + , newProcessGroup + , Session (..) + , setSession + + -- ** Posix Only Modifiers + -- | These options have no effect on Windows. + , interruptChildOnly + , setUserId + , setGroupId + + -- ** Windows Only Modifiers + -- | These options have no effect on Posix. + , waitForDescendants + + -- * Generation + , toChunks + , toChunksWith + , toBytes + , toChars + , toLines + + -- * Effects + , toString + , toStdout + , toNull + + -- * Transformation + , pipeChunks + , pipeChunksWith + , pipeBytes + + -- * Including Stderr Stream + -- | Like other "Generation" routines but along with stdout, stderr is also + -- included in the output stream. stdout is converted to 'Right' values in + -- the output stream and stderr is converted to 'Left' values. + , toBytesEither + , toChunksEither + , toChunksEitherWith + , pipeBytesEither + , pipeChunksEither + , pipeChunksEitherWith + + -- * Non-streaming Processes + -- | These processes do not attach the IO streams with other processes. + , foreground + , daemon + , standalone + + -- * Deprecated + , processChunks + , processBytes + ) +where + +import Streamly.Internal.System.Process + +#include "DocTestProcess.hs" diff --git a/streamly-process.cabal b/streamly-process.cabal --- a/streamly-process.cabal +++ b/streamly-process.cabal @@ -1,6 +1,6 @@ cabal-version: 2.2 name: streamly-process -version: 0.3.0 +version: 0.3.1 synopsis: Use OS processes as stream transformation functions description: Use operating system (OS) commands in Haskell programs as if they were @@ -30,6 +30,8 @@ NOTICE README.md design/proposal.md + src/DocTestCommand.hs + src/DocTestProcess.hs test/data/failExec.bat test/data/failExec.sh test/data/passExec.bat @@ -81,6 +83,7 @@ library import: compile-options, optimization-options + include-dirs: src hs-source-dirs: src exposed-modules: Streamly.System.Process @@ -94,8 +97,8 @@ base >= 4.8 && < 5 , exceptions >= 0.8 && < 0.11 -- Uses internal APIs - , streamly == 0.9.0.* - , streamly-core == 0.1.0 + , streamly >= 0.9 && < 0.10 + , streamly-core >= 0.1 && < 0.2 if !flag(use-native) build-depends: process >= 1.0 && < 1.7 else @@ -123,7 +126,7 @@ , base >= 4.8 && < 5 , directory >= 1.2.2 && < 1.4 -- Uses internal APIs - , streamly-core == 0.1.0.* + , streamly-core , tasty-bench >= 0.2.5 && < 0.4 if flag(fusion-plugin) && !impl(ghc < 8.6) @@ -146,4 +149,4 @@ , hspec >= 2.0 && < 3 , QuickCheck >= 2.10 && < 2.15 -- Uses internal APIs - , streamly-core == 0.1.0.* + , streamly-core diff --git a/test/Streamly/System/Process.hs b/test/Streamly/System/Process.hs --- a/test/Streamly/System/Process.hs +++ b/test/Streamly/System/Process.hs @@ -16,7 +16,7 @@ import System.Directory (removeFile, findExecutable, doesFileExist) import System.Exit (exitSuccess) import System.IO (IOMode(..), openFile, hClose) -import Test.Hspec (hspec, describe) +import Test.Hspec (describe, hspec, it, shouldBe) import Test.Hspec.QuickCheck import Test.QuickCheck ( forAll @@ -31,10 +31,10 @@ import qualified Streamly.Data.Fold as Fold import qualified Streamly.Data.Stream as S import qualified Streamly.System.Process as Proc +import qualified Streamly.Data.Stream as Stream import qualified Streamly.Internal.FileSystem.Handle as FH (putBytes, read) -import qualified Streamly.Internal.System.Process as Proc - (pipeChunksEither, pipeBytesEither, toChunksEither, toBytesEither) +import qualified Streamly.Internal.System.Command as Cmd (quotedWord) newtype SimpleError = SimpleError String deriving Show @@ -466,6 +466,14 @@ checkFailAction = catch action failAction +quotedWordTest :: String -> [String] -> IO () +quotedWordTest inp expected = do + res <- + Stream.fold Fold.toList + $ Stream.catRights + $ Stream.parseMany Cmd.quotedWord $ Stream.fromList inp + res `shouldBe` expected + main :: IO () main = do -- Nix does not have "/usr/bin/env", so the execution of test executables @@ -474,7 +482,7 @@ -- https://github.com/haskell/cabal/issues/7577 . r <- doesFileExist "/usr/bin/env" unless r $ do - putStrLn $ "/usr/bin/env does not exist, skipping tests." + putStrLn "/usr/bin/env does not exist, skipping tests." exitSuccess hspec $ do @@ -522,3 +530,15 @@ describe "toBytesEither" $ do prop "toBytesEither cat = FH.toBytes" toBytes1 prop "toBytesEither on failing executable" toBytes2 + + describe "quotedWord" $ do + it "Single quote test" $ + quotedWordTest "'hello\\\\\"world'" ["hello\\\\\"world"] + it "Double quote test" $ + quotedWordTest + "\"hello\\\"\\\\w\\'orld\"" + ["hello\"\\w\\'orld"] + -- TODO: We need to let the escape character be at the end + -- "wordWithQuotes" needs to be fixed! + -- it "Double quote test" $ + -- quotedWordTest "'hello\'" ["hello\\"]