shelltestrunner 0.8 → 0.9
raw patch · 2 files changed
+144/−105 lines, 2 filesdep +regex-tdfadep +utf8-stringdep −pcre-lightdep ~FileManipCompatdep ~HUnitdep ~cmdargs
Dependencies added: regex-tdfa, utf8-string
Dependencies removed: pcre-light
Dependency ranges changed: FileManipCompat, HUnit, cmdargs, directory, filepath, parsec, process, test-framework, test-framework-hunit
Files
- shelltest.hs +66/−35
- shelltestrunner.cabal +78/−70
shelltest.hs view
@@ -1,6 +1,6 @@ #!/usr/bin/env runhaskell {-# LANGUAGE DeriveDataTypeable #-}-{-+{- | shelltest - a tool for testing command-line programs. @@ -8,19 +8,26 @@ (c) Simon Michael 2009-2010, released under GNU GPLv3 +We try to handle non-ascii content in test file paths, test commands, and+expected test results. For this we assume:+- ghc 6.12+- on unix, all file names, arguments, environment variables etc. are utf-8 encoded+ -} module Main where+import Codec.Binary.UTF8.String as UTF8 (decodeString, encodeString, isUTF8Encoded) import Control.Concurrent (forkIO) import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar) import Control.Monad (liftM,when,unless) import Data.List (intercalate, nub, isPrefixOf)-import Data.Maybe (isNothing,isJust,fromJust,maybe,catMaybes)+import Data.Maybe (isNothing,isJust,fromJust,catMaybes) import qualified Test.HUnit (Test) import System.Console.CmdArgs hiding (args) import qualified System.Console.CmdArgs as CmdArgs (args) import System.Exit (ExitCode(..), exitWith)+import System.Info (os) import System.IO (Handle, hGetContents, hPutStr) import System.Process (StdStream (CreatePipe), shell, createProcess, CreateProcess (..), waitForProcess, ProcessHandle) import Test.Framework (defaultMainWithArgs)@@ -28,7 +35,7 @@ import Test.HUnit hiding (Test) import Text.ParserCombinators.Parsec import Text.Printf (printf)-import Text.Regex.PCRE.Light.Char8+import Text.Regex.TDFA ((=~)) import Debug.Trace import System.Directory (doesDirectoryExist) import System.FilePath (takeDirectory)@@ -39,7 +46,7 @@ strace a = trace (show a) a -version = "0.8" -- keep synced with shelltestrunner.cabal+version = "0.9" -- keep synced with cabal file progname = "shelltest" progversion = progname ++ " " ++ version version, progname, progversion :: String@@ -47,6 +54,7 @@ data Args = Args { debug :: Bool ,debugparse :: Bool+ ,color :: Bool ,execdir :: Bool ,extension :: String ,implicit :: String@@ -57,13 +65,14 @@ nullargs :: Args-nullargs = Args False False False "" "" "" [] []+nullargs = Args False False False False "" "" "" [] [] argmodes :: [Mode Args] argmodes = [ mode (Args{ debug = def &= text "show debug messages" ,debugparse = def &= flag "debug-parse" & explicit & text "show parsing debug messages and stop"+ ,color = def &= text "display with ANSI color codes" ,execdir = def &= text "run tests in same directory as test file" ,extension = ".test" &= typ "EXT" & text "extension of test files when dirs specified" ,implicit = "exit" &= typ "none|exit|all" & text "provide implicit tests"@@ -89,8 +98,7 @@ ,"output is matched, or 0 or more data lines, in which case the output" ,"must match these exactly. The expected exit status (>>>=) field can have" ,"either a numeric exit code or a /regexp/. A ! preceding a /regexp/ or exit"- ,"code negates the match. The regular expression syntax is that of the"- ,"pcre-light library with the dotall flag."+ ,"code negates the match." ,"" ,"By default there is an implicit test for exit status=0, but no implicit test" ,"for stdout or stderr. You can change this with -i/--implicit-tests."@@ -132,10 +140,7 @@ data TestCommand = ReplaceableCommand String | FixedCommand String--instance Show TestCommand where- show (ReplaceableCommand s) = s- show (FixedCommand s) = " " ++ s+ deriving Show type Regexp = String @@ -155,13 +160,13 @@ if isdir then findWithHandler (\_ e->fail (show e)) always (Find.extension ==? extension args) p else return [p]) paths- loud <- isLoud- when loud $ do- printf "executable: %s\n" (with args)- printf "test files: %s\n" (intercalate ", " $ testfiles)- parseresults <- mapM (parseShellTestFile args) testfiles + verbose <- isLoud+ when verbose $ do+ printf "executable: %s\n" (fromPlatformString $ with args)+ printf "test files: %s\n" (intercalate ", " $ map fromPlatformString $ testfiles)+ parseresults <- mapM (parseShellTestFile args) testfiles unless (debugparse args) $- defaultMainWithArgs (concatMap (hUnitTestToTests.testFileParseToHUnitTest args) parseresults) (otheropts args)+ defaultMainWithArgs (concatMap (hUnitTestToTests.testFileParseToHUnitTest args) parseresults) (otheropts args ++ if color args then [] else ["--plain"]) -- parsing @@ -173,7 +178,7 @@ let ts' | length ts > 1 = [t{testname=testname t++":"++show n} | (n,t) <- zip ([1..]::[Int]) ts] | otherwise = ts when (debug args || debugparse args) $ do- printf "parsed %s:\n" f+ printf "parsed %s:\n" $ fromPlatformString f mapM_ (putStrLn.(' ':).show) ts' return $ Right ts' Left _ -> return p@@ -188,7 +193,7 @@ shelltestp :: Parser ShellTest shelltestp = do st <- getParserState- let f = sourceName $ statePos st+ let f = fromPlatformString $ sourceName $ statePos st many $ try commentlinep c <- commandp <?> "command line" i <- optionMaybe inputp <?> "input"@@ -285,7 +290,8 @@ n ~: do let e = with args cmd = case (e,c) of (_:_, ReplaceableCommand s) -> e ++ " " ++ dropWhile (/=' ') s- _ -> show c+ (_, ReplaceableCommand s) -> s+ (_, FixedCommand s) -> s (o_expected',e_expected',x_expected') = case (implicit args) of "all" -> (case o_expected of@@ -300,12 +306,12 @@ _ -> Just $ Numeric "0") _ -> (o_expected,e_expected,x_expected) dir = if execdir args then Just $ takeDirectory n else Nothing- when (debug args) $ printf "command: %s\n" (show cmd)- (o_actual, e_actual, x_actual) <- runCommandWithInput dir cmd i+ when (debug args) $ printf "command was: %s\n" (show cmd)+ (o_actual, e_actual, x_actual) <- runCommandWithInput dir (toPlatformString cmd) i when (debug args) $ do- printf "stdout: %s\n" (show $ trim o_actual)- printf "stderr: %s\n" (show $ trim e_actual)- printf "exit: %s\n" (show $ trim $ show x_actual)+ printf "stdout was : %s\n" (show $ trim o_actual)+ printf "stderr was : %s\n" (show $ trim e_actual)+ printf "exit was : %s\n" (show $ trim $ show x_actual) if (x_actual == 127) -- catch bad executable - should work on posix systems at least then ioError $ userError e_actual -- XXX still a test failure; should be an error else assertString $ addnewline $ intercalate "\n" $ filter (not . null) [@@ -324,6 +330,7 @@ -- | Run a shell command line, passing it standard input if provided, -- and return the standard output, standard error output and exit code.+-- Note on unix, at least with ghc 6.12, command (and filepath) are assumed to be utf8-encoded. runCommandWithInput :: Maybe FilePath -> String -> Maybe String -> IO (String, String, Int) runCommandWithInput wd cmd input = do -- this has to be done carefully@@ -360,7 +367,7 @@ showExpectedActual :: String -> Matcher -> String -> String showExpectedActual field e a =- printf "**Expected %s: %s\n**Got %s: %s" field (show e) field (show $ trim a)+ printf "**Expected %s: %s\n**Got %s: %s" field (show e) field (show $ trim a) instance Show Matcher where show (PositiveRegex r) = "/"++(trim r)++"/"@@ -404,16 +411,15 @@ rstrip = reverse . dropws . reverse dropws = dropWhile (`elem` " \t") --- | Does string contain this regular expression ?--- A malformed regexp will cause a runtime error.+-- | Test if a string contains a regular expression. A malformed regexp+-- (or a regexp larger than 300 characters, to avoid a regex-tdfa memory leak)+-- will cause a runtime error. This version uses regex-tdfa and no regexp+-- options. containsRegex :: String -> String -> Bool-containsRegex s r =- case compileM r [- dotall- -- ,utf8 -- shows no obvious benefit with 6.10, review situation with 6.12- ] of- Right regex -> isJust $ match regex s []- Left e -> error $ printf "bad regexp, %s: %s" e (trim $ r)+containsRegex s "" = containsRegex s "^"+containsRegex s r+ | length r <= 300 = s =~ r+ | otherwise = error "please avoid regexps larger than 300 characters, they are currently problematic" -- | Replace occurrences of old list with new list within a larger list. replace::(Eq a) => [a] -> [a] -> [a] -> [a]@@ -425,3 +431,28 @@ then new ++ replace' (drop len l) else h : replace' ts len = length old++-- | A platform string is a string value from or for the operating system,+-- such as a file path or command-line argument (or environment variable's+-- name or value ?). On some platforms (such as unix) these are not real+-- unicode strings but have some encoding such as UTF-8. This alias does+-- no type enforcement but aids code clarity.+type PlatformString = String++-- | Convert a possibly encoded platform string to a real unicode string.+-- We decode the UTF-8 encoding recommended for unix systems+-- (cf http://www.dwheeler.com/essays/fixing-unix-linux-filenames.html)+-- and leave anything else unchanged.+fromPlatformString :: PlatformString -> String+fromPlatformString s = if UTF8.isUTF8Encoded s then UTF8.decodeString s else s++-- | Convert a unicode string to a possibly encoded platform string.+-- On unix we encode with the recommended UTF-8+-- (cf http://www.dwheeler.com/essays/fixing-unix-linux-filenames.html)+-- and elsewhere we leave it unchanged.+toPlatformString :: String -> PlatformString+toPlatformString = case os of+ "unix" -> UTF8.encodeString+ "linux" -> UTF8.encodeString+ "darwin" -> UTF8.encodeString+ _ -> id
shelltestrunner.cabal view
@@ -1,65 +1,72 @@ name: shelltestrunner-version: 0.8+-- keep synced with shelltest.hs:+version: 0.9 category: Testing synopsis: A tool for testing command-line programs. description:- .- Run a command-line program through \"shell tests\" defined in one or- more test files. Each test specifies command-line arguments, some input,- and expected output, stderr output and/or exit status.- We use test-framework's test runner, so can run tests in parallel.- shelltestrunner was inspired by the tests in John Wiegley's ledger project.- .- Usage: + This tool aims to help with repeatable testing of a command-line program+ (or any shell command), via declarative \"shell tests\" which are easier+ to write than procedural shell scripts. Tests are defined in one or more+ test files, which typically have the .test suffix and live in a tests/+ subdirectory. Each test specifies a command line, optional standard input,+ and expected standard output, error output and/or exit status. Tests can+ be run in parallel for greater speed. shelltestrunner was inspired by the+ tests in John Wiegley's ledger project.+ . + Compatibility: should work on microsoft windows as well as unix; not well+ tested on windows. Should build with ghc 6.10; for unicode support,+ requires ghc 6.12. .- > shelltest [FLAG] [TESTFILES|TESTDIRS]- >- > - > -? --help[=FORMAT] Show usage information (optional format)- > -V --version Show version information- > -v --verbose Higher verbosity- > -q --quiet Lower verbosity- > -d --debug show debug messages- > --debug-parse show parsing debug messages and stop- > --execdir run tests in same directory as test file- > --extension=EXT extension of test files when dirs specified (default=.test)- > -i --implicit=none|exit|all provide implicit tests (default=exit)- > -w --with=EXECUTABLE alternate executable, replaces the first word of test commands- > =OTHER FLAGS any other flags are passed to test runner- >- > A test file contains one or more shell tests, which look like this:- > - > # optional comment lines- > a one-line shell command to be tested- > <<<- > stdin lines- > >>> [/regexp to match in stdout/]- > [or expected stdout lines- > >>>2 [/regexp to match in stderr/]- > [or expected stderr lines]- > >>>= expected exit status or /regexp/- > - > The command line is required; all other fields are optional.- > The expected stdout (>>>) and expected stderr (>>>2) fields can have either- > a regular expression match pattern, in which case the test passes if the- > output is matched, or 0 or more data lines, in which case the output- > must match these exactly. The expected exit status (>>>=) field can have- > either a numeric exit code or a /regexp/. A ! preceding a /regexp/ or exit- > code negates the match. The regular expression syntax is that of the- > pcre-light library with the dotall flag.- > - > By default there is an implicit test for exit status=0, but no implicit test- > for stdout or stderr. You can change this with -i/--implicit-tests.- > - > The command runs in your current directory unless you use --execdir.- > You can use --with/-w to replace the first word of command lines- > (everything up to the first space) with something else, eg to test a- > different version of your program. To prevent this, start the command line- > with a space.- > - > Any unrecognised options will be passed through to test-framework's runner.- > You may be able to get a big speedup by running tests in parallel: try -j8.+ Usage: .+ > shelltest [FLAG] [TESTFILES|TESTDIRS]+ > + > -? --help[=FORMAT] Show usage information (optional format)+ > -V --version Show version information+ > -v --verbose Higher verbosity+ > -q --quiet Lower verbosity+ > -d --debug show debug messages+ > --debug-parse show parsing debug messages and stop+ > -c --color display with ANSI color codes+ > --execdir run tests in same directory as test file+ > --extension=EXT extension of test files when dirs specified (default=.test)+ > -i --implicit=none|exit|all provide implicit tests (default=exit)+ > -w --with=EXECUTABLE alternate executable, replaces the first word of test commands+ > =OTHER FLAGS any other flags are passed to test runner+ > + > A test file contains one or more shell tests, which look like this:+ > + > # optional comment lines+ > a one-line shell command to be tested+ > <<<+ > stdin lines+ > >>> [/regexp to match in stdout/]+ > [or expected stdout lines+ > >>>2 [/regexp to match in stderr/]+ > [or expected stderr lines]+ > >>>= expected exit status or /regexp/+ > + > The command line is required; all other fields are optional.+ > The expected stdout (>>>) and expected stderr (>>>2) fields can have either+ > a regular expression match pattern, in which case the test passes if the+ > output is matched, or 0 or more data lines, in which case the output+ > must match these exactly. The expected exit status (>>>=) field can have+ > either a numeric exit code or a /regexp/. A ! preceding a /regexp/ or exit+ > code negates the match. The regular expression syntax is that of the+ > pcre-light library with the dotall flag.+ > + > By default there is an implicit test for exit status=0, but no implicit test+ > for stdout or stderr. You can change this with -i/--implicit-tests.+ > + > The command runs in your current directory unless you use --execdir.+ > You can use --with/-w to replace the first word of command lines+ > (everything up to the first space) with something else, eg to test a+ > different version of your program. To prevent this, start the command line+ > with a space.+ > + > Any unrecognised options will be passed through to test-framework's runner.+ > You may be able to get a big speedup by running tests in parallel: try -j8.+ license: GPL license-file: LICENSE author: Simon Michael <simon@joyful.com>@@ -67,7 +74,7 @@ homepage: http://joyful.com/darcsweb/darcsweb.cgi?r=shelltestrunner bug-reports: mailto:simon@joyful.com stability: beta-tested-with: GHC==6.10+tested-with: GHC==6.12 cabal-version: >= 1.6 build-type: Simple @@ -76,20 +83,21 @@ executable shelltest main-is: shelltest.hs- ghc-options: -Wall -threaded+ ghc-options: -threaded -W -fwarn-tabs build-depends:- base >= 3 && < 5- ,parsec- ,process- ,HUnit- ,test-framework- ,test-framework-hunit >= 0.2- ,cmdargs >= 0.1- ,pcre-light >= 0.3.1- ,directory- ,filepath- ,FileManipCompat+ base >= 3 && < 5+ ,FileManipCompat >= 0.15 && < 0.16+ ,HUnit < 1.3+ ,cmdargs >= 0.1 && < 0.2+ ,directory == 1.0.*+ ,filepath >= 1.0 && < 1.2+ ,parsec < 3.2+ ,regex-tdfa >= 1.1 && < 1.2+ ,process < 1.1+ ,test-framework >= 0.3.2 && < 0.4+ ,test-framework-hunit >= 0.2 && < 0.3+ ,utf8-string >= 0.3.5 && < 0.4 source-repository head type: darcs- location: http://joyful.com/repos/hledger+ location: http://joyful.com/repos/shelltestrunner