diff --git a/shelltest.hs b/shelltest.hs --- a/shelltest.hs +++ b/shelltest.hs @@ -1,6 +1,6 @@ #!/usr/bin/env runhaskell {-# LANGUAGE DeriveDataTypeable #-} -{- +{- | shelltest - a tool for testing command-line programs. @@ -8,19 +8,26 @@ (c) Simon Michael 2009-2010, released under GNU GPLv3 +We try to handle non-ascii content in test file paths, test commands, and +expected test results. For this we assume: +- ghc 6.12 +- on unix, all file names, arguments, environment variables etc. are utf-8 encoded + -} module Main where +import Codec.Binary.UTF8.String as UTF8 (decodeString, encodeString, isUTF8Encoded) import Control.Concurrent (forkIO) import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar) import Control.Monad (liftM,when,unless) import Data.List (intercalate, nub, isPrefixOf) -import Data.Maybe (isNothing,isJust,fromJust,maybe,catMaybes) +import Data.Maybe (isNothing,isJust,fromJust,catMaybes) import qualified Test.HUnit (Test) import System.Console.CmdArgs hiding (args) import qualified System.Console.CmdArgs as CmdArgs (args) import System.Exit (ExitCode(..), exitWith) +import System.Info (os) import System.IO (Handle, hGetContents, hPutStr) import System.Process (StdStream (CreatePipe), shell, createProcess, CreateProcess (..), waitForProcess, ProcessHandle) import Test.Framework (defaultMainWithArgs) @@ -28,7 +35,7 @@ import Test.HUnit hiding (Test) import Text.ParserCombinators.Parsec import Text.Printf (printf) -import Text.Regex.PCRE.Light.Char8 +import Text.Regex.TDFA ((=~)) import Debug.Trace import System.Directory (doesDirectoryExist) import System.FilePath (takeDirectory) @@ -39,7 +46,7 @@ strace a = trace (show a) a -version = "0.8" -- keep synced with shelltestrunner.cabal +version = "0.9" -- keep synced with cabal file progname = "shelltest" progversion = progname ++ " " ++ version version, progname, progversion :: String @@ -47,6 +54,7 @@ data Args = Args { debug :: Bool ,debugparse :: Bool + ,color :: Bool ,execdir :: Bool ,extension :: String ,implicit :: String @@ -57,13 +65,14 @@ nullargs :: Args -nullargs = Args False False False "" "" "" [] [] +nullargs = Args False False False False "" "" "" [] [] argmodes :: [Mode Args] argmodes = [ mode (Args{ debug = def &= text "show debug messages" ,debugparse = def &= flag "debug-parse" & explicit & text "show parsing debug messages and stop" + ,color = def &= text "display with ANSI color codes" ,execdir = def &= text "run tests in same directory as test file" ,extension = ".test" &= typ "EXT" & text "extension of test files when dirs specified" ,implicit = "exit" &= typ "none|exit|all" & text "provide implicit tests" @@ -89,8 +98,7 @@ ,"output is matched, or 0 or more data lines, in which case the output" ,"must match these exactly. The expected exit status (>>>=) field can have" ,"either a numeric exit code or a /regexp/. A ! preceding a /regexp/ or exit" - ,"code negates the match. The regular expression syntax is that of the" - ,"pcre-light library with the dotall flag." + ,"code negates the match." ,"" ,"By default there is an implicit test for exit status=0, but no implicit test" ,"for stdout or stderr. You can change this with -i/--implicit-tests." @@ -132,10 +140,7 @@ data TestCommand = ReplaceableCommand String | FixedCommand String - -instance Show TestCommand where - show (ReplaceableCommand s) = s - show (FixedCommand s) = " " ++ s + deriving Show type Regexp = String @@ -155,13 +160,13 @@ if isdir then findWithHandler (\_ e->fail (show e)) always (Find.extension ==? extension args) p else return [p]) paths - loud <- isLoud - when loud $ do - printf "executable: %s\n" (with args) - printf "test files: %s\n" (intercalate ", " $ testfiles) - parseresults <- mapM (parseShellTestFile args) testfiles + verbose <- isLoud + when verbose $ do + printf "executable: %s\n" (fromPlatformString $ with args) + printf "test files: %s\n" (intercalate ", " $ map fromPlatformString $ testfiles) + parseresults <- mapM (parseShellTestFile args) testfiles unless (debugparse args) $ - defaultMainWithArgs (concatMap (hUnitTestToTests.testFileParseToHUnitTest args) parseresults) (otheropts args) + defaultMainWithArgs (concatMap (hUnitTestToTests.testFileParseToHUnitTest args) parseresults) (otheropts args ++ if color args then [] else ["--plain"]) -- parsing @@ -173,7 +178,7 @@ let ts' | length ts > 1 = [t{testname=testname t++":"++show n} | (n,t) <- zip ([1..]::[Int]) ts] | otherwise = ts when (debug args || debugparse args) $ do - printf "parsed %s:\n" f + printf "parsed %s:\n" $ fromPlatformString f mapM_ (putStrLn.(' ':).show) ts' return $ Right ts' Left _ -> return p @@ -188,7 +193,7 @@ shelltestp :: Parser ShellTest shelltestp = do st <- getParserState - let f = sourceName $ statePos st + let f = fromPlatformString $ sourceName $ statePos st many $ try commentlinep c <- commandp "command line" i <- optionMaybe inputp "input" @@ -285,7 +290,8 @@ n ~: do let e = with args cmd = case (e,c) of (_:_, ReplaceableCommand s) -> e ++ " " ++ dropWhile (/=' ') s - _ -> show c + (_, ReplaceableCommand s) -> s + (_, FixedCommand s) -> s (o_expected',e_expected',x_expected') = case (implicit args) of "all" -> (case o_expected of @@ -300,12 +306,12 @@ _ -> Just $ Numeric "0") _ -> (o_expected,e_expected,x_expected) dir = if execdir args then Just $ takeDirectory n else Nothing - when (debug args) $ printf "command: %s\n" (show cmd) - (o_actual, e_actual, x_actual) <- runCommandWithInput dir cmd i + when (debug args) $ printf "command was: %s\n" (show cmd) + (o_actual, e_actual, x_actual) <- runCommandWithInput dir (toPlatformString cmd) i when (debug args) $ do - printf "stdout: %s\n" (show $ trim o_actual) - printf "stderr: %s\n" (show $ trim e_actual) - printf "exit: %s\n" (show $ trim $ show x_actual) + printf "stdout was : %s\n" (show $ trim o_actual) + printf "stderr was : %s\n" (show $ trim e_actual) + printf "exit was : %s\n" (show $ trim $ show x_actual) if (x_actual == 127) -- catch bad executable - should work on posix systems at least then ioError $ userError e_actual -- XXX still a test failure; should be an error else assertString $ addnewline $ intercalate "\n" $ filter (not . null) [ @@ -324,6 +330,7 @@ -- | Run a shell command line, passing it standard input if provided, -- and return the standard output, standard error output and exit code. +-- Note on unix, at least with ghc 6.12, command (and filepath) are assumed to be utf8-encoded. runCommandWithInput :: Maybe FilePath -> String -> Maybe String -> IO (String, String, Int) runCommandWithInput wd cmd input = do -- this has to be done carefully @@ -360,7 +367,7 @@ showExpectedActual :: String -> Matcher -> String -> String showExpectedActual field e a = - printf "**Expected %s: %s\n**Got %s: %s" field (show e) field (show $ trim a) + printf "**Expected %s: %s\n**Got %s: %s" field (show e) field (show $ trim a) instance Show Matcher where show (PositiveRegex r) = "/"++(trim r)++"/" @@ -404,16 +411,15 @@ rstrip = reverse . dropws . reverse dropws = dropWhile (`elem` " \t") --- | Does string contain this regular expression ? --- A malformed regexp will cause a runtime error. +-- | Test if a string contains a regular expression. A malformed regexp +-- (or a regexp larger than 300 characters, to avoid a regex-tdfa memory leak) +-- will cause a runtime error. This version uses regex-tdfa and no regexp +-- options. containsRegex :: String -> String -> Bool -containsRegex s r = - case compileM r [ - dotall - -- ,utf8 -- shows no obvious benefit with 6.10, review situation with 6.12 - ] of - Right regex -> isJust $ match regex s [] - Left e -> error $ printf "bad regexp, %s: %s" e (trim $ r) +containsRegex s "" = containsRegex s "^" +containsRegex s r + | length r <= 300 = s =~ r + | otherwise = error "please avoid regexps larger than 300 characters, they are currently problematic" -- | Replace occurrences of old list with new list within a larger list. replace::(Eq a) => [a] -> [a] -> [a] -> [a] @@ -425,3 +431,28 @@ then new ++ replace' (drop len l) else h : replace' ts len = length old + +-- | A platform string is a string value from or for the operating system, +-- such as a file path or command-line argument (or environment variable's +-- name or value ?). On some platforms (such as unix) these are not real +-- unicode strings but have some encoding such as UTF-8. This alias does +-- no type enforcement but aids code clarity. +type PlatformString = String + +-- | Convert a possibly encoded platform string to a real unicode string. +-- We decode the UTF-8 encoding recommended for unix systems +-- (cf http://www.dwheeler.com/essays/fixing-unix-linux-filenames.html) +-- and leave anything else unchanged. +fromPlatformString :: PlatformString -> String +fromPlatformString s = if UTF8.isUTF8Encoded s then UTF8.decodeString s else s + +-- | Convert a unicode string to a possibly encoded platform string. +-- On unix we encode with the recommended UTF-8 +-- (cf http://www.dwheeler.com/essays/fixing-unix-linux-filenames.html) +-- and elsewhere we leave it unchanged. +toPlatformString :: String -> PlatformString +toPlatformString = case os of + "unix" -> UTF8.encodeString + "linux" -> UTF8.encodeString + "darwin" -> UTF8.encodeString + _ -> id diff --git a/shelltestrunner.cabal b/shelltestrunner.cabal --- a/shelltestrunner.cabal +++ b/shelltestrunner.cabal @@ -1,65 +1,72 @@ name: shelltestrunner -version: 0.8 +-- keep synced with shelltest.hs: +version: 0.9 category: Testing synopsis: A tool for testing command-line programs. description: - . - Run a command-line program through \"shell tests\" defined in one or - more test files. Each test specifies command-line arguments, some input, - and expected output, stderr output and/or exit status. - We use test-framework's test runner, so can run tests in parallel. - shelltestrunner was inspired by the tests in John Wiegley's ledger project. - . - Usage: + This tool aims to help with repeatable testing of a command-line program + (or any shell command), via declarative \"shell tests\" which are easier + to write than procedural shell scripts. Tests are defined in one or more + test files, which typically have the .test suffix and live in a tests/ + subdirectory. Each test specifies a command line, optional standard input, + and expected standard output, error output and/or exit status. Tests can + be run in parallel for greater speed. shelltestrunner was inspired by the + tests in John Wiegley's ledger project. + . + Compatibility: should work on microsoft windows as well as unix; not well + tested on windows. Should build with ghc 6.10; for unicode support, + requires ghc 6.12. . - > shelltest [FLAG] [TESTFILES|TESTDIRS] - > - > - > -? --help[=FORMAT] Show usage information (optional format) - > -V --version Show version information - > -v --verbose Higher verbosity - > -q --quiet Lower verbosity - > -d --debug show debug messages - > --debug-parse show parsing debug messages and stop - > --execdir run tests in same directory as test file - > --extension=EXT extension of test files when dirs specified (default=.test) - > -i --implicit=none|exit|all provide implicit tests (default=exit) - > -w --with=EXECUTABLE alternate executable, replaces the first word of test commands - > =OTHER FLAGS any other flags are passed to test runner - > - > A test file contains one or more shell tests, which look like this: - > - > # optional comment lines - > a one-line shell command to be tested - > <<< - > stdin lines - > >>> [/regexp to match in stdout/] - > [or expected stdout lines - > >>>2 [/regexp to match in stderr/] - > [or expected stderr lines] - > >>>= expected exit status or /regexp/ - > - > The command line is required; all other fields are optional. - > The expected stdout (>>>) and expected stderr (>>>2) fields can have either - > a regular expression match pattern, in which case the test passes if the - > output is matched, or 0 or more data lines, in which case the output - > must match these exactly. The expected exit status (>>>=) field can have - > either a numeric exit code or a /regexp/. A ! preceding a /regexp/ or exit - > code negates the match. The regular expression syntax is that of the - > pcre-light library with the dotall flag. - > - > By default there is an implicit test for exit status=0, but no implicit test - > for stdout or stderr. You can change this with -i/--implicit-tests. - > - > The command runs in your current directory unless you use --execdir. - > You can use --with/-w to replace the first word of command lines - > (everything up to the first space) with something else, eg to test a - > different version of your program. To prevent this, start the command line - > with a space. - > - > Any unrecognised options will be passed through to test-framework's runner. - > You may be able to get a big speedup by running tests in parallel: try -j8. + Usage: . + > shelltest [FLAG] [TESTFILES|TESTDIRS] + > + > -? --help[=FORMAT] Show usage information (optional format) + > -V --version Show version information + > -v --verbose Higher verbosity + > -q --quiet Lower verbosity + > -d --debug show debug messages + > --debug-parse show parsing debug messages and stop + > -c --color display with ANSI color codes + > --execdir run tests in same directory as test file + > --extension=EXT extension of test files when dirs specified (default=.test) + > -i --implicit=none|exit|all provide implicit tests (default=exit) + > -w --with=EXECUTABLE alternate executable, replaces the first word of test commands + > =OTHER FLAGS any other flags are passed to test runner + > + > A test file contains one or more shell tests, which look like this: + > + > # optional comment lines + > a one-line shell command to be tested + > <<< + > stdin lines + > >>> [/regexp to match in stdout/] + > [or expected stdout lines + > >>>2 [/regexp to match in stderr/] + > [or expected stderr lines] + > >>>= expected exit status or /regexp/ + > + > The command line is required; all other fields are optional. + > The expected stdout (>>>) and expected stderr (>>>2) fields can have either + > a regular expression match pattern, in which case the test passes if the + > output is matched, or 0 or more data lines, in which case the output + > must match these exactly. The expected exit status (>>>=) field can have + > either a numeric exit code or a /regexp/. A ! preceding a /regexp/ or exit + > code negates the match. The regular expression syntax is that of the + > pcre-light library with the dotall flag. + > + > By default there is an implicit test for exit status=0, but no implicit test + > for stdout or stderr. You can change this with -i/--implicit-tests. + > + > The command runs in your current directory unless you use --execdir. + > You can use --with/-w to replace the first word of command lines + > (everything up to the first space) with something else, eg to test a + > different version of your program. To prevent this, start the command line + > with a space. + > + > Any unrecognised options will be passed through to test-framework's runner. + > You may be able to get a big speedup by running tests in parallel: try -j8. + license: GPL license-file: LICENSE author: Simon Michael @@ -67,7 +74,7 @@ homepage: http://joyful.com/darcsweb/darcsweb.cgi?r=shelltestrunner bug-reports: mailto:simon@joyful.com stability: beta -tested-with: GHC==6.10 +tested-with: GHC==6.12 cabal-version: >= 1.6 build-type: Simple @@ -76,20 +83,21 @@ executable shelltest main-is: shelltest.hs - ghc-options: -Wall -threaded + ghc-options: -threaded -W -fwarn-tabs build-depends: - base >= 3 && < 5 - ,parsec - ,process - ,HUnit - ,test-framework - ,test-framework-hunit >= 0.2 - ,cmdargs >= 0.1 - ,pcre-light >= 0.3.1 - ,directory - ,filepath - ,FileManipCompat + base >= 3 && < 5 + ,FileManipCompat >= 0.15 && < 0.16 + ,HUnit < 1.3 + ,cmdargs >= 0.1 && < 0.2 + ,directory == 1.0.* + ,filepath >= 1.0 && < 1.2 + ,parsec < 3.2 + ,regex-tdfa >= 1.1 && < 1.2 + ,process < 1.1 + ,test-framework >= 0.3.2 && < 0.4 + ,test-framework-hunit >= 0.2 && < 0.3 + ,utf8-string >= 0.3.5 && < 0.4 source-repository head type: darcs - location: http://joyful.com/repos/hledger + location: http://joyful.com/repos/shelltestrunner