shelltestrunner 0.5 → 0.6
raw patch · 2 files changed
+66/−61 lines, 2 files
Files
- shelltestrunner.cabal +15/−16
- shelltestrunner.hs +51/−45
shelltestrunner.cabal view
@@ -1,12 +1,12 @@ name: shelltestrunner-version: 0.5+version: 0.6 category: Testing synopsis: A tool for testing command-line programs. description: . Run a given program through \"shell\" tests specifed by one or more test- files, each of which can specify: command-line arguments, input, expected- output, expected stderr output, and expected exit code. This was+ files, where each test can specify: command-line arguments, input, expected+ output, expected stderr output, and/or expected exit code. This was extracted from the hledger project, inspired by the tests in John Wiegley's ledger project, and uses test-framework's test runner. .@@ -20,41 +20,40 @@ . Test file format: .+ A test file contains 0 or more shell tests, each of which looks like this: + . > # 0 or more comment lines beginning with #- > -opt1 -opt2 arg1 arg2 # command line args, executable will be prepended+ > -opt1 -opt2 arg1 arg2 # one line of command line args, executable will be prepended > <<< > 0 or more lines of input > >>> [/regexp/] > [..or 0 or more lines of expected output] > >>>2 [/regexp/] > [..or 0 or more lines of expected error output]- > >>>= [/regexp/]- > [..or expected numeric exit code]+ > >>>= [expected numeric exit code or /regexp/] . Each expected field can have either a regular expression match expression, in which case the test passes if the output is matched, or 0 or more data lines, in which case the output must match these exactly. A- ! preceding a /regexp/ negates the match. The regular expression syntax+ ! preceding a \/regexp\/ negates the match. The regular expression syntax is that supported by the regexpr library. .- Apart from the command line, all fields are optional. Only fields- specified in the test will be tested, unless you use the- -i/--implicit-tests flag, which will test for empty stdout, empty stderr,- or 0 exit code where fields are omitted.+ Apart from the command line, all fields are optional. Only the fields you+ specify will be tested, unless you use the -i/--implicit-tests flag,+ which adds default tests (empty stdout, empty stderr, and 0 exit code)+ for omitted fields. . Issues: .- - can't put / in a regexp- . - can't test input/output which does not end with newline .- - option processing is weak+ - can't use / in regexps .+ - option processing and --help output could be better+ . Wishlist: . - configurable delimiters- .- - multiple tests per file . license: GPL license-file: LICENSE
shelltestrunner.hs view
@@ -15,7 +15,7 @@ import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar) import Control.Monad (liftM,when) import Data.List (partition, intercalate)-import Data.Maybe (fromJust,isJust,maybe)+import Data.Maybe (fromJust,isJust,maybe,isNothing,catMaybes) import qualified Test.HUnit (Test) import System.Console.ParseArgs hiding (args) import System.Environment (withArgs)@@ -34,7 +34,7 @@ version :: String-version = "0.5" -- keep synced with .cabal+version = "0.6" -- keep synced with .cabal data ArgId = HelpFlag | VersionFlag@@ -88,7 +88,7 @@ type Regex = String -data Matcher = Exact String+data Matcher = Lines String | Numeric String | PositiveRegex Regex | NegativeRegex Regex@@ -105,7 +105,7 @@ putStrLn $ "testing executable: " ++ (show $ fromJust $ getArgString args ExecutableArg) putStrLn $ "unprocessed options: " ++ show unprocessedopts putStrLn $ "test files: " ++ show testfiles- shelltests <- mapM (parseShellTest args) testfiles+ shelltests <- liftM concat $ mapM (parseShellTestFile args) testfiles withArgs unprocessedopts $ defaultMain $ concatMap (hUnitTestToTests.shellTestToHUnitTest args) shelltests printVersion :: IO ()@@ -116,13 +116,14 @@ -- parsing -parseShellTest :: Args ArgId -> FilePath -> IO ShellTest-parseShellTest args f = do- t <- liftM (either (error.show) id) $ parseFromFile shelltestp f- when (args `gotArg` DebugFlag) $ do- putStrLn $ "parsing file: " ++ f- putStrLn $ "parsed test: " ++ show t- return t+parseShellTestFile :: Args ArgId -> FilePath -> IO [ShellTest]+parseShellTestFile args f = do+ when (args `gotArg` DebugFlag) $ putStrLn $ "parsing file: " ++ f+ ts <- liftM (either (error.show) id) $ parseFromFile (many shelltestp) f+ let ts' | length ts > 1 = [t{testname=testname t++":"++show n} | (n,t) <- zip ([1..]::[Int]) ts]+ | otherwise = ts+ when (args `gotArg` DebugFlag) $ putStrLn $ "parsed tests: " ++ show ts'+ return ts' shelltestp :: Parser ShellTest shelltestp = do@@ -134,6 +135,7 @@ o <- optionMaybe expectedoutputp e <- optionMaybe expectederrorp x <- optionMaybe expectedexitcodep+ when (null c && (isNothing i) && (null $ catMaybes [o,e,x])) $ fail "empty test file" return ShellTest{testname=f,commandargs=c,stdin=i,stdoutExpected=o,stderrExpected=e,exitCodeExpected=x} linep,commentlinep,commandargsp,delimiterp,inputp,whitespacep :: Parser String@@ -146,25 +148,25 @@ delimiterp = choice [try $ string "<<<", try $ string ">>>", (eof >> return "")] -inputp = string "<<<\n" >> (liftM unlines) (linep `manyTill` (lookAhead delimiterp))+inputp = string "<<<" >> newline >> (liftM unlines) (linep `manyTill` (lookAhead delimiterp)) expectedoutputp :: Parser Matcher expectedoutputp = try $ do- string ">>>" >> optional (char '1')- whitespacep- choice [positiveregexmatcherp, negativeregexmatcherp, datalinesp]+ string ">>>" >> optional (char '1')+ whitespacep+ choice [positiveregexmatcherp, negativeregexmatcherp, linesmatcherp] expectederrorp :: Parser Matcher expectederrorp = try $ do- string ">>>2"- whitespacep- choice [positiveregexmatcherp, negativeregexmatcherp, datalinesp]+ string ">>>2"+ whitespacep+ choice [positiveregexmatcherp, negativeregexmatcherp, linesmatcherp] expectedexitcodep :: Parser Matcher expectedexitcodep = do- string ">>>="- whitespacep- choice [positiveregexmatcherp, negativeregexmatcherp, numericdatalinesp]+ string ">>>="+ whitespacep+ choice [positiveregexmatcherp, negativeregexmatcherp, numericmatcherp] negativeregexmatcherp :: Parser Matcher negativeregexmatcherp = do@@ -181,18 +183,16 @@ newline return $ PositiveRegex r -datalinesp :: Parser Matcher-datalinesp = do+linesmatcherp :: Parser Matcher+linesmatcherp = do whitespacep newline- (liftM $ Exact . unlines) (linep `manyTill` (lookAhead delimiterp))+ (liftM $ Lines . unlines) (linep `manyTill` (lookAhead delimiterp)) -numericdatalinesp :: Parser Matcher-numericdatalinesp = do- whitespacep- newline- whitespacep+numericmatcherp :: Parser Matcher+numericmatcherp = do s <- many1 $ oneOf "0123456789"+ whitespacep newline return $ Numeric s @@ -210,9 +210,9 @@ case (args `gotArg` ImplicitTestsFlag) of True -> (case o_expected of Just m -> Just m- _ -> Just $ Exact ""+ _ -> Just $ Lines "" ,case e_expected of Just m -> Just m- _ -> Just $ Exact ""+ _ -> Just $ Lines "" ,case x_expected of Just m -> Just m _ -> Just $ Numeric "0") False -> (o_expected,e_expected,x_expected)@@ -228,21 +228,27 @@ e <- newEmptyMVar forkIO $ oh `hGetContentsStrictlyAnd` putMVar o forkIO $ eh `hGetContentsStrictlyAnd` putMVar e- x_actual <- waitForProcess ph+ x_actual <- waitForProcess ph >>= return.fromExitCode o_actual <- takeMVar o e_actual <- takeMVar e - assertString $ addnewline $ intercalate "\n" $ filter (not . null)- [if (maybe True (o_actual `matches`) o_expected')- then "" - else showExpectedActual "stdout" (fromJust o_expected') o_actual- ,if (maybe True (e_actual `matches`) e_expected')- then "" - else showExpectedActual "stderr" (fromJust e_expected') e_actual- ,if (maybe True ((show $ fromExitCode x_actual) `matches`) x_expected')- then ""- else showExpectedActual "exit code" (fromJust x_expected') (show $ fromExitCode x_actual)- ]+ let failuremsg+ -- when a bad exe is provided, we don't want to show all the test+ -- failures. This should work on all posix systems at least.+ -- Really we should report a test error here, not a failure.+ | x_actual == 127 = cmd ++ ": command not found"+ | otherwise = addnewline $ intercalate "\n" $ filter (not . null) [+ if (maybe True (o_actual `matches`) o_expected')+ then "" + else showExpectedActual "stdout" (fromJust o_expected') o_actual+ ,if (maybe True (e_actual `matches`) e_expected')+ then "" + else showExpectedActual "stderr" (fromJust e_expected') e_actual+ ,if (maybe True (show x_actual `matches`) x_expected')+ then ""+ else showExpectedActual "exit code" (fromJust x_expected') (show x_actual)+ ]+ assertString failuremsg where addnewline "" = "" addnewline s = "\n"++s @@ -253,7 +259,7 @@ matches s (PositiveRegex r) = s `containsRegex` r matches s (NegativeRegex r) = not $ s `containsRegex` r matches s (Numeric p) = s == p-matches s (Exact p) = s == p+matches s (Lines p) = s == p showExpectedActual :: String -> Matcher -> String -> String showExpectedActual field e a =@@ -263,7 +269,7 @@ showMatcher (PositiveRegex r) = " /"++r++"/\n" showMatcher (NegativeRegex r) = " !/"++r++"/\n" showMatcher (Numeric s) = "\n"++s++"\n"-showMatcher (Exact s) = "\n"++s+showMatcher (Lines s) = "\n"++s -- utils