diff --git a/MuCheck.cabal b/MuCheck.cabal --- a/MuCheck.cabal +++ b/MuCheck.cabal @@ -1,5 +1,5 @@ name: MuCheck -version: 0.1.0.1 +version: 0.1.1.0 synopsis: Automated Mutation Testing description: This package defines a mutation analysis library for haskell programs. It does this by parsing the haskell source, and @@ -12,28 +12,27 @@ license-file: LICENSE author: Duc Lee Rahul Gopinath -maintainer: ledu@onid.oregonstate.edu - rahul@gopinath.org +maintainer: rahul@gopinath.org -- copyright: category: Testing build-type: Simple cabal-version: >= 1.10 -source-repository head +source-repository this type: git - location: https://rgopinath@bitbucket.org/osu-testing/mucheck.git + location: https://bitbucket.org/osu-testing/mucheck.git tag: 0.1.0.1 executable mucheck main-is: Main.hs - build-depends: base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath + build-depends: base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath, hspec >= 2.0, hspec-core >= 2.0 default-language: Haskell2010 hs-source-dirs: src library exposed-modules: MuCheck.MuOp, MuCheck.StdArgs, MuCheck.Interpreter, MuCheck.Mutation, MuCheck.Operators, MuCheck.Utils.Syb, MuCheck.Utils.Common, MuCheck.Utils.Print -- other-modules: - build-depends: base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath + build-depends: base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath, hspec>= 2.0, hspec-core >= 2.0 default-language: Haskell2010 hs-source-dirs: src @@ -42,5 +41,5 @@ hs-source-dirs: test main-is: Spec.hs default-language: Haskell2010 - build-depends: base >= 4 && <5, MuCheck, hspec >= 2.0, QuickCheck >= 2.6, filepath + build-depends: base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath, hspec>= 2.0, hspec-core >= 2.0, MuCheck diff --git a/src/Main.hs b/src/Main.hs --- a/src/Main.hs +++ b/src/Main.hs @@ -8,21 +8,29 @@ import MuCheck.Mutation import MuCheck.Operators import MuCheck.Utils.Common +import MuCheck.Utils.Print -process :: String -> String -> String -> [String] -> IO () -process fn file modulename args = do +process :: String -> String -> String -> String -> [String] -> IO () +process t fn file modulename args = do numMutants <- genMutants fn file - checkPropsOnMutants (take numMutants $ genFileNames file) modulename args "./test.log" - return () + case t of + "qcheck" -> checkQuickCheckOnMutants (take numMutants $ genFileNames file) modulename args "./qcheck.log" >> return () + "hspec" -> checkHspecOnMutants (take numMutants $ genFileNames file) modulename args "./hspec.log" >> return () + "hunit" -> checkHUnitOnMutants (take numMutants $ genFileNames file) modulename args "./hunit.log" >> return () + _ -> error "Unexpected test type" main :: IO () main = do val <- getArgs case val of - ("-help" : _ ) -> help - (fn : file : modulename : args) -> process fn file modulename args - _ -> error "Need function file modulename [args]" + ("-h" : _ ) -> help + (t: fn : file : modulename : args) -> withArgs [] $ process t fn file modulename args + _ -> error "Need [qcheck|hunit|hspec] function file modulename [args]\n\tUse -h to get help" help :: IO () -help = putStrLn "mucheck function file modulename [args]" +help = putStrLn ("mucheck type function file modulename [args]\n" ++ (showAS ["E.g:", + " ./mucheck qcheck qsort Examples/QuickCheckTest.hs Examples.QuickCheckTest 'quickCheckResult idEmpProp' 'quickCheckResult revProp' 'quickCheckResult modelProp'", + " ./mucheck hunit qsort Examples/HUnitTest.hs Examples.HUnitTest 'runTestTT tests'", + " ./mucheck hspec qsort Examples/HspecTest.hs Examples.HspecTest 'spec (with \\\"qsort1\\\")'"])) + diff --git a/src/MuCheck/Interpreter.hs b/src/MuCheck/Interpreter.hs --- a/src/MuCheck/Interpreter.hs +++ b/src/MuCheck/Interpreter.hs @@ -5,47 +5,66 @@ import Control.Monad.Trans ( liftIO ) import qualified Test.QuickCheck.Test as Qc import qualified Test.HUnit as HUnit +import qualified Test.Hspec.Core.Runner as Hspec import Data.Typeable -import qualified MuCheck.Utils.Print as Mu -import Data.Either -import Data.List((\\), groupBy, sortBy) +import MuCheck.Utils.Print (showA, showAS, (./.)) +import Data.Either (partitionEithers, rights) +import Data.List((\\), groupBy, sortBy, intercalate, isInfixOf) import Data.Time.Clock deriving instance Typeable Qc.Result deriving instance Typeable HUnit.Counts +deriving instance Typeable Hspec.Summary type InterpreterOutput a = Either I.InterpreterError (String, a) -checkPropsOnMutants :: [String] -> String -> [String] -> String -> IO [Qc.Result] -checkPropsOnMutants = mutantCheckSummary +checkQuickCheckOnMutants :: [String] -> String -> [String] -> String -> IO [Qc.Result] +checkQuickCheckOnMutants = mutantCheckSummary -checkTestSuiteOnMutants :: [String] -> String -> [String] -> String -> IO [HUnit.Counts] -checkTestSuiteOnMutants = mutantCheckSummary +checkHUnitOnMutants :: [String] -> String -> [String] -> String -> IO [HUnit.Counts] +checkHUnitOnMutants = mutantCheckSummary +checkHspecOnMutants :: [String] -> String -> [String] -> String -> IO [Hspec.Summary] +checkHspecOnMutants = mutantCheckSummary + -- main entry point mutantCheckSummary :: Summarizable a => [String] -> String -> [String] -> FilePath -> IO [a] mutantCheckSummary mutantFiles topModule evalSrcLst logFile = do results <- mapM (runCodeOnMutants mutantFiles topModule) evalSrcLst - let delim = "\n" ++ (take 25 (repeat '=')) ++ "\n" - singleTestSummaries = map (singleSummary mutantFiles) results - (terminalSummary,logSummary) = multipleSummary results - evalSrcLst' = map (delim ++) evalSrcLst + let singleTestSummaries = zip evalSrcLst $ map (testSummary mutantFiles) results + tssum = suiteSummary results -- print results to terminal putStrLn $ delim ++ "Overall Results:" - putStrLn terminalSummary - putStrLn $ Mu.showAS $ zipWith (++) evalSrcLst' $ map fst singleTestSummaries + putStrLn $ terminalSummary tssum + putStrLn $ showAS $ map showBrief singleTestSummaries putStr delim -- print results to logfile - appendFile logFile $ "OVERALL RESULTS:\n" ++ logSummary ++ Mu.showAS (zipWith (++) evalSrcLst' $ map snd singleTestSummaries) + appendFile logFile $ "OVERALL RESULTS:\n" ++ (tssum_log tssum) ++ (showAS $ map showDetail singleTestSummaries) -- hacky solution to avoid printing entire results to stdout and to give -- guidance to the type checker in picking specific Summarizable instances return $ tail [head $ (map snd) $ snd $ partitionEithers $ head results] + where showDetail (method, msum) = delim ++ showBrief (method, msum) ++ "\n" ++ detail msum + showBrief (method, msum) = showAS [method, + "\tTotal number of mutants:\t" ++ show (tsum_numMutants msum), + "\tFailed to Load:\t" ++ show (tsum_loadError msum), + "\tNot Killed:\t" ++ show (tsum_notKilled msum), + "\tKilled:\t" ++ show (tsum_killed msum), + "\tOthers:\t" ++ show (tsum_others msum), + ""] + detail msum = tsum_log msum + terminalSummary tssum = showAS ["Total number of mutants:\t" ++ show (tssum_numMutants tssum), + "Total number of alive mutants:\t" ++ show (tssum_alive tssum), + "Total number of load errors:\t" ++ show (tssum_errors tssum), + ""] + delim = "\n" ++ (take 25 (repeat '=')) ++ "\n" -- Interpreter Functionalities -- Examples -- t = runInterpreter (evalMethod "Examples/Quicksort.hs" "Quicksort" "quickCheckResult idEmp") runCodeOnMutants mutantFiles topModule evalStr = mapM (evalMyStr evalStr) mutantFiles - where evalMyStr evalStr file = I.runInterpreter (evalMethod file topModule evalStr) + where evalMyStr evalStr file = do putStrLn $ ">" ++ ":" ++ file ++ ":" ++ topModule ++ ":" ++ evalStr ++ ">" + x <- I.runInterpreter (evalMethod file topModule evalStr) + return x -- Given the filename, modulename, method to evaluate, evaluate, and return -- result as a pair. @@ -53,88 +72,109 @@ evalMethod fileName topModule evalStr = do I.loadModules [fileName] I.setTopLevelModules [topModule] - I.setImports ["Prelude", "Test.QuickCheck", "Test.HUnit"] result <- I.interpret evalStr (I.as :: (Typeable a => IO a)) >>= liftIO return (fileName, result) +data TSum = TSum {tsum_numMutants::Int, tsum_loadError::Int, tsum_notKilled::Int, tsum_killed::Int, tsum_others::Int, tsum_log::String} +data TSSum = TSSum {tssum_numMutants::Int, tssum_alive::Int, tssum_errors::Int, tssum_log::String} -- Class/Instance declaration type MutantFilename = String -type TerminalSummary = String -type LogSummary = String class Typeable s => Summarizable s where - singleSummary :: [MutantFilename] -> [InterpreterOutput s] -> (TerminalSummary, LogSummary) - multipleSummary :: [[InterpreterOutput s]] -> (TerminalSummary, LogSummary) + testSummary :: [MutantFilename] -> [InterpreterOutput s] -> TSum + suiteSummary :: [[InterpreterOutput s]] -> TSSum + isSuccess :: s -> Bool instance Summarizable HUnit.Counts where - singleSummary mutantFiles results = (terminalMsg, logMsg) - where (loadingErrorCases, executedCases) = partitionEithers results - loadingErrorFiles = mutantFiles \\ map fst executedCases - successCases = filter ((\c -> (HUnit.cases c == HUnit.tried c) && HUnit.failures c == 0 && HUnit.errors c == 0) . snd) executedCases - failuresCases = filter ((>0) . HUnit.failures . snd) executedCases - runningErrorCases = (filter ((>0) . HUnit.errors . snd) executedCases) \\ failuresCases - failToFullyTryCases = filter ((\c -> HUnit.cases c > HUnit.tried c) . snd) executedCases - r = length results - le = length loadingErrorCases - [s, fl, re, ftc] = map length [successCases, failuresCases, runningErrorCases, failToFullyTryCases] - terminalMsg = "\n\nTotal number of mutants: " ++ show r ++ - "\n\nFailed to load: " ++ show le ++ - "\nSuccesses (not killed): " ++ show s ++ - "\nFailures (killed): " ++ show fl ++ - "\nError while running: " ++ show re ++ - "\nIncompletely tested (may include failures and running errors): " ++ show ftc - logMsg = terminalMsg ++ "\n\nDetails: \n\nLoading error files:\n" ++ Mu.showA loadingErrorFiles - ++ "\n\nLoading error messages:\n" ++ Mu.showA loadingErrorCases - ++ "\n\nSuccesses:\n" ++ Mu.showA successCases - ++ "\n\nFailures:\n" ++ Mu.showA failuresCases - ++ "\n\nError while running:\n" ++ Mu.showA runningErrorCases - ++ "\n\nIncompletely tested (may include failures and running errors):\n" - ++ Mu.showA failToFullyTryCases ++ "\n" - multipleSummary = multipleCheckSummary (\c -> (HUnit.cases c == HUnit.tried c) && HUnit.failures c == 0 && HUnit.errors c == 0) + testSummary mutantFiles results = TSum { + tsum_numMutants = r, + tsum_loadError = le, + tsum_notKilled = s, + tsum_killed = fl, + tsum_others = re + ftc, + tsum_log = logMsg} + where (loadingErrorCases, executedCases) = partitionEithers results + loadingErrorFiles = mutantFiles \\ map fst executedCases + successCases = filter (isSuccess . snd) executedCases + failuresCases = filter ((>0) . HUnit.failures . snd) executedCases + runningErrorCases = (filter ((>0) . HUnit.errors . snd) executedCases) \\ failuresCases + failToFullyTryCases = filter ((\c -> HUnit.cases c > HUnit.tried c) . snd) executedCases + r = length results + le = length loadingErrorCases + [s, fl, re, ftc] = map length [successCases, failuresCases, runningErrorCases, failToFullyTryCases] + logMsg = showAS ["Details:", + "Loading error files:",showA loadingErrorFiles, + "Loading error messages:",showA loadingErrorCases, + "Successes:", showA successCases, + "Failures:", showA failuresCases, + "Error while running:", showA runningErrorCases, + "Incompletely tested (may include failures and running errors):",showA failToFullyTryCases] + suiteSummary = multipleCheckSummary (isSuccess . snd) + isSuccess = (\c -> (HUnit.cases c == HUnit.tried c) && HUnit.failures c == 0 && HUnit.errors c == 0) instance Summarizable Qc.Result where - singleSummary mutantFiles results = (terminalMsg, logMsg) + testSummary mutantFiles results = TSum { + tsum_numMutants = r, + tsum_loadError = e, + tsum_notKilled = s, + tsum_killed = f, + tsum_others = g, + tsum_log = logMsg} + where (errorCases, executedCases) = partitionEithers results + [successCases, failureCases, gaveUpCases] = map (\c -> filter (c . snd) executedCases) [isSuccess, isFailure, isGaveUp] + r = length results + e = length errorCases + [s,f,g] = map length [successCases, failureCases, gaveUpCases] + errorFiles = mutantFiles \\ map fst executedCases + logMsg = showAS ["Details:", + "Loading error files:", showA errorFiles, + "Loading error messages:", showA errorCases, + "Successes:", showA successCases, + "Failure:", showA failureCases, + "Gaveups:", showA gaveUpCases] + isFailure :: Qc.Result -> Bool + isFailure Qc.Failure{} = True + isFailure _ = False + isGaveUp :: Qc.Result -> Bool + isGaveUp Qc.GaveUp{} = True + isGaveUp _ = False + + suiteSummary = multipleCheckSummary (isSuccess . snd) + isSuccess = Qc.isSuccess + +instance Summarizable Hspec.Summary where + testSummary mutantFiles results = TSum { + tsum_numMutants = r, + tsum_loadError = e, + tsum_notKilled = s, + tsum_killed = f, + tsum_others = 0, + tsum_log = logMsg} where (errorCases, executedCases) = partitionEithers results - [successCases, failureCases, gaveUpCases] = map (\c -> filter (c . snd) executedCases) [Qc.isSuccess, isFailure, isGaveUp] r = length results e = length errorCases - [s,f,g] = map length [successCases, failureCases, gaveUpCases] + [successCases, failureCases] = map (\c -> filter (c . snd) executedCases) [isSuccess, isFailure] + [s,f] = map length [successCases, failureCases] errorFiles = mutantFiles \\ map fst executedCases - terminalMsg = "\n\nTotal number of mutants: " ++ show r ++ - "\n\nErrors: " ++ show e ++ Mu.showPerCent (e `Mu.percent` r) ++ - "\nSuccesses (not killed): " ++ show s ++ Mu.showPerCent (s `Mu.percent` r) ++ - "\nFailures (killed): " ++ show f ++ Mu.showPerCent (f `Mu.percent` r) ++ - "\nGaveups: " ++ show g ++ Mu.showPerCent (g `Mu.percent` r) - logMsg = terminalMsg ++ "\n\nDetails:\n\nLoading error files:\n" ++ Mu.showA errorFiles - ++ "\n\nLoading error messages:\n " ++ Mu.showA errorCases - ++ "\n\nSUCCESSES:\n " ++ Mu.showA successCases - ++ "\n\nFAILURE:\n " ++ Mu.showA failureCases - ++ "\n\nGAVEUPs:\n " ++ Mu.showA gaveUpCases ++ "\n" - isFailure :: Qc.Result -> Bool - isFailure Qc.Failure{} = True - isFailure _ = False - isGaveUp :: Qc.Result -> Bool - isGaveUp Qc.GaveUp{} = True - isGaveUp _ = False - - multipleSummary = multipleCheckSummary Qc.isSuccess + logMsg = showAS ["Details:", + "Loading error files:", showA errorFiles, + "Loading error messages:", showA errorCases, + "Successes:", showA successCases, + "Failure:", showA failureCases] + isFailure = not . isSuccess + suiteSummary = multipleCheckSummary (isSuccess . snd) + isSuccess (Hspec.Summary { Hspec.summaryExamples = se, Hspec.summaryFailures = sf } ) = sf == 0 -- we assume that checking each prop results in the same number of errorCases and executedCases -multipleCheckSummary :: Show a => (a -> Bool) -> [[InterpreterOutput a]] -> (String, String) multipleCheckSummary isSuccessFunction results | not (checkLength results) = error "Output lengths differ for some properties." - | otherwise = (terminalMsg, logMsg) - where executedCases = groupBy (\x y -> fst x == fst y) . sortBy (\x y -> fst x `compare` fst y) . rights $ concat results - allSuccesses = [rs | rs <- executedCases, length rs == length results, all (isSuccessFunction . snd) rs] - countAlive = length allSuccesses - countErrors = countMutants - length executedCases - terminalMsg = "\nTotal number of mutants: " ++ show countMutants - ++ "\nTotal number of alive and error-free mutants: " ++ show countAlive - ++ Mu.showPerCent (countAlive `Mu.percent` countMutants) ++ "\n" - ++ "Total number of erroneous mutants (failed to be loaded): " ++ show countErrors ++ Mu.showPerCent (countErrors `Mu.percent` countMutants) ++ "\n" - logMsg = terminalMsg ++ "\nDetails:\n\n" ++ Mu.showA allSuccesses ++ "\n" - - checkLength results = and $ map ((==countMutants) . length) results ++ map ((==countExecutedCases) . length) executedCases - countExecutedCases = length . head $ executedCases - countMutants = length . head $ results + | otherwise = TSSum {tssum_numMutants = countMutants, tssum_alive = countAlive, tssum_errors= countErrors, tssum_log = logMsg} + where executedCases = groupBy (\x y -> fst x == fst y) . sortBy (\x y -> fst x `compare` fst y) . rights $ concat results + allSuccesses = [rs | rs <- executedCases, length rs == length results, all isSuccessFunction rs] + countAlive = length allSuccesses + countErrors = countMutants - length executedCases + logMsg = showA allSuccesses + checkLength results = and $ map ((==countMutants) . length) results ++ map ((==countExecutedCases) . length) executedCases + countExecutedCases = length . head $ executedCases + countMutants = length . head $ results diff --git a/src/MuCheck/Utils/Common.hs b/src/MuCheck/Utils/Common.hs --- a/src/MuCheck/Utils/Common.hs +++ b/src/MuCheck/Utils/Common.hs @@ -2,27 +2,27 @@ import System.FilePath (splitExtension) --- choose [1,2,3,4,5] 4 --- = [[2,3,4,5],[1,3,4,5],[1,2,4,5],[1,2,3,5],[1,2,3,4]] +-- | The `choose` function generates subsets of a given size choose :: [b] -> Int -> [[b]] _ `choose` 0 = [[]] [] `choose` _ = [] (x:xs) `choose` k = (x:) `fmap` (xs `choose` (k-1)) ++ xs `choose` k --- generating mutant files names --- e.g.: "Quicksort.hs" ==> "Quicksort_1.hs", "Quicksort_2.hs", etc. +-- | The `genFileNames` function lazily generates filenames of mutants genFileNames :: String -> [String] genFileNames s = map newname [1..] where (name, ext) = splitExtension s newname i= name ++ "_" ++ show i ++ ext --- replace first element in a list given old and new values +-- | The `replace` function replaces first element in a list given old and new values as a pair replace :: Eq a => (a,a) -> [a] -> [a] replace (o,n) lst = map replaceit lst where replaceit v | v == o = n | otherwise = v +-- | The `safeHead` function safely extracts head of a list using Maybe safeHead :: [a] -> Maybe a safeHead [] = Nothing safeHead (x:xs) = Just x + diff --git a/src/MuCheck/Utils/Print.hs b/src/MuCheck/Utils/Print.hs --- a/src/MuCheck/Utils/Print.hs +++ b/src/MuCheck/Utils/Print.hs @@ -2,13 +2,14 @@ import Data.List(intercalate) --- utils for interpreter -showPerCent x = "(" ++ show x ++ "%)" -n `percent` t = 100 * n `div` t +-- | simple wrapper for adding a % at the end. +n ./. t = "(" ++ show (n * 100 `div` t) ++ "%)" +-- | join lines together showAS :: [String] -> String showAS = intercalate "\n" +-- | make lists into lines in text. showA :: Show a => [a] -> String showA = showAS . map show