diff --git a/MuCheck.cabal b/MuCheck.cabal --- a/MuCheck.cabal +++ b/MuCheck.cabal @@ -1,5 +1,5 @@ name: MuCheck -version: 0.3.0.0 +version: 0.3.0.4 synopsis: Automated Mutation Testing description: This package defines a mutation analysis library for haskell programs. It does this by parsing the haskell source, and @@ -25,7 +25,7 @@ source-repository this type: git location: https://bitbucket.org/osu-testing/mucheck.git - tag: 0.3.0.0 + tag: 0.3.0.4 library exposed-modules: Test.MuCheck, diff --git a/src/Test/MuCheck.hs b/src/Test/MuCheck.hs --- a/src/Test/MuCheck.hs +++ b/src/Test/MuCheck.hs @@ -8,8 +8,21 @@ import Test.MuCheck.TestAdapter import Test.MuCheck.AnalysisSummary --- | Perform mutation analysis -mucheck :: (Summarizable a, Show a) => (Mutant -> TestStr -> InterpreterOutput a -> Summary) -> String -> String -> [TestStr] -> IO (MAnalysisSummary, [MutantSummary]) +-- | Perform mutation analysis using any of the test frameworks that support +-- Summarizable (essentially, after running it on haskell, we should be able to +-- distinguish a successful run without failures from one with failures.) +-- E.g. using the mucheck-quickcheck adapter +-- +-- > tFn :: Mutant -> TestStr -> InterpreterOutput QuickCheckSummary` +-- > tFn = testSummary +-- > mucheck tFn "qsort" "Examples/QuickCheckTest.hs" ["quickCheckResult revProp"] + +mucheck :: (Summarizable a, Show a) => + (Mutant -> TestStr -> InterpreterOutput a -> Summary) -- ^ The summarization function to use on test results + -> String -- ^ The mutating function we are checking the test adequacy of. + -> String -- ^ The module file where the mutating function was declared + -> [TestStr] -- ^ The tests we can use to kill mutants + -> IO (MAnalysisSummary, [MutantSummary]) -- ^ Returns a tuple of full summary, and individual mutant results. mucheck resFn mutatingFn moduleFile tests = do mutants <- genMutants mutatingFn moduleFile >>= rSample (maxNumMutants defaultConfig) evaluateMutants resFn mutants tests diff --git a/src/Test/MuCheck/AnalysisSummary.hs b/src/Test/MuCheck/AnalysisSummary.hs --- a/src/Test/MuCheck/AnalysisSummary.hs +++ b/src/Test/MuCheck/AnalysisSummary.hs @@ -1,19 +1,23 @@ +{-# LANGUAGE RecordWildCards #-} +-- | The AnalysisSummary declares the mutation result datatype, and its +-- instances. module Test.MuCheck.AnalysisSummary where import Test.MuCheck.Utils.Print -- | Datatype to hold results of the entire run data MAnalysisSummary = MAnalysisSummary { - maNumMutants::Int, - maAlive::Int, - maKilled::Int, - maErrors::Int} + _maNumMutants::Int -- ^ The number of mutants tested + , _maAlive::Int -- ^ The number of mutants that were alive after the mutation run + , _maKilled::Int -- ^ The number of mutants that were killed + , _maErrors::Int -- ^ The number of non-viable mutants. +} +-- | The show instance for MAnalysisSummary instance Show MAnalysisSummary where - show ma = let total = maNumMutants ma - noerrors = total - maErrors ma - in showAS ["Total mutants: " ++ show total, - "\terrors: " ++ show (maErrors ma) ++ " "++ maErrors ma ./. total, - "\talive: " ++ show (maAlive ma) ++ "/" ++ show noerrors, - "\tkilled: " ++ show (maKilled ma) ++ "/" ++ show noerrors ++ " " ++ maKilled ma ./. noerrors] + show (MAnalysisSummary{..}) = let noerrors = _maNumMutants - _maErrors + in showAS ["Total mutants: " ++ show _maNumMutants, + "\terrors: " ++ show _maErrors ++ " "++ _maErrors ./. _maNumMutants, + "\talive: " ++ show _maAlive ++ "/" ++ show noerrors, + "\tkilled: " ++ show _maKilled ++ "/" ++ show noerrors ++ " " ++ _maKilled ./. noerrors] diff --git a/src/Test/MuCheck/Config.hs b/src/Test/MuCheck/Config.hs --- a/src/Test/MuCheck/Config.hs +++ b/src/Test/MuCheck/Config.hs @@ -37,19 +37,17 @@ -- -- becomes -- --- > if (not True) then 1 else 0 +-- > if True then 0 else 1 , doNegateIfElse :: Rational --- | negate guarded booleans in case statements, that is +-- | negate guarded booleans in guarded definitions -- --- > case v of --- > True -> 1 --- > otherwise -> 0 +-- > myFn x | x == 1 = True +-- > myFn | otherwise = False -- -- becomes -- --- > case v of --- > (not True) -> 1 --- > otherwise -> 0 +-- > myFn x | not (x == 1) = True +-- > myFn | otherwise = False , doNegateGuards :: Rational -- | Maximum number of mutants to generate. , maxNumMutants :: Int diff --git a/src/Test/MuCheck/Interpreter.hs b/src/Test/MuCheck/Interpreter.hs --- a/src/Test/MuCheck/Interpreter.hs +++ b/src/Test/MuCheck/Interpreter.hs @@ -1,5 +1,6 @@ {-# LANGUAGE TupleSections, MultiWayIf, DeriveDataTypeable #-} --- | The entry point for mucheck +-- | The Interpreter module is responible for invoking the Hint interpreter to +-- evaluate mutants. module Test.MuCheck.Interpreter (evaluateMutants, evalMethod, evalMutant, evalTest, summarizeResults, MutantSummary(..)) where import qualified Language.Haskell.Interpreter as I @@ -17,21 +18,31 @@ -- | Data type to hold results of a single test execution -data MutantSummary = MSumError Mutant String [Summary] -- errorStr - | MSumAlive Mutant [Summary] - | MSumKilled Mutant [Summary] - | MSumOther Mutant [Summary] +data MutantSummary = MSumError Mutant String [Summary] -- ^ Capture the error if one occured + | MSumAlive Mutant [Summary] -- ^ The mutant was alive + | MSumKilled Mutant [Summary] -- ^ The mutant was kileld + | MSumOther Mutant [Summary] -- ^ Undetermined - we will treat it as killed as it is not a success. deriving (Show, Typeable) -- | Given the list of tests suites to check, run the test suite on mutants. -evaluateMutants :: (Summarizable a, Show a) => (Mutant -> TestStr -> InterpreterOutput a -> Summary) -> [Mutant] -> [String] -> IO (MAnalysisSummary, [MutantSummary]) +evaluateMutants :: (Summarizable a, Show a) => + (Mutant -> TestStr -> InterpreterOutput a -> Summary) -- ^ The summary function + -> [Mutant] -- ^ The mutants to be evaluated + -> [TestStr] -- ^ The tests to be used by mutation analysis + -> IO (MAnalysisSummary, [MutantSummary]) -- ^ Returns a tuple of full run summary and individual mutant summary evaluateMutants testSummaryFn mutants tests = do results <- mapM (evalMutant tests) mutants -- [InterpreterOutput t] let singleTestSummaries = map (summarizeResults testSummaryFn tests) $ zip mutants results ma = fullSummary tests results return (ma, singleTestSummaries) -summarizeResults :: Summarizable a => (Mutant -> TestStr -> InterpreterOutput a -> Summary) -> [String] -> (Mutant, [InterpreterOutput a]) -> MutantSummary +-- | The `summarizeResults` function evaluates the results of a test run +-- using the supplied `isSuccess` and `testSummaryFn` functions from the adapters +summarizeResults :: Summarizable a => + (Mutant -> TestStr -> InterpreterOutput a -> Summary) -- ^ The summary function + -> [TestStr] -- ^ Tests we used to run analysis + -> (Mutant, [InterpreterOutput a]) -- ^ The mutant and its corresponding output of test runs. + -> MutantSummary -- ^ Returns a summary of the run for the mutant summarizeResults testSummaryFn tests (mutant, ioresults) = case last results of -- the last result should indicate status because we dont run if there is error. Left err -> MSumError mutant (show err) logS Right out -> myresult out @@ -43,7 +54,10 @@ logS = zipWith (testSummaryFn mutant) tests ioresults -- | Run all tests on a mutant -evalMutant :: (Typeable t, Summarizable t) => [TestStr] -> Mutant -> IO [InterpreterOutput t] +evalMutant :: (Typeable t, Summarizable t) => + [TestStr] -- ^ The tests to be used + -> Mutant -- ^ Mutant being tested + -> IO [InterpreterOutput t] -- ^ Returns the result of test runs evalMutant tests mutant = do -- Hint does not provide us a way to evaluate the module without -- writing it to disk first, so we go for this hack. @@ -55,8 +69,12 @@ let logF = mutantFile ++ ".log" stopFast (evalTest mutantFile logF) tests --- | Stop mutant runs at the first sign of problems. -stopFast :: (Typeable t, Summarizable t) => (String -> IO (InterpreterOutput t)) -> [TestStr] -> IO [InterpreterOutput t] +-- | Stop mutant runs at the first sign of problems (invalid mutants or test +-- failure). +stopFast :: (Typeable t, Summarizable t) => + (String -> IO (InterpreterOutput t)) -- ^ The function that given a test, runs it, and returns the result + -> [TestStr] -- ^ The tests to be run + -> IO [InterpreterOutput t] -- ^ Returns the output of all tests. If there is an error, then it will be at the last test. stopFast _ [] = return [] stopFast fn (x:xs) = do v <- fn x @@ -67,7 +85,11 @@ else return [v] -- test failed (mutant detected) -- | Run one single test on a mutant -evalTest :: (Typeable a, Summarizable a) => String -> String -> String -> IO (InterpreterOutput a) +evalTest :: (Typeable a, Summarizable a) => + String -- ^ The mutant _file_ that we have to evaluate (_not_ the content) + -> String -- ^ The file where we will write the stdout and stderr during the run. + -> TestStr -- ^ The test to be run + -> IO (InterpreterOutput a) -- ^ Returns the output of given test run evalTest mutantFile logF test = do val <- withArgs [] $ catchOutput logF $ I.runInterpreter (evalMethod mutantFile test) return Io {_io = val, _ioLog = logF} @@ -77,7 +99,10 @@ -- > t = I.runInterpreter (evalMethod -- > "Examples/QuickCheckTest.hs" -- > "quickCheckResult idEmp") -evalMethod :: (I.MonadInterpreter m, Typeable t) => String -> String -> m t +evalMethod :: (I.MonadInterpreter m, Typeable t) => + String -- ^ The mutant _file_ to load + -> TestStr -- ^ The test to be run + -> m t -- ^ Returns the monadic computation to be run by I.runInterpreter evalMethod fileName evalStr = do I.loadModules [fileName] ms <- I.getLoadedModules @@ -86,12 +111,15 @@ -- | Summarize the entire run. Passed results are per mutant -fullSummary :: (Show b, Summarizable b) => [TestStr] -> [[InterpreterOutput b]] -> MAnalysisSummary +fullSummary :: (Show b, Summarizable b) => + [TestStr] -- ^ The list of tests we used + -> [[InterpreterOutput b]] -- ^ The test ouput (per mutant, (per test)) + -> MAnalysisSummary -- ^ Returns the full summary of the run fullSummary _tests results = MAnalysisSummary { - maNumMutants = length results, - maAlive = length alive, - maKilled = length fails, - maErrors= length errors} + _maNumMutants = length results, + _maAlive = length alive, + _maKilled = length fails, + _maErrors= length errors} where res = map (map _io) results lasts = map last res -- get the last test runs (errors, completed) = partitionEithers lasts diff --git a/src/Test/MuCheck/MuOp.hs b/src/Test/MuCheck/MuOp.hs --- a/src/Test/MuCheck/MuOp.hs +++ b/src/Test/MuCheck/MuOp.hs @@ -23,8 +23,7 @@ | G (GuardedRhs, GuardedRhs) deriving Eq --- boilerplate code - +-- | Apply the given function on the tuple inside MuOp apply :: (forall a. (Eq a, G.Typeable a, Show a) => (a,a) -> c) -> MuOp -> c apply f (N m) = f m apply f (QN m) = f m @@ -50,8 +49,6 @@ -- | MuOp instance for Show instance Show MuOp where show = apply showM - --- end boilerplate code -- | Mutation operation representing translation from one fn to another fn. class Mutable a where diff --git a/src/Test/MuCheck/Mutation.hs b/src/Test/MuCheck/Mutation.hs --- a/src/Test/MuCheck/Mutation.hs +++ b/src/Test/MuCheck/Mutation.hs @@ -1,5 +1,5 @@ {-# LANGUAGE ImpredicativeTypes #-} --- | Mutation happens here. +-- | This module handles the mutation of different patterns. module Test.MuCheck.Mutation where import Language.Haskell.Exts(Literal(Int, Char, Frac, String, PrimInt, PrimChar, PrimFloat, PrimDouble, PrimWord, PrimString), @@ -20,13 +20,21 @@ -- | The `genMutants` function is a wrapper to genMutantsWith with standard -- configuraton -genMutants :: String -> FilePath -> IO [Mutant] +genMutants :: + String -- ^ The mutating function under test + -> FilePath -- ^ The module where the mutating function is declared + -> IO [Mutant] -- ^ Returns the mutants produced. genMutants = genMutantsWith defaultConfig -- | The `genMutantsWith` function takes configuration function to mutate, --- filename the function is defined in, and produces mutants in the same --- directory as the filename, and returns the number of mutants produced. -genMutantsWith :: Config -> String -> FilePath -> IO [Mutant] +-- function to mutate, filename the function is defined in, and produces +-- mutants in the same directory as the filename, and returns the number +-- of mutants produced. +genMutantsWith :: + Config -- ^ The configuration to be used + -> String -- ^ The mutating function + -> FilePath -- ^ The module file where mutating function was declared + -> IO [Mutant] -- ^ Returns the mutants produced genMutantsWith args func filename = do g <- genRandomSeed f <- readFile filename @@ -34,13 +42,23 @@ -- | Wrapper around sampleF that returns correct sampling ratios according to -- configuration passed. -sampler :: RandomGen g => Config -> g -> MuVars -> [t] -> [t] +sampler :: RandomGen g => + Config -- ^ Configuration + -> g -- ^ The random seed + -> MuVars -- ^ What kind of a mutation are we interested in? + -> [t] -- ^ The original list of mutation operators + -> [t] -- ^ Returns the sampled mutation operators sampler args g m = sampleF g (getSample m args) -- | The `genMutantsForSrc` takes the function name to mutate, source where it -- is defined, and a sampling function, and returns the mutated sources selected -- using sampling function. -genMutantsForSrc :: Config -> String -> String -> (MuVars -> [MuOp] -> [MuOp]) -> [Mutant] +genMutantsForSrc :: + Config -- ^ Configuration + -> String -- ^ The mutating function + -> String -- ^ The module where mutating function was declared + -> (MuVars -> [MuOp] -> [MuOp]) -- ^ The sampling function + -> [Mutant] -- ^ Returns the sampled mutants genMutantsForSrc args funcname src sampleFn = map prettyPrint programMutants where astMod = getASTFromStr src f = getFunc funcname astMod @@ -172,6 +190,13 @@ convert (PrimWord i) = map PrimWord $ nub [i + 1, i - 1, 0, 1] -- | Convert Boolean Literals +-- +-- > (True, False) +-- +-- becomes +-- +-- > (False, True) + selectBLitOps :: Decl -> [MuOp] selectBLitOps m = selectValOps isLit convert m where isLit (Ident "True") = True @@ -182,6 +207,13 @@ convert _ = [] -- | Negating boolean in if/else statements +-- +-- > if True then 1 else 0 +-- +-- becomes +-- +-- > if True then 0 else 1 + selectIfElseBoolNegOps :: Decl -> [MuOp] selectIfElseBoolNegOps m = selectValOps isIf convert m where isIf If{} = True @@ -190,6 +222,16 @@ convert _ = [] -- | Negating boolean in Guards +-- | negate guarded booleans in guarded definitions +-- +-- > myFn x | x == 1 = True +-- > myFn | otherwise = False +-- +-- becomes +-- +-- > myFn x | not (x == 1) = True +-- > myFn | otherwise = False + selectGuardedBoolNegOps :: Decl -> [MuOp] selectGuardedBoolNegOps m = selectValOps isGuardedRhs convert m where isGuardedRhs GuardedRhs{} = True