MuCheck 0.3.0.0 → 0.3.0.4

raw patch · 7 files changed

+132/−50 lines, 7 filesdep ~directorydep ~hashabledep ~hintPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: directory, hashable, hint, hspec, mtl, random, syb, temporary, time

API changes (from Hackage documentation)

Files

• MuCheck.cabal +2/−2
• src/Test/MuCheck.hs +15/−2
• src/Test/MuCheck/AnalysisSummary.hs +14/−10
• src/Test/MuCheck/Config.hs +6/−8
• src/Test/MuCheck/Interpreter.hs +45/−17
• src/Test/MuCheck/MuOp.hs +1/−4
• src/Test/MuCheck/Mutation.hs +49/−7

@@ -1,5 +1,5 @@ name:                MuCheck-version:             0.3.0.0+version:             0.3.0.4 synopsis:            Automated Mutation Testing description:         This package defines a mutation analysis library for haskell                      programs. It does this by parsing the haskell source, and@@ -25,7 +25,7 @@ source-repository    this   type:              git   location:          https://bitbucket.org/osu-testing/mucheck.git-  tag:               0.3.0.0+  tag:               0.3.0.4  library   exposed-modules:  Test.MuCheck,

@@ -8,8 +8,21 @@ import Test.MuCheck.TestAdapter import Test.MuCheck.AnalysisSummary --- | Perform mutation analysis-mucheck :: (Summarizable a, Show a) => (Mutant -> TestStr -> InterpreterOutput a -> Summary) -> String -> String -> [TestStr] -> IO (MAnalysisSummary, [MutantSummary])+-- | Perform mutation analysis using any of the test frameworks that support+-- Summarizable (essentially, after running it on haskell, we should be able to+-- distinguish a successful run without failures from one with failures.)+-- E.g. using the mucheck-quickcheck adapter+--+-- > tFn :: Mutant -> TestStr -> InterpreterOutput QuickCheckSummary`+-- > tFn = testSummary+-- > mucheck tFn "qsort" "Examples/QuickCheckTest.hs" ["quickCheckResult revProp"]++mucheck :: (Summarizable a, Show a) =>+     (Mutant -> TestStr -> InterpreterOutput a -> Summary) -- ^ The summarization function to use on test results+  -> String                                                -- ^ The mutating function we are checking the test adequacy of.+  -> String                                                -- ^ The module file where the mutating function was declared+  -> [TestStr]                                             -- ^ The tests we can use to kill mutants+  -> IO (MAnalysisSummary, [MutantSummary])                -- ^ Returns a tuple of full summary, and individual mutant results. mucheck resFn mutatingFn moduleFile tests = do   mutants <- genMutants mutatingFn moduleFile >>= rSample (maxNumMutants defaultConfig)   evaluateMutants resFn mutants tests

@@ -1,19 +1,23 @@+{-# LANGUAGE RecordWildCards #-}+-- | The AnalysisSummary declares the mutation result datatype, and its+-- instances. module Test.MuCheck.AnalysisSummary where  import Test.MuCheck.Utils.Print  -- | Datatype to hold results of the entire run data MAnalysisSummary = MAnalysisSummary {-    maNumMutants::Int,-    maAlive::Int,-    maKilled::Int,-    maErrors::Int}+    _maNumMutants::Int     -- ^ The number of mutants tested+  , _maAlive::Int          -- ^ The number of mutants that were alive after the mutation run+  , _maKilled::Int         -- ^ The number of mutants that were killed+  , _maErrors::Int         -- ^ The number of non-viable mutants.+} +-- | The show instance for MAnalysisSummary instance Show MAnalysisSummary where-  show ma = let total = maNumMutants ma-                noerrors = total - maErrors ma-            in showAS ["Total mutants: " ++ show total,-                       "\terrors: " ++  show (maErrors ma) ++ "  "++ maErrors ma ./. total,-                       "\talive: " ++  show (maAlive ma) ++ "/" ++ show noerrors,-                       "\tkilled: " ++  show (maKilled ma) ++ "/" ++ show noerrors ++ " " ++ maKilled ma ./. noerrors]+  show (MAnalysisSummary{..}) = let noerrors = _maNumMutants - _maErrors+    in showAS ["Total mutants: " ++ show _maNumMutants,+               "\terrors: " ++  show _maErrors ++ "  "++ _maErrors ./. _maNumMutants,+               "\talive: " ++  show _maAlive  ++ "/" ++ show noerrors,+               "\tkilled: " ++  show _maKilled ++ "/" ++ show noerrors ++ " " ++ _maKilled ./. noerrors] 

@@ -37,19 +37,17 @@ -- -- becomes ----- > if (not True) then 1 else 0+-- > if True then 0 else 1   , doNegateIfElse :: Rational--- | negate guarded booleans in case statements, that is+-- | negate guarded booleans in guarded definitions ----- > case v of--- >   True -> 1--- >   otherwise -> 0+-- > myFn x | x == 1 = True+-- > myFn   | otherwise = False -- -- becomes ----- > case v of--- >  (not True) -> 1--- >  otherwise -> 0+-- > myFn x | not (x == 1) = True+-- > myFn   | otherwise = False   , doNegateGuards :: Rational -- | Maximum number of mutants to generate.   , maxNumMutants :: Int

@@ -1,5 +1,6 @@ {-# LANGUAGE TupleSections, MultiWayIf, DeriveDataTypeable #-}--- | The entry point for mucheck+-- | The Interpreter module is responible for invoking the Hint interpreter to+-- evaluate mutants. module Test.MuCheck.Interpreter (evaluateMutants, evalMethod, evalMutant, evalTest, summarizeResults, MutantSummary(..)) where  import qualified Language.Haskell.Interpreter as I@@ -17,21 +18,31 @@   -- | Data type to hold results of a single test execution-data MutantSummary = MSumError Mutant String [Summary]         -- errorStr-                   | MSumAlive Mutant [Summary]-                   | MSumKilled Mutant [Summary]-                   | MSumOther Mutant [Summary]+data MutantSummary = MSumError Mutant String [Summary]         -- ^ Capture the error if one occured+                   | MSumAlive Mutant [Summary]                -- ^ The mutant was alive+                   | MSumKilled Mutant [Summary]               -- ^ The mutant was kileld+                   | MSumOther Mutant [Summary]                -- ^ Undetermined - we will treat it as killed as it is not a success.                    deriving (Show, Typeable)  -- | Given the list of tests suites to check, run the test suite on mutants.-evaluateMutants :: (Summarizable a, Show a) => (Mutant -> TestStr -> InterpreterOutput a -> Summary) -> [Mutant] -> [String] -> IO (MAnalysisSummary, [MutantSummary])+evaluateMutants :: (Summarizable a, Show a) =>+    (Mutant -> TestStr -> InterpreterOutput a -> Summary)           -- ^ The summary function+ -> [Mutant]                                                        -- ^ The mutants to be evaluated+ -> [TestStr]                                                       -- ^ The tests to be used by mutation analysis+ -> IO (MAnalysisSummary, [MutantSummary])                          -- ^ Returns a tuple of full run summary and individual mutant summary evaluateMutants testSummaryFn mutants tests = do   results <- mapM (evalMutant tests) mutants -- [InterpreterOutput t]   let singleTestSummaries = map (summarizeResults testSummaryFn tests) $ zip mutants results       ma  = fullSummary tests results   return (ma, singleTestSummaries) -summarizeResults :: Summarizable a => (Mutant -> TestStr -> InterpreterOutput a -> Summary) -> [String] -> (Mutant, [InterpreterOutput a]) -> MutantSummary+-- | The `summarizeResults` function evaluates the results of a test run+-- using the supplied `isSuccess` and `testSummaryFn` functions from the adapters+summarizeResults :: Summarizable a =>+     (Mutant -> TestStr -> InterpreterOutput a -> Summary)        -- ^ The summary function+  -> [TestStr]                                                    -- ^ Tests we used to run analysis+  -> (Mutant, [InterpreterOutput a])                              -- ^ The mutant and its corresponding output of test runs.+  -> MutantSummary                                                -- ^ Returns a summary of the run for the mutant summarizeResults testSummaryFn tests (mutant, ioresults) = case last results of -- the last result should indicate status because we dont run if there is error.   Left err -> MSumError mutant (show err) logS   Right out -> myresult out@@ -43,7 +54,10 @@         logS = zipWith (testSummaryFn mutant) tests ioresults  -- | Run all tests on a mutant-evalMutant :: (Typeable t, Summarizable t) => [TestStr] -> Mutant -> IO [InterpreterOutput t]+evalMutant :: (Typeable t, Summarizable t) =>+    [TestStr]                                                     -- ^ The tests to be used+  -> Mutant                                                       -- ^ Mutant being tested+  -> IO [InterpreterOutput t]                                     -- ^ Returns the result of test runs evalMutant tests mutant = do   -- Hint does not provide us a way to evaluate the module without   -- writing it to disk first, so we go for this hack.@@ -55,8 +69,12 @@   let logF = mutantFile ++ ".log"   stopFast (evalTest mutantFile logF) tests --- | Stop mutant runs at the first sign of problems.-stopFast :: (Typeable t, Summarizable t) => (String -> IO (InterpreterOutput t)) -> [TestStr] -> IO [InterpreterOutput t]+-- | Stop mutant runs at the first sign of problems (invalid mutants or test+-- failure).+stopFast :: (Typeable t, Summarizable t) =>+     (String -> IO (InterpreterOutput t))  -- ^ The function that given a test, runs it, and returns the result+  -> [TestStr]                             -- ^ The tests to be run+  -> IO [InterpreterOutput t]              -- ^ Returns the output of all tests. If there is an error, then it will be at the last test. stopFast _ [] = return [] stopFast fn (x:xs) = do   v <- fn x@@ -67,7 +85,11 @@       else return [v] -- test failed (mutant detected)  -- | Run one single test on a mutant-evalTest :: (Typeable a, Summarizable a) => String -> String -> String -> IO (InterpreterOutput a)+evalTest :: (Typeable a, Summarizable a) =>+    String                                 -- ^ The mutant _file_ that we have to evaluate (_not_ the content)+ -> String                                 -- ^ The file where we will write the stdout and stderr during the run. + -> TestStr                                -- ^ The test to be run+ -> IO (InterpreterOutput a)               -- ^ Returns the output of given test run evalTest mutantFile logF test = do   val <- withArgs [] $ catchOutput logF $ I.runInterpreter (evalMethod mutantFile test)   return Io {_io = val, _ioLog = logF}@@ -77,7 +99,10 @@ -- > t = I.runInterpreter (evalMethod -- >        "Examples/QuickCheckTest.hs" -- >        "quickCheckResult idEmp")-evalMethod :: (I.MonadInterpreter m, Typeable t) => String -> String -> m t+evalMethod :: (I.MonadInterpreter m, Typeable t) =>+     String                               -- ^ The mutant _file_ to load+  -> TestStr                              -- ^ The test to be run+  -> m t                                  -- ^ Returns the monadic computation to be run by I.runInterpreter evalMethod fileName evalStr = do   I.loadModules [fileName]   ms <- I.getLoadedModules@@ -86,12 +111,15 @@   -- | Summarize the entire run. Passed results are per mutant-fullSummary :: (Show b, Summarizable b) => [TestStr] -> [[InterpreterOutput b]] -> MAnalysisSummary+fullSummary :: (Show b, Summarizable b) =>+     [TestStr]                              -- ^ The list of tests we used+  -> [[InterpreterOutput b]]                -- ^ The test ouput (per mutant, (per test))+  -> MAnalysisSummary                       -- ^ Returns the full summary of the run fullSummary _tests results = MAnalysisSummary {-  maNumMutants = length results,-  maAlive = length alive,-  maKilled = length fails,-  maErrors= length errors}+  _maNumMutants = length results,+  _maAlive = length alive,+  _maKilled = length fails,+  _maErrors= length errors}   where res = map (map _io) results         lasts = map last res -- get the last test runs         (errors, completed) = partitionEithers lasts

@@ -23,8 +23,7 @@           | G  (GuardedRhs, GuardedRhs)   deriving Eq --- boilerplate code-+-- | Apply the given function on the tuple inside MuOp apply :: (forall a. (Eq a, G.Typeable a, Show a) => (a,a) -> c) -> MuOp -> c apply f (N  m) = f m apply f (QN m) = f m@@ -50,8 +49,6 @@ -- | MuOp instance for Show instance Show MuOp where   show = apply showM---- end boilerplate code  -- | Mutation operation representing translation from one fn to another fn. class Mutable a where

@@ -1,5 +1,5 @@ {-# LANGUAGE ImpredicativeTypes #-}--- | Mutation happens here.+-- | This module handles the mutation of different patterns. module Test.MuCheck.Mutation where  import Language.Haskell.Exts(Literal(Int, Char, Frac, String, PrimInt, PrimChar, PrimFloat, PrimDouble, PrimWord, PrimString),@@ -20,13 +20,21 @@  -- | The `genMutants` function is a wrapper to genMutantsWith with standard -- configuraton-genMutants :: String -> FilePath -> IO [Mutant]+genMutants ::+     String           -- ^ The mutating function under test+  -> FilePath         -- ^ The module where the mutating function is declared+  -> IO [Mutant]      -- ^ Returns the mutants produced. genMutants = genMutantsWith defaultConfig  -- | The `genMutantsWith` function takes configuration function to mutate,--- filename the function is defined in, and produces mutants in the same--- directory as the filename, and returns the number of mutants produced.-genMutantsWith :: Config -> String -> FilePath -> IO [Mutant]+-- function to mutate, filename the function is defined in, and produces+-- mutants in the same directory as the filename, and returns the number+-- of mutants produced.+genMutantsWith ::+     Config                     -- ^ The configuration to be used+  -> String                     -- ^ The mutating function+  -> FilePath                   -- ^ The module file where mutating function was declared+  -> IO [Mutant]                -- ^ Returns the mutants produced genMutantsWith args func filename  = do       g <- genRandomSeed       f <- readFile filename@@ -34,13 +42,23 @@  -- | Wrapper around sampleF that returns correct sampling ratios according to -- configuration passed.-sampler :: RandomGen g => Config -> g -> MuVars -> [t] -> [t]+sampler :: RandomGen g =>+     Config                   -- ^ Configuration+  -> g                        -- ^ The random seed+  -> MuVars                   -- ^ What kind of a mutation are we interested in?+  -> [t]                      -- ^ The original list of mutation operators+  -> [t]                      -- ^ Returns the sampled mutation operators sampler args g m = sampleF g (getSample m args)  -- | The `genMutantsForSrc` takes the function name to mutate, source where it -- is defined, and a sampling function, and returns the mutated sources selected -- using sampling function.-genMutantsForSrc :: Config -> String -> String -> (MuVars -> [MuOp] -> [MuOp]) -> [Mutant]+genMutantsForSrc ::+     Config                   -- ^ Configuration+  -> String                   -- ^ The mutating function+  -> String                   -- ^ The module where mutating function was declared+  -> (MuVars -> [MuOp] -> [MuOp]) -- ^ The sampling function+  -> [Mutant]                 -- ^ Returns the sampled mutants genMutantsForSrc args funcname src sampleFn = map prettyPrint programMutants   where astMod = getASTFromStr src         f = getFunc funcname astMod@@ -172,6 +190,13 @@         convert (PrimWord i) = map PrimWord $ nub [i + 1, i - 1, 0, 1]  -- | Convert Boolean Literals+--+-- > (True, False)+--+-- becomes+--+-- > (False, True)+ selectBLitOps :: Decl -> [MuOp] selectBLitOps m = selectValOps isLit convert m   where isLit (Ident "True") = True@@ -182,6 +207,13 @@         convert _ = []  -- | Negating boolean in if/else statements+--+-- > if True then 1 else 0+--+-- becomes+--+-- > if True then 0 else 1+ selectIfElseBoolNegOps :: Decl -> [MuOp] selectIfElseBoolNegOps m = selectValOps isIf convert m   where isIf If{} = True@@ -190,6 +222,16 @@         convert _ = []  -- | Negating boolean in Guards+-- | negate guarded booleans in guarded definitions+--+-- > myFn x | x == 1 = True+-- > myFn   | otherwise = False+--+-- becomes+--+-- > myFn x | not (x == 1) = True+-- > myFn   | otherwise = False+ selectGuardedBoolNegOps :: Decl -> [MuOp] selectGuardedBoolNegOps m = selectValOps isGuardedRhs convert m   where isGuardedRhs GuardedRhs{} = True