MuCheck 0.1.0.1 → 0.1.1.0

raw patch · 5 files changed

+152/−104 lines, 5 filesdep +hspec-coredep ~haskell-src-extsPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: hspec-core

Dependency ranges changed: haskell-src-exts

API changes (from Hackage documentation)

Files

• MuCheck.cabal +7/−8
• src/Main.hs +16/−8
• src/MuCheck/Interpreter.hs +120/−80
• src/MuCheck/Utils/Common.hs +5/−5
• src/MuCheck/Utils/Print.hs +4/−3

@@ -1,5 +1,5 @@ name:                MuCheck-version:             0.1.0.1+version:             0.1.1.0 synopsis:            Automated Mutation Testing description:         This package defines a mutation analysis library for haskell                      programs. It does this by parsing the haskell source, and@@ -12,28 +12,27 @@ license-file:        LICENSE author:              Duc Lee <ledu@onid.oregonstate.edu>                      Rahul Gopinath <rahul@gopinath.org>-maintainer:          ledu@onid.oregonstate.edu-                     rahul@gopinath.org+maintainer:          rahul@gopinath.org -- copyright:            category:            Testing build-type:          Simple cabal-version:       >= 1.10 -source-repository    head+source-repository    this   type:              git-  location:          https://rgopinath@bitbucket.org/osu-testing/mucheck.git+  location:          https://bitbucket.org/osu-testing/mucheck.git   tag:               0.1.0.1  executable mucheck   main-is:          Main.hs-  build-depends:    base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath+  build-depends:    base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath, hspec >= 2.0, hspec-core >= 2.0   default-language: Haskell2010   hs-source-dirs:   src  library   exposed-modules:  MuCheck.MuOp, MuCheck.StdArgs, MuCheck.Interpreter, MuCheck.Mutation, MuCheck.Operators, MuCheck.Utils.Syb, MuCheck.Utils.Common, MuCheck.Utils.Print   -- other-modules:       -  build-depends:    base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath+  build-depends:    base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath, hspec>= 2.0, hspec-core >= 2.0   default-language: Haskell2010   hs-source-dirs:   src @@ -42,5 +41,5 @@   hs-source-dirs:   test   main-is:          Spec.hs   default-language: Haskell2010-  build-depends: base >= 4 && <5, MuCheck, hspec >= 2.0, QuickCheck >= 2.6, filepath+  build-depends:    base >=4 && <5, haskell-src-exts >=1.13, syb >= 0.4.0, time >= 1.4.0.1, QuickCheck>=2.6, hint >= 0.3.1.0, mtl>=2.1.2, HUnit >= 1.0, filepath, hspec>= 2.0, hspec-core >= 2.0, MuCheck 

@@ -8,21 +8,29 @@ import MuCheck.Mutation import MuCheck.Operators import MuCheck.Utils.Common+import MuCheck.Utils.Print -process :: String -> String -> String -> [String] -> IO ()-process fn file modulename args = do+process :: String -> String -> String -> String -> [String] -> IO ()+process t fn file modulename args = do   numMutants <- genMutants fn file-  checkPropsOnMutants (take numMutants $ genFileNames file) modulename args "./test.log"-  return ()+  case t of+    "qcheck" -> checkQuickCheckOnMutants (take numMutants $ genFileNames file) modulename args "./qcheck.log" >> return ()+    "hspec" ->  checkHspecOnMutants (take numMutants $ genFileNames file) modulename args "./hspec.log" >> return ()+    "hunit" ->  checkHUnitOnMutants (take numMutants $ genFileNames file) modulename args "./hunit.log" >> return ()+    _ -> error "Unexpected test type"   main :: IO () main = do   val <- getArgs   case val of-    ("-help" : _ ) -> help-    (fn : file : modulename : args) -> process fn file modulename args-    _ -> error "Need function file modulename [args]"+    ("-h" : _ ) -> help+    (t: fn : file : modulename : args) -> withArgs [] $ process t fn file modulename args+    _ -> error "Need [qcheck|hunit|hspec] function file modulename [args]\n\tUse -h to get help"  help :: IO ()-help = putStrLn "mucheck function file modulename [args]"+help = putStrLn ("mucheck type function file modulename [args]\n" ++ (showAS ["E.g:",+       " ./mucheck qcheck qsort Examples/QuickCheckTest.hs Examples.QuickCheckTest 'quickCheckResult idEmpProp' 'quickCheckResult revProp' 'quickCheckResult modelProp'",+       " ./mucheck hunit  qsort Examples/HUnitTest.hs Examples.HUnitTest 'runTestTT tests'",+       " ./mucheck hspec  qsort Examples/HspecTest.hs Examples.HspecTest 'spec (with \\\"qsort1\\\")'"]))+

@@ -5,47 +5,66 @@ import Control.Monad.Trans ( liftIO ) import qualified Test.QuickCheck.Test as Qc import qualified Test.HUnit as HUnit+import qualified Test.Hspec.Core.Runner as Hspec import Data.Typeable-import qualified MuCheck.Utils.Print as Mu-import Data.Either-import Data.List((\\), groupBy, sortBy)+import MuCheck.Utils.Print (showA, showAS, (./.))+import Data.Either (partitionEithers, rights)+import Data.List((\\), groupBy, sortBy, intercalate, isInfixOf) import Data.Time.Clock  deriving instance Typeable Qc.Result deriving instance Typeable HUnit.Counts+deriving instance Typeable Hspec.Summary  type InterpreterOutput a = Either I.InterpreterError (String, a) -checkPropsOnMutants :: [String] -> String -> [String] -> String -> IO [Qc.Result]-checkPropsOnMutants = mutantCheckSummary+checkQuickCheckOnMutants :: [String] -> String -> [String] -> String -> IO [Qc.Result]+checkQuickCheckOnMutants = mutantCheckSummary -checkTestSuiteOnMutants :: [String] -> String -> [String] -> String -> IO [HUnit.Counts]-checkTestSuiteOnMutants = mutantCheckSummary+checkHUnitOnMutants :: [String] -> String -> [String] -> String -> IO [HUnit.Counts]+checkHUnitOnMutants = mutantCheckSummary +checkHspecOnMutants :: [String] -> String -> [String] -> String -> IO [Hspec.Summary]+checkHspecOnMutants = mutantCheckSummary+ -- main entry point mutantCheckSummary :: Summarizable a => [String] -> String -> [String] -> FilePath -> IO [a] mutantCheckSummary mutantFiles topModule evalSrcLst logFile  = do   results <- mapM (runCodeOnMutants mutantFiles topModule) evalSrcLst-  let delim = "\n" ++ (take 25 (repeat '=')) ++ "\n"-      singleTestSummaries = map (singleSummary mutantFiles) results-      (terminalSummary,logSummary) = multipleSummary results-      evalSrcLst' = map (delim ++) evalSrcLst+  let singleTestSummaries = zip evalSrcLst $ map (testSummary mutantFiles) results+      tssum  = suiteSummary results   -- print results to terminal   putStrLn $ delim ++ "Overall Results:"-  putStrLn terminalSummary-  putStrLn $ Mu.showAS $ zipWith (++) evalSrcLst' $ map fst singleTestSummaries+  putStrLn $ terminalSummary tssum+  putStrLn $ showAS $ map showBrief singleTestSummaries   putStr delim   -- print results to logfile-  appendFile logFile $ "OVERALL RESULTS:\n" ++ logSummary ++ Mu.showAS (zipWith (++) evalSrcLst' $ map snd singleTestSummaries)+  appendFile logFile $ "OVERALL RESULTS:\n" ++ (tssum_log tssum) ++ (showAS $ map showDetail singleTestSummaries)   -- hacky solution to avoid printing entire results to stdout and to give   -- guidance to the type checker in picking specific Summarizable instances   return $ tail [head $ (map snd) $ snd $ partitionEithers $ head results]+  where showDetail (method, msum) = delim ++ showBrief (method, msum) ++ "\n" ++ detail msum+        showBrief (method, msum) = showAS [method,+                                           "\tTotal number of mutants:\t" ++ show (tsum_numMutants msum),+                                           "\tFailed to Load:\t" ++ show (tsum_loadError msum),+                                           "\tNot Killed:\t" ++ show (tsum_notKilled msum),+                                           "\tKilled:\t" ++ show (tsum_killed msum),+                                           "\tOthers:\t" ++ show (tsum_others msum),+                                           ""]+        detail msum = tsum_log msum+        terminalSummary tssum = showAS ["Total number of mutants:\t" ++ show (tssum_numMutants tssum),+                                        "Total number of alive mutants:\t" ++ show (tssum_alive tssum),+                                        "Total number of load errors:\t" ++ show (tssum_errors tssum),+                                        ""]+        delim = "\n" ++ (take 25 (repeat '=')) ++ "\n"  -- Interpreter Functionalities -- Examples -- t = runInterpreter (evalMethod "Examples/Quicksort.hs" "Quicksort" "quickCheckResult idEmp") runCodeOnMutants mutantFiles topModule evalStr = mapM (evalMyStr evalStr) mutantFiles-  where evalMyStr evalStr file = I.runInterpreter (evalMethod file topModule evalStr)+  where evalMyStr evalStr file = do putStrLn $ ">" ++ ":" ++ file ++ ":" ++ topModule ++ ":" ++ evalStr ++ ">"+                                    x <- I.runInterpreter (evalMethod file topModule evalStr)+                                    return x  -- Given the filename, modulename, method to evaluate, evaluate, and return -- result as a pair.@@ -53,88 +72,109 @@ evalMethod fileName topModule evalStr = do   I.loadModules [fileName]   I.setTopLevelModules [topModule]-  I.setImports ["Prelude", "Test.QuickCheck", "Test.HUnit"]   result <- I.interpret evalStr (I.as :: (Typeable a => IO a)) >>= liftIO   return (fileName, result) +data TSum = TSum {tsum_numMutants::Int, tsum_loadError::Int, tsum_notKilled::Int, tsum_killed::Int, tsum_others::Int, tsum_log::String}+data TSSum = TSSum {tssum_numMutants::Int, tssum_alive::Int, tssum_errors::Int, tssum_log::String}  -- Class/Instance declaration type MutantFilename = String-type TerminalSummary = String-type LogSummary = String class Typeable s => Summarizable s where-    singleSummary :: [MutantFilename] -> [InterpreterOutput s] -> (TerminalSummary, LogSummary)-    multipleSummary :: [[InterpreterOutput s]] -> (TerminalSummary, LogSummary)+  testSummary :: [MutantFilename] -> [InterpreterOutput s] -> TSum+  suiteSummary :: [[InterpreterOutput s]] -> TSSum+  isSuccess :: s -> Bool  instance Summarizable HUnit.Counts where-    singleSummary mutantFiles results = (terminalMsg, logMsg)-        where (loadingErrorCases, executedCases) = partitionEithers results-              loadingErrorFiles = mutantFiles \\ map fst executedCases-              successCases = filter ((\c -> (HUnit.cases c == HUnit.tried c) && HUnit.failures c == 0 && HUnit.errors c == 0) . snd) executedCases-              failuresCases = filter ((>0) . HUnit.failures . snd) executedCases-              runningErrorCases = (filter ((>0) . HUnit.errors . snd) executedCases) \\ failuresCases-              failToFullyTryCases = filter ((\c -> HUnit.cases c > HUnit.tried c) . snd) executedCases-              r = length results-              le = length loadingErrorCases-              [s, fl, re, ftc] = map length [successCases, failuresCases, runningErrorCases, failToFullyTryCases]-              terminalMsg = "\n\nTotal number of mutants: " ++ show r ++-                            "\n\nFailed to load: " ++ show le ++-                            "\nSuccesses (not killed): " ++ show s ++-                            "\nFailures (killed): " ++ show fl ++-                            "\nError while running: " ++ show re ++-                            "\nIncompletely tested (may include failures and running errors): " ++ show ftc-              logMsg = terminalMsg ++ "\n\nDetails: \n\nLoading error files:\n" ++ Mu.showA loadingErrorFiles-                       ++ "\n\nLoading error messages:\n" ++ Mu.showA loadingErrorCases-                       ++ "\n\nSuccesses:\n" ++ Mu.showA successCases-                       ++ "\n\nFailures:\n" ++ Mu.showA failuresCases-                       ++ "\n\nError while running:\n" ++ Mu.showA runningErrorCases-                       ++ "\n\nIncompletely tested (may include failures and running errors):\n"-                       ++ Mu.showA failToFullyTryCases ++ "\n"-    multipleSummary = multipleCheckSummary (\c -> (HUnit.cases c == HUnit.tried c) && HUnit.failures c == 0 && HUnit.errors c == 0)+  testSummary mutantFiles results = TSum {+    tsum_numMutants = r,+    tsum_loadError = le,+    tsum_notKilled = s,+    tsum_killed = fl,+    tsum_others = re + ftc,+    tsum_log = logMsg}+    where (loadingErrorCases, executedCases) = partitionEithers results+          loadingErrorFiles = mutantFiles \\ map fst executedCases+          successCases = filter (isSuccess . snd) executedCases+          failuresCases = filter ((>0) . HUnit.failures . snd) executedCases+          runningErrorCases = (filter ((>0) . HUnit.errors . snd) executedCases) \\ failuresCases+          failToFullyTryCases = filter ((\c -> HUnit.cases c > HUnit.tried c) . snd) executedCases+          r = length results+          le = length loadingErrorCases+          [s, fl, re, ftc] = map length [successCases, failuresCases, runningErrorCases, failToFullyTryCases]+          logMsg = showAS ["Details:",+                           "Loading error files:",showA loadingErrorFiles,+                           "Loading error messages:",showA loadingErrorCases,+                           "Successes:", showA successCases,+                           "Failures:", showA failuresCases,+                           "Error while running:", showA runningErrorCases,+                           "Incompletely tested (may include failures and running errors):",showA failToFullyTryCases]+  suiteSummary = multipleCheckSummary (isSuccess . snd)+  isSuccess = (\c -> (HUnit.cases c == HUnit.tried c) && HUnit.failures c == 0 && HUnit.errors c == 0)  instance Summarizable Qc.Result where-    singleSummary mutantFiles results = (terminalMsg, logMsg)+  testSummary mutantFiles results = TSum {+    tsum_numMutants = r,+    tsum_loadError = e,+    tsum_notKilled = s,+    tsum_killed = f,+    tsum_others = g,+    tsum_log = logMsg}+    where (errorCases, executedCases) = partitionEithers results+          [successCases, failureCases, gaveUpCases] = map (\c -> filter (c . snd) executedCases) [isSuccess, isFailure, isGaveUp]+          r = length results+          e = length errorCases+          [s,f,g] = map length [successCases, failureCases, gaveUpCases]+          errorFiles = mutantFiles \\ map fst executedCases+          logMsg = showAS ["Details:",+                           "Loading error files:", showA errorFiles,+                           "Loading error messages:", showA errorCases,+                           "Successes:", showA successCases,+                           "Failure:", showA failureCases,+                           "Gaveups:", showA gaveUpCases]+          isFailure :: Qc.Result -> Bool+          isFailure Qc.Failure{} = True+          isFailure _         = False+          isGaveUp :: Qc.Result -> Bool+          isGaveUp Qc.GaveUp{} = True+          isGaveUp _        = False++  suiteSummary = multipleCheckSummary (isSuccess . snd)+  isSuccess = Qc.isSuccess++instance Summarizable Hspec.Summary where+  testSummary mutantFiles results = TSum {+    tsum_numMutants = r,+    tsum_loadError = e,+    tsum_notKilled = s,+    tsum_killed = f,+    tsum_others = 0,+    tsum_log = logMsg}       where (errorCases, executedCases) = partitionEithers results-            [successCases, failureCases, gaveUpCases] = map (\c -> filter (c . snd) executedCases) [Qc.isSuccess, isFailure, isGaveUp]             r = length results             e = length errorCases-            [s,f,g] = map length [successCases, failureCases, gaveUpCases]+            [successCases, failureCases] = map (\c -> filter (c . snd) executedCases) [isSuccess, isFailure]+            [s,f] = map length [successCases, failureCases]             errorFiles = mutantFiles \\ map fst executedCases-            terminalMsg = "\n\nTotal number of mutants: " ++ show r ++-                          "\n\nErrors: " ++ show e ++ Mu.showPerCent (e `Mu.percent` r) ++-                          "\nSuccesses (not killed): " ++ show s ++ Mu.showPerCent (s `Mu.percent` r) ++-                          "\nFailures (killed): " ++ show f ++ Mu.showPerCent (f `Mu.percent` r) ++-                          "\nGaveups: " ++ show g ++ Mu.showPerCent (g `Mu.percent` r)-            logMsg = terminalMsg ++ "\n\nDetails:\n\nLoading error files:\n" ++ Mu.showA errorFiles-                    ++ "\n\nLoading error messages:\n " ++ Mu.showA errorCases-                    ++ "\n\nSUCCESSES:\n " ++ Mu.showA successCases-                    ++ "\n\nFAILURE:\n " ++ Mu.showA failureCases-                    ++ "\n\nGAVEUPs:\n " ++ Mu.showA gaveUpCases ++ "\n"-            isFailure :: Qc.Result -> Bool-            isFailure Qc.Failure{} = True-            isFailure _         = False-            isGaveUp :: Qc.Result -> Bool-            isGaveUp Qc.GaveUp{} = True-            isGaveUp _        = False--    multipleSummary = multipleCheckSummary Qc.isSuccess+            logMsg = showAS ["Details:",+                             "Loading error files:", showA errorFiles,+                             "Loading error messages:", showA errorCases,+                             "Successes:", showA successCases,+                             "Failure:", showA failureCases]+            isFailure = not . isSuccess+  suiteSummary = multipleCheckSummary (isSuccess . snd)+  isSuccess (Hspec.Summary { Hspec.summaryExamples = se, Hspec.summaryFailures = sf } ) = sf == 0  -- we assume that checking each prop results in the same number of errorCases and executedCases-multipleCheckSummary :: Show a => (a -> Bool) -> [[InterpreterOutput a]] -> (String, String) multipleCheckSummary isSuccessFunction results   | not (checkLength results) = error "Output lengths differ for some properties."-  | otherwise = (terminalMsg, logMsg)-   where executedCases = groupBy (\x y -> fst x == fst y) . sortBy (\x y -> fst x `compare` fst y) . rights $ concat results-         allSuccesses = [rs | rs <- executedCases, length rs == length results, all (isSuccessFunction . snd) rs]-         countAlive = length allSuccesses-         countErrors = countMutants - length executedCases-         terminalMsg = "\nTotal number of mutants: " ++ show countMutants-                    ++ "\nTotal number of alive and error-free mutants: " ++ show countAlive-                    ++ Mu.showPerCent (countAlive `Mu.percent` countMutants) ++ "\n"-                    ++ "Total number of erroneous mutants (failed to be loaded): " ++ show countErrors ++ Mu.showPerCent (countErrors `Mu.percent` countMutants) ++ "\n"-         logMsg = terminalMsg ++ "\nDetails:\n\n" ++ Mu.showA allSuccesses ++ "\n"--         checkLength results = and $ map ((==countMutants) . length) results ++ map ((==countExecutedCases) . length) executedCases-         countExecutedCases = length . head $ executedCases-         countMutants = length . head $ results+  | otherwise = TSSum {tssum_numMutants = countMutants, tssum_alive = countAlive, tssum_errors= countErrors, tssum_log = logMsg}+  where executedCases = groupBy (\x y -> fst x == fst y) . sortBy (\x y -> fst x `compare` fst y) . rights $ concat results+        allSuccesses = [rs | rs <- executedCases, length rs == length results, all isSuccessFunction rs]+        countAlive = length allSuccesses+        countErrors = countMutants - length executedCases+        logMsg = showA allSuccesses+        checkLength results = and $ map ((==countMutants) . length) results ++ map ((==countExecutedCases) . length) executedCases+        countExecutedCases = length . head $ executedCases+        countMutants = length . head $ results 

@@ -2,27 +2,27 @@  import System.FilePath (splitExtension) --- choose [1,2,3,4,5] 4---  = [[2,3,4,5],[1,3,4,5],[1,2,4,5],[1,2,3,5],[1,2,3,4]]+-- | The `choose` function generates subsets of a given size choose :: [b] -> Int -> [[b]] _      `choose` 0       = [[]] []     `choose` _       =  [] (x:xs) `choose` k       =  (x:) `fmap` (xs `choose` (k-1)) ++ xs `choose` k --- generating mutant files names--- e.g.: "Quicksort.hs" ==> "Quicksort_1.hs", "Quicksort_2.hs", etc.+-- | The `genFileNames` function lazily generates filenames of mutants genFileNames :: String -> [String] genFileNames s =  map newname [1..]     where (name, ext) = splitExtension s           newname i= name ++ "_" ++ show i ++ ext --- replace first element in a list given old and new values+-- | The `replace` function replaces first element in a list given old and new values as a pair replace :: Eq a => (a,a) -> [a] -> [a] replace (o,n) lst = map replaceit lst   where replaceit v           | v == o = n           | otherwise = v +-- | The `safeHead` function safely extracts head of a list using Maybe safeHead :: [a] -> Maybe a safeHead [] = Nothing safeHead (x:xs) = Just x+

@@ -2,13 +2,14 @@  import Data.List(intercalate) --- utils for interpreter-showPerCent x = "(" ++ show x ++ "%)"-n `percent` t = 100 * n `div` t+-- | simple wrapper for adding a % at the end.+n ./. t =  "(" ++ show (n * 100 `div` t) ++ "%)" +-- | join lines together showAS :: [String] -> String showAS = intercalate "\n" +-- | make lists into lines in text. showA :: Show a => [a] -> String showA =  showAS . map show