diff --git a/FitSpec.hs b/FitSpec.hs --- a/FitSpec.hs +++ b/FitSpec.hs @@ -88,7 +88,7 @@ -- * Re-export modules , module FitSpec.TestTypes - , module Test.Check + , module Test.LeanCheck ) where @@ -103,4 +103,4 @@ import FitSpec.Derive import FitSpec.TestTypes -import Test.Check +import Test.LeanCheck diff --git a/FitSpec/Derive.hs b/FitSpec/Derive.hs --- a/FitSpec/Derive.hs +++ b/FitSpec/Derive.hs @@ -11,14 +11,14 @@ , deriveMutableE , module FitSpec.Mutable , module FitSpec.ShowMutable - , module Test.Check + , module Test.LeanCheck ) where import FitSpec.Mutable import FitSpec.ShowMutable -import Test.Check +import Test.LeanCheck import Language.Haskell.TH import Control.Monad (when, unless, liftM, liftM2) diff --git a/FitSpec/Engine.hs b/FitSpec/Engine.hs --- a/FitSpec/Engine.hs +++ b/FitSpec/Engine.hs @@ -27,7 +27,7 @@ ) where -import Test.Check.Error +import Test.LeanCheck.Error import FitSpec.Utils import Data.Maybe (catMaybes, listToMaybe) import Data.List ((\\),union,transpose) diff --git a/FitSpec/Mutable.hs b/FitSpec/Mutable.hs --- a/FitSpec/Mutable.hs +++ b/FitSpec/Mutable.hs @@ -6,10 +6,10 @@ ) where -import Test.Check +import Test.LeanCheck import Data.List (intercalate, delete) import Data.Maybe -import Test.Check.Error (errorToNothing) +import Test.LeanCheck.Error (errorToNothing) -- | This typeclass is similar to 'Listable'. -- diff --git a/FitSpec/Mutable/Tuples.hs b/FitSpec/Mutable/Tuples.hs --- a/FitSpec/Mutable/Tuples.hs +++ b/FitSpec/Mutable/Tuples.hs @@ -5,7 +5,7 @@ module FitSpec.Mutable.Tuples () where import FitSpec.Mutable -import Test.Check (productWith) +import Test.LeanCheck (productWith) instance (Mutable a, Mutable b, Mutable c, Mutable d, Mutable e, Mutable f, Mutable g) diff --git a/FitSpec/ShowMutable.hs b/FitSpec/ShowMutable.hs --- a/FitSpec/ShowMutable.hs +++ b/FitSpec/ShowMutable.hs @@ -12,7 +12,7 @@ where import FitSpec.PrettyPrint -import Test.Check.Error (errorToNothing, Listable(..)) +import Test.LeanCheck.Error (errorToNothing, Listable(..)) import Data.Maybe (mapMaybe,isNothing) import Control.Monad (join) import Data.List (intercalate,tails) diff --git a/FitSpec/TestTypes.hs b/FitSpec/TestTypes.hs --- a/FitSpec/TestTypes.hs +++ b/FitSpec/TestTypes.hs @@ -6,11 +6,11 @@ -- This module basically re-exports LeanCheck's Test.Types module -- and defines Mutable and ShowMutable instances for the types -- defined there. -module FitSpec.TestTypes (module Test.Types) where +module FitSpec.TestTypes (module Test.LeanCheck.Utils.Types) where import FitSpec.Mutable import FitSpec.ShowMutable -import Test.Types +import Test.LeanCheck.Utils.Types -- {- Standard implementation: instance Mutable Nat where mutiers = mutiersEq diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -138,14 +138,14 @@ (TODO: link to a possible future FitSpec paper goes here) -[Listable]: https://hackage.haskell.org/package/leancheck/docs/Test-Check.html#t:Listable -[Mutable]: https://hackage.haskell.org/package/fitspec/docs/FitSpec.html#t:Mutable +[Listable]: https://hackage.haskell.org/package/leancheck/docs/Test-LeanCheck.html#t:Listable +[Mutable]: https://hackage.haskell.org/package/fitspec/docs/FitSpec.html#t:Mutable [ShowMutable]: https://hackage.haskell.org/package/fitspec/docs/FitSpec.html#t:ShowMutable [FitSpec API]: https://hackage.haskell.org/package/fitspec/docs/FitSpec.html [leancheck]: https://hackage.haskell.org/package/leancheck -[cmdargs]: https://hackage.haskell.org/package/cmdargs -[pretty]: https://hackage.haskell.org/package/pretty +[cmdargs]: https://hackage.haskell.org/package/cmdargs +[pretty]: https://hackage.haskell.org/package/pretty -[TH]: https://wiki.haskell.org/Template_Haskell +[TH]: https://wiki.haskell.org/Template_Haskell [Cabal]: https://www.haskell.org/cabal diff --git a/bench/avltrees.hs b/bench/avltrees.hs --- a/bench/avltrees.hs +++ b/bench/avltrees.hs @@ -1,6 +1,5 @@ {-# LANGUAGE CPP #-} import FitSpec -import Test.Check import AVLTree import Data.List (sort,nubBy) diff --git a/bench/list.hs b/bench/list.hs --- a/bench/list.hs +++ b/bench/list.hs @@ -1,7 +1,6 @@ import System.Console.CmdArgs hiding (args) import FitSpec import Data.List -import Test.Check type Cons a = a -> [a] -> [a] type Head a = [a] -> a diff --git a/bench/pretty.hs b/bench/pretty.hs --- a/bench/pretty.hs +++ b/bench/pretty.hs @@ -1,6 +1,5 @@ {-# LANGUAGE CPP #-} import FitSpec -import Test.Check import Text.PrettyPrint #if __GLASGOW_HASKELL__ < 710 diff --git a/doc/modules.md b/doc/modules.md new file mode 100644 --- /dev/null +++ b/doc/modules.md @@ -0,0 +1,22 @@ +Important modules +----------------- + +* [FitSpec](FitSpec.hs): + the entry point, import this to use FitSpec; + +* [FitSpec.Engine](FitSpec/Engine.hs): + main engine that tests mutants against properties; + +* [FitSpec.Report](FitSpec/Report.hs): + gather results from the engine and build textual reports; + +* [FitSpec.Mutable](FitSpec/Mutable.hs): + list mutations of a given function without repetitions; + +* [FitSpec.ShowMutable](FitSpec/ShowMutable.hs): + show mutations; + +* [example benchmarks](bench): + example use cases for FitSpec, + some are customizable using command line arguments + (sorting, booleans, lists, pretty-printing, etc). diff --git a/doc/tutorial-property-creation.md b/doc/tutorial-property-creation.md new file mode 100644 --- /dev/null +++ b/doc/tutorial-property-creation.md @@ -0,0 +1,166 @@ +Using FitSpec to guide property creation +---------------------------------------- + +Suppose we want to write test properties for the function `sort`, +but we do not know where to start. +We can use FitSpec to guide property creation. + + +We first import what is needed: + + import FitSpec + import Data.List (sort) + + +Then we need a property list function: given a sorting implementation, return +the properties applied to *that* implementation. Since we don't have any +properties, we will start by returning and empty list: + + properties :: (Show a, Ord a, Listable a) + => ([a] -> [a]) -> [Properties] + properties sort' = + [] + + +Then, we need a main function, that calls the FitSpec's `report` function, +which will report the results of mutation testing. +It needs a function to be mutated and the property list. + + main = report (sort::[Int]->[Int]) properties + +Optionally, for a nicer output, you might want to use the reportWith function, +which allows specifying function and argument names (among other options): + + main = reportWith args { callNames = ["sort xs"] } + (sort::[Int]->[Int]) properties + +By having the three sections above in a file called sorting.hs, +we then compile and run: + + $ ghc -ipath/to/leancheck:path/to/fitspec sorting.hs + [9 of 9] Compiling Main ( sorting.hs, sorting.o ) + Linking sorting ... + + $ ./sorting + Results based on at most 4000 test cases for each of 2000 mutant variations. + + Property #Survivors Smallest or simplest + sets (%Killed) surviving mutant + + [] 2000 (0%) \xs -> case xs of + [] -> [0] + _ -> sort xs + +The output is self-explanatory. Obviously, our empty property set `[]` did not +kill any mutant (`0%`). In other words, all of the `2000` mutants survived. +(The actual number of mutants tested will vary depending on your machine, it +will probably be higher than 2000 *in this case*, by default FitSpec runs for +at least 5 seconds.) + +The surviving mutant shown on the third column is clearly not a valid +implementation of sort. For the empty list, it returns `[0]`. We should +improve our property set by killing that mutant. Lets start very simple by +adding a property stating that sorting an empty list must yield an empty list: + + properties sort' = + [ property $ sort' [] == [] + ] + +Above, we need to apply the function `property` to each property in the list. +Now: + + $ ./sorting + Results based on at most 4000 test cases for each of 2000 mutant variations. + + Property #Survivors Smallest or simplest + sets (%Killed) surviving mutant + + [1] 984 (49%) \xs -> case xs of + [0] -> [] + _ -> sort xs + + [] 2000 (0%) \xs -> case xs of + [] -> [0] + _ -> sort xs + +The last row of results is the same as before (all mutants still obviously +survive the empty property set). The *first row* show that there are `984` +*surviving mutants* (`49%`) for the first property `[1]`: the smallest one is +shown on the third column. It sorts `[0]` to `[]`, which is not valid. Lets +still be very simple -- sorting a list with one value must yield a list with +the same value: + + properties sort' = + [ property $ sort' [] == [] + , property $ \x -> sort' [x] == [x] + ] + +Note that, our new property (2) has a free variable. Now: + + $ ./sorting + Results based on at most 1000 test cases for each of 500 mutant variations. + + Property #Survivors Smallest or simplest + sets (%Killed) surviving mutant + + [1,2] 134 (73%) \xs -> case xs of + [0,0] -> [] + _ -> sort xs + ... + +Only 27% of mutants to go, perhaps a property stating that the length of the +sorted list should not change? + + properties sort' = + [ property $ sort' [] == [] + , property $ \x -> sort' [x] == [x] + , property $ \xs -> length (sort' xs) == length xs + ] + +Now: + + $ ./sorting + Results based on at most 1000 test cases for each of 500 mutant variations. + + Property #Survivors Smallest or simplest + sets (%Killed) surviving mutant + + [2,3] 12 (97%) \xs -> case xs of + [0,0] -> [0,1] + _ -> sort xs + ... + + Conjectures based on at most 1000 test cases for each of 500 mutant variations: + [3] ==> [1] 95% killed (likely) + +The first row show that the current candidate minimal-complete propety-set +kills all but `4` mutants and is composed only by properties 2 and 3 (`[2,3]`). +When possible, FitSpec also reports *conjectures* based on test results. In +this case, that property `sort [] == []` (1) follows from the length property +(3). Since that is *clearly* true, we can safely remove that property. + + properties sort' = + [ property $ \x -> sort' [x] == [x] + , property $ \xs -> length (sort' xs) == length xs + , property $ \x xs -> elem x (sort' xs) == elem x xs + ] + +Now: + + $ ./sorting + Property #Survivors Smallest or simplest + sets (%Killed) surviving mutant + + [2,3] 2 (99%) \xs -> case xs of + [0,1] -> [1,0] + _ -> sort xs + ... + Conjectures based on at most 1000 test cases for each of 500 mutant variations: + [2,3] ==> [1] 99% killed (possible+) + +We could go on, but *at this point, you probably got how it works*. As an +exercise you can try to improve our property-set over `sort` by killing the +above mutant by adding a new property. Later, you can try to improve the +results by increasing the time limit (`minimumTime = 10` on args). + + diff --git a/fitspec.cabal b/fitspec.cabal --- a/fitspec.cabal +++ b/fitspec.cabal @@ -1,5 +1,5 @@ name: fitspec -version: 0.2.0 +version: 0.2.1 synopsis: refining property sets for testing Haskell programs description: FitSpec provides automated assistance in the task of refining test properties @@ -9,10 +9,10 @@ property set, recording any surviving mutants that pass all tests. FitSpec then reports: . - * *surviving mutants:* indicating incompleteness of properties, + * surviving mutants: indicating incompleteness of properties, prompting the user to amend a property or to add a new one; . - * *conjectures:* indicating redundancy in the property set, + * conjectures: indicating redundancy in the property set, prompting the user to remove properties so to reduce the cost of testing. homepage: https://github.com/rudymatela/fitspec#readme @@ -24,7 +24,10 @@ build-type: Simple cabal-version: >=1.10 -extra-source-files: README.md CREDITS.md +extra-doc-files: README.md + , CREDITS.md + , doc/modules.md + , doc/tutorial-property-creation.md source-repository head type: git @@ -33,7 +36,7 @@ source-repository this type: git location: https://github.com/rudymatela/fitspec - tag: v0.2.0 + tag: v0.2.1 library @@ -51,7 +54,7 @@ other-modules: FitSpec.Utils , FitSpec.PrettyPrint , FitSpec.Dot - build-depends: base >= 4 && < 5, leancheck, cmdargs, template-haskell + build-depends: base >= 4 && < 5, leancheck >= 0.4, cmdargs, template-haskell hs-source-dirs: . default-language: Haskell2010 diff --git a/tests/test-mutate.hs b/tests/test-mutate.hs --- a/tests/test-mutate.hs +++ b/tests/test-mutate.hs @@ -4,8 +4,8 @@ import FitSpec import FitSpec.Utils (contained) -import Test.Check.Error (errorToNothing, errorToFalse) -import Test.Check.Function.ListsOfPairs (functionPairs, defaultFunPairsToFunction) +import Test.LeanCheck.Error (errorToNothing, errorToFalse) +import Test.LeanCheck.Function.ListsOfPairs (functionPairs, defaultFunPairsToFunction) diff --git a/tests/test-showmutable.hs b/tests/test-showmutable.hs --- a/tests/test-showmutable.hs +++ b/tests/test-showmutable.hs @@ -7,8 +7,8 @@ import FitSpec import FitSpec.ShowMutable import FitSpec.PrettyPrint -import Test.Check.Error (errorToNothing) -import Test.TypeBinding +import Test.LeanCheck.Error (errorToNothing) +import Test.LeanCheck.Utils.TypeBinding main :: IO () main = do