fitspec 0.2.0 → 0.2.1

raw patch · 16 files changed

+217/−29 lines, 16 filesdep ~leancheck

Dependency ranges changed: leancheck

Files

• FitSpec.hs +2/−2
• FitSpec/Derive.hs +2/−2
• FitSpec/Engine.hs +1/−1
• FitSpec/Mutable.hs +2/−2
• FitSpec/Mutable/Tuples.hs +1/−1
• FitSpec/ShowMutable.hs +1/−1
• FitSpec/TestTypes.hs +2/−2
• README.md +5/−5
• bench/avltrees.hs +0/−1
• bench/list.hs +0/−1
• bench/pretty.hs +0/−1
• doc/modules.md +22/−0
• doc/tutorial-property-creation.md +166/−0
• fitspec.cabal +9/−6
• tests/test-mutate.hs +2/−2
• tests/test-showmutable.hs +2/−2

@@ -88,7 +88,7 @@    -- * Re-export modules   , module FitSpec.TestTypes-  , module Test.Check+  , module Test.LeanCheck   ) where @@ -103,4 +103,4 @@ import FitSpec.Derive import FitSpec.TestTypes -import Test.Check+import Test.LeanCheck

@@ -11,14 +11,14 @@   , deriveMutableE   , module FitSpec.Mutable   , module FitSpec.ShowMutable-  , module Test.Check+  , module Test.LeanCheck   ) where  import FitSpec.Mutable import FitSpec.ShowMutable -import Test.Check+import Test.LeanCheck import Language.Haskell.TH import Control.Monad (when, unless, liftM, liftM2) 

@@ -27,7 +27,7 @@   ) where -import Test.Check.Error+import Test.LeanCheck.Error import FitSpec.Utils import Data.Maybe (catMaybes, listToMaybe) import Data.List ((\\),union,transpose)

@@ -6,10 +6,10 @@   ) where -import Test.Check+import Test.LeanCheck import Data.List (intercalate, delete) import Data.Maybe-import Test.Check.Error (errorToNothing)+import Test.LeanCheck.Error (errorToNothing)  -- | This typeclass is similar to 'Listable'. --

@@ -5,7 +5,7 @@ module FitSpec.Mutable.Tuples () where  import FitSpec.Mutable-import Test.Check (productWith)+import Test.LeanCheck (productWith)  instance (Mutable a, Mutable b, Mutable c, Mutable d,           Mutable e, Mutable f, Mutable g)

@@ -12,7 +12,7 @@ where  import FitSpec.PrettyPrint-import Test.Check.Error (errorToNothing, Listable(..))+import Test.LeanCheck.Error (errorToNothing, Listable(..)) import Data.Maybe (mapMaybe,isNothing) import Control.Monad (join) import Data.List (intercalate,tails)

@@ -6,11 +6,11 @@ -- This module basically re-exports LeanCheck's Test.Types module -- and defines Mutable and ShowMutable instances for the types -- defined there.-module FitSpec.TestTypes (module Test.Types) where+module FitSpec.TestTypes (module Test.LeanCheck.Utils.Types) where  import FitSpec.Mutable import FitSpec.ShowMutable-import Test.Types+import Test.LeanCheck.Utils.Types  -- {- Standard implementation: instance Mutable Nat   where mutiers = mutiersEq

@@ -138,14 +138,14 @@ (TODO: link to a possible future FitSpec paper goes here)  -[Listable]: https://hackage.haskell.org/package/leancheck/docs/Test-Check.html#t:Listable-[Mutable]: https://hackage.haskell.org/package/fitspec/docs/FitSpec.html#t:Mutable+[Listable]:    https://hackage.haskell.org/package/leancheck/docs/Test-LeanCheck.html#t:Listable+[Mutable]:     https://hackage.haskell.org/package/fitspec/docs/FitSpec.html#t:Mutable [ShowMutable]: https://hackage.haskell.org/package/fitspec/docs/FitSpec.html#t:ShowMutable [FitSpec API]: https://hackage.haskell.org/package/fitspec/docs/FitSpec.html  [leancheck]: https://hackage.haskell.org/package/leancheck-[cmdargs]: https://hackage.haskell.org/package/cmdargs-[pretty]: https://hackage.haskell.org/package/pretty+[cmdargs]:   https://hackage.haskell.org/package/cmdargs+[pretty]:    https://hackage.haskell.org/package/pretty -[TH]: https://wiki.haskell.org/Template_Haskell+[TH]:    https://wiki.haskell.org/Template_Haskell [Cabal]: https://www.haskell.org/cabal

@@ -1,6 +1,5 @@ {-# LANGUAGE CPP #-} import FitSpec-import Test.Check import AVLTree import Data.List (sort,nubBy) 

@@ -1,7 +1,6 @@ import System.Console.CmdArgs hiding (args) import FitSpec import Data.List-import Test.Check  type Cons a = a -> [a] -> [a] type Head a = [a] -> a

@@ -1,6 +1,5 @@ {-# LANGUAGE CPP #-} import FitSpec-import Test.Check import Text.PrettyPrint  #if __GLASGOW_HASKELL__ < 710

@@ -0,0 +1,22 @@+Important modules+-----------------++* [FitSpec](FitSpec.hs):+  the entry point, import this to use FitSpec;++* [FitSpec.Engine](FitSpec/Engine.hs):+  main engine that tests mutants against properties;++* [FitSpec.Report](FitSpec/Report.hs):+  gather results from the engine and build textual reports;++* [FitSpec.Mutable](FitSpec/Mutable.hs):+  list mutations of a given function without repetitions;++* [FitSpec.ShowMutable](FitSpec/ShowMutable.hs):+  show mutations;++* [example benchmarks](bench):+  example use cases for FitSpec,+  some are customizable using command line arguments+  (sorting, booleans, lists, pretty-printing, etc).

@@ -0,0 +1,166 @@+Using FitSpec to guide property creation+----------------------------------------++Suppose we want to write test properties for the function `sort`,+but we do not know where to start.+We can use FitSpec to guide property creation.+++We first import what is needed:++	import FitSpec+	import Data.List (sort)+++Then we need a property list function: given a sorting implementation, return+the properties applied to *that* implementation.  Since we don't have any+properties, we will start by returning and empty list:++	properties :: (Show a, Ord a, Listable a)+	           => ([a] -> [a]) -> [Properties]+	properties sort' =+	  []+++Then, we need a main function, that calls the FitSpec's `report` function,+which will report the results of mutation testing.+It needs a function to be mutated and the property list.++	main = report (sort::[Int]->[Int]) properties++Optionally, for a nicer output, you might want to use the reportWith function,+which allows specifying function and argument names (among other options):++	main = reportWith args { callNames = ["sort xs"] }+	                  (sort::[Int]->[Int]) properties++By having the three sections above in a file called sorting.hs,+we then compile and run:++	$ ghc -ipath/to/leancheck:path/to/fitspec sorting.hs+	[9 of 9] Compiling Main             ( sorting.hs, sorting.o )+	Linking sorting ...++	$ ./sorting+	Results based on at most 4000 test cases for each of 2000 mutant variations.++	Property   #Survivors    Smallest or simplest+	 sets       (%Killed)     surviving mutant++	[]         2000 (0%)     \xs -> case xs of+	                                  [] -> [0]+	                                  _ -> sort xs++The output is self-explanatory.  Obviously, our empty property set `[]` did not+kill any mutant (`0%`).  In other words, all of the `2000` mutants survived.+(The actual number of mutants tested will vary depending on your machine, it+will probably be higher than 2000 *in this case*, by default FitSpec runs for+at least 5 seconds.)++The surviving mutant shown on the third column is clearly not a valid+implementation of sort.  For the empty list, it returns `[0]`.  We should+improve our property set by killing that mutant.  Lets start very simple by+adding a property stating that sorting an empty list must yield an empty list:++	properties sort' =+	  [ property $ sort' [] == []+	  ]++Above, we need to apply the function `property` to each property in the list.+Now:++	$ ./sorting+	Results based on at most 4000 test cases for each of 2000 mutant variations.++	Property   #Survivors    Smallest or simplest+	 sets       (%Killed)     surviving mutant++	[1]        984 (49%)     \xs -> case xs of+	                                  [0] -> []+	                                  _ -> sort xs++	[]         2000 (0%)     \xs -> case xs of+	                                  [] -> [0]+	                                  _ -> sort xs++The last row of results is the same as before (all mutants still obviously+survive the empty property set).  The *first row* show that there are `984`+*surviving mutants* (`49%`) for the first property `[1]`: the smallest one is+shown on the third column.  It sorts `[0]` to `[]`, which is not valid.  Lets+still be very simple -- sorting a list with one value must yield a list with+the same value:++	properties sort' =+	  [ property $        sort' [] == []+	  , property $ \x -> sort' [x] == [x]+	  ]++Note that, our new property (2) has a free variable.  Now:++	$ ./sorting+	Results based on at most 1000 test cases for each of 500 mutant variations.++	Property   #Survivors   Smallest or simplest+	 sets       (%Killed)    surviving mutant++	[1,2]      134 (73%)    \xs -> case xs of+	                                 [0,0] -> []+	                                 _ -> sort xs+	...++Only 27% of mutants to go, perhaps a property stating that the length of the+sorted list should not change?++	properties sort' =+	  [ property $                 sort' [] == []+	  , property $ \x  ->         sort' [x] == [x]+	  , property $ \xs -> length (sort' xs) == length xs+	  ]++Now:++	$ ./sorting+	Results based on at most 1000 test cases for each of 500 mutant variations.++	Property   #Survivors   Smallest or simplest+	 sets       (%Killed)    surviving mutant++	[2,3]      12 (97%)     \xs -> case xs of+	                                 [0,0] -> [0,1]+	                                 _ -> sort xs+	...++	Conjectures based on at most 1000 test cases for each of 500 mutant variations:+	[3] ==> [1]     95% killed (likely)++The first row show that the current candidate minimal-complete propety-set+kills all but `4` mutants and is composed only by properties 2 and 3 (`[2,3]`).+When possible, FitSpec also reports *conjectures* based on test results.  In+this case, that property `sort [] == []` (1) follows from the length property+(3).  Since that is *clearly* true, we can safely remove that property.++	properties sort' =+	  [ property $ \x    ->         sort' [x] == [x]+	  , property $ \xs   -> length (sort' xs) == length xs+	  , property $ \x xs -> elem x (sort' xs) == elem x xs+	  ]++Now:++	$ ./sorting+	Property   #Survivors   Smallest or simplest+	 sets       (%Killed)    surviving mutant++	 [2,3]      2 (99%)      \xs -> case xs of+	                                  [0,1] -> [1,0]+	                                  _ -> sort xs+	...+	Conjectures based on at most 1000 test cases for each of 500 mutant variations:+	[2,3] ==> [1]     99% killed (possible+)++We could go on, but *at this point, you probably got how it works*.  As an+exercise you can try to improve our property-set over `sort` by killing the+above mutant by adding a new property.  Later, you can try to improve the+results by increasing the time limit (`minimumTime = 10` on args).++

@@ -1,5 +1,5 @@ name:                fitspec-version:             0.2.0+version:             0.2.1 synopsis:            refining property sets for testing Haskell programs description:   FitSpec provides automated assistance in the task of refining test properties@@ -9,10 +9,10 @@   property set, recording any surviving mutants that pass all tests. FitSpec   then reports:   .-  * *surviving mutants:* indicating incompleteness of properties,+  * surviving mutants: indicating incompleteness of properties,     prompting the user to amend a property or to add a new one;   .-  * *conjectures:* indicating redundancy in the property set,+  * conjectures: indicating redundancy in the property set,     prompting the user to remove properties so to reduce the cost of testing.  homepage:            https://github.com/rudymatela/fitspec#readme@@ -24,7 +24,10 @@ build-type:          Simple cabal-version:       >=1.10 -extra-source-files:  README.md CREDITS.md+extra-doc-files: README.md+               , CREDITS.md+               , doc/modules.md+               , doc/tutorial-property-creation.md  source-repository head   type:            git@@ -33,7 +36,7 @@ source-repository this   type:            git   location:        https://github.com/rudymatela/fitspec-  tag:             v0.2.0+  tag:             v0.2.1   library@@ -51,7 +54,7 @@   other-modules: FitSpec.Utils                , FitSpec.PrettyPrint                , FitSpec.Dot-  build-depends: base >= 4 && < 5, leancheck, cmdargs, template-haskell+  build-depends: base >= 4 && < 5, leancheck >= 0.4, cmdargs, template-haskell   hs-source-dirs:    .   default-language:  Haskell2010 

@@ -4,8 +4,8 @@  import FitSpec import FitSpec.Utils (contained)-import Test.Check.Error (errorToNothing, errorToFalse)-import Test.Check.Function.ListsOfPairs (functionPairs, defaultFunPairsToFunction)+import Test.LeanCheck.Error (errorToNothing, errorToFalse)+import Test.LeanCheck.Function.ListsOfPairs (functionPairs, defaultFunPairsToFunction)   

@@ -7,8 +7,8 @@ import FitSpec import FitSpec.ShowMutable import FitSpec.PrettyPrint-import Test.Check.Error (errorToNothing)-import Test.TypeBinding+import Test.LeanCheck.Error (errorToNothing)+import Test.LeanCheck.Utils.TypeBinding  main :: IO () main = do