diff --git a/.travis.yml b/.travis.yml --- a/.travis.yml +++ b/.travis.yml @@ -47,8 +47,8 @@ env: GHCVER=head CABALVER=head addons: {apt: {packages: [ghc-head, cabal-install-head], sources: hvr-ghc}} - ghc: '8.6' - env: GHCVER=8.6.1 CABALVER=2.4 - addons: {apt: {packages: [ghc-8.6.1, cabal-install-2.4], sources: hvr-ghc}} + env: GHCVER=8.6.3 CABALVER=2.4 + addons: {apt: {packages: [ghc-8.6.3, cabal-install-2.4], sources: hvr-ghc}} - ghc: '8.4' env: GHCVER=8.4.4 CABALVER=2.2 addons: {apt: {packages: [ghc-8.4.4, cabal-install-2.2], sources: hvr-ghc}} diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -115,10 +115,14 @@ --ignore "Redundant bracket" \ . -markdown: - pandoc README.md -o README.html - pandoc doc/tutorial.md -o doc/tutorial.html - pandoc doc/data-invariant.md -o doc/data-invariant.html +markdown: \ + README.html \ + doc/tutorial.html \ + doc/data-invariant.html \ + doc/faq.html + +%.html: %.md + pandoc $< -o $@ # NOTE: (very hacky!) the following target allows parallel compilation (-jN) of # eg and tests programs so long as they don't share dependencies _not_ stored diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ [LeanCheck provider for Tasty]: https://hackage.haskell.org/package/tasty-leancheck [LeanCheck provider for test-framework]: https://hackage.haskell.org/package/test-framework-leancheck [LeanCheck provider for Hspec]: https://hackage.haskell.org/package/hspec-leancheck -[leancheck-instances]: https://hackage.haskell.org/package/hspec-leancheck +[leancheck-instances]: https://hackage.haskell.org/package/leancheck-instances [the intentional exception of a few types]: https://hackage.haskell.org/package/leancheck/docs/Test-LeanCheck-Basic.html [Haskell 2010 Language Report]: https://www.haskell.org/onlinereport/haskell2010/ [Haskell Platform]: https://www.haskell.org/platform/ @@ -220,8 +220,8 @@ [build-log]: https://travis-ci.org/rudymatela/leancheck [hackage-version]: https://img.shields.io/hackage/v/leancheck.svg [leancheck-on-hackage]: https://hackage.haskell.org/package/leancheck -[stackage-lts-badge]: http://stackage.org/package/leancheck/badge/lts -[stackage-nightly-badge]: http://stackage.org/package/leancheck/badge/nightly -[leancheck-on-stackage]: http://stackage.org/package/leancheck -[leancheck-on-stackage-lts]: http://stackage.org/lts/package/leancheck -[leancheck-on-stackage-nightly]: http://stackage.org/nightly/package/leancheck +[stackage-lts-badge]: https://stackage.org/package/leancheck/badge/lts +[stackage-nightly-badge]: https://stackage.org/package/leancheck/badge/nightly +[leancheck-on-stackage]: https://stackage.org/package/leancheck +[leancheck-on-stackage-lts]: https://stackage.org/lts/package/leancheck +[leancheck-on-stackage-nightly]: https://stackage.org/nightly/package/leancheck diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,14 @@ Changelog for LeanCheck ======================= +v0.9.1 +------ + +* fix bug in `genericTiers` where using it bound to a recursive datatype could + cause an infinite loop; +* minor improvements in documentation and tests. + + v0.9.0 ------ diff --git a/doc/faq.md b/doc/faq.md new file mode 100644 --- /dev/null +++ b/doc/faq.md @@ -0,0 +1,177 @@ +LeanCheck FAQ +============= + + +What is LeanCheck? +------------------ + +[LeanCheck] is a (property-based) testing tool for Haskell. It provides a +`check` function that takes a property, tests it by automatically generating +test values, then reports the results. + +More details on the [LeanCheck tutorial]. + + +What is property-based testing (PBT)? +------------------------------------- + +In property-based testing, tests are defined as functions returning a boolean +value which should be true for all choices of argument values. We call these +tests properties. A property-based testing tool can then generate arguments +automatically to search for a failing test case. + +Property-based testing is also known as: +* property testing; +* parameterized unit testing. + +More details on the [LeanCheck tutorial]. + + +What are the differences between QuickCheck and LeanCheck? +---------------------------------------------------------- + +[QuickCheck] and [LeanCheck] are both property-based testing tools for Haskell. +Some of the differences follow: + +__Test case generation.__ In QuickCheck, test cases are generated randomly for +types that are instances of the [`Arbitrary`] typeclass. In LeanCheck, test +cases are generated enumeratively for types that are instances of the +[`Listable`] typeclass. The next section in this FAQ expands on this +difference. + +__Purity.__ Differently from QuickCheck, LeanCheck generators are purely +functional. To get a list of all booleans you simply do: + + > list :: [Bool] + [False, True] + +To get a (infinite) list of all integers, you simply do: + + > list :: [Int] + [0, 1, 2, 3, 4, 5, 6, 7, ...] + +__Memory consumption.__ LeanCheck is more memory intensive when compared to +QuickCheck. With LeanCheck you may run out of memory when you're running tens +of millions of tests depending on your datatype. + + + +What are the differences between enumerative and random testing? +---------------------------------------------------------------- + +__Counterexample size.__ When using enumerative testing, reported +counterexamples are guaranteed to be the smallest or simplest as possible. +Most of the times, the smaller the counterexample, the easier it is to find the +cause of the fault. When doing random testing, you may sometimes get a bigger +counterexample and you'll have to resort to [shrinking] which is enabled by +default on [`Arbitrary`] instances that define a [`shrink`] function. +Depending on your datatype, shrinking can be effective. + + > import Test.LeanCheck + > check $ \xs ys -> xs `union` ys == ys `union` (xs :: [Int]) + *** Failed! Falsifiable (after 4 tests): + [] [0,0] + + > import Test.QuickCheck + > quickCheck . noShrinking $ \xs ys -> xs `union` ys == ys `union` (xs :: [Int]) + *** Failed! Falsifiable (after 3 tests): + [2,-1] + [0,2] + + > quickCheck $ \xs ys -> xs `union` ys == ys `union` (xs :: [Int]) + *** Failed! Falsifiable (after 3 tests): + [] + [2,2] + +__Existential properties.__ Enumerative testing tools allow for the definition +of existential properties: + + prop_lessThanExists :: Natural -> Natural -> Bool + prop_lessThanExists x y = x <= y ==> exists 1000 $ \z -> x + z == y + +Random testing does not allow it without the use of sometimes-not-so-obvious +process of [Skolemization]. + +__Coverage.__ Random testing always hits different test cases, so in the long +term you may get more test coverage. With enumerative testing you only get +more coverage when you configure more tests. + +__Ease of use.__ In my humble opinion, writing generators for enumerative +testing tools is easier than on random testing tools. Nevertheless, the +availability of automatic derivation makes this a minor issue. + + +What are the differences between LeanCheck and Feat? +---------------------------------------------------- + +[LeanCheck] and [Feat] are both size-bounded enumerative testing tools. There +are a few differences. Some are detailed in the following paragraphs. + +__Enumeration.__ Choices when defining enumeration were different on LeanCheck +and Feat. For some types, like `Bool` and `[Bool]` the enumeration is +isomorphic (has the same order). For other types, like `Int` and `[Int]`, the +enumeration is different. In LeanCheck, [`tiers`] of values are as thin as +possible, this makes the enumeration less likely to "blow-up" when tupling. +Take for example, the enumeration for trios of integers, +`(Int,Int,Int)`: + + > import Test.LeanCheck + > take 10 $ map length (tiers :: [[(Int,Int,Int)]]) + [1,3,6,10,15,21,28,36,45,55] + + > import Test.Feat + > take 10 $ map fst ( values :: [(Integer, [(Int,Int,Int)])] ) + [8,24,72,200,528,1344,3328,8064,19200,45056] + +As you nest, the difference increases: + + > import Test.LeanCheck + > take 10 $ map length (tiers :: [[ [(Int,Int,Int)] ]]) + [1,1,4,13,41,129,406,1278,4023,12664] + + > import Test.Feat + > take 10 $ map fst ( values :: [(Integer, [[(Int,Int,Int)]] )] ) + [0,1,8,88,968,10632,116752,1282048,14078080,154590400] + +__Interface.__ The API of these tools differ in several ways. Two examples: + +* LeanCheck supports ([`Testable`]) properties with multiple arguments + out-of-the-box whereas Feat requires, at least by default, properties to be + uncurried. +* LeanCheck is configured by the number of tests whereas Feat is configured by + the size of tests. + +__Memory consumption.__ LeanCheck is more memory intensive than Feat. + +__Random testing.__ Feat is able to do random testing. LeanCheck is not (at +least not with good performance). + +__Implementation.__ The [implementation and internals of LeanCheck] are +simpler than those of Feat. + +__Tool support.__ Tool support for LeanCheck and Feat is different. +[LeanCheck has bindings] for [Tasty], [test-framework] and [Hspec]. As of Jan +2019, [Feat only has bindings for test-framework], nevertheless adding support +for Tasty and Hspec would be simple. + +LeanCheck facilitates the use of [Extrapolate] to generalize counterexamples. + + +[LeanCheck]: https://hackage.haskell.org/package/leancheck +[QuickCheck]: https://hackage.haskell.org/package/QuickCheck +[Feat]: https://hackage.haskell.org/package/testing-feat +[Extrapolate]: https://hackage.haskell.org/package/extrapolate +[LeanCheck tutorial]: doc/tutorial.md +[`Listable`]: https://hackage.haskell.org/package/leancheck/docs/Test-LeanCheck.html#t:Listable +[`Testable`]: https://hackage.haskell.org/package/leancheck/docs/Test-LeanCheck.html#t:Testable +[`tiers`]: https://hackage.haskell.org/package/leancheck/docs/Test-LeanCheck.html#v:tiers +[`Arbitrary`]: https://hackage.haskell.org/package/QuickCheck/docs/Test-QuickCheck.html#t:Arbitrary +[shrinking]: https://hackage.haskell.org/package/QuickCheck/docs/Test-QuickCheck.html#v:shrink +[`shrink`]: https://hackage.haskell.org/package/QuickCheck/docs/Test-QuickCheck.html#v:shrink +[Skolemization]: https://en.wikipedia.org/wiki/Skolem_normal_form +[Tasty]: https://github.com/feuerbach/tasty#readme +[test-framework]: https://haskell.github.io/test-framework/ +[Hspec]: https://hspec.github.io/ +[LeanCheck has bindings]: https://github.com/rudymatela/leancheck#providers-for-tasty-test-framework-and-hspec +[Feat only has bindings for test-framework]: https://hackage.haskell.org/package/test-framework-testing-feat +[implementation and internals of LeanCheck]: https://hackage.haskell.org/package/leancheck/docs/src/Test.LeanCheck.Core.html diff --git a/doc/tutorial.md b/doc/tutorial.md --- a/doc/tutorial.md +++ b/doc/tutorial.md @@ -203,9 +203,16 @@ LeanCheck also provides the function [`deriveListable`] to automatically derive [`Listable`] instances for types that do not follow a data invariant (precondition). +The above [`Listable`] instance could be replaced by simply: + {-# LANGUAGE TemplateHaskell #-} + ... + ... + deriveListable ''Stack + + Advantages of property-based testing ------------------------------------ @@ -215,7 +222,7 @@ after making a small change to a program, we might [`checkFor`] `50` tests; before making a major release, we may [`checkFor`] `1000` tests. A continuous integration system can be configured to run more test than what - is usual on developers machines. + is usual on the development environment. * (+) documentation: properties serve as a clear documentation of behaviour; @@ -239,6 +246,7 @@ ---------------------------------------------- * [QuickCheck] : randomized +* [Hedgehog] : randomized * [SmallCheck] : enumerative, depth-bounded * [Lazy SmallCheck] : enumerative, depth-bounded, lazy, demand-driven * [Feat] : enumerative, size-bounded @@ -260,6 +268,7 @@ [`deriveListable`]: https://hackage.haskell.org/package/leancheck/docs/Test-LeanCheck.html#v:deriveListable [QuickCheck]: https://hackage.haskell.org/package/QuickCheck +[Hedgehog]: https://hackage.haskell.org/package/hedgehog [SmallCheck]: https://hackage.haskell.org/package/smallcheck [Lazy SmallCheck]: https://hackage.haskell.org/package/lazysmallcheck [Feat]: https://hackage.haskell.org/package/testing-feat diff --git a/leancheck.cabal b/leancheck.cabal --- a/leancheck.cabal +++ b/leancheck.cabal @@ -11,7 +11,7 @@ -- this cabal file too complicated. -- Rudy name: leancheck -version: 0.9.0 +version: 0.9.1 synopsis: Enumerative property-based testing description: LeanCheck is a simple enumerative property-based testing library. @@ -40,6 +40,7 @@ , TODO.md , changelog.md , doc/data-invariant.md + , doc/faq.md , doc/tutorial.md , doc/leancheck.svg extra-source-files: .gitignore @@ -74,7 +75,7 @@ source-repository this type: git location: https://github.com/rudymatela/leancheck - tag: v0.9.0 + tag: v0.9.1 library exposed-modules: Test.LeanCheck diff --git a/mk/depend.mk b/mk/depend.mk --- a/mk/depend.mk +++ b/mk/depend.mk @@ -382,8 +382,11 @@ test/Test.hs \ test/derive.hs \ src/Test/LeanCheck/Utils/Types.hs \ + src/Test/LeanCheck/Utils/TypeBinding.hs \ + src/Test/LeanCheck/Utils.hs \ src/Test/LeanCheck/Utils/Operators.hs \ src/Test/LeanCheck/Tiers.hs \ + src/Test/LeanCheck/Stats.hs \ src/Test/LeanCheck.hs \ src/Test/LeanCheck/IO.hs \ src/Test/LeanCheck/Derive.hs \ @@ -451,8 +454,11 @@ test/Test.hs \ test/generic.hs \ src/Test/LeanCheck/Utils/Types.hs \ + src/Test/LeanCheck/Utils/TypeBinding.hs \ + src/Test/LeanCheck/Utils.hs \ src/Test/LeanCheck/Utils/Operators.hs \ src/Test/LeanCheck/Tiers.hs \ + src/Test/LeanCheck/Stats.hs \ src/Test/LeanCheck.hs \ src/Test/LeanCheck/IO.hs \ src/Test/LeanCheck/Generic.hs \ diff --git a/src/Test/LeanCheck/Function.hs b/src/Test/LeanCheck/Function.hs --- a/src/Test/LeanCheck/Function.hs +++ b/src/Test/LeanCheck/Function.hs @@ -43,8 +43,8 @@ -- -- The 'Listable' and 'Show' function instance are defined in, respectively: -- --- * "Test.LeanCheck.Function.Listable"; --- * "Test.LeanCheck.Function.Show". +-- * "Test.LeanCheck.Function.Listable" +-- * "Test.LeanCheck.Function.Show" -- -- The 'Show' instance will work for all functions whose return types are -- instances of ShowFunction from "Test.LeanCheck.Function.ShowFunction". diff --git a/src/Test/LeanCheck/Generic.hs b/src/Test/LeanCheck/Generic.hs --- a/src/Test/LeanCheck/Generic.hs +++ b/src/Test/LeanCheck/Generic.hs @@ -1,4 +1,7 @@ -{-# LANGUAGE FlexibleContexts, TypeOperators #-} +{-# LANGUAGE CPP, FlexibleContexts, FlexibleInstances, TypeOperators #-} +#if __GLASGOW_HASKELL__ < 710 +{-# LANGUAGE OverlappingInstances #-} +#endif -- | -- Module : Test.LeanCheck.Generic -- Copyright : (c) 2018 Rudy Matela @@ -61,5 +64,25 @@ instance (Listable' a, Listable' b) => Listable' (a :*: b) where tiers' = productWith (:*:) tiers' tiers' -instance Listable' f => Listable' (M1 i c f) where +instance Listable' f => Listable' (S1 c f) where tiers' = mapT M1 tiers' + +instance Listable' f => Listable' (D1 c f) where + tiers' = mapT M1 tiers' + +#if __GLASGOW_HASKELL__ >= 710 +-- don't delay when there is a constructor with 0 arguments +instance {-# OVERLAPPING #-} Listable' (C1 c U1) where + tiers' = mapT M1 tiers' + +-- delay when there is a constructor with 1 or more arguments +instance {-# OVERLAPPABLE #-} Listable' f => Listable' (C1 c f) where + tiers' = delay $ mapT M1 tiers' +#else + +instance Listable' (C1 c U1) + where tiers' = mapT M1 tiers' + +instance Listable' f => Listable' (C1 c f) + where tiers' = delay $ mapT M1 tiers' +#endif diff --git a/stack.yaml b/stack.yaml --- a/stack.yaml +++ b/stack.yaml @@ -3,7 +3,7 @@ # resolver: nightly-2015-09-21 # resolver: ghc-7.10.2 -resolver: lts-12.9 +resolver: lts-13.6 packages: - . diff --git a/test/derive.hs b/test/derive.hs --- a/test/derive.hs +++ b/test/derive.hs @@ -6,7 +6,7 @@ import Test.LeanCheck.Derive import System.Exit (exitFailure) import Data.List (elemIndices,sort) -import Test.LeanCheck.Utils.Operators +import Test.LeanCheck.Utils data D0 = D0 deriving Show data D1 a = D1 a deriving Show @@ -24,6 +24,17 @@ deriveListable ''C2 deriveListable ''I +-- recursive datatypes +data Peano = Zero | Succ Peano deriving Show +data List a = a :- List a | Nil deriving Show +data Bush a = Bush a :-: Bush a | Leaf a deriving (Show, Eq) +data Tree a = Node (Tree a) a (Tree a) | Null deriving (Show, Eq) + +deriveListable ''Peano +deriveListable ''List +deriveListable ''Bush +deriveListable ''Tree + -- Nested datatype cascade data Nested = Nested N0 (N1 Int) (N2 Int Int) data N0 = R0 Int @@ -86,6 +97,32 @@ , map unD2 list == (list :: [(Bool,Bool)]) , map unD3 list == (list :: [(Bool,Bool,Bool)]) + , map peanoToNat list =| n |= list + , map listToList list =| n |= (list :: [[Bool]]) + , map listToList list =| n |= (list :: [[Int]]) + + , mapT peanoToNat tiers =| 6 |= tiers + , mapT listToList tiers =| 6 |= (tiers :: [[ [Bool] ]]) + , mapT listToList tiers =| 6 |= (tiers :: [[ [Int] ]]) + + , take 6 (list :: [Bush Bool]) + == [ Leaf False + , Leaf True + , Leaf False :-: Leaf False + , Leaf False :-: Leaf True + , Leaf True :-: Leaf False + , Leaf True :-: Leaf True + ] + + , take 6 (list :: [Tree Bool]) + == [ Null + , Node Null False Null + , Node Null True Null + , Node Null False (Node Null False Null) + , Node Null False (Node Null True Null) + , Node Null True (Node Null False Null) + ] + , (tiers :: [[ Bool ]]) =| 6 |= $(deriveTiers ''Bool) , (tiers :: [[ [Int] ]]) =| 6 |= $(deriveTiers ''[]) , (tiers :: [[ [Bool] ]]) =| 6 |= $(deriveTiers ''[]) @@ -104,3 +141,11 @@ unD1 (D1 x) = (x) unD2 (D2 x y) = (x,y) unD3 (D3 x y z) = (x,y,z) + +peanoToNat :: Peano -> Nat +peanoToNat Zero = 0 +peanoToNat (Succ n) = 1 + peanoToNat n + +listToList :: List a -> [a] +listToList Nil = [] +listToList (x :- xs) = x : listToList xs diff --git a/test/generic.hs b/test/generic.hs --- a/test/generic.hs +++ b/test/generic.hs @@ -1,12 +1,13 @@ -- Copyright (c) 2015-2018 Rudy Matela. -- Distributed under the 3-Clause BSD licence (see the file LICENSE). -{-# LANGUAGE DeriveGeneric, StandaloneDeriving #-} +{-# LANGUAGE DeriveGeneric, StandaloneDeriving, TemplateHaskell #-} import Test import Test.LeanCheck import Test.LeanCheck.Generic +import Test.LeanCheck.Derive (deriveTiers) import System.Exit (exitFailure) import Data.List (elemIndices,sort) -import Test.LeanCheck.Utils.Operators +import Test.LeanCheck.Utils import GHC.Generics (Generic) data D0 = D0 deriving (Eq, Show, Generic) @@ -35,6 +36,18 @@ instance (Listable a, Listable b) => Listable (I a b) where tiers = genericTiers +-- recursive datatypes +data Peano = Zero | Succ Peano deriving (Show, Generic) +data List a = a :- List a | Nil deriving (Show, Generic) +data Bush a = Bush a :-: Bush a | Leaf a deriving (Show, Eq, Generic) +data Tree a = Node (Tree a) a (Tree a) | Null deriving (Show, Eq, Generic) + +instance Listable Peano where tiers = genericTiers +instance Listable a => Listable (List a) where tiers = genericTiers +instance Listable a => Listable (Bush a) where tiers = genericTiers +instance Listable a => Listable (Tree a) where tiers = genericTiers + + main :: IO () main = case elemIndices False (tests 100) of @@ -57,9 +70,64 @@ , map unD1 list == (list :: [Bool]) , map unD2 list == (list :: [(Bool,Bool)]) , map unD3 list == (list :: [(Bool,Bool,Bool)]) + + , map peanoToNat list =| n |= list + , map listToList list =| n |= (list :: [[Bool]]) + , map listToList list =| n |= (list :: [[Int]]) + + , mapT peanoToNat tiers =| 6 |= tiers + , mapT listToList tiers =| 6 |= (tiers :: [[ [Bool] ]]) + , mapT listToList tiers =| 6 |= (tiers :: [[ [Int] ]]) + + , take 6 (list :: [Bush Bool]) + == [ Leaf False + , Leaf True + , Leaf False :-: Leaf False + , Leaf False :-: Leaf True + , Leaf True :-: Leaf False + , Leaf True :-: Leaf True + ] + + , take 6 (list :: [Tree Bool]) + == [ Null + , Node Null False Null + , Node Null True Null + , Node Null False (Node Null False Null) + , Node Null False (Node Null True Null) + , Node Null True (Node Null False Null) + ] + + , (tiers :: [[ Bool ]]) =| 6 |= genericTiers + , (tiers :: [[ [Int] ]]) =| 6 |= genericTiers + , (tiers :: [[ [Bool] ]]) =| 6 |= genericTiers + , (tiers :: [[ Maybe Int ]]) =| 6 |= genericTiers + , (tiers :: [[ Maybe Bool ]]) =| 6 |= genericTiers + , ([]:tiers :: [[Either Bool Int]]) =$ map sort . take 6 $= genericTiers + + , (list :: [ Bool ]) =| n |= genericList + , (list :: [ [Int] ]) =| n |= genericList + , (list :: [ [Bool] ]) =| n |= genericList + , (list :: [ Maybe Int ]) =| n |= genericList + , (list :: [ Maybe Bool ]) =| n |= genericList + + -- test consistency with deriveTiers + , (genericTiers :: [[ Bool ]]) =| 6 |= $(deriveTiers ''Bool) + , (genericTiers :: [[ [Int] ]]) =| 6 |= $(deriveTiers ''[]) + , (genericTiers :: [[ [Bool] ]]) =| 6 |= $(deriveTiers ''[]) + , (genericTiers :: [[ Maybe Int ]]) =| 6 |= $(deriveTiers ''Maybe) + , (genericTiers :: [[ Maybe Bool ]]) =| 6 |= $(deriveTiers ''Maybe) + , (genericTiers :: [[ Either Bool Int ]]) =| 6 |= $(deriveTiers ''Either) ] where unD0 (D0) = () unD1 (D1 x) = (x) unD2 (D2 x y) = (x,y) unD3 (D3 x y z) = (x,y,z) + +peanoToNat :: Peano -> Nat +peanoToNat Zero = 0 +peanoToNat (Succ n) = 1 + peanoToNat n + +listToList :: List a -> [a] +listToList Nil = [] +listToList (x :- xs) = x : listToList xs