diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,8 @@ +# 1.1.16 + +* New command-line option `--rerun-all-on-success`. +* New command-line shortcut `--rerun`. + # 1.1.15 * Bump upper bound of base. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# tasty-rerun + +This `Ingredient` for [`tasty`](https://hackage.haskell.org/package/tasty) testing framework +allows to filter a test tree depending +on an outcome of the previous run. +This may be useful in many scenarios, +especially when a test suite grows large. + +For example, `tasty-rerun` allows: + +* Rerun only tests, which failed during the last run (`--rerun`). + Combined with live reloading (e. g., using `ghcid` or `stack test --file-watch`), + it gives an ultimate power to focus on broken parts + and put them back in shape, enjoying a tight feedback loop. +* Rerun only tests, which have beed added since the last saved test run. + This comes handy when writing a new module, which does not affect other + parts of the system, or adding new test cases. +* Rerun only tests, which passed during the last saved test run. + Sometimes a part of the test suite is consistently failing + (e. g., an external service is temporarily down), but you want be sure + that you are not breaking anything else in course of your work. + +Add it to your test suite as follows: + +```haskell +import Test.Tasty +import Test.Tasty.Runners +import Test.Tasty.Ingredients.Rerun + +main :: IO () +main = + defaultMainWithIngredients + [ rerunningTests [ listingTests, consoleTestReporter ] ] + tests + +tests :: TestTree +tests = undefined +``` + +Use `--help` to list command-line options: + +* `--rerun` + + Rerun only tests, which failed during the last run. + If the last run was successful, execute a full test + suite afresh. A shortcut for `--rerun-update + --rerun-filter failures,exceptions + --rerun-all-on-success`. + +* `--rerun-update` + + Update the log file to reflect latest test outcomes. + +* `--rerun-filter CATEGORIES` + + Read the log file and rerun only tests from a given + comma-separated list of categories: `failures`, + `exceptions`, `new`, `successful`. If this option is + omitted or the log file is missing, rerun everything. + +* `--rerun-all-on-success` + + If according to the log file and `--rerun-filter` there + is nothing left to rerun, run all tests. This comes + especially handy in `stack test --file-watch` or + `ghcid` scenarios. + +* `--rerun-log-file FILE` + + Location of the log file (default: `.tasty-rerun-log`). +``` diff --git a/src/Test/Tasty/Ingredients/Rerun.hs b/src/Test/Tasty/Ingredients/Rerun.hs --- a/src/Test/Tasty/Ingredients/Rerun.hs +++ b/src/Test/Tasty/Ingredients/Rerun.hs @@ -13,7 +13,7 @@ import Data.List (intercalate) import Data.List.Split (endBy) import Data.Maybe (fromMaybe) -import Data.Monoid (mempty) +import Data.Monoid (Any(..), mempty) import Data.Proxy (Proxy(..)) import Data.Typeable (Typeable) import System.IO.Error (catchIOError, isDoesNotExistError) @@ -78,16 +78,54 @@ optionCLParser = Tasty.mkOptionCLParser (OptParse.metavar "CATEGORIES") -------------------------------------------------------------------------------- +newtype AllOnSuccess = AllOnSuccess Bool + deriving (Typeable) + +instance Tasty.IsOption AllOnSuccess where + optionName = return "rerun-all-on-success" + optionHelp = return "If according to the log file and --rerun-filter there is nothing left to rerun, run all tests. This comes especially handy in `stack test --file-watch` or `ghcid` scenarios." + defaultValue = AllOnSuccess False + parseValue = fmap AllOnSuccess . Tasty.safeReadBool + optionCLParser = Tasty.mkFlagCLParser mempty (AllOnSuccess True) + +-------------------------------------------------------------------------------- +newtype Rerun = Rerun { unRerun :: Bool } + deriving (Typeable) + +instance Tasty.IsOption Rerun where + optionName = return "rerun" + optionHelp = return "Rerun only tests, which failed during the last run. If the last run was successful, execute a full test suite afresh. A shortcut for --rerun-update --rerun-filter failures,exceptions --rerun-all-on-success" + defaultValue = Rerun False + parseValue = fmap Rerun . Tasty.safeReadBool + optionCLParser = Tasty.mkFlagCLParser mempty (Rerun True) + +rerunMeaning :: (UpdateLog, AllOnSuccess, FilterOption) +rerunMeaning = (UpdateLog True, AllOnSuccess True, FilterOption (Set.fromList [Failures, Exceptions])) + +-------------------------------------------------------------------------------- data TestResult = Completed Bool | ThrewException deriving (Read, Show) -------------------------------------------------------------------------------- --- | This 'Tasty.Ingredient' transformer allows to control --- which tests to run depending on their previous outcomes. --- For example, you can rerun only failing tests or only new tests. +-- | +-- This ingredient +-- for testing framework +-- allows to filter a test tree depending +-- on an outcome of the previous run. +-- This may be useful in many scenarios, +-- especially when a test suite grows large. +-- -- The behaviour is controlled by command-line options: -- +-- * @--rerun@ @ @ +-- +-- Rerun only tests, which failed during the last run. +-- If the last run was successful, execute a full test +-- suite afresh. A shortcut for @--rerun-update@ +-- @--rerun-filter failures,exceptions@ +-- @--rerun-all-on-success@. +-- -- * @--rerun-update@ @ @ -- -- Update the log file to reflect latest test outcomes. @@ -99,6 +137,13 @@ -- @exceptions@, @new@, @successful@. If this option is -- omitted or the log file is missing, rerun everything. -- +-- * @--rerun-all-on-success@ @ @ +-- +-- If according to the log file and @--rerun-filter@ there +-- is nothing left to rerun, run all tests. This comes +-- especially handy in @stack test --file-watch@ or +-- @ghcid@ scenarios. +-- -- * @--rerun-log-file@ @FILE@ -- -- Location of the log file (default: @.tasty-rerun-log@). @@ -122,10 +167,15 @@ Tasty.TestManager (rerunOptions ++ Tasty.ingredientsOptions ingredients) $ \options testTree -> Just $ do let RerunLogFile stateFile = Tasty.lookupOption options - UpdateLog updateLog = Tasty.lookupOption options - FilterOption filter = Tasty.lookupOption options + (UpdateLog updateLog, AllOnSuccess allOnSuccess, FilterOption filter) + | unRerun (Tasty.lookupOption options) = rerunMeaning + | otherwise = (Tasty.lookupOption options, Tasty.lookupOption options, Tasty.lookupOption options) - filteredTestTree <- maybe testTree (filterTestTree testTree filter) + let nonEmptyFold = Tasty.trivialFold { Tasty.foldSingle = \_ _ _ -> Any True } + nullTestTree = not . getAny . Tasty.foldTestTree nonEmptyFold options + recoverFromEmpty t = if allOnSuccess && nullTestTree t then testTree else t + + filteredTestTree <- maybe testTree (recoverFromEmpty . filterTestTree testTree filter) <$> tryLoadStateFrom stateFile let tryAndRun (Tasty.TestReporter _ f) = do @@ -159,12 +209,15 @@ -- simply run the above constructed IO action. Just e -> e where - rerunOptions = [ Tasty.Option (Proxy :: Proxy UpdateLog) + rerunOptions = [ Tasty.Option (Proxy :: Proxy Rerun) + , Tasty.Option (Proxy :: Proxy UpdateLog) , Tasty.Option (Proxy :: Proxy FilterOption) + , Tasty.Option (Proxy :: Proxy AllOnSuccess) , Tasty.Option (Proxy :: Proxy RerunLogFile) ] ------------------------------------------------------------------------------ + filterTestTree :: Tasty.TestTree -> Set.Set Filter -> Map.Map [String] TestResult -> Tasty.TestTree filterTestTree testTree filter lastRecord = let go prefix (Tasty.SingleTest name t) = let requiredFilter = case Map.lookup (prefix ++ [name]) lastRecord of @@ -193,6 +246,7 @@ in go [] testTree + tryLoadStateFrom :: FilePath -> IO (Maybe (Map.Map [String] TestResult)) tryLoadStateFrom filePath = do fileContents <- (Just <$> readFile filePath) `catchIOError` (\e -> if isDoesNotExistError e @@ -201,10 +255,14 @@ return (read <$> fileContents) ------------------------------------------------------------------------------ + saveStateTo :: FilePath -> IO (Map.Map [String] TestResult) -> IO () saveStateTo filePath getTestResults = getTestResults >>= (show >>> writeFile filePath) ------------------------------------------------------------------------------ + observeResults + :: IntMap.IntMap (STM.TVar Tasty.Status) + -> Tasty.TreeFold (Tasty.Traversal (Functor.Compose (State.StateT Int IO) (Const (Map.Map [String] TestResult)))) observeResults statusMap = let foldSingle _ name _ = Tasty.Traversal $ Functor.Compose $ do i <- State.get diff --git a/tasty-rerun.cabal b/tasty-rerun.cabal --- a/tasty-rerun.cabal +++ b/tasty-rerun.cabal @@ -1,5 +1,5 @@ name: tasty-rerun -version: 1.1.15 +version: 1.1.16 homepage: http://github.com/ocharles/tasty-rerun license: BSD3 license-file: LICENSE @@ -12,63 +12,18 @@ cabal-version: >=1.10 extra-source-files: Changelog.md + README.md synopsis: Rerun only tests which failed in a previous test run description: - This ingredient adds the ability to run tests by first filtering the test tree - based on the result of a previous test run. For example, you can use this to - run only those tests that failed in the last run, or to run only tests that - have been added since tests were last ran. - . - This ingredient is specifically an ingredient *transformer* - given a list of - 'Tasty.Ingredient's, 'rerunningTests' adds the ability for all of these - ingredients to run against a filtered test tree. This transformer can be - applied as follows: - . - > import Test.Tasty - > import Test.Tasty.Runners - > import Test.Tasty.Ingredients.Rerun - > - > main :: IO () - > main = - > defaultMainWithIngredients - > [ rerunningTests [ listingTests, consoleTestReporter ] ] - > tests - > - > tests :: TestTree - > tests = undefined - . - This ingredient adds three command line parameters: - . - [@--rerun-update@] If specified the results of this test run will be saved to - the log file at @--rerun-log-file@. If the ingredient does not execute tests - (for example, @--list-tests@ is used) then the log file will not be - updated. This option is not enabled by default. This option does not require - a value. - . - [@--rerun-log-file@] The path to the log file to read previous test - information from, and where to write new information to (if @--rerun-update@ - is specified). This option defaults to @.tasty-rerun-log@. - . - [@--rerun-filter@] Which filters to apply to the 'Tasty.TestTree' based on - previous test runs. The value of this option is a comma separated list of the - following options: - . - * @failures@: Only run tests that failed on the previous run. - . - * @exceptions@: Only run tests that threw an exception on the previous run. - . - * @new@: Only run tests that are new since the previous test run. - . - * @successful@: Only run tests that were successful in the previous run. - . - Multiple options can be combined and will be taken under disjunction - so - @--rerun-filter=failures,exceptions@ will run only tests that failed *or* - threw an exception on the last run. - . - Defaults to all filters, which means all tests will be ran. + This ingredient + for testing framework + allows to filter a test tree depending + on an outcome of the previous run. + This may be useful in many scenarios, + especially when a test suite grows large. tested-with: GHC==8.8.1, GHC==8.6.5, GHC==8.4.4, GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4, GHC==7.6.3