diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [0.4.0.0] - 2026-10-01 + +### Fixed + +* `assert-score` no longer fails on a mutation that ran out of time, and nor + does a `--fail-fast` run. Such a mutation is a kill again -- mutating a + loop's exit condition turns it into `while True`, and a suite that never + finishes is one that noticed -- so there is nothing left to fail on. See + sydtest 0.33.0.0. + + ## [0.3.0.0] - 2026-10-01 ### Changed diff --git a/src/Test/Syd/Mutation/Driver.hs b/src/Test/Syd/Mutation/Driver.hs --- a/src/Test/Syd/Mutation/Driver.hs +++ b/src/Test/Syd/Mutation/Driver.hs @@ -33,11 +33,12 @@ ( AugmentedManifest (..), AugmentedMutationGroup (..), AugmentedMutationRecord (..), + MutationRunReport (..), + MutationTally (..), RunSoundness (..), filterAugmentedManifestByIds, readAndUnionBaselineDirs, readAndUnionCoverageDirs, - runFoundFailure, runSoundness, writeAugmentedManifestFile, ) @@ -122,8 +123,13 @@ case runSoundness report of RunUnsound -> exitWith (ExitFailure 1) RunSound -> pure () - when (mutationDriverSettingFailFast && runFoundFailure report) $ - exitWith (ExitFailure 1) + when + ( mutationDriverSettingFailFast + && ( mutationTallySurvived (mutationRunReportMutations report) > 0 + || mutationTallyUncovered (mutationRunReportMutations report) > 0 + ) + ) + $ exitWith (ExitFailure 1) -- | Assemble the augmented manifest the mutation phase reads from pre-computed -- per-package coverage directories, instead of running the coverage phase. diff --git a/src/Test/Syd/Mutation/Driver/AssertScore.hs b/src/Test/Syd/Mutation/Driver/AssertScore.hs --- a/src/Test/Syd/Mutation/Driver/AssertScore.hs +++ b/src/Test/Syd/Mutation/Driver/AssertScore.hs @@ -6,8 +6,8 @@ -- | Implementation of the @assert-score@ subcommand: read @report.json@ -- from a report directory, print a one-line pass\/fail header followed -- by the rendered report body, and exit 0 on success, 1 on a failed --- assertion (any survived, any timed out, or any uncovered when the --- uncovered assertion is enabled). +-- assertion (any survived, or any uncovered when the uncovered assertion +-- is enabled). module Test.Syd.Mutation.Driver.AssertScore ( runAssertScore, assertScoreResult, @@ -43,8 +43,8 @@ -- | The decision 'assert-score' renders from a 'MutationRunReport'. data AssertScoreResult = AssertScoreResult { -- | True when the assertion is violated: at least one survivor, at least - -- one mutation that ran out of time, at least one failed control, or - -- (when uncovered is also asserted) at least one uncovered mutation. + -- one failed control, or (when uncovered is also asserted) at least one + -- uncovered mutation. assertScoreFailed :: !Bool, -- | Pass/fail header line (e.g. @PASS: All 17 mutation(s) accounted for.@). assertScoreHeader :: ![Chunk] @@ -65,17 +65,13 @@ -- transient noise. Treat it as an assertion failure like a survivor. failed = mutationTallySurvived > 0 - || mutationTallyTimedOut > 0 || (assertNoneUncovered && mutationTallyUncovered > 0) || controlTallyFailed > 0 - -- Timed-out mutations used to be inside the killed count, which is why - -- this sum did not name them. Now that they are their own category it - -- has to add them back, or the count a FAIL is measured against leaves - -- out part of what was run. + -- Timed-out mutations are inside the killed count, so this sum already + -- includes them. total = mutationTallyKilled + mutationTallySurvived - + mutationTallyTimedOut + mutationTallyUncovered -- A count is good (green) when it's zero, bad (red) when it isn't. -- The header colour reflects the overall verdict, but each @@ -94,14 +90,6 @@ countChunk mutationTallyUncovered, chunk " uncovered" ] - -- Named only when one ran out of time, so the usual - -- survivor/uncovered header reads as it always has. Without - -- it a run that failed on nothing else reads "0 surviving, 0 - -- uncovered" and leaves the reader to guess what failed. - ++ ( if mutationTallyTimedOut > 0 - then [chunk ", ", countChunk mutationTallyTimedOut, chunk " timed out"] - else [] - ) -- Only mention controls when one actually failed, so the common -- survivor/uncovered FAIL header reads exactly as before. ++ ( if controlTallyFailed > 0 diff --git a/src/Test/Syd/Mutation/Driver/Mutate.hs b/src/Test/Syd/Mutation/Driver/Mutate.hs --- a/src/Test/Syd/Mutation/Driver/Mutate.hs +++ b/src/Test/Syd/Mutation/Driver/Mutate.hs @@ -304,6 +304,13 @@ -- without favouring either outcome. That is worth doing here rather than -- leaving to whoever re-runs the check, because the run this belongs to -- costs the better part of an hour. + -- + -- [tag:TimeoutIsAKill] An overrun that survives these attempts is scored + -- as a kill, in 'tallyGroups', and left alone by 'isMutationFailure'. + -- Both of those are sound only because of the retry here: one overrun + -- says nothing, since a busy machine exceeds a budget derived from an + -- idle one, and crediting that would let load raise the score. Weakening + -- or removing these attempts means revisiting how an overrun is scored. runSuitesRetryingOverruns record mid suiteNames attemptsLeft = do outcomes <- mapM (runOneSuite record mid) suiteNames case classifyOutcomes record outcomes of diff --git a/sydtest-mutation-driver.cabal b/sydtest-mutation-driver.cabal --- a/sydtest-mutation-driver.cabal +++ b/sydtest-mutation-driver.cabal @@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: sydtest-mutation-driver -version: 0.3.0.0 +version: 0.4.0.0 synopsis: Out-of-process mutation testing driver for sydtest. description: Standalone driver executable that orchestrates the coverage and mutation phases of sydtest's mutation testing infrastructure. Spawns instrumented sydtest test suite executables as children. category: Testing