sydtest-mutation-driver 0.2.0.0 → 0.3.0.0

raw patch · 5 files changed

+86/−23 lines, 5 files

Files

• CHANGELOG.md +12/−0
• src/Test/Syd/Mutation/Driver.hs +23/−14
• src/Test/Syd/Mutation/Driver/AssertScore.hs +18/−4
• src/Test/Syd/Mutation/Driver/Mutate.hs +32/−4
• sydtest-mutation-driver.cabal +1/−1

@@ -1,5 +1,17 @@ # Changelog +## [0.3.0.0] - 2026-10-01++### Changed++* A mutation that only overran its budget is given another go, up to three+  attempts.+* A run with a failed control exits non-zero, so its verdict is not cached+  and the run can simply be retried.+* `assert-score` fails on a mutation that ran out of time, and so does a+  `--fail-fast` run.++ ## [0.2.0.0] - 2026-09-11  ### Fixed

@@ -33,12 +33,12 @@   ( AugmentedManifest (..),     AugmentedMutationGroup (..),     AugmentedMutationRecord (..),-    ControlTally (..),-    MutationRunReport (..),-    MutationTally (..),+    RunSoundness (..),     filterAugmentedManifestByIds,     readAndUnionBaselineDirs,     readAndUnionCoverageDirs,+    runFoundFailure,+    runSoundness,     writeAugmentedManifestFile,   ) import Test.Syd.Mutation.Driver.AssertScore (runAssertScore)@@ -94,9 +94,8 @@   -- 'SuiteConfig' map, not just the exes), so no parent-side 'cd' is needed   -- here.  It prints the report to stdout and writes report.json +   -- report.txt + per-suite *.log files to the out dir.  It returns the run-  -- report; under --fail-fast we then exit non-zero ourselves, preserving-  -- the historical behaviour of the @run@ subcommand (a downstream-  -- @assert-score@ step is the gate when --fail-fast is off).+  -- report; we then decide the exit code from it below (a downstream+  -- @assert-score@ step is the gate for survivors when --fail-fast is off).   report <-     runMutationMode       mutationDriverSettingFailFast@@ -107,14 +106,24 @@       mutationDriverSettingMutationJobs       suites   hFlush stdout-  when-    ( mutationDriverSettingFailFast-        && ( mutationTallySurvived (mutationRunReportMutations report) > 0-               || mutationTallyUncovered (mutationRunReportMutations report) > 0-               || controlTallyFailed (mutationRunReportControls report) > 0-           )-    )-    $ exitWith (ExitFailure 1)+  -- A failed control fails this run even without --fail-fast, which a+  -- survivor deliberately does not.+  --+  -- The difference is whether the verdict can be reproduced. Survivors can+  -- be, so writing them to a report that a later assert-score step reads is+  -- what we want: the expensive run is done once and the cheap gate reports+  -- it as often as asked. A failed control cannot be, because the thing that+  -- fails a control is a suite nobody can rely on: flaky, or too slow for its+  -- budget on the machine it happened to land on. Succeeding here would store+  -- that verdict in a build artefact, and every later attempt would read the+  -- stored copy rather than run anything, so the run could not be retried+  -- without hunting down and deleting the artefact. Failing leaves nothing+  -- behind to read, and the next attempt runs the mutations again.+  case runSoundness report of+    RunUnsound -> exitWith (ExitFailure 1)+    RunSound -> pure ()+  when (mutationDriverSettingFailFast && runFoundFailure report) $+    exitWith (ExitFailure 1)  -- | Assemble the augmented manifest the mutation phase reads from pre-computed -- per-package coverage directories, instead of running the coverage phase.

@@ -6,8 +6,8 @@ -- | Implementation of the @assert-score@ subcommand: read @report.json@ -- from a report directory, print a one-line pass\/fail header followed -- by the rendered report body, and exit 0 on success, 1 on a failed--- assertion (any survived, or any uncovered when the uncovered assertion--- is enabled).+-- assertion (any survived, any timed out, or any uncovered when the+-- uncovered assertion is enabled). module Test.Syd.Mutation.Driver.AssertScore   ( runAssertScore,     assertScoreResult,@@ -43,8 +43,8 @@ -- | The decision 'assert-score' renders from a 'MutationRunReport'. data AssertScoreResult = AssertScoreResult   { -- | True when the assertion is violated: at least one survivor, at least-    -- one failed control, or (when uncovered is also asserted) at least one-    -- uncovered mutation.+    -- one mutation that ran out of time, at least one failed control, or+    -- (when uncovered is also asserted) at least one uncovered mutation.     assertScoreFailed :: !Bool,     -- | Pass/fail header line (e.g. @PASS: All 17 mutation(s) accounted for.@).     assertScoreHeader :: ![Chunk]@@ -65,11 +65,17 @@       -- transient noise.  Treat it as an assertion failure like a survivor.       failed =         mutationTallySurvived > 0+          || mutationTallyTimedOut > 0           || (assertNoneUncovered && mutationTallyUncovered > 0)           || controlTallyFailed > 0+      -- Timed-out mutations used to be inside the killed count, which is why+      -- this sum did not name them.  Now that they are their own category it+      -- has to add them back, or the count a FAIL is measured against leaves+      -- out part of what was run.       total =         mutationTallyKilled           + mutationTallySurvived+          + mutationTallyTimedOut           + mutationTallyUncovered       -- A count is good (green) when it's zero, bad (red) when it isn't.       -- The header colour reflects the overall verdict, but each@@ -88,6 +94,14 @@               countChunk mutationTallyUncovered,               chunk " uncovered"             ]+              -- Named only when one ran out of time, so the usual+              -- survivor/uncovered header reads as it always has.  Without+              -- it a run that failed on nothing else reads "0 surviving, 0+              -- uncovered" and leaves the reader to guess what failed.+              ++ ( if mutationTallyTimedOut > 0+                     then [chunk ", ", countChunk mutationTallyTimedOut, chunk " timed out"]+                     else []+                 )               -- Only mention controls when one actually failed, so the common               -- survivor/uncovered FAIL header reads exactly as before.               ++ ( if controlTallyFailed > 0

@@ -272,10 +272,10 @@           Nothing -> pure (MutationUncovered (UncoveredMutation record))           Just suiteNames -> do             -- Run one child per covering suite.  The mutation is killed-            -- if any child exits non-zero; timed out (counted as killed)-            -- if any child exceeded its budget without any other child-            -- killing it first; otherwise survived.-            outcomes <- mapM (runOneSuite record mid) suiteNames+            -- if any child exits non-zero; timed out if any child exceeded+            -- its budget without any other child killing it first;+            -- otherwise survived.+            outcomes <- runSuitesRetryingOverruns record mid suiteNames attemptsOnOverrun             -- A control (no-op) mutation is expected to survive.  Reinterpret             -- its raw outcome: survival is the control passing, a kill or             -- timeout is the control failing (the suite is unsound).@@ -283,6 +283,34 @@               if isControlOperator (augmentedMutationRecordOperator record)                 then asControlResult outcomes (classifyOutcomes record outcomes)                 else classifyOutcomes record outcomes++    -- How many times a mutation may overrun its budget before the overrun is+    -- taken at face value.+    attemptsOnOverrun :: Word+    attemptsOnOverrun = 3++    -- Run every covering suite, repeating only while the run tells us+    -- nothing.+    --+    -- A kill and a survival are both observations and stand on the first run.+    -- Repeating either would bias the score towards whichever outcome we+    -- repeated into, which is the trap sydtest's own 'flaky' warns about from+    -- the other side: retrying a survivor until something fails scores the+    -- mutation as caught on the strength of an accidental failure.+    --+    -- An overrun is not an observation. The budget is ten times a baseline+    -- measured on an idle machine, so a loaded one exceeds it with nothing+    -- wrong, and re-running replaces a non-observation with a real one+    -- without favouring either outcome. That is worth doing here rather than+    -- leaving to whoever re-runs the check, because the run this belongs to+    -- costs the better part of an hour.+    runSuitesRetryingOverruns record mid suiteNames attemptsLeft = do+      outcomes <- mapM (runOneSuite record mid) suiteNames+      case classifyOutcomes record outcomes of+        MutationTimedOut _+          | attemptsLeft > 1 ->+              runSuitesRetryingOverruns record mid suiteNames (attemptsLeft - 1)+        _ -> pure outcomes      -- Map a control mutation's raw classification onto the control-specific     -- results.  An uncovered control never reaches here (it short-circuits

@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           sydtest-mutation-driver-version:        0.2.0.0+version:        0.3.0.0 synopsis:       Out-of-process mutation testing driver for sydtest. description:    Standalone driver executable that orchestrates the coverage and mutation phases of sydtest's mutation testing infrastructure. Spawns instrumented sydtest test suite executables as children. category:       Testing