packages feed

kenshou-remote-0.1.0.0: test/PerfAttemptSpec.hs

module PerfAttemptSpec (spec) where

import Data.Aeson
import Data.ByteString qualified as Bytes
import Data.IORef
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict qualified as Map
import Data.Text qualified as Text
import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)
import Data.Word (Word64)
import Kenshou.Core.Id (mkSeed, newRunId, renderRunId)
import Kenshou.Core.Manifest (writeManifest)
import Kenshou.Core.Phase (zeroPhases)
import Kenshou.Measure.Clock (Origin (..))
import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..))
import Kenshou.Measure.Compare.Ordering (PairedOrdering (..))
import Kenshou.Measure.Compare.Policy qualified as Policy
import Kenshou.Measure.Histogram qualified as Histogram
import Kenshou.Measure.Histogram.Codec (encodeHistogram)
import Kenshou.Measure.Samples
import Kenshou.Measure.Summary (summarizeRunDir)
import Kenshou.Perf.Plan
import Kenshou.Perf.Result
import Kenshou.Perf.Result qualified as Configuration (ConfigurationResult (..))
import Kenshou.Plan.RunPlan (PlannedRun, RunPlan (..), TrialInfo (..))
import Kenshou.Plan.RunPlan qualified as Plan (PlannedRun (..))
import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)
import Kenshou.Remote.Pair
import Kenshou.Remote.Perf.Attempt
import Kenshou.Remote.Perf.Journal
import Kenshou.Remote.Perf.Runner
import System.Directory (createDirectoryIfMissing, renameDirectory)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec

spec :: Spec
spec = describe "performance paired attempts" do
  it "reconstructs five raw pairs, preserves an exit-zero defect and replays portable paths" $ withSystemTempDirectory "perf-raw-pairs" \directory -> do
    (entry, schedule, policy) <- fixture
    identifier <- newRunId
    calls <- newIORef (0 :: Int)
    blockers <- newIORef ([] :: [Text.Text])
    let attempt = directory </> "original"
        moved = directory </> "relocated"
        callbacks = ComparisonHooks (\_ -> expectationFailure "stable five-pair control must not replace pairs") (\reason -> modifyIORef' blockers (<> [reason]))
        execute run = do
          modifyIORef' calls (+ 1)
          let path = attempt </> "retained" </> Text.unpack (renderRunId run.runId)
          writeRawRun run (run.ordinal == 2) 1000000 path
          pure (Right path)
    result <- executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) attempt callbacks execute
    result.performanceVerdict `shouldBe` "pass"
    result.infrastructureState `shouldBe` "healthy"
    result.validPairs `shouldBe` 5
    result.runtimeFailures `shouldBe` ["runtime assertions failed (including nonblocking known defects)"]
    readIORef calls `shouldReturn` 10
    readIORef blockers `shouldReturn` result.runtimeFailures
    verifyAttempt entry schedule policy (VaryControl :| []) attempt result `shouldReturn` True
    encodeFile (attempt </> "comparison.json") result.comparison
    originalBytes <- Bytes.readFile (attempt </> "comparison.json")
    renameDirectory attempt moved
    verifyAttempt entry schedule policy (VaryControl :| []) moved result `shouldReturn` True
    Bytes.readFile (moved </> "comparison.json") `shouldReturn` originalBytes
    plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure
    aggregate <- either (fail . show) pure (aggregateResult (plan {selected = [entry]} :: PerfPlan) ("sha256:" <> Text.replicate 64 "a") True [result])
    aggregate.exitCode `shouldBe` 1
  it "replaces an invalid raw pair once and detects a genuine retained latency regression" $ withSystemTempDirectory "perf-raw-regression" \directory -> do
    (entry, schedule, policy) <- fixture
    identifier <- newRunId
    calls <- newIORef (0 :: Int)
    replacements <- newIORef (0 :: Int)
    let callbacks = ComparisonHooks (\_ -> modifyIORef' replacements (+ 1)) (\_ -> pure ())
        execute run = do
          modifyIORef' calls (+ 1)
          let path = directory </> "retained" </> Text.unpack (renderRunId run.runId)
              latency = if maybe False ((== "candidate") . (.arm)) run.trial then 3000000 else 1000000
          writeRawRun run False latency path
          if run.ordinal == 1 then Bytes.appendFile (path </> "samples/publish-to-finalize.raw") "invalid" else pure ()
          pure (Right path)
    result <- executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) directory callbacks execute
    result.performanceVerdict `shouldBe` "regression"
    result.infrastructureState `shouldBe` "healthy"
    result.validPairs `shouldBe` 5
    readIORef calls `shouldReturn` 12
    readIORef replacements `shouldReturn` 1
    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True
  it "bounds replacements and reuses retained invalid paths without resubmitting trials" $ withSystemTempDirectory "perf-attempt" \directory -> do
    (entry, schedule, policy) <- fixture
    identifier <- newRunId
    calls <- newIORef (0 :: Int)
    replacements <- newIORef (0 :: Int)
    let lease = LeaseIdentity "alpha" identifier
        callbacks = ComparisonHooks (\_ -> modifyIORef' replacements (+ 1)) (\_ -> pure ())
        execute run = do
          modifyIORef' calls (+ 1)
          let path = directory </> "retained" </> Text.unpack (renderRunId run.runId)
          createDirectoryIfMissing True path
          encodeFile (path </> "run-result.json") (object [])
          encodeFile (path </> "run-spec.json") (object [])
          pure (Right path)
    result <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute
    result.infrastructureState `shouldBe` "failed"
    result.validPairs `shouldBe` 0
    readIORef calls `shouldReturn` 14
    readIORef replacements `shouldReturn` 2
    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True
    verifyAttempt entry schedule policy (VaryControl :| []) directory result {Configuration.attempts = []} `shouldReturn` False
    _ <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute
    readIORef calls `shouldReturn` 14
    another <- newRunId
    verifyAttemptInputs lease entry schedule directory
    verifyAttemptInputs (LeaseIdentity "alpha" another) entry schedule directory `shouldThrow` anyIOException
    executeAttempt (LeaseIdentity "alpha" another) entry schedule policy (VaryControl :| []) directory callbacks execute `shouldThrow` anyIOException
  it "checkpoints terminal executor failures and replays them without new work" $ withSystemTempDirectory "perf-attempt-failures" \directory -> do
    (entry, schedule, policy) <- fixture
    identifier <- newRunId
    calls <- newIORef (0 :: Int)
    let callbacks = ComparisonHooks (\_ -> pure ()) (\_ -> pure ())
        execute _ = modifyIORef' calls (+ 1) >> pure (Left "sealed outer health failed")
        lease = LeaseIdentity "alpha" identifier
    result <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute
    readIORef calls `shouldReturn` 14
    repeated <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute
    repeated `shouldBe` result
    readIORef calls `shouldReturn` 14
    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True
    encodeFile (directory </> "trial-failures.json") (Map.singleton ("unplanned" :: Text.Text) ("invalid" :: Text.Text))
    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldThrow` anyIOException
  it "rejects an incomplete schedule and a forged retained path before dispatch" $ withSystemTempDirectory "perf-attempt-rejection" \directory -> do
    (entry, schedule, policy) <- fixture
    identifier <- newRunId
    let callbacks = ComparisonHooks (\_ -> pure ()) (\_ -> pure ())
        execute _ = expectationFailure "must not dispatch an invalid attempt" >> pure (Left "invalid")
    executeAttempt (LeaseIdentity "alpha" identifier) entry schedule {runs = drop 1 schedule.runs} policy (VaryControl :| []) (directory </> "incomplete") callbacks execute `shouldThrow` anyIOException
    createDirectoryIfMissing True (directory </> "forged")
    encodeFile (directory </> "forged/trial-paths.json") (Map.singleton ("unplanned" :: Text.Text) ("../escape" :: FilePath))
    executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) (directory </> "forged") callbacks execute `shouldThrow` anyIOException

fixture :: IO (PlannedProfile, RunPlan, Policy.Policy)
fixture = do
  plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure
  entry <- case (plan :: PerfPlan).selected of value : _ -> pure value; _ -> fail "empty profile fixture"
  identifier <- newRunId
  seed <- either (fail . show) pure (mkSeed 37)
  source <-
    either
      (fail . show)
      pure
      ( decodeWorkPlan
          ( object
              [ "schema" .= ("kenshou.run-plan/v1" :: Text.Text),
                "planId" .= identifier,
                "runs"
                  .= [ object
                         [ "ordinal" .= (0 :: Int),
                           "runId" .= identifier,
                           "estimateMinutes" .= (1 :: Int),
                           "spec"
                             .= object
                               [ "schema" .= ("kenshou.run-spec/v1" :: Text.Text),
                                 "runId" .= identifier,
                                 "scenario" .= entry.profile.scenario,
                                 "scenarioRevision" .= entry.profile.scenarioRevision,
                                 "knobs" .= entry.profile.knobs,
                                 "dimensions" .= entry.profile.dimensions,
                                 "seed" .= seed,
                                 "phases" .= zeroPhases,
                                 "timeoutSeconds" .= entry.profile.timeoutSeconds
                               ]
                         ]
                     ]
              ]
          )
      )
  schedule <- either (fail . show) pure (pairPlan (PairRequest "candidate" "baseline" 5 ABBA 37 2) source)
  policy <- Bytes.readFile "../policies/default.json" >>= either (fail . show) pure . Policy.decodePolicy
  pure (entry, schedule, policy)

-- Synthetic protocol evidence only; this is not workload calibration.
writeRawRun :: PlannedRun -> Bool -> Word64 -> FilePath -> IO ()
writeRawRun run knownFailure latency directory = do
  createDirectoryIfMissing True (directory </> "samples")
  createDirectoryIfMissing True (directory </> "series")
  Bytes.writeFile (directory </> "series/load.csv") "mono_ns,wall,phase\n1000,0,steady\n5000001000,0,drain\n"
  encodeFile (directory </> "run-spec.json") run.spec
  mutable <- Histogram.newHistogram Histogram.defaultHistogramConfig
  mapM_ (const (Histogram.recordValue mutable latency)) [1 :: Int .. 1000]
  histogram <- Histogram.freeze mutable
  let base = directory </> "samples/publish-to-finalize"
  Bytes.writeFile (base <> ".hist") (encodeHistogram histogram)
  Bytes.writeFile (base <> ".service.hist") (encodeHistogram histogram)
  writer <- openSampleWriter (base <> ".raw") (SampleHeader (Origin 0 0) MonotonicClock "publish-to-finalize" "synthetic")
  writeSampleBlock writer 0 [SampleRecord start start (start + latency) 0 1 | index <- [0 .. 999], let start = 1000 + index * 5000000]
  closeSampleWriter writer
  encodeFile (base <> ".meta.json") (object ["operation" .= ("publish-to-finalize" :: Text.Text), "latencyBasis" .= ("intended-start" :: Text.Text), "rawSamples" .= ("full" :: Text.Text), "phaseCounts" .= object ["steady" .= object ["successes" .= (1000 :: Int), "failures" .= (0 :: Int), "units" .= (1000 :: Int)]]])
  measurements <- summarizeRunDir directory >>= either (fail . show) pure
  let origin = UTCTime (fromGregorian 2026 10 10) (secondsToDiffTime 0)
  encodeFile
    (directory </> "run-result.json")
    (object ["runId" .= run.runId, "outcome" .= (if knownFailure then "failed" else "passed" :: Text.Text), "exitCode" .= (0 :: Int), "blocking" .= False, "compatibility" .= object ["inputs" .= object ["cohortPlanHash" .= ("synthetic-control" :: Text.Text)]], "fingerprint" .= object ["machineProfile" .= ("synthetic" :: Text.Text)], "timings" .= object ["startedAt" .= addUTCTime (fromIntegral run.ordinal) origin], "summaries" .= object ["measurements" .= object ["measurements" .= measurements]]])
  manifest <- writeManifest directory run.runId Map.empty
  encodeFile (directory </> "manifest.json") manifest