packages feed

kenshou-core-0.1.0.0: test/Kenshou/PerfCalibrationSpec.hs

module Kenshou.PerfCalibrationSpec (spec) where

import Data.Aeson
import Data.Either (isLeft)
import Data.Text qualified as Text
import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)
import Kenshou.Core.Id (newRunId)
import Kenshou.Perf.Calibration
import Test.Hspec

spec :: Spec
spec = describe "performance calibration admission" do
  it "accepts twenty independently replayed slots and preserves frozen protocol roundtrips" do
    (definition, acceptance, aa, slowdown) <- fixture
    validateCalibration now now definition acceptance aa slowdown `shouldBe` Right 90
    eitherDecode (encode definition) `shouldBe` Right definition
    eitherDecode (encode acceptance) `shouldBe` Right acceptance
  it "never drops a completed statistical outcome to replace it with a better result" do
    (definition, acceptance, aa, slowdown) <- fixture
    replacementId <- newRunId
    let slot = first acceptance.aa
        replaced = slot {receipts = slot.receipts <> [CalibrationReceipt "gs://fixture/replacement/receipt.json" "sha256:replacement"]}
        statisticalRetry = first aa <> [(first (first aa)) {sessionId = replacementId}]
    validateCalibration now now definition acceptance {aa = replaced : drop 1 acceptance.aa} (statisticalRetry : drop 1 aa) slowdown `shouldSatisfy` isLeft
  it "requires at least nineteen expected outcomes and zero opposite outcomes" do
    (definition, acceptance, aa, slowdown) <- fixture
    let change verdict = map (\trial -> trial {verdict})
    validateCalibration now now definition acceptance (change "inconclusive" (first aa) : drop 1 aa) slowdown `shouldBe` Right 90
    validateCalibration now now definition acceptance (change "regression" (first aa) : drop 1 aa) slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance aa (change "pass" (first slowdown) : drop 1 slowdown) `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (map (change "inconclusive") (take 2 aa) <> drop 2 aa) slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (change "not-run" (first aa) : drop 1 aa) slowdown `shouldSatisfy` isLeft
  it "invalidates expired, drifted, duplicate, missing and runtime-blocked batches" do
    (definition, acceptance, aa, slowdown) <- fixture
    validateCalibration (addUTCTime 3601 now) now definition acceptance aa slowdown `shouldSatisfy` isLeft
    validateCalibration now (addUTCTime 1 now) definition acceptance aa slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (drop 1 aa) slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (first aa : first aa : drop 2 aa) slowdown `shouldSatisfy` isLeft
    let corrupt :: (VerifiedCalibrationTrial -> VerifiedCalibrationTrial) -> [[VerifiedCalibrationTrial]]
        corrupt change = map change (first aa) : drop 1 aa
    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialHealthPolicySha256 = "relaxed-health-limits"})) slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialMachineProfile = "different-cell-image"})) slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialToolchainSha256 = "changed"})) slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialHarnessSemanticsSha256 = "changed"})) slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition acceptance (corrupt (\trial -> trial {runtimeBlockers = ["known exit-zero defect"]})) slowdown `shouldSatisfy` isLeft
  it "retains infrastructure retries under the frozen cap and includes their time" do
    (definition, acceptance, aa, slowdown) <- fixture
    extra <- newRunId
    let failed = (first (first aa)) {sessionId = extra, infrastructureState = "failed", verdict = "not-run", elapsedSeconds = 850}
        original = first acceptance.aa
        slot = original {receipts = CalibrationReceipt "gs://fixture/failed/receipt.json" "sha256:failed" : original.receipts}
        revised = acceptance {aa = slot : drop 1 acceptance.aa}
        trials = (failed : first aa) : drop 1 aa
    validateCalibration now now definition {infrastructureRetryCap = 0} revised trials slowdown `shouldSatisfy` isLeft
    validateCalibration now now definition revised trials slowdown `shouldBe` Right 90
    let late = map (map (\trial -> trial {elapsedSeconds = 901})) aa
    validateCalibration now now definition acceptance late slowdown `shouldSatisfy` isLeft

fixture :: IO (CalibrationDefinition, CalibrationAcceptance, [[VerifiedCalibrationTrial]], [[VerifiedCalibrationTrial]])
fixture = do
  identifier <- newRunId
  let seeds = [1 .. 20]
      definition = CalibrationDefinition identifier "mori://fixture/runtime" "core" "synthetic-machine" (digest "d") (digest "a") (digest "b") (digest "c") (Text.replicate 40 "a") (Text.replicate 40 "b") "delay at operation boundary" "op.latency.p99" 0.5 1000000 seeds 1 (addUTCTime 3600 now)
      slot control seed = CalibrationSlot seed [CalibrationReceipt ("gs://fixture/" <> control <> "/" <> Text.pack (show seed) <> "/receipt.json") "sha256:receipt"]
      acceptance = CalibrationAcceptance identifier (map (slot "aa") seeds) (map (slot "slowdown") seeds)
      trial control verdict seed = do
        session <- newRunId
        pure [VerifiedCalibrationTrial session seed control definition.project definition.baselineRevision (if control == "aa" then definition.baselineRevision else definition.slowdownRevision) definition.profile definition.machineProfile definition.healthPolicySha256 definition.workloadSha256 definition.toolchainSha256 definition.harnessSemanticsSha256 verdict "healthy" [] now 90]
  aa <- traverse (trial "aa" "pass") seeds
  slowdown <- traverse (trial "none" "regression") seeds
  pure (definition, acceptance, aa, slowdown)
  where
    digest letter = "sha256:" <> Text.replicate 64 letter

now :: UTCTime
now = UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0)

first :: [a] -> a
first (value : _) = value
first [] = error "empty synthetic calibration fixture"