packages feed

kenshou-core-0.1.0.0: test/Kenshou/PerfSpec.hs

{-# LANGUAGE FieldSelectors #-}

module Kenshou.PerfSpec (spec) where

import Control.Monad (forM_)
import Data.Aeson
import Data.ByteString qualified as ByteString
import Data.Either (isLeft)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Cohort qualified as Cohort
import Kenshou.Core.Dimension
import Kenshou.Core.Id (parseScenarioId)
import Kenshou.Core.Knob
import Kenshou.Core.Knob (mkKnobName)
import Kenshou.Core.Scenario (Placement (..), Tier (..))
import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)
import Kenshou.Perf.Plan qualified as Perf
import Kenshou.Perf.Request qualified as Request
import Kenshou.Perf.Source qualified as Source
import Kenshou.Plan.Catalog (ScenarioInfo (..))
import Kenshou.Plan.Change
import Kenshou.Plan.Components qualified as Graph
import Test.Hspec hiding (context)

spec :: Spec
spec = do
  describe "performance requests" do
    it "normalizes defaults and round-trips explicit request fields" do
      eitherDecode (encode request) `shouldBe` Right request
    it "binds scope, purpose, control and budgets to request identity" do
      map Request.requestDigest [request {Request.coverageMode = "affected"}, request {Request.purpose = "gating"}, request {Request.control = "none"}, request {Request.budgetMinutes = 31}]
        `shouldSatisfy` all (/= Request.requestDigest request)
    it "sorts set fields without changing the digest" do
      Request.requestDigest request {Request.profiles = reverse request.profiles} `shouldBe` Request.requestDigest request
    it "rejects abbreviated, mutable, uppercase and placeholder revisions" do
      mapM_ (\revision -> Request.validateRequest request {Request.baselineRevision = revision} `shouldSatisfy` isLeft) ["main", "HEAD", "BASE_SHA", "abcd", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]
    it "rejects cross-project artifact URIs and malformed canonical project URIs" do
      mapM_ (\project -> Request.validateRequest request {Request.project = project} `shouldSatisfy` isLeft) ["acme", "mori://shinzui/acme/packages/acme-core", "mori://shinzui/../acme"]
    it "rejects A/A with distinct revisions and A/A without explicit profiles" do
      Request.validateRequest request {Request.candidateRevision = shaB} `shouldSatisfy` isLeft
      Request.validateRequest request {Request.profiles = []} `shouldSatisfy` isLeft
    it "rejects duplicate target IDs, empty scope and invalid budgets" do
      mapM_
        (\value -> Request.validateRequest value `shouldSatisfy` isLeft)
        [request {Request.targetComponents = ["acme", "acme"]}, request {Request.profiles = []}, request {Request.budgetMinutes = 0}, request {Request.waitSeconds = -1}]
    it "rejects unknown wire fields instead of silently ignoring a typo" do
      let malformed = object ["schema" .= ("kenshou.perf-request/v1" :: Text), "purpos" .= ("calibration" :: Text)]
      (eitherDecode (encode malformed) :: Either String Request.PerfRequest) `shouldSatisfy` isLeft
  describe "performance source identities for generated cohorts" do
    it "reuses an unchanged cohort across candidate revisions, calibration seeds and measurement scope" do
      descriptor <- template
      manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")
      metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.3\n")
      snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])
      baseline <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)
      let otherCandidate = request {Request.control = "none", Request.candidateRevision = shaB}
          revised = [request {Request.seed = request.seed + 1}, request {Request.cells = ["beta"]}, request {Request.budgetMinutes = 31}, request {Request.waitSeconds = 0}, request {Request.profiles = take 1 request.profiles}, request {Request.purpose = "gating"}, otherCandidate, otherCandidate {Request.candidateRevision = Text.replicate 40 "c"}]
      forM_ revised \selection -> Source.descriptorForSnapshot selection descriptor "baseline" snapshot `shouldBe` Right baseline
      Source.descriptorForSnapshot otherCandidate {Request.baselineRevision = shaB, Request.candidateRevision = shaA} descriptor "candidate" snapshot `shouldBe` Right baseline
      different <- right (Source.sourceSnapshot otherCandidate descriptor upstream shaB [manifest, metrics])
      Source.descriptorForSnapshot otherCandidate descriptor "candidate" different `shouldNotBe` Right baseline
    it "invalidates cohort identity when compiler, index state or fixed dependency inputs change" do
      descriptor <- template
      manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")
      metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.3\n")
      snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])
      original <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)
      let altered = descriptor {Cohort.descriptorComponents = map (\component -> if component.componentId == Cohort.ComponentId "acme" then component else component {Cohort.componentPackages = map (\pin -> pin {Cohort.pinVersion = "99.0.0"}) component.componentPackages}) descriptor.descriptorComponents}
      forM_ [descriptor {Cohort.descriptorCompiler = "ghc-9.14.1"}, descriptor {Cohort.descriptorIndexState = "2026-10-11T00:00:00Z"}, altered] \changed -> Source.descriptorForSnapshot request changed "baseline" snapshot `shouldNotBe` Right original
    it "reads manifest version and package directory independently in each revision" do
      descriptor <- template
      manifest <- right (Source.parsePackageManifest "moved/acme-core.cabal" "name: acme-core\nversion: 0.9.0.4\n")
      metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.4\n")
      snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])
      generated <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)
      otherArm <- right (Source.descriptorForSnapshot request descriptor "candidate" snapshot)
      generated `shouldBe` otherArm
      target <- first (filter ((== Cohort.ComponentId "acme") . (.componentId)) generated.descriptorComponents)
      target.componentSource `shouldBe` Cohort.GitSource upstream shaA
      map (.pinVersion) target.componentPackages `shouldBe` ["0.9.0.4", "0.9.0.4"]
      map (.pinSubdir) target.componentPackages `shouldBe` [Just "moved", Just "acme-metrics"]
      filter ((/= Cohort.ComponentId "acme") . (.componentId)) generated.descriptorComponents
        `shouldBe` filter ((/= Cohort.ComponentId "acme") . (.componentId)) descriptor.descriptorComponents
    it "rejects unknown descriptor IDs and targets owned by a different project" do
      descriptor <- template
      Source.targetPackages request {Request.targetComponents = ["unknown"]} descriptor `shouldSatisfy` isLeft
      Source.targetPackages request {Request.targetComponents = ["acme-quill-adapter"]} descriptor `shouldSatisfy` isLeft
    it "rejects missing and duplicate packages at a requested revision" do
      descriptor <- template
      manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")
      Source.sourceSnapshot request descriptor upstream shaA [manifest] `shouldSatisfy` isLeft
      Source.sourceSnapshot request descriptor upstream shaA [manifest, manifest] `shouldSatisfy` isLeft
    it "rejects mutable template source pins" do
      descriptor <- template
      let components = map (\component -> component {Cohort.componentSource = Cohort.GitSource upstream "main"}) descriptor.descriptorComponents
      Source.targetPackages request descriptor {Cohort.descriptorComponents = components} `shouldSatisfy` isLeft
    it "rejects forged wire paths and manifest digests" do
      manifest <- right (Source.parsePackageManifest "safe/package.cabal" "name: package\nversion: 1.0\n")
      (eitherDecode (encode manifest {Source.path = "../outside.cabal"}) :: Either String Source.PackageManifest) `shouldSatisfy` isLeft
      (eitherDecode (encode manifest {Source.sha256 = "sha256:fake"}) :: Either String Source.PackageManifest) `shouldSatisfy` isLeft
    it "rejects ambiguous identity and paths that escape generated inputs" do
      Source.parsePackageManifest "../bad.cabal" "name: bad\nversion: 1.0\n" `shouldSatisfy` isLeft
      Source.parsePackageManifest "bad.cabal" "name: bad\nname: other\nversion: 1.0\n" `shouldSatisfy` isLeft
  describe "bounded performance planning" do
    it "retains explicit A/A profiles in calibration mode with an empty diff" do
      (graph, profiles, catalog) <- context
      plan <- right (Perf.planPerformance syntheticContracts request graph catalog profiles [] [] [] policyDigests Map.empty)
      length plan.selected `shouldBe` 2
      plan.coverage `shouldBe` "complete"
      plan.gatingEligible `shouldBe` False
      plan.estimateSeconds `shouldBe` 1540
      eitherDecode (encode plan) `shouldBe` Right plan
      map (.reasons) plan.selected `shouldSatisfy` all (not . null)
    it "cannot admit a catalog's unverified calibration claim to a gating plan" do
      (graph, profiles, catalog) <- context
      let claimed = profiles {Perf.profiles = map (\profile -> profile {Perf.calibration = Just (object ["calibrated" .= True])}) profiles.profiles}
      plan <- right (Perf.planPerformance syntheticContracts request {Request.purpose = "gating"} graph catalog claimed [] [] [] policyDigests Map.empty)
      plan.coverage `shouldBe` "incomplete"
      map snd plan.excluded `shouldBe` ["uncalibrated", "uncalibrated"]
      plan.gatingEligible `shouldBe` False
    it "binds workload identity without making calibration promotion self-referential" do
      (_, profiles, _) <- context
      profile <- first profiles.profiles
      Perf.workloadDigest profile "sha256:policy" `shouldBe` Perf.workloadDigest profile {Perf.calibration = Just (String "evidence")} "sha256:policy"
      Perf.workloadDigest profile "sha256:policy" `shouldNotBe` Perf.workloadDigest profile {Perf.scenarioRevision = 3} "sha256:policy"
    it "defers an entire configuration when the paired estimate exceeds budget" do
      (graph, profiles, catalog) <- context
      plan <- right (Perf.planPerformance syntheticContracts request {Request.budgetMinutes = 13} graph catalog profiles [] [] [] policyDigests Map.empty)
      length plan.selected `shouldBe` 1
      map snd plan.excluded `shouldBe` ["over-budget"]
      plan.coverage `shouldBe` "incomplete"
    it "reports transitive dependent and uncovered correctness paths" do
      (graph, profiles, catalog) <- context
      reference <- right (Graph.parseRef graph "acme-core:contract")
      let changes = [Change reference Named "core contract changed"]
      plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none", Request.coverageMode = "affected", Request.profiles = []} graph catalog profiles changes [] [] policyDigests Map.empty)
      plan.affectedScenarios `shouldContain` ["acme/quill-adapter/benchmark/end-to-end-throughput-latency"]
      plan.uncoveredScenarios `shouldContain` ["acme/metrics/correctness/test"]
      plan.coverage `shouldBe` "incomplete"
    it "separates selected-profile completion from whole-change coverage" do
      (graph, profiles, catalog) <- context
      reference <- right (Graph.parseRef graph "acme-core:contract")
      plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none"} graph catalog profiles [Change reference Named "core changed"] [] [] policyDigests Map.empty)
      plan.coverage `shouldBe` "complete"
      plan.changeCoverage `shouldBe` "incomplete"
    it "does not certify a telemetry change with disabled telemetry" do
      (graph, profiles, catalog) <- context
      reference <- right (Graph.parseRef graph "acme-core:telemetry")
      plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none", Request.coverageMode = "affected", Request.profiles = []} graph catalog profiles [Change reference Named "telemetry changed"] [] [] policyDigests Map.empty)
      plan.uncoveredScenarios `shouldContain` ["acme/core-ordering/benchmark/concurrency-sweep"]
      plan.coverage `shouldBe` "incomplete"
    it "selects conservatively for unmapped build inputs" do
      (graph, _, _) <- context
      let (changes, unknown, documentation) = Perf.conservativeChanges graph "acme" ["cabal.project", "docs/guide.md"] []
      map (.source) changes `shouldBe` [Everything]
      unknown `shouldBe` ["cabal.project"]
      documentation `shouldBe` ["docs/guide.md"]
    it "requires a repository-scoped documentation proof for not-applicable" do
      (graph, profiles, catalog) <- context
      let normal = request {Request.control = "none", Request.profiles = [], Request.coverageMode = "affected"}
      empty <- right (Perf.planPerformance syntheticContracts normal graph catalog profiles [] [] [] policyDigests Map.empty)
      empty.coverage `shouldBe` "incomplete"
      proved <- right (Perf.planPerformance syntheticContracts normal graph catalog profiles [] [] ["docs/guide.md"] policyDigests Map.empty)
      proved.coverage `shouldBe` "not-applicable"
      proved.gatingEligible `shouldBe` False
    it "rejects phases the pinned scenario revision cannot honor" do
      (_, profiles, catalog) <- context
      profile <- first profiles.profiles
      Perf.validateProfile syntheticContracts catalog profile {Perf.warmupSeconds = 1} `shouldSatisfy` isLeft
      Perf.validateProfile syntheticContracts catalog profile {Perf.steadyCount = 2000} `shouldSatisfy` isLeft
      Perf.validateProfile syntheticContracts catalog profile {Perf.timeoutSeconds = 181} `shouldSatisfy` isLeft
    it "rejects unknown profile names and raw sample suppression" do
      (graph, profiles, catalog) <- context
      Perf.planPerformance syntheticContracts request {Request.profiles = ["missing"]} graph catalog profiles [] [] [] policyDigests Map.empty `shouldSatisfy` isLeft
      profile <- first profiles.profiles
      Perf.validateProfile syntheticContracts catalog profile {Perf.knobs = Map.insert "measure.raw-samples" (String "off") profile.knobs} `shouldSatisfy` isLeft

shaA, shaB, upstream :: Text
shaA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
shaB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
upstream = "https://github.com/shinzui/acme.git"

request :: Request.PerfRequest
request = Request.PerfRequest "mori://shinzui/acme" shaA shaA "cohort/released.json" ["acme"] "change-performance" "selected-profiles" "calibration" "aa" ["acme-core-quick", "acme-quill-quick"] 20261010 ["alpha"] 90 30

policyDigests :: Map.Map Text Text
policyDigests = Map.singleton "policies/default.json" "sha256:policy"

context :: IO (Graph.ComponentGraph, Perf.ProfileCatalog, [ScenarioInfo])
context = do
  graph <- Graph.loadGraphFile "test/fixtures/plan/components.synthetic.json" >>= right
  profiles <- load "catalog.v1.json"
  catalog <- traverse makeScenario [(profile.scenario, profile.scenarioRevision, True) | profile <- profiles.profiles]
  correctness <- makeScenario ("acme/metrics/correctness/test", 1, False)
  pure (graph, profiles, catalog <> [correctness])
  where
    makeScenario (name, revision, benchmark) = do
      identifier <- right (parseScenarioId name)
      let dimensions = if name == "acme/core-ordering/benchmark/concurrency-sweep" then allTelemetryArms noDimensions else postgresDimensions (PgDurable :| []) (Pg18 :| [Pg17]) noDimensions
      knobs <- traverse makeKnob [("bench.arm", KnobText, VText "configured"), ("bench.rate", KnobInt, VInt 200), ("acme.messages", KnobInt, VInt 1000), ("measure.raw-samples", KnobText, VText "full")]
      pure (ScenarioInfo identifier revision "fixture" TierStandard PlaceEither (if benchmark then knobs else []) dimensions Nothing)
    makeKnob (name, kind, def) = do
      key <- right (mkKnobName name)
      pure (KnobSpec key "fixture" kind def AnyValue [])

template :: IO Cohort.CohortDescriptor
template = load "cohort.json"

load :: (FromJSON a) => FilePath -> IO a
load path = ByteString.readFile ("test/fixtures/perf/" <> path) >>= right . eitherDecodeStrict'

right :: (Show e) => Either e a -> IO a
right (Right value) = pure value
right (Left problem) = expectationFailure (show problem) >> fail "expected Right"

first :: [a] -> IO a
first (value : _) = pure value
first [] = expectationFailure "expected a nonempty fixture" >> fail "empty fixture"

-- The two synthetic benchmark revisions the catalog's profiles pin implement
-- count-bound steady work; their steady count is the acme.messages knob.
syntheticContracts :: PerfContracts
syntheticContracts =
  noPerfContracts
    { reviewedPhaseContracts = [("acme/core-ordering/benchmark/concurrency-sweep", 2), ("acme/quill-adapter/benchmark/end-to-end-throughput-latency", 1)],
      steadyCountKnob = either (const Nothing) Just (mkKnobName "acme.messages")
    }