packages feed

kenshou-cli-0.1.0.0: test/PerfSessionSpec.hs

module PerfSessionSpec (PerfSessionSpec.spec) where

import Control.Concurrent (forkIO, killThread, threadDelay)
import Control.Exception (bracket, finally)
import Control.Monad (forM, forM_)
import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as Bytes
import Data.ByteString.Lazy qualified as Lazy
import Data.Map.Strict qualified as Map
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, addUTCTime, getCurrentTime)
import Kenshou.Cli (runWithArgsWith, selfTestAssembly)
import Kenshou.Core.Canonical (sha256Hex)
import Kenshou.Core.Id (renderRunId)
import Kenshou.Core.Manifest (writeManifest)
import Kenshou.Core.RunSpec (RunSpec (..))
import Kenshou.Measure.Clock (Origin (..))
import Kenshou.Measure.Histogram qualified as Histogram
import Kenshou.Measure.Histogram.Codec (encodeHistogram)
import Kenshou.Measure.Samples
import Kenshou.Measure.Summary (summarizeRunDir)
import Kenshou.Perf.Artifacts (sealPerfDirectory)
import Kenshou.Perf.Plan (PerfPlan (..), PerfProfile (..), PlannedProfile (..))
import Kenshou.Perf.Prepared qualified as Prepared
import Kenshou.Perf.Result qualified as Result
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))
import Kenshou.Remote.Cell.Docs
import Kenshou.Remote.Cell.Lease qualified as Lease
import Kenshou.Remote.Cell.Prepare (cellMachineProfile)
import Kenshou.Remote.Cell.Submit (workObjectFor)
import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)
import Kenshou.Remote.Payload (Bundle (..), PayloadDescriptor (..))
import Kenshou.Remote.Payload qualified as Payload
import Kenshou.Remote.Perf.Journal qualified as Journal
import Kenshou.Remote.Store
import Kenshou.Remote.Store.File (newFileStore)
import System.Directory (createDirectoryIfMissing, doesFileExist, listDirectory, removeFile, withCurrentDirectory)
import System.Environment (lookupEnv, setEnv, unsetEnv)
import System.Exit (ExitCode (..))
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import System.Timeout (timeout)
import Test.Hspec

runWithArgs :: [String] -> IO ExitCode
runWithArgs = runWithArgsWith selfTestAssembly

spec :: Spec
-- The prepared fixture pins scenarios of a consumer layer that this repository does not
-- register, so dispatch cannot resolve them. The kernel plan that regenerates the perf
-- fixtures from synthetic scenarios removes this guard:
-- docs/plans/2-open-the-kernel-layer-identity-and-schema-composition-to-registered-suites.md (M4).
spec = describe "file-backed performance CLI integration" $ before_ (pendingWith "perf-prepared fixture names unregistered consumer scenarios; regenerated in docs/plans/2-… M4") do
  forM_ [(False, "executes two profiles under one lease and replays the full sealed graph without redispatch"), (True, "retains an exit-zero known defect through complete report replay and resume")] \(knownFault, title) -> it title $
    withSystemTempDirectory "perf-cli-healthy" \root -> projectRoot do
      let preparation = root </> "preparation"
          output = root </> "session"
          storeRoot = root </> "store"
          payloadBytes = "synthetic protocol payload; no runtime calibration is claimed"
          digest = Text.drop 7 (sha256Hex (Lazy.toStrict payloadBytes))
          bundle = Bundle ("gs://tan-nb-exp-cells-control/payloads/sha256/" <> digest <> ".nar.zst") digest (Lazy.length payloadBytes)
          replacePayload arm = case fromJSON arm.payload of
            Error problem -> fail problem
            Success payload -> pure arm {Prepared.payload = toJSON (payload {Payload.cell = payload.cell {Payload.bundle = bundle}} :: PayloadDescriptor)}
      original <- readJSON "fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json"
      baseline <- replacePayload (original :: Prepared.PerfPrepared).baseline
      candidate <- replacePayload original.candidate
      let prepared = original {Prepared.baseline = baseline, Prepared.candidate = candidate}
          -- The captured preparation's profiles, in selection order.
          profileNames = [entry.profile.name | entry <- prepared.plan.selected]
          (firstProfile, lastProfile) = case profileNames of
            first : rest -> (Text.unpack first, last (first : rest))
            [] -> error "the captured preparation selects no profile"
      createDirectoryIfMissing True preparation
      encodeFile (preparation </> "prepared.json") prepared
      sealPerfDirectory preparation >>= either (fail . Text.unpack) pure
      descriptor <- readJSON "kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
      policy <- Lazy.readFile "kenshou-remote/test/golden/cell/cell.policy.v1.json"
      store <- newFileStore storeRoot
      let control = Bucket descriptor.buckets.control
          put bucket name bytes = store.putObject bucket (ObjectName name) "application/json" DoesNotExist bytes >>= requireWritten
      put control ("payloads/sha256/" <> digest <> ".nar.zst") payloadBytes
      put control "cells/alpha/descriptor.json" (encode descriptor)
      put control "cells/alpha/policy.json" policy
      agent <- forkIO (sealMarkers knownFault store descriptor root Set.empty)
      ( withSetting "KENSHOU_CELL_STORE" ("file:" <> storeRoot) $
          withSetting "KENSHOU_CELL_CAPABILITIES_DIR" (root </> "capabilities") do
            execution <- timeout 90000000 (runWithArgs ["perf", "run", "--prepared", preparation </> "prepared.json", "--out", output, "--evidence-base-uri", "gs://fixture-evidence/performance"])
            let expectedExit = if knownFault then ExitFailure 1 else ExitSuccess
            execution `shouldBe` Just expectedExit
            result <- readJSON (output </> "result.json")
            (result :: Result.PerfResult).performanceVerdict `shouldBe` "pass"
            result.gatingEligible `shouldBe` False
            length result.configurations `shouldBe` 2
            map (.validPairs) result.configurations `shouldBe` [5, 5]
            null result.blockingRuntimeFailures `shouldBe` not knownFault
            store.getObject control (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing
            initialMarkers <- store.listObjects control "cells/alpha/submissions/"
            let markers = filter (Text.isSuffixOf "/submission.json" . (.unObjectName) . fst) initialMarkers
            length markers `shouldBe` 20
            submissions <- forM markers \(name, _) -> do
              retained <- store.getObject control name
              maybe (fail "marker vanished") (either fail pure . eitherDecode . fst) retained
            Set.size (Set.fromList [submission.leaseId | submission :: Submission <- submissions]) `shouldBe` 1
            retainedResult <- Lazy.readFile (output </> "result.json")
            forM_ [[], ["--offline"]] \flags -> runWithArgs (["perf", "verify", "--session", output] <> flags) `shouldReturn` expectedExit
            runWithArgs ["perf", "resume", "--session", output] `shouldReturn` expectedExit
            Lazy.readFile (output </> "result.json") `shouldReturn` retainedResult
            store.listObjects control "cells/alpha/submissions/" `shouldReturn` initialMarkers
            -- A second snapshot is forbidden when completed evidence is reused.
            snapshots <- listDirectory (output </> "snapshots")
            length snapshots `shouldBe` 1
            report <- Bytes.readFile (output </> "report.md")
            report `shouldSatisfy` Bytes.isInfixOf "within policy limits"
            report `shouldSatisfy` Bytes.isInfixOf "Comparison reasons: none"
            -- Reconstruct an interruption after the last comparison file was
            -- saved, before its attempt result and root completion checkpoint.
            -- Every sealed cell object and paired schedule stays unchanged.
            saved <- Journal.readPerfSession (output </> "session.json") >>= either (fail . Text.unpack) pure
            let beforeCompletion = takeWhile (\event -> case event.action of Journal.ComparisonCompleted item -> item.profile /= lastProfile; _ -> True) saved.events
                attempt = output </> "configurations" </> Text.unpack lastProfile </> "attempt-0"
            identity <- case [one | event <- beforeCompletion, Journal.LeaseAcquired one <- [event.action]] of
              [one] -> pure one
              _ -> fail "expected exactly one original lease"
            beforeComparison <- Lazy.readFile (attempt </> "comparison.json")
            -- Fixture construction only: the production writer correctly
            -- refuses to rewind an already committed append-only history.
            encodeFile (output </> "session.json") saved {Journal.events = beforeCompletion}
            removeFile (output </> "receipt.json")
            removeFile (attempt </> "result.json")
            Lazy.writeFile (attempt </> "result.json.next") "interrupted JSON write"
            now <- getCurrentTime
            let activeLease = Lease.Lease identity.leaseId identity.cell "kenshou-perf" "pinned runtime performance validation" 180 now now False 20
            put control "cells/alpha/lease.json" (encode activeLease)
            runWithArgs ["perf", "resume", "--session", output] `shouldReturn` expectedExit
            Lazy.readFile (attempt </> "comparison.json") `shouldReturn` beforeComparison
            store.listObjects control "cells/alpha/submissions/" `shouldReturn` initialMarkers
            forM_ [[], ["--offline"]] \flags -> runWithArgs (["perf", "verify", "--session", output] <> flags) `shouldReturn` expectedExit
            -- Even a locally resealed inventory and matching receipt cannot
            -- make an unlinked cell tree part of the interpreted evidence.
            receipt <- readJSON (output </> "receipt.json")
            fields <- case receipt of Object value -> pure value; _ -> fail "invalid receipt"
            relative <- case KeyMap.lookup "snapshot" fields of Just (String value) -> pure value; _ -> fail "receipt omits snapshot"
            let snapshot = output </> Text.unpack relative
                orphan = snapshot </> "configurations" </> firstProfile </> "attempt-0/trials/unlinked-cell/tree"
            createDirectoryIfMissing True orphan
            encodeFile (orphan </> "run-result.json") (object ["outcome" .= ("passed" :: Text)])
            sealPerfDirectory snapshot >>= either (fail . Text.unpack) pure
            changedManifest <- Bytes.readFile (snapshot </> "manifest.json")
            encodeFile (output </> "receipt.json") (Object (KeyMap.insert "manifestSha256" (String (sha256Hex changedManifest)) fields))
            runWithArgs ["perf", "verify", "--session", output, "--offline"] `shouldReturn` ExitFailure 4
        )
        `finally` killThread agent

-- Implements only the immutable protocol in a file store. Its measurements and
-- machine samples are synthetic controls, never acceptance calibration evidence.
sealMarkers :: Bool -> ObjectStore -> CellDescriptor -> FilePath -> Set.Set ObjectName -> IO ()
sealMarkers knownFault store descriptor root seen = do
  let control = Bucket descriptor.buckets.control
  objects <- store.listObjects control "cells/alpha/submissions/"
  let fresh = [name | (name, _) <- objects, "/submission.json" `Text.isSuffixOf` name.unObjectName, name `Set.notMember` seen]
  forM_ fresh \name -> do
    (markerBytes, _) <- store.getObject control name >>= maybe (fail "missing marker") pure
    submission <- either fail pure (eitherDecode markerBytes)
    let prefix = Text.dropEnd (Text.length "submission.json") name.unObjectName
    (workBytes, _) <- store.getObject control (ObjectName (prefix <> "work")) >>= maybe (fail "missing work") pure
    document <- either fail pure (eitherDecode workBytes)
    plan <- either (fail . Text.unpack) pure (decodeWorkPlan document)
    run <- case plan.runs of [one] -> pure one; _ -> fail "expected exactly one paired trial"
    let local = root </> "agent-output" </> Text.unpack (renderRunId submission.runId)
        nested = local </> "output" </> Text.unpack (renderRunId run.runId)
    writeRawRun (knownFault && Set.null seen) descriptor submission run nested
    nestedFiles <- readTree (local </> "output")
    now <- getCurrentTime
    let sequenceNumber = Set.size seen + length (takeWhile (/= name) fresh) + 1
        files =
          [("submission/work", workBytes), ("submission/submission.json", markerBytes), ("cell/result.json", encode (CellRunResult submission.runId "alpha" submission.leaseId sequenceNumber Completed (Just 0) Nothing []))]
            <> healthFiles descriptor submission
            <> [("output/" <> path, bytes) | (path, bytes) <- nestedFiles]
        artifactFor (path, bytes) = let info = workObjectFor "application/json" bytes in Artifact path info.sha256 info.bytes "application/json"
        manifest = CellManifest submission.runId "alpha" submission.leaseId sequenceNumber now "0.1.0" (ManifestPayload submission.payload.bundle.sha256 submission.payload.storePath) Completed 3600 (map artifactFor files)
        manifestBytes = encode manifest
        put bucket path bytes = store.putObject bucket (ObjectName path) "application/json" DoesNotExist bytes >>= requireWritten
        resultPrefix = "runs/" <> renderRunId submission.runId <> "/"
    forM_ files \(path, bytes) -> put (Bucket descriptor.buckets.results) (resultPrefix <> path) bytes
    put (Bucket descriptor.buckets.results) (resultPrefix <> "manifest.json") manifestBytes
    put control (prefix <> "status.json") (encode (CellStatus submission.runId Sealed (Just sequenceNumber) now Nothing (LogChunks 0 0) (Just Completed) (Just (workObjectFor "application/json" manifestBytes).sha256) (Just [])))
  threadDelay 10000
  sealMarkers knownFault store descriptor root (Set.union seen (Set.fromList fresh))

writeRawRun :: Bool -> CellDescriptor -> Submission -> PlannedRun -> FilePath -> IO ()
writeRawRun knownFault descriptor submission run directory = do
  createDirectoryIfMissing True (directory </> "samples")
  createDirectoryIfMissing True (directory </> "series")
  Bytes.writeFile (directory </> "series/load.csv") "mono_ns,wall,phase\n1000,0,steady\n5000001000,0,drain\n"
  encodeFile (directory </> "run-spec.json") run.spec
  mutable <- Histogram.newHistogram Histogram.defaultHistogramConfig
  forM_ [1 :: Int .. 1000] (const (Histogram.recordValue mutable 1000000))
  histogram <- Histogram.freeze mutable
  let operation = "publish-to-finalize"
      base = directory </> "samples" </> Text.unpack operation
  Bytes.writeFile (base <> ".hist") (encodeHistogram histogram)
  Bytes.writeFile (base <> ".service.hist") (encodeHistogram histogram)
  writer <- openSampleWriter (base <> ".raw") (SampleHeader (Origin 0 0) MonotonicClock operation "synthetic")
  writeSampleBlock writer 0 [SampleRecord start start (start + 1000000) 0 1 | index <- [0 .. 999], let start = 1000 + index * 5000000]
  closeSampleWriter writer
  encodeFile (base <> ".meta.json") (object ["operation" .= operation, "latencyBasis" .= ("intended-start" :: Text), "rawSamples" .= ("full" :: Text), "phaseCounts" .= object ["steady" .= object ["successes" .= (1000 :: Int), "failures" .= (0 :: Int), "units" .= (1000 :: Int)]]])
  measurements <- summarizeRunDir directory >>= either (fail . show) pure
  encodeFile (directory </> "run-result.json") (object ["runId" .= run.runId, "scenario" .= run.spec.scenario, "outcome" .= (if knownFault then "failed" else "passed" :: Text), "exitCode" .= (0 :: Int), "blocking" .= False, "compatibility" .= object ["inputs" .= object ["cohortPlanHash" .= ("synthetic-control" :: Text)]], "fingerprint" .= object ["machineProfile" .= cellMachineProfile descriptor, "cell" .= object ["payload" .= object ["bundleSha256" .= submission.payload.bundle.sha256]]], "timings" .= object ["startedAt" .= addUTCTime (fromIntegral run.ordinal) (at 0)], "summaries" .= object ["measurements" .= object ["measurements" .= measurements]]])
  writeManifest directory run.runId Map.empty >>= encodeFile (directory </> "manifest.json")

healthFiles :: CellDescriptor -> Submission -> [(Text, Lazy.ByteString)]
healthFiles descriptor submission =
  [ ("cell/execution-window.json", encode (object ["schema" .= ("cell.execution-window/v1" :: Text), "runId" .= submission.runId, "resetStarted" .= at 0, "runStarted" .= at 10, "runFinished" .= at 60])),
    ("cell/health.json", encode (object ["schema" .= ("cell.health/v1" :: Text), "runId" .= submission.runId, "passed" .= True, "windowStart" .= at 0, "windowEnd" .= at 60, "gates" .= concatMap gates roles])),
    ("cell/reset-evidence.json", encode (object ["schema" .= ("cell.reset-evidence/v1" :: Text), "runId" .= submission.runId, "cell" .= ("alpha" :: Text), "cachePolicy" .= ("cold" :: Text), "verified" .= True, "startedAt" .= at 0, "finishedAt" .= at 5, "steps" .= [object ["name" .= ("stop-driver-processes" :: Text), "status" .= ("ok" :: Text), "startedAt" .= at 0, "finishedAt" .= at 5]]]))
  ]
    <> [("cell/observations/" <> role <> ".json", encode (object ["schema" .= ("cell.observations/v1" :: Text), "machine" .= machine, "image" .= image, "agentVersion" .= ("0.1.0" :: Text), "systemClosure" .= ("/nix/store/synthetic-system-identity" :: Text), "from" .= at 0, "to" .= at 60, "samples" .= [sample i | i <- [0 .. 60]], "maintenanceEvents" .= ([] :: [Value]), "samplingErrors" .= ([] :: [Value])])) | (role, machine, image) <- roles]
  where
    roles = [("driver", machine, descriptor.images.driver) | machine <- take 1 descriptor.instances.drivers] <> [("postgres", descriptor.instances.postgres, descriptor.images.postgres), ("monitoring", descriptor.instances.monitoring, descriptor.images.monitoring)]
    sample i = object ["at" .= at i, "cpuTotalTicks" .= (1000 + i * 100), "cpuIdleTicks" .= (990 + i * 99), "cpuStealTicks" .= (0 :: Int), "diskFreePercent" .= (if i <= 10 then 80 else 75 :: Int)]
    gates (role, machine, _) =
      [ gate "host-maintenance" (String "NONE") (String "NONE"),
        gate "background-load" (object ["settleCpuPercent" .= (if role == "monitoring" then 0 else 1 :: Int), "runStealCpuPercent" .= (0 :: Int)]) (object ["settleCpuMaxPercent" .= (5 :: Int), "runStealCpuMaxPercent" .= (2 :: Int)]),
        gate "storage-pressure" (object ["startFreePercent" .= (80 :: Int), "runFreePercent" .= (75 :: Int), "outputExceeded" .= False]) (object ["startFreeMinPercent" .= (20 :: Int), "runFreeMinPercent" .= (5 :: Int), "outputExceeded" .= False]),
        gate "incomplete-reset" (Bool True) (Bool True),
        gate "version-mismatch" (Bool True) (Bool True)
      ]
      where
        gate name observed threshold = object ["name" .= (name :: Text), "machine" .= machine, "status" .= ("ok" :: Text), "at" .= at 60, "observed" .= observed, "threshold" .= threshold]

at :: Int -> UTCTime
at seconds = addUTCTime (fromIntegral seconds) (read "2026-10-11 00:00:00 UTC")

readTree :: FilePath -> IO [(Text, Lazy.ByteString)]
readTree directory = do
  names <- listDirectory directory
  concat <$> forM names \name -> do
    let path = directory </> name
    regular <- doesFileExist path
    if regular
      then do bytes <- Lazy.readFile path; pure [(Text.pack name, bytes)]
      else map (\(relative, bytes) -> (Text.pack name <> "/" <> relative, bytes)) <$> readTree path

readJSON :: (FromJSON a) => FilePath -> IO a
readJSON path = eitherDecodeFileStrict' path >>= either fail pure

requireWritten :: PutOutcome -> IO ()
requireWritten (Written _) = pure ()
requireWritten PreconditionFailed = fail "synthetic protocol object already exists"

projectRoot :: IO a -> IO a
projectRoot action = do
  root <- doesFileExist "cabal.project"
  withCurrentDirectory (if root then "." else "..") action

withSetting :: String -> String -> IO a -> IO a
withSetting name value action = bracket (lookupEnv name) restore \_ -> setEnv name value >> action
  where
    restore Nothing = unsetEnv name
    restore (Just previous) = setEnv name previous