packages feed

kenshou-cli-0.1.0.0: src/Kenshou/Cli/Command/Perf/Session.hs

module Kenshou.Cli.Command.Perf.Session (sessionCommands, replayCalibrationReceipt, verifyCalibrationAdmission) where

import Control.Concurrent (threadDelay)
import Control.Exception (Exception, IOException, catch, displayException, throwIO, try)
import Control.Monad (unless, when)
import Data.Aeson hiding (decode)
import Data.Aeson.Types (parseEither)
import Data.ByteString qualified as Bytes
import Data.ByteString.Lazy.Char8 qualified as Lazy
import Data.List (find, isSuffixOf)
import Data.List qualified as List
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as TextIO
import Data.Time (UTCTime, diffUTCTime, getCurrentTime)
import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
import Kenshou.Cli.Command.Cell (startCellInstances)
import Kenshou.Core.Bundle (registryRoutingRules)
import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)
import Kenshou.Core.Cli (CliEnv (..))
import Kenshou.Core.Id
import Kenshou.Core.Phase (zeroPhases)
import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..))
import Kenshou.Measure.Compare.Ordering (PairedOrdering (ABBA))
import Kenshou.Measure.Compare.Policy (Direction (..), MetricRule (..), Policy, matchesRule)
import Kenshou.Measure.Compare.Policy qualified as Policy
import Kenshou.Perf.Artifacts
import Kenshou.Perf.Calibration qualified as Calibration
import Kenshou.Perf.Contracts (PerfContracts)
import Kenshou.Perf.Plan
import Kenshou.Perf.Plan qualified as PerfPlan
import Kenshou.Perf.Prepared qualified as Prepared
import Kenshou.Perf.Request (PerfRequest (..), strictObject)
import Kenshou.Perf.Result
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))
import Kenshou.Remote.Cell.Control
import Kenshou.Remote.Cell.Docs (CachePolicy (Cold), CellBuckets (..), CellDescriptor (..), Submission (..), WorkObject (..))
import Kenshou.Remote.Cell.Lease qualified as Lease
import Kenshou.Remote.Cell.Prepare
import Kenshou.Remote.Cell.RouteRules
import Kenshou.Remote.Cell.Session.Build (BuildOptions (..), buildSession)
import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..), SliceState (..), readSessionJournal)
import Kenshou.Remote.Cell.Session.Resume (resumeObservedSlices)
import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)
import Kenshou.Remote.Pair
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))
import Kenshou.Remote.Payload.Publish (digestFile)
import Kenshou.Remote.Perf.Attempt
import Kenshou.Remote.Perf.Cell
import Kenshou.Remote.Perf.Cleanup
import Kenshou.Remote.Perf.Conditions (validateHealthPolicy)
import Kenshou.Remote.Perf.Journal
import Kenshou.Remote.Perf.Lock
import Kenshou.Remote.Perf.Publication
import Kenshou.Remote.Perf.Runner
import Kenshou.Remote.Store
import Kenshou.Remote.Store.File (newFileStore)
import Kenshou.Remote.Store.Gcs (newGcsStore, newTokenProvider)
import Options.Applicative hiding (Success, action)
import System.Directory
import System.Environment (lookupEnv)
import System.Exit (ExitCode (..))
import System.FilePath (splitDirectories, takeDirectory, (</>))
import System.IO (stderr)
import System.IO.Temp (withSystemTempDirectory)
import Text.Printf (printf)

data SessionFailure = SessionFailure Int Text deriving stock (Show)

instance Exception SessionFailure

data PublicationReceipt = PublicationReceipt
  { snapshot :: Text,
    manifestUri :: Text,
    manifestSha256 :: Text,
    resultSha256 :: Text,
    payloadsReadyAt :: UTCTime,
    publicationStartedAt :: UTCTime,
    verifiedReportAt :: UTCTime,
    totalElapsedSeconds :: Double
  }
  deriving stock (Eq, Show)

instance ToJSON PublicationReceipt where
  toJSON receipt = object ["schema" .= ("kenshou.perf-publication/v1" :: Text), "snapshot" .= receipt.snapshot, "manifestUri" .= receipt.manifestUri, "manifestSha256" .= receipt.manifestSha256, "resultSha256" .= receipt.resultSha256, "payloadsReadyAt" .= receipt.payloadsReadyAt, "publicationStartedAt" .= receipt.publicationStartedAt, "verifiedReportAt" .= receipt.verifiedReportAt, "totalElapsedSeconds" .= receipt.totalElapsedSeconds]

instance FromJSON PublicationReceipt where
  parseJSON = withObject "PerformanceReceipt" \v -> do
    strictObject ["schema", "snapshot", "manifestUri", "manifestSha256", "resultSha256", "payloadsReadyAt", "publicationStartedAt", "verifiedReportAt", "totalElapsedSeconds"] v
    schema <- v .: "schema"
    unless (schema == ("kenshou.perf-publication/v1" :: Text)) (fail "unsupported publication receipt")
    receipt <- PublicationReceipt <$> v .: "snapshot" <*> v .: "manifestUri" <*> v .: "manifestSha256" <*> v .: "resultSha256" <*> v .: "payloadsReadyAt" <*> v .: "publicationStartedAt" <*> v .: "verifiedReportAt" <*> v .: "totalElapsedSeconds"
    unless (receipt.payloadsReadyAt <= receipt.publicationStartedAt && receipt.publicationStartedAt <= receipt.verifiedReportAt && abs (receipt.totalElapsedSeconds - realToFrac (diffUTCTime receipt.verifiedReportAt receipt.payloadsReadyAt)) < 0.000001) (fail "publication receipt timing mismatch")
    pure receipt

data ExecutionInputs = ExecutionInputs
  { controlBucket :: Text,
    storeLabel :: Text,
    allowedProjects :: [Text],
    start :: Bool,
    healthPolicies :: Map.Map Text Value,
    cells :: [(CellDescriptor, Maybe CellCapabilities)]
  }

instance ToJSON ExecutionInputs where
  toJSON inputs = object ["controlBucket" .= inputs.controlBucket, "storeLabel" .= inputs.storeLabel, "allowedProjects" .= inputs.allowedProjects, "start" .= inputs.start, "healthPolicies" .= inputs.healthPolicies, "cells" .= [object ["descriptor" .= descriptor, "capabilities" .= capabilities] | (descriptor, capabilities) <- inputs.cells]]

instance FromJSON ExecutionInputs where
  parseJSON = withObject "PerformanceExecutionInputs" \v -> do
    strictObject ["controlBucket", "storeLabel", "allowedProjects", "start", "healthPolicies", "cells"] v
    entries <-
      v .: "cells"
        >>= traverse
          ( withObject "PinnedCell" \c -> do
              strictObject ["descriptor", "capabilities"] c
              (,) <$> c .: "descriptor" <*> c .:? "capabilities"
          )
    ExecutionInputs <$> v .: "controlBucket" <*> v .: "storeLabel" <*> v .: "allowedProjects" <*> v .: "start" <*> (v .:? "healthPolicies" .!= Map.empty) <*> pure entries

sessionCommands :: Mod CommandFields (CliEnv -> IO ExitCode)
sessionCommands =
  command "run" (info (runParser <**> helper) (progDesc "Execute a pinned performance preparation with persistent budgets"))
    <> command "status" (info (statusParser <**> helper) (progDesc "Read persistent performance session state without cell actions"))
    <> command "resume" (info (resumeParser <**> helper) (progDesc "Verify inputs and reconcile a retained performance session"))
    <> command "verify" (info (verifyParser <**> helper) (progDesc "Replay retained comparisons and verify published evidence without executing trials"))
    <> command "calibration" (info (calibrationParser <**> helper) (progDesc "Freeze calibration batches and verify their retained acceptance evidence"))
  where
    session = strOption (long "session" <> metavar "DIRECTORY")
    runParser = runNew <$> strOption (long "prepared" <> metavar "FILE") <*> strOption (long "evidence-base-uri" <> metavar "GS_URI") <*> strOption (long "out" <> metavar "DIRECTORY") <*> strOption (long "control-bucket" <> value "tan-nb-exp-cells-control" <> showDefault) <*> switch (long "start" <> help "Start stopped instances after acquiring their cell lease")
    statusParser = runStatus <$> session
    resumeParser = runResume <$> session
    verifyParser = runVerify <$> session <*> switch (long "offline" <> help "Verify the retained sealed snapshot and receipt without contacting storage")
    calibrationParser =
      hsubparser
        ( command "declare" (info (runCalibrationPublish True <$> strOption (long "definition" <> metavar "FILE") <*> strOption (long "uri" <> metavar "GS_URI")) (progDesc "Create and retrieve a frozen definition before any acceptance trials"))
            <> command "fingerprint" (info (runCalibrationFingerprint <$> strOption (long "descriptor" <> metavar "FILE") <*> strOption (long "cell-policy" <> metavar "FILE")) (progDesc "Read the exact machine/image profile and health-policy digest before a batch"))
            <> command "publish" (info (runCalibrationPublish False <$> strOption (long "acceptance" <> metavar "FILE") <*> strOption (long "uri" <> metavar "GS_URI")) (progDesc "Create and retrieve the complete acceptance receipt inventory"))
            <> command "verify" (info (runCalibrationVerify <$> strOption (long "reference" <> metavar "FILE") <*> strOption (long "prepared" <> metavar "FILE")) (progDesc "Replay every calibration receipt against one prepared profile"))
        )

runCalibrationFingerprint :: FilePath -> FilePath -> CliEnv -> IO ExitCode
runCalibrationFingerprint descriptorPath policyPath _ = guarded do
  descriptor <- readJSON descriptorPath
  policy <- readJSON policyPath
  input (validateHealthPolicy policy)
  Lazy.putStrLn (encode (object ["machineProfile" .= cellMachineProfile descriptor, "healthPolicySha256" .= sha256Hex (canonicalEncode policy)]))
  pure ExitSuccess

runNew :: FilePath -> Text -> FilePath -> Text -> Bool -> CliEnv -> IO ExitCode
runNew preparedPath uri destination bucket start environment = guarded do
  prepared <- eitherDecodeFileStrict' preparedPath >>= either (throwIO . SessionFailure 2 . Text.pack) pure
  input (Prepared.validatePrepared environment.perf prepared)
  _ <- input (parseDestination uri)
  input (initialSessionPlaceholder prepared uri)
  verifyPerfManifest (takeDirectory preparedPath) >>= require
  exists <- doesPathExist destination
  when exists (throwIO (SessionFailure 2 "performance output already exists"))
  verifyGatingCalibration environment.perf prepared
  output <- makeAbsolute destination
  (store, label) <- openStore (Bucket bucket)
  projects <- maybe ["tan-nb-exp"] (map Text.strip . Text.splitOn "," . Text.pack) <$> lookupEnv "KENSHOU_GCP_ALLOWED_PROJECTS"
  capabilityDirectory <- fromMaybe ".dev/cells" <$> lookupEnv "KENSHOU_CELL_CAPABILITIES_DIR"
  pinned <- traverse (loadCell store projects capabilityDirectory bucket) prepared.plan.request.cells
  healthPolicies <- if null prepared.plan.selected then pure Map.empty else Map.fromList <$> traverse (\(descriptor, _) -> (descriptor.name,) <$> loadHealthPolicy store bucket descriptor.name) pinned
  let inputs = ExecutionInputs bucket label projects start healthPolicies pinned
  checkCalibratedPolicies prepared inputs
  payloads <- loadPayloads store prepared
  payloadsReady <- getCurrentTime
  identifier <- newRunId
  preflight environment store inputs payloads prepared identifier
  -- The create-only probe is retained as evidence of the actual destination
  -- permissions. It needs object creation and retrieval, never deletion.
  accessProbe store uri identifier prepared
  session <- require (initialSession identifier (Prepared.preparedDigest prepared) prepared.plan uri payloadsReady)
  createDirectoryIfMissing True (takeDirectory output)
  createDirectory output
  copyTree (takeDirectory preparedPath) (output </> "preparation")
  encodeFile (output </> "prepared.json") prepared
  encodeFile (output </> "request.json") prepared.plan.request
  encodeFile (output </> "plan.json") prepared.plan
  writePerfSession (output </> "session.json") session {executionInputs = Just (toJSON inputs)}
  withPerformanceLock output (execute environment store inputs payloads prepared output)

-- Validate the destination using the same immutable journal admission rules
-- before obtaining an ID, writing output or performing any remote operation.
initialSessionPlaceholder :: Prepared.PerfPrepared -> Text -> Either Text ()
initialSessionPlaceholder prepared uri = do
  identifier <- parseRunId "00000000-0000-7000-8000-000000000000"
  let epoch = posixSecondsToUTCTime 0
  _ <- initialSession identifier (Prepared.preparedDigest prepared) prepared.plan uri epoch
  pure ()

runResume :: FilePath -> CliEnv -> IO ExitCode
runResume destination environment = guarded do
  output <- makeAbsolute destination
  withPerformanceLock output do
    prepared <- loadPreparation environment.perf (output </> "preparation/prepared.json")
    session <- readPerfSession (output </> "session.json") >>= require
    require (validateSessionInputs environment.perf prepared session)
    checkRootInputs output prepared
    verifyGatingCalibration environment.perf prepared
    inputs <- require (parseInputs session)
    checkCalibratedPolicies prepared inputs
    state <- require (journalState session)
    receiptExists <- doesFileExist (output </> "receipt.json")
    if receiptExists && state.lease == Nothing && state.pendingClaim == Nothing && state.activeComparison == Nothing && Map.size state.completed == length session.plan.selected
      then do
        (result, _) <- verifySessionEvidence environment.perf output False
        receipt <- readJSON (output </> "receipt.json")
        retained <- readPerfSession (output </> Text.unpack (receipt :: PublicationReceipt).snapshot </> "session.json") >>= require
        unless (retained == session) (ioError (userError "published report does not describe the current session"))
        Lazy.putStrLn (encode result)
        pure (exit result.exitCode)
      else do
        (store, label) <- openStore (Bucket inputs.controlBucket)
        unless (inputs.storeLabel == label) (ioError (userError "object store differs from the immutable performance session"))
        mapM_ (checkCell store inputs) inputs.cells
        payloads <- loadPayloads store prepared
        preflight environment store inputs payloads prepared session.sessionId
        execute environment store inputs payloads prepared output

runStatus :: FilePath -> CliEnv -> IO ExitCode
runStatus output _ = guarded do
  session <- readPerfSession (output </> "session.json") >>= require
  state <- require (journalState session)
  now <- getCurrentTime
  Lazy.putStrLn (encode (object ["session" .= session.sessionId, "requestSha256" .= session.plan.requestSha256, "admissionSpentSeconds" .= state.admissionSpentSeconds, "executionDeadline" .= state.executionDeadline, "cleanupDeadline" .= state.cleanupDeadline, "lease" .= state.lease, "pendingClaim" .= state.pendingClaim, "activeComparison" .= state.activeComparison, "attempts" .= state.attempts, "replacements" .= state.replacements, "completed" .= Map.elems state.completed, "runtimeBlockers" .= state.runtimeBlockers, "totalElapsedSeconds" .= (realToFrac (diffUTCTime now session.createdAt) :: Double)]))
  pure ExitSuccess

loadPreparation :: PerfContracts -> FilePath -> IO Prepared.PerfPrepared
loadPreparation contracts path = do
  verifyPerfManifest (takeDirectory path) >>= require
  prepared <- readJSON path
  require (Prepared.validatePrepared contracts prepared)
  pure prepared

loadCell :: ObjectStore -> [Text] -> FilePath -> Text -> Text -> IO (CellDescriptor, Maybe CellCapabilities)
loadCell store projects directory bucket name = do
  snapshot <- readCellSnapshot store (Lease.CellRef name (Bucket bucket)) projects >>= require
  capabilities <- loadCellCapabilities directory snapshot.descriptor >>= require
  pure (snapshot.descriptor, capabilities)

loadHealthPolicy :: ObjectStore -> Text -> Text -> IO Value
loadHealthPolicy store bucket cell = do
  retained <- store.getObject (Bucket bucket) (ObjectName ("cells/" <> cell <> "/policy.json"))
  bytes <- maybe (ioError (userError "cell health policy is missing")) (pure . fst) retained
  policy <- either (ioError . userError) pure (eitherDecode bytes)
  require (validateHealthPolicy policy)
  pure policy

checkCell :: ObjectStore -> ExecutionInputs -> (CellDescriptor, Maybe CellCapabilities) -> IO ()
checkCell store inputs (descriptor, _) = do
  snapshot <- readCellSnapshot store (Lease.CellRef descriptor.name (Bucket inputs.controlBucket)) inputs.allowedProjects >>= require
  case Map.lookup descriptor.name inputs.healthPolicies of
    Nothing -> pure ()
    Just frozen -> do
      policy <- loadHealthPolicy store inputs.controlBucket descriptor.name
      unless (policy == frozen) (ioError (userError "cell health policy changed since performance admission"))
  unless (snapshot.descriptor == descriptor) (ioError (userError "cell descriptor changed since performance preparation"))

parseInputs :: PerfSession -> Either Text ExecutionInputs
parseInputs session = do
  inputs <- maybe (Left "performance session has no frozen execution inputs") decode session.executionInputs
  unless (map ((.name) . fst) inputs.cells == session.plan.request.cells && not (Text.null inputs.controlBucket) && not (null inputs.allowedProjects)) (Left "frozen cell pool differs from the request")
  unless (null session.plan.selected || Map.keys inputs.healthPolicies == List.sort (map ((.name) . fst) inputs.cells)) (Left "frozen cell health policies are missing or differ from the pool")
  mapM_ validateHealthPolicy (Map.elems inputs.healthPolicies)
  mapM_
    ( \(descriptor, capabilities) -> do
        unless (descriptor.buckets.control == inputs.controlBucket && descriptor.project `elem` inputs.allowedProjects) (Left "frozen cell bucket or project is invalid")
        case capabilities of
          Nothing -> pure ()
          Just record -> unless (record.cell == descriptor.name && record.descriptorSha256 == descriptorDigest descriptor) (Left "frozen capabilities differ from the cell descriptor")
    )
    inputs.cells
  pure inputs

checkRootInputs :: FilePath -> Prepared.PerfPrepared -> IO ()
checkRootInputs directory prepared = mapM_ check [("prepared.json", toJSON prepared), ("plan.json", toJSON prepared.plan), ("request.json", toJSON prepared.plan.request)]
  where
    check (name, expected) = do
      actual <- readJSON (directory </> name)
      unless (actual == expected) (ioError (userError ("retained " <> name <> " differs from immutable preparation")))

loadPayloads :: ObjectStore -> Prepared.PerfPrepared -> IO (Map.Map Text PayloadDescriptor)
loadPayloads store prepared = do
  entries <-
    traverse
      ( \(label, arm) -> do
          payload <- require (decode arm.payload)
          let prefix = "gs://"
              (bucket, suffix) = Text.breakOn "/" (Text.drop (Text.length prefix) payload.cell.bundle.uri)
          unless (prefix `Text.isPrefixOf` payload.cell.bundle.uri && not (Text.null bucket) && suffix == "/payloads/sha256/" <> payload.cell.bundle.sha256 <> ".nar.zst") (ioError (userError "invalid pinned payload location"))
          withSystemTempDirectory "kenshou-perf-payload-check" \temporary -> do
            let path = temporary </> "bundle.nar.zst"
            present <- store.downloadTo (Bucket bucket) (ObjectName (Text.drop 1 suffix)) path
            unless (present /= Nothing) (ioError (userError "pinned payload is missing"))
            (digest, bytes) <- digestFile path
            unless (digest == payload.cell.bundle.sha256 && bytes == payload.cell.bundle.bytes) (ioError (userError "pinned payload retrieval digest differs"))
          pure (label, payload)
      )
      [("baseline", prepared.baseline), ("candidate", prepared.candidate)]
  pure (Map.fromList entries)

options :: BuildOptions
options = BuildOptions (PrepareOptions False False [] Cold) GranularityRun False NullSink Nothing (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0"

adapter :: CliEnv -> ObjectStore -> ExecutionInputs -> Map.Map Text PayloadDescriptor -> (CellDescriptor, Maybe CellCapabilities) -> CellAdapter
adapter environment store inputs payloads (descriptor, capabilities) = CellAdapter environment.registry store (Lease.CellRef descriptor.name (Bucket inputs.controlBucket)) descriptor (Map.findWithDefault Null descriptor.name inputs.healthPolicies) capabilities (registryRoutingRules environment.registry) payloads options inputs.storeLabel (\event -> TextIO.hPutStrLn stderr (Text.pack (show event)))

preflight :: CliEnv -> ObjectStore -> ExecutionInputs -> Map.Map Text PayloadDescriptor -> Prepared.PerfPrepared -> RunId -> IO ()
preflight environment store inputs payloads prepared identifier = mapM_ check prepared.plan.selected
  where
    check entry = do
      schedule <- makeSchedule prepared entry identifier
      compatible <-
        traverse
          ( \pinned -> do
              let a = adapter environment store inputs payloads pinned
              built <- buildSession a.registry a.descriptor a.capabilities a.rules a.payloads a.options a.storeLabel identifier (encode schedule)
              pure (either (const False) (const (prepared.plan.request.purpose == "calibration" || entry.profile.machineProfile == cellMachineProfile a.descriptor)) built)
          )
          inputs.cells
      unless (or compatible) (ioError (userError ("no pinned cell supports profile " <> Text.unpack entry.profile.name)))

makeSchedule :: Prepared.PerfPrepared -> PlannedProfile -> RunId -> IO RunPlan
makeSchedule prepared entry identifier = do
  let profile = entry.profile
      source = object ["schema" .= ("kenshou.run-plan/v1" :: Text), "planId" .= identifier, "inputs" .= object ["preparedSha256" .= Prepared.preparedDigest prepared, "workloadSha256" .= entry.workloadSha256], "runs" .= [object ["ordinal" .= (1 :: Int), "runId" .= identifier, "estimateMinutes" .= (1 :: Int), "spec" .= object ["schema" .= ("kenshou.run-spec/v1" :: Text), "runId" .= identifier, "scenario" .= profile.scenario, "scenarioRevision" .= profile.scenarioRevision, "knobs" .= profile.knobs, "dimensions" .= profile.dimensions, "seed" .= prepared.plan.request.seed, "phases" .= zeroPhases, "timeoutSeconds" .= profile.timeoutSeconds, "environment" .= object ["placement" .= ("cell" :: Text), "machineProfile" .= profile.machineProfile]]]]]
  projected <- require (decodeWorkPlan source)
  require (pairPlan (PairRequest "candidate" "baseline" profile.minimumPairs ABBA (fromIntegral prepared.plan.request.seed) profile.replacementPairs) projected)

policyFor :: Prepared.PerfPrepared -> PlannedProfile -> Either Text Policy
policyFor prepared entry = do
  policies <- either (Left . Text.pack) Right (parseEither (withObject "PinnedInputs" (.: "policies")) prepared.inputs)
  document <- maybe (Left "pinned comparison policy missing") Right (Map.lookup entry.profile.policy (policies :: Map.Map FilePath Value))
  decode document

axesFor :: Prepared.PerfPrepared -> NonEmpty VaryingAxis
axesFor prepared = (if prepared.plan.request.control == "aa" then VaryControl else VaryCohort) :| []

attemptDirectory :: FilePath -> PlannedProfile -> Int -> FilePath
attemptDirectory output entry index = output </> "configurations" </> Text.unpack entry.profile.name </> "attempt-" <> show index

execute :: CliEnv -> ObjectStore -> ExecutionInputs -> Map.Map Text PayloadDescriptor -> Prepared.PerfPrepared -> FilePath -> IO ExitCode
execute environment store inputs payloads prepared output = do
  let ref identity = Lease.CellRef identity.cell (Bucket inputs.controlBucket)
      pinned identity = maybe (ioError (userError "leased cell is outside frozen pool")) pure (find ((== identity.cell) . (.name) . fst) inputs.cells)
      handle identity = Lease.reattachActiveLease store (ref identity) identity.leaseId >>= maybe (ioError (userError "performance lease is no longer active")) pure
      supports entry p = do
        planned <- makeSchedule prepared entry =<< newRunId
        let a = adapter environment store inputs payloads p
        built <- buildSession a.registry a.descriptor a.capabilities a.rules a.payloads a.options a.storeLabel planned.planId (encode planned)
        pure (either (const False) (const (prepared.plan.request.purpose == "calibration" || entry.profile.machineProfile == cellMachineProfile a.descriptor)) built)
      admission entry seconds pending callbacks = case pending of
        Just (name, identity, _) -> do
          unless (name == entry.profile.name) (ioError (userError "proposed claim belongs to another profile"))
          active <- Lease.reattachActiveLease store (ref identity) identity.leaseId
          pure (maybe (AdmissionFailed "proposed cell claim is unresolved, expired or cancelled; no new claim issued") (const (Admitted identity)) active)
        Nothing -> waitFor entry seconds callbacks
      waitFor entry left callbacks = do
        granted <- findLease entry inputs.cells callbacks
        case granted of
          Occupied | left > 1 -> threadDelay 1000000 >> waitFor entry (left - 1) callbacks
          _ -> pure granted
      findLease _ [] _ = pure Occupied
      findLease entry (p@(descriptor, _) : rest) callbacks = do
        compatible <- supports entry p
        if not compatible
          then findLease entry rest callbacks
          else do
            checkCell store inputs p
            identifier <- newRunId
            let identity = LeaseIdentity descriptor.name identifier
            callbacks.proposeClaim identity
            granted <- Lease.acquireLeaseWithId store (Lease.CellRef descriptor.name (Bucket inputs.controlBucket)) (Lease.LeaseRequest "kenshou-perf" "pinned runtime performance validation" 180) identifier
            case granted of
              Lease.Acquired _ -> pure (Admitted identity)
              Lease.Busy _ -> callbacks.withdrawClaim >> findLease entry rest callbacks
              Lease.Quarantined _ -> pure (AdmissionFailed "cell was quarantined during the proposed claim")
      compareOne identity entry index _ callbacks = do
        p <- pinned identity
        acquired <- handle identity
        Lease.withHeartbeat store (ref identity) acquired \_ -> do
          when inputs.start (startCellInstances (fst p))
          let directory = attemptDirectory output entry index
              schedulePath = directory </> "schedule.json"
          exists <- doesFileExist schedulePath
          schedule <-
            if exists
              then readJSON schedulePath >>= require . decodeWorkPlan
              else do
                identifier <- newRunId
                planned <- makeSchedule prepared entry identifier
                createDirectoryIfMissing True directory
                encodeFile (schedulePath <> ".next") planned
                renameFile (schedulePath <> ".next") schedulePath
                pure planned
          policy <- require (policyFor prepared entry)
          computed <- executeAttempt identity entry schedule policy (axesFor prepared) directory callbacks (executeCellTrial (adapter environment store inputs payloads p) acquired schedule directory entry.profile callbacks.recordRuntimeFailure)
          retainAttemptResult directory computed
      verify result = verifyConfiguration inputs prepared output result
      hooks =
        RunnerHooks
          environment.perf
          getCurrentTime
          admission
          (\identity -> maybe False (const True) <$> Lease.reattachActiveLease store (ref identity) identity.leaseId)
          (\identity entry -> pinned identity >>= supports entry)
          compareOne
          verify
          ( \identity -> do
              _ <- Lease.requestCancel store (ref identity) (Just identity.leaseId) False
              pure ()
          )
          ( \identity -> do
              files <- treeFiles (output </> "configurations")
              journals <- traverse (\path -> readSessionJournal (output </> "configurations" </> path) >>= require) [path | path <- files, [_, _, "trials", _, "session.json"] <- [splitDirectories path]]
              -- Fence first, then inspect markers. A marker arriving after
              -- this fence cannot start work under the old lease identity.
              _ <- Lease.requestCancel store (ref identity) (Just identity.leaseId) False
              settled <- ownedWorkSettled store (ref identity) identity journals
              unless settled do
                _ <- Lease.requestCancel store (ref identity) (Just identity.leaseId) False
                waitForOwnedWork store (ref identity) identity journals
              owned <- Lease.reattachLease store (ref identity) identity.leaseId
              case owned of
                Just acquired -> Lease.releaseLease store (ref identity) acquired
                Nothing -> do
                  history <- readPerfSession (output </> "session.json") >>= require
                  -- An absent ambiguous proposal must stay unresolved. A known
                  -- lost lease with terminal owned work needs no deletion.
                  pure (any ((== LeaseAcquired identity) . (.action)) history.events)
          )
          (recoverSealedRuntime store inputs prepared output)
          (describeReport output)
          (publishSnapshot store inputs prepared output)
  createDirectoryIfMissing True (output </> "configurations")
  auditAllAttempts True inputs prepared output
  result <- runPerformance hooks prepared (output </> "session.json")
  encodeFile (output </> "result.json") result
  TextIO.writeFile (output </> "report.md") (renderReport result)
  Lazy.putStrLn (encode result)
  pure (exit result.exitCode)

verifyConfiguration :: ExecutionInputs -> Prepared.PerfPrepared -> FilePath -> ConfigurationResult -> IO Bool
verifyConfiguration inputs prepared output result = do
  entry <- maybe (ioError (userError "result names an unselected profile")) pure (find ((== result.profile) . (.name) . (.profile)) prepared.plan.selected)
  auditProfileAttempts False inputs prepared output entry (Just result)

recoverSealedRuntime :: ObjectStore -> ExecutionInputs -> Prepared.PerfPrepared -> FilePath -> LeaseIdentity -> IO [(Text, Text)]
recoverSealedRuntime store inputs prepared output identity = fmap (List.nub . concat) (traverse profile prepared.plan.selected)
  where
    profile entry = do
      let base = output </> "configurations" </> Text.unpack entry.profile.name
      exists <- doesDirectoryExist base
      attempts <- if exists then listDirectory base else pure []
      fmap concat (traverse (attempt entry base) attempts)
    attempt entry base name = do
      let directory = base </> name
          schedulePath = directory </> "schedule.json"
      exists <- doesFileExist schedulePath
      if not exists
        then pure []
        else do
          schedule <- readJSON schedulePath >>= require . decodeWorkPlan
          fmap concat (traverse (trial entry directory) schedule.runs)
    trial entry directory run = do
      let trialDirectory = directory </> "trials" </> Text.unpack (renderRunId run.runId)
          path = trialDirectory </> "session.json"
      exists <- doesFileExist path
      if not exists
        then pure []
        else do
          retained <- readSessionJournal path >>= require
          if retained.leaseId /= identity.leaseId || retained.cell /= identity.cell
            then pure []
            else do
              observed <- resumeObservedSlices store (Lease.CellRef identity.cell (Bucket inputs.controlBucket)) (Bucket retained.resultsBucket) path (\event -> TextIO.hPutStrLn stderr (Text.pack (show event)))
              journal <- either (ioError . userError . show) pure observed
              reasons <- retainedRuntimeProblems entry.profile run trialDirectory journal
              pure [(entry.profile.name, reason) | reason <- reasons]

auditAllAttempts :: Bool -> ExecutionInputs -> Prepared.PerfPrepared -> FilePath -> IO ()
auditAllAttempts recover inputs prepared output = do
  names <- listDirectory (output </> "configurations")
  unless (all (`elem` map (Text.unpack . (.name) . (.profile)) prepared.plan.selected) names) (ioError (userError "retained evidence contains an unselected profile"))
  mapM_ (\entry -> auditProfileAttempts recover inputs prepared output entry Nothing >>= \valid -> unless valid (ioError (userError "retained attempt audit failed"))) prepared.plan.selected

-- Every retained attempt is checked, including abandoned and partially written
-- attempts. Only a complete matching attempt supplies the final comparison.
auditProfileAttempts :: Bool -> ExecutionInputs -> Prepared.PerfPrepared -> FilePath -> PlannedProfile -> Maybe ConfigurationResult -> IO Bool
auditProfileAttempts recover inputs prepared output entry wanted = do
  session <- readPerfSession (output </> "session.json") >>= require
  let (_, identities) = foldl' remember (Nothing, Map.empty) session.events
      base = output </> "configurations" </> Text.unpack entry.profile.name
      remember (lease, known) event = case event.action of
        LeaseAcquired identity -> (Just identity, known)
        LeaseLost _ -> (Nothing, known)
        LeaseReleased -> (Nothing, known)
        ComparisonStarted profile index -> (lease, maybe known (\identity -> Map.insert (profile, "attempt-" <> show index) identity known) lease)
        _ -> (lease, known)
  exists <- doesDirectoryExist base
  names <- if exists then listDirectory base else pure []
  baseline <- require (decode prepared.baseline.payload)
  candidate <- require (decode prepared.candidate.payload)
  let payloads = Map.fromList [("baseline", baseline), ("candidate", candidate)]
  candidates <-
    traverse
      ( \name -> do
          identity <- maybe (ioError (userError "retained attempt has no journaled lease identity")) pure (Map.lookup (entry.profile.name, name) identities)
          let directory = base </> name
          files <- treeFiles directory
          scheduleExists <- doesFileExist (directory </> "schedule.json")
          if not scheduleExists
            then do
              unless (all (== "schedule.json.next") files) (ioError (userError "retained attempt has evidence before its paired schedule"))
              pure False
            else do
              schedule <- readJSON (directory </> "schedule.json") >>= require . decodeWorkPlan
              expected <- makeSchedule prepared entry schedule.planId
              unless (toJSON schedule == toJSON expected) (ioError (userError "retained paired schedule differs from its pinned request and seed"))
              metadataExists <- doesFileExist (directory </> "attempt.json")
              if not metadataExists
                then do
                  unless (all (`elem` ["schedule.json", "attempt.json.next"]) files) (ioError (userError "retained attempt has evidence but no immutable attempt header"))
                  pure False
                else do
                  verifyAttemptInputs identity entry schedule directory
                  pinned <- maybe (ioError (userError "attempt cell is outside frozen pool")) pure (find ((== identity.cell) . (.name) . fst) inputs.cells)
                  pathsExists <- doesFileExist (directory </> "trial-paths.json")
                  paths <- if pathsExists then readJSON (directory </> "trial-paths.json") else pure Map.empty
                  failureExists <- doesFileExist (directory </> "trial-failures.json")
                  failures <- if failureExists then readJSON (directory </> "trial-failures.json") else pure (Map.empty :: Map.Map Text Text)
                  let journalPaths = [path | path <- files, ["trials", _, "session.json"] <- [splitDirectories path]]
                      expectedPaths = ["trials" </> Text.unpack (renderRunId run.runId) </> "session.json" | run <- schedule.runs]
                  unless (all (`elem` expectedPaths) journalPaths) (ioError (userError "retained cell journal names an unplanned trial"))
                  journals <- traverse (\path -> readSessionJournal (directory </> path) >>= require) journalPaths
                  let linkedDirectories = map (Text.unpack . renderRunId . (.runId)) schedule.runs <> [Text.unpack (renderRunId slice.cellRun) | journal <- journals, slice <- journal.slices]
                      attemptFiles = ["schedule.json", "attempt.json", "trial-paths.json", "trial-failures.json", "comparison.json", "result.json"]
                      interpreted path = case splitDirectories path of
                        [name] -> name `elem` attemptFiles || name `elem` map (<> ".next") attemptFiles
                        "trials" : identifier : _ -> identifier `elem` linkedDirectories
                        _ -> False
                  unless (all interpreted files) (ioError (userError "retained attempt contains unlinked evidence outside its scheduled trials and cell journals"))
                  mapM_
                    ( \run -> do
                        let identifier = renderRunId run.runId
                            trial = directory </> "trials" </> Text.unpack identifier
                            journalPath = trial </> "session.json"
                            planBytes = encode schedule {runs = [run], estimateMinutes = run.estimateMinutes}
                        journalExists <- doesFileExist journalPath
                        if not journalExists
                          then unless (Map.notMember identifier (paths :: Map.Map Text FilePath)) (ioError (userError "retained accepted trial has no cell journal"))
                          else do
                            journal <- readSessionJournal journalPath >>= require
                            unless (journal.leaseId == identity.leaseId && journal.cell == identity.cell && journal.controlBucket == inputs.controlBucket && journal.resultsBucket == (fst pinned).buckets.results && journal.payloads == payloads && journal.planSha256 == Text.drop 7 (sha256Hex (Lazy.toStrict planBytes))) (ioError (userError "retained cell journal differs from pinned payloads, trial or lease"))
                            storedPlan <- Lazy.readFile (trial </> "plan.json")
                            unless (storedPlan == planBytes) (ioError (userError "retained trial plan bytes differ"))
                            mapM_
                              ( \slice -> do
                                  work <- Lazy.readFile (trial </> slice.workPath)
                                  unless (slice.submission.leaseId == identity.leaseId && slice.submission.runId == slice.cellRun && slice.runIds == [run.runId] && slice.submission.work.sha256 == Text.drop 7 (sha256Hex (Lazy.toStrict work)) && slice.submission.work.bytes == Lazy.length work) (ioError (userError "retained submission work or identity differs"))
                              )
                              journal.slices
                            when (not recover && any (\slice -> slice.state `notElem` [SlicePlanned, SliceRejected, SliceVerified]) journal.slices) (ioError (userError "retained report contains unresolved submitted cell work"))
                            verified <- verifyCellTrial (Map.findWithDefault Null identity.cell inputs.healthPolicies) (fst pinned) run trial journal
                            runtimeProblems <- retainedRuntimeProblems entry.profile run trial journal
                            mapM_
                              ( \reason -> do
                                  current <- readPerfSession (output </> "session.json") >>= require
                                  state <- require (journalState current)
                                  unless (reason `elem` Map.findWithDefault [] entry.profile.name state.runtimeBlockers) do
                                    unless recover (ioError (userError "sealed runtime blocker is missing from the session"))
                                    now <- getCurrentTime
                                    changed <- require (appendEvent (SessionEvent now (RuntimeBlocked entry.profile.name reason)) current)
                                    writePerfSession (output </> "session.json") changed
                              )
                              runtimeProblems
                            case Map.lookup identifier paths of
                              Just path -> unless (verified == Right (directory </> path)) (ioError (userError "retained accepted trial outer evidence differs"))
                              Nothing -> when (Map.member identifier failures && either (const False) (const True) verified) (ioError (userError "retained failed trial has healthy completed outer evidence"))
                    )
                    schedule.runs
                  present <- doesFileExist (directory </> "result.json")
                  if not present
                    then pure False
                    else do
                      record <- readJSON (directory </> "result.json")
                      policy <- require (policyFor prepared entry)
                      replayed <- verifyAttempt entry schedule policy (axesFor prepared) directory record
                      unless replayed (ioError (userError "retained attempt result replay differs"))
                      comparisonExists <- doesFileExist (directory </> "comparison.json")
                      comparison <- if comparisonExists then Just <$> readJSON (directory </> "comparison.json") else pure Nothing
                      unless (comparison == record.comparison) (ioError (userError "retained ordinary comparison differs from attempt result"))
                      pure (maybe False (\result -> record.comparison == result.comparison && record.validPairs == result.validPairs && record.attempts == result.attempts && all (`elem` result.runtimeFailures) record.runtimeFailures) wanted)
      )
      names
  pure (maybe True (\result -> or candidates || (result.comparison == Nothing && result.validPairs == 0 && result.infrastructureState == "pending" && result.performanceVerdict == "inconclusive")) wanted)

describeReport :: FilePath -> PerfSession -> PerfResult -> IO PerfResult
describeReport output session result = do
  identifier <- newRunId
  now <- getCurrentTime
  let summaryPath = output </> "preparation/preparation-summary.json"
  summaryExists <- doesFileExist summaryPath
  preparationTimes <- if summaryExists then readJSON summaryPath >>= either (ioError . userError) pure . parseEither (withObject "PreparationSummary" (.: "stageSeconds")) else pure Map.empty
  let base = session.evidenceBaseUri <> "/sessions/" <> renderRunId session.sessionId <> "/snapshots/" <> renderRunId identifier
      leaseTimes = [event.at | event <- session.events, LeaseAcquired _ <- [event.action]]
      elapsed from to = realToFrac (diffUTCTime to from) :: Double
      timings =
        Map.fromList
          ( [("totalAtReportAssembly", elapsed session.createdAt now)] <> case leaseTimes of
              [] -> [("initialAdmission", elapsed session.createdAt now), ("measurementAndCleanupToAssembly", 0)]
              first : _ -> [("initialAdmission", elapsed session.createdAt first), ("measurementAndCleanupToAssembly", elapsed first now)]
          )
  files <- treeFiles (output </> "configurations")
  let relativeFiles = ["prepared.json", "request.json", "plan.json", "session.json"] <> ["configurations/" <> path | path <- files, "comparison.json" `isSuffixOf` path || "manifest.json" `isSuffixOf` path]
  references <-
    traverse
      ( \path -> do
          (digest, _) <- digestFile (output </> path)
          pure (ArtifactReference (if "manifest.json" `isSuffixOf` path then "cell-or-runtime-manifest" else if "comparison.json" `isSuffixOf` path then "comparison" else Text.pack path) (base <> "/" <> Text.pack path) (Just ("sha256:" <> digest)))
      )
      relativeFiles
  pure result {stageSeconds = timings <> Map.mapKeys ("preparation." <>) preparationTimes, artifactReferences = ArtifactReference "aggregate-manifest" (base <> "/manifest.json") Nothing : ArtifactReference "publication-receipt" (base <> "/receipt.json") Nothing : references}

treeFiles :: FilePath -> IO [FilePath]
treeFiles root = do
  names <- listDirectory root
  fmap
    concat
    ( traverse
        ( \name -> do
            let path = root </> name
            linked <- pathIsSymbolicLink path
            when linked (ioError (userError "performance evidence contains a symbolic link"))
            directory <- doesDirectoryExist path
            if directory then map (name </>) <$> treeFiles path else pure [name]
        )
        names
    )

publishSnapshot :: ObjectStore -> ExecutionInputs -> Prepared.PerfPrepared -> FilePath -> PerfSession -> PerfResult -> IO Bool
publishSnapshot store inputs prepared output session result = do
  auditAllAttempts False inputs prepared output
  publicationStarted <- getCurrentTime
  manifestReference <- case filter ((== "aggregate-manifest") . (.kind)) result.artifactReferences of
    [reference] -> pure reference.uri
    _ -> ioError (userError "report has no unique aggregate manifest reference")
  relative <- maybe (ioError (userError "aggregate reference is outside its pinned session")) pure (Text.stripPrefix (session.evidenceBaseUri <> "/sessions/" <> renderRunId session.sessionId <> "/") manifestReference >>= Text.stripSuffix "/manifest.json")
  let directory = output </> Text.unpack relative
  createDirectoryIfMissing True directory
  mapM_ (\name -> copyTree (output </> name) (directory </> name)) ["preparation", "configurations"]
  mapM_ (\name -> copyFile (output </> name) (directory </> name)) ["prepared.json", "request.json", "plan.json", "session.json"]
  encodeFile (directory </> "result.json") result
  TextIO.writeFile (directory </> "report.md") (renderReport result)
  sealPerfDirectory directory >>= require
  (bucket, prefix) <- require (parseDestination session.evidenceBaseUri)
  published <- publishEvidence store bucket (prefix <> "/sessions/" <> renderRunId session.sessionId <> "/" <> relative) directory
  case published of
    Left problem -> TextIO.hPutStrLn stderr problem >> pure False
    Right uri -> do
      manifest <- Bytes.readFile (directory </> "manifest.json")
      verified <- getCurrentTime
      state <- require (journalState session)
      unless (maybe True (verified <=) state.executionDeadline) (ioError (userError "publication verification exceeded the original execution deadline"))
      let receipt = PublicationReceipt relative uri (sha256Hex manifest) (sha256Hex (canonicalEncode (toJSON result))) session.createdAt publicationStarted verified (realToFrac (diffUTCTime verified session.createdAt))
          bytes = encode receipt
          name = ObjectName (prefix <> "/sessions/" <> renderRunId session.sessionId <> "/" <> relative <> "/receipt.json")
      _ <- store.putObject bucket name "application/json" DoesNotExist bytes
      retained <- store.getObject bucket name
      unless (fmap fst retained == Just bytes) (ioError (userError "durable publication receipt retrieval differs"))
      receiptRetrievedAt <- getCurrentTime
      unless (maybe True (receiptRetrievedAt <=) state.executionDeadline) (ioError (userError "receipt retrieval exceeded the original execution deadline"))
      atomicBytes (output </> "receipt.json") bytes
      pure True

-- An interruption after comparison publication but before the root checkpoint
-- must preserve that comparison's original identity and bytes on reanalysis.
retainAttemptResult :: FilePath -> ConfigurationResult -> IO ConfigurationResult
retainAttemptResult directory computed = do
  let comparisonPath = directory </> "comparison.json"
      resultPath = directory </> "result.json"
  comparisonExists <- doesFileExist comparisonPath
  comparison <- case (comparisonExists, computed.comparison) of
    (False, Nothing) -> pure Nothing
    (True, Nothing) -> ioError (userError "retained comparison no longer matches completed trial evidence")
    (False, Just document) -> atomicBytes comparisonPath (encode document) >> pure (Just document)
    (True, Just document) -> do
      original <- readJSON comparisonPath
      retained <- require (semanticComparison original)
      replayed <- require (semanticComparison document)
      unless (retained == replayed) (ioError (userError "retained comparison differs from interrupted attempt replay"))
      pure (Just original)
  let result = computed {comparison = comparison}
  exists <- doesFileExist resultPath
  if exists
    then do
      original <- readJSON resultPath
      unless (original == result) (ioError (userError "retained attempt result differs from interrupted replay"))
      pure original
    else atomicBytes resultPath (encode result) >> pure result

atomicBytes :: FilePath -> Lazy.ByteString -> IO ()
atomicBytes path bytes = Lazy.writeFile (path <> ".next") bytes >> renameFile (path <> ".next") path

runCalibrationPublish :: Bool -> FilePath -> Text -> CliEnv -> IO ExitCode
runCalibrationPublish definition path uri _ = guarded do
  bytes <- Lazy.readFile path
  (bucket, name) <- input (parseDestination uri)
  now <- getCurrentTime
  if definition
    then do
      document <- either (throwIO . SessionFailure 2 . Text.pack) pure (eitherDecode bytes :: Either String Calibration.CalibrationDefinition)
      input (Calibration.validateCalibrationDefinition now now document)
    else do
      document <- either (throwIO . SessionFailure 2 . Text.pack) pure (eitherDecode bytes :: Either String Calibration.CalibrationAcceptance)
      unless (length document.aa == 20 && length document.slowdown == 20 && all (not . null . (.receipts)) (document.aa <> document.slowdown)) (throwIO (SessionFailure 2 "acceptance inventory needs both twenty-slot experiments"))
  (store, _) <- openStore bucket
  _ <- store.putObject bucket (ObjectName name) "application/json" DoesNotExist bytes
  retained <- store.getObject bucket (ObjectName name)
  (retrieved, metadata) <- maybe (ioError (userError "calibration publication retrieval failed")) pure retained
  unless (retrieved == bytes) (ioError (userError "create-only calibration publication differs"))
  Lazy.putStrLn (encode (object ["uri" .= uri, "sha256" .= sha256Hex (Lazy.toStrict bytes), "declaredAt" .= metadata.updated]))
  pure ExitSuccess

runCalibrationVerify :: FilePath -> FilePath -> CliEnv -> IO ExitCode
runCalibrationVerify referencePath preparedPath environment = guarded do
  prepared <- loadPreparation environment.perf preparedPath
  reference <- readJSON referencePath :: IO Calibration.CalibrationReference
  entry <- case prepared.plan.selected of [selected] -> pure selected; _ -> throwIO (SessionFailure 2 "calibration verification needs one prepared profile")
  (semantics, policies) <- require (either (Left . Text.pack) Right (parseEither (withObject "PinnedInputs" \v -> (,) <$> v .: "harnessSemanticsSha256" <*> v .: "policies") prepared.inputs))
  policy <- maybe (ioError (userError "missing pinned calibration policy")) pure (Map.lookup entry.profile.policy (policies :: Map.Map FilePath Value))
  digest <- verifyCalibrationAdmission environment.perf entry {PerfPlan.profile = entry.profile {calibration = Just (toJSON reference)}} (Calibration.toolchainDigest prepared.baseline.buildInventory) semantics policy >>= require
  Lazy.putStrLn (encode (object ["verified" .= True, "profile" .= entry.profile.name, "acceptanceSha256" .= digest, "gatingEligible" .= False]))
  pure ExitSuccess

verifyCalibrationAdmission :: PerfContracts -> PlannedProfile -> Text -> Text -> Value -> IO (Either Text Text)
verifyCalibrationAdmission contracts entry expectedToolchain semantics policyValue = do
  outcome <- try @IOException do
    reference <- require (either (Left . Text.pack) Right (parseEither (parseJSON @Calibration.CalibrationReference) (fromMaybe Null entry.profile.calibration)))
    (definitionBytes, declared) <- readPinned reference.definitionUri reference.definitionSha256
    (acceptanceBytes, _) <- readPinned reference.acceptanceUri reference.acceptanceSha256
    definition <- either (ioError . userError) pure (eitherDecode definitionBytes :: Either String Calibration.CalibrationDefinition)
    acceptance <- either (ioError . userError) pure (eitherDecode acceptanceBytes :: Either String Calibration.CalibrationAcceptance)
    unless ((definition.profile, definition.machineProfile, definition.workloadSha256, definition.toolchainSha256, definition.harnessSemanticsSha256) == (entry.profile.name, entry.profile.machineProfile, entry.workloadSha256, expectedToolchain, semantics)) (ioError (userError "calibration definition differs from the current workload, toolchain or harness"))
    unless (not ("pending" `Text.isInfixOf` entry.profile.machineProfile)) (ioError (userError "calibration requires an actual designated machine profile"))
    policy <- require (either (Left . Text.pack) Right (parseEither parseJSON policyValue))
    rule <- case filter (\rule -> matchesRule rule.match definition.targetMetric) (policy :: Policy).metricRules of
      [metricRule] -> pure metricRule
      _ -> ioError (userError "injected metric needs one unambiguous policy rule")
    unless (definition.targetMetric `elem` map (.name) entry.profile.requiredMetrics && rule.direction == LowerIsBetter && definition.targetRelativeEffect > rule.relativeLimit && definition.targetAbsoluteEffect > rule.absoluteFloor) (ioError (userError "injected latency effect must exceed both frozen policy thresholds"))
    aa <- traverse (traverse (replayCalibrationReceipt contracts) . (.receipts)) acceptance.aa
    slowdown <- traverse (traverse (replayCalibrationReceipt contracts) . (.receipts)) acceptance.slowdown
    now <- getCurrentTime
    _ <- require (Calibration.validateCalibration now declared definition acceptance aa slowdown)
    pure reference.acceptanceSha256
  pure (either (Left . Text.pack . displayException) Right outcome)
  where
    readPinned uri digest = do
      (bucket, name) <- require (parseDestination uri)
      (store, _) <- openStore bucket
      retained <- store.getObject bucket (ObjectName name)
      (bytes, metadata) <- maybe (ioError (userError "calibration record is missing")) pure retained
      unless (sha256Hex (Lazy.toStrict bytes) == digest) (ioError (userError "calibration record digest differs"))
      pure (bytes, metadata.updated)

-- A change in the owner's health thresholds invalidates the batch even when
-- hardware and image names stayed the same. Check only compatible pool cells.
checkCalibratedPolicies :: Prepared.PerfPrepared -> ExecutionInputs -> IO ()
checkCalibratedPolicies prepared inputs = when prepared.plan.gatingEligible do
  mapM_
    ( \entry -> do
        definition <- calibrationDefinition entry
        let compatible = [descriptor | (descriptor, _) <- inputs.cells, cellMachineProfile descriptor == entry.profile.machineProfile]
        unless (not (null compatible) && all (\descriptor -> ((sha256Hex . canonicalEncode) <$> Map.lookup descriptor.name inputs.healthPolicies) == Just definition.healthPolicySha256) compatible) (ioError (userError "cell health policy differs from the calibrated policy"))
    )
    prepared.plan.selected

calibrationDefinition :: PlannedProfile -> IO Calibration.CalibrationDefinition
calibrationDefinition entry = do
  reference <- require (either (Left . Text.pack) Right (parseEither (parseJSON @Calibration.CalibrationReference) (fromMaybe Null entry.profile.calibration)))
  (bucket, name) <- require (parseDestination reference.definitionUri)
  (store, _) <- openStore bucket
  retained <- store.getObject bucket (ObjectName name)
  bytes <- maybe (ioError (userError "calibration declaration is missing")) (pure . fst) retained
  unless (sha256Hex (Lazy.toStrict bytes) == reference.definitionSha256) (ioError (userError "calibration declaration digest differs"))
  either (ioError . userError) pure (eitherDecode bytes)

verifyGatingCalibration :: PerfContracts -> Prepared.PerfPrepared -> IO ()
verifyGatingCalibration contracts prepared = when prepared.plan.gatingEligible do
  (semantics, policies, admissions) <- require (either (Left . Text.pack) Right (parseEither (withObject "PinnedInputs" \v -> (,,) <$> v .: "harnessSemanticsSha256" <*> v .: "policies" <*> v .: "calibrationAdmissions") prepared.inputs))
  mapM_
    ( \entry -> do
        policy <- maybe (ioError (userError "pinned calibration policy is absent")) pure (Map.lookup entry.profile.policy (policies :: Map.Map FilePath Value))
        evidence <- verifyCalibrationAdmission contracts entry (Calibration.toolchainDigest prepared.baseline.buildInventory) semantics policy >>= require
        unless (Map.lookup entry.workloadSha256 (admissions :: Map.Map Text Text) == Just evidence) (ioError (userError "pinned calibration admission differs"))
    )
    prepared.plan.selected

replayCalibrationReceipt :: PerfContracts -> Calibration.CalibrationReceipt -> IO Calibration.VerifiedCalibrationTrial
replayCalibrationReceipt contracts reference = do
  (bucket, objectName) <- require (parseDestination reference.uri)
  prefix <- maybe (ioError (userError "calibration must reference a publication receipt")) pure (Text.stripSuffix "/receipt.json" objectName)
  (store, _) <- openStore bucket
  retained <- store.getObject bucket (ObjectName objectName)
  bytes <- maybe (ioError (userError "calibration receipt is missing")) (pure . fst) retained
  unless (sha256Hex (Lazy.toStrict bytes) == reference.sha256) (ioError (userError "calibration receipt digest differs"))
  receipt <- either (ioError . userError) pure (eitherDecode bytes :: Either String PublicationReceipt)
  unless (receipt.manifestUri == "gs://" <> bucket.unBucket <> "/" <> prefix <> "/manifest.json") (ioError (userError "calibration receipt destination differs"))
  unless ("snapshots/" `Text.isPrefixOf` receipt.snapshot && Text.count "/" receipt.snapshot == 1 && Text.all (\c -> c `elem` (['a' .. 'z'] <> ['0' .. '9'] <> ['-', '/'])) receipt.snapshot) (ioError (userError "unsafe calibration snapshot"))
  withSystemTempDirectory "kenshou-calibration-replay-" \root -> do
    let directory = root </> Text.unpack receipt.snapshot
    retrieveEvidence store bucket prefix receipt.manifestSha256 directory >>= require
    Lazy.writeFile (root </> "receipt.json") bytes
    (result, _) <- verifySessionEvidence contracts root True
    prepared <- loadPreparation contracts (directory </> "preparation/prepared.json")
    session <- readPerfSession (directory </> "session.json") >>= require
    unless (result.purpose == "calibration" && not result.gatingEligible && length result.configurations == 1 && length prepared.plan.selected == 1) (ioError (userError "calibration needs a single-profile non-gating report"))
    configuration <- case result.configurations of [retainedConfiguration] -> pure retainedConfiguration; _ -> ioError (userError "missing calibration configuration")
    semantics <- require (either (Left . Text.pack) Right (parseEither (withObject "PinnedInputs" (.: "harnessSemanticsSha256")) prepared.inputs))
    inputs <- require (parseInputs session)
    entry <- case prepared.plan.selected of [selected] -> pure selected; _ -> ioError (userError "missing calibration profile")
    let leasedCells = [identity.cell | event <- session.events, LeaseAcquired identity <- [event.action]]
    actualProfiles <-
      traverse
        (\cell -> maybe (ioError (userError "calibration lease is outside the frozen pool")) (pure . cellMachineProfile . fst) (find ((== cell) . (.name) . fst) inputs.cells))
        leasedCells
    unless (all (== entry.profile.machineProfile) actualProfiles && (result.infrastructureState /= "healthy" || not (null actualProfiles))) (ioError (userError "calibration ran on a different machine or image profile"))
    policies <- traverse (\cell -> maybe (ioError (userError "calibration health policy is absent")) pure (Map.lookup cell inputs.healthPolicies)) leasedCells
    policy <- case policies of first : rest | all (== first) rest -> pure first; [] | result.infrastructureState /= "healthy" -> maybe (ioError (userError "calibration health policy is absent")) pure (Map.lookup (case prepared.plan.request.cells of cell : _ -> cell; [] -> "") inputs.healthPolicies); _ -> ioError (userError "calibration health policy changed across leases")
    pure (Calibration.VerifiedCalibrationTrial session.sessionId prepared.plan.request.seed prepared.plan.request.control prepared.plan.request.project prepared.plan.request.baselineRevision prepared.plan.request.candidateRevision configuration.profile entry.profile.machineProfile (sha256Hex (canonicalEncode policy)) configuration.workloadSha256 (Calibration.toolchainDigest prepared.baseline.buildInventory) semantics configuration.performanceVerdict result.infrastructureState result.blockingRuntimeFailures session.createdAt receipt.totalElapsedSeconds)

runVerify :: FilePath -> Bool -> CliEnv -> IO ExitCode
runVerify output offline environment = guarded do
  (result, uri) <- verifySessionEvidence environment.perf output offline
  Lazy.putStrLn (encode (object ["verified" .= True, "offline" .= offline, "manifestUri" .= uri, "result" .= result]))
  pure (exit result.exitCode)

verifySessionEvidence :: PerfContracts -> FilePath -> Bool -> IO (PerfResult, Text)
verifySessionEvidence contracts output offline = do
  receipt <- readJSON (output </> "receipt.json")
  let relative = (receipt :: PublicationReceipt).snapshot
      uri = receipt.manifestUri
      manifestDigest = receipt.manifestSha256
      resultDigest = receipt.resultSha256
  unless ("snapshots/" `Text.isPrefixOf` relative && Text.all (\c -> c `elem` (['a' .. 'z'] <> ['0' .. '9'] <> ['-', '/'])) relative && Text.count "/" relative == 1) (ioError (userError "unsafe retained snapshot"))
  let directory = output </> Text.unpack relative
  verifyPerfManifest directory >>= require
  manifest <- Bytes.readFile (directory </> "manifest.json")
  unless (sha256Hex manifest == manifestDigest) (ioError (userError "retained manifest differs from durable receipt"))
  prepared <- loadPreparation contracts (directory </> "preparation/prepared.json")
  checkRootInputs directory prepared
  session <- readPerfSession (directory </> "session.json") >>= require
  require (validateSessionInputs contracts prepared session)
  unless offline (verifyGatingCalibration contracts prepared)
  unless (receipt.payloadsReadyAt == session.createdAt) (ioError (userError "receipt timing origin differs from its pinned session"))
  unless (uri == session.evidenceBaseUri <> "/sessions/" <> renderRunId session.sessionId <> "/" <> relative <> "/manifest.json") (ioError (userError "receipt destination differs from its pinned session"))
  inputs <- require (parseInputs session)
  unless offline (checkCalibratedPolicies prepared inputs)
  result <- readJSON (directory </> "result.json")
  unless (sha256Hex (canonicalEncode (toJSON (result :: PerfResult))) == resultDigest) (ioError (userError "retained aggregate differs from durable receipt"))
  unless (maybe False (<= receipt.totalElapsedSeconds) (Map.lookup "totalAtReportAssembly" result.stageSeconds)) (ioError (userError "report timing exceeds its verified publication time"))
  let base = Text.dropEnd (Text.length ("/manifest.json" :: Text)) uri
  unless ([reference.uri | reference <- result.artifactReferences, reference.kind == "aggregate-manifest"] == [uri] && [reference.uri | reference <- result.artifactReferences, reference.kind == "publication-receipt"] == [base <> "/receipt.json"]) (ioError (userError "report does not reference its own publication and receipt"))
  mapM_
    ( \reference -> case reference.sha256 of
        Nothing -> unless (reference.kind `elem` ["aggregate-manifest", "publication-receipt"]) (ioError (userError "artifact reference has no digest"))
        Just expected -> do
          path <- maybe (ioError (userError "artifact reference is outside the sealed snapshot")) pure (Text.stripPrefix (base <> "/") reference.uri)
          unless (all (\part -> not (Text.null part) && part `notElem` [".", ".."]) (Text.splitOn "/" path) && not (Text.any (`elem` ['\\', '\n', '\r']) path)) (ioError (userError "unsafe artifact reference"))
          (actual, _) <- digestFile (directory </> Text.unpack path)
          unless (expected == "sha256:" <> actual) (ioError (userError "artifact reference digest differs"))
    )
    result.artifactReferences
  state <- require (journalState session)
  auditAllAttempts False inputs prepared directory
  unless (maybe True (receipt.verifiedReportAt <=) state.executionDeadline) (ioError (userError "publication receipt exceeds the original execution deadline"))
  let completed = Map.elems state.completed
  valid <- traverse (verifyConfiguration inputs prepared directory) completed
  unless (and valid) (ioError (userError "retained comparison replay differs"))
  recomputed <- require (aggregateResult session.plan session.preparedSha256 result.evidenceVerified result.configurations)
  unless (recomputed == result {stageSeconds = Map.empty, artifactReferences = []} && all (\entry -> any (\retained -> retained.profile == entry.profile && retained.comparison == entry.comparison && retained.runtimeFailures == entry.runtimeFailures) result.configurations) completed) (ioError (userError "aggregate differs from the pinned session results"))
  unless offline do
    destination <- maybe (ioError (userError "receipt has no manifest URI")) pure (Text.stripSuffix "/manifest.json" uri)
    (bucket, prefix) <- require (parseDestination destination)
    (store, _) <- openStore bucket
    verifyPublication store bucket prefix directory >>= require
    stored <- store.getObject bucket (ObjectName (prefix <> "/receipt.json"))
    bytes <- Lazy.readFile (output </> "receipt.json")
    unless (fmap fst stored == Just bytes) (ioError (userError "retained publication receipt differs from durable storage"))
  pure (result, uri)

accessProbe :: ObjectStore -> Text -> RunId -> Prepared.PerfPrepared -> IO ()
accessProbe store uri identifier prepared = do
  (bucket, prefix) <- require (parseDestination uri)
  let name = ObjectName (prefix <> "/access-checks/" <> renderRunId identifier <> ".json")
      bytes = encode (object ["session" .= identifier, "preparedSha256" .= Prepared.preparedDigest prepared])
  outcome <- store.putObject bucket name "application/json" DoesNotExist bytes
  unless (case outcome of Written _ -> True; _ -> False) (ioError (userError "performance evidence destination failed create-only access check"))
  found <- store.getObject bucket name
  unless (fmap fst found == Just bytes) (ioError (userError "performance evidence destination failed retrieval access check"))

parseDestination :: Text -> Either Text (Bucket, Text)
parseDestination uri = do
  suffix <- maybe (Left "evidence destination must be gs://BUCKET/PREFIX") Right (Text.stripPrefix "gs://" uri)
  let (bucket, path) = Text.breakOn "/" suffix
      prefix = Text.drop 1 path
  unless (not (Text.null bucket) && Text.all (\c -> c `elem` ['a' .. 'z'] <> ['0' .. '9'] <> ['-', '.']) bucket && not (Text.null prefix) && all (\part -> not (Text.null part) && part `notElem` [".", ".."]) (Text.splitOn "/" prefix) && not (Text.any (`elem` ['\\', '\n', '\r', ' ', '?', '#']) prefix)) (Left "unsafe evidence destination")
  pure (Bucket bucket, prefix)

copyTree :: FilePath -> FilePath -> IO ()
copyTree source destination = do
  link <- pathIsSymbolicLink source
  when link (ioError (userError "performance artifacts may not contain symbolic links"))
  directory <- doesDirectoryExist source
  if directory
    then do
      createDirectoryIfMissing True destination
      names <- listDirectory source
      mapM_ (\name -> copyTree (source </> name) (destination </> name)) names
    else copyFile source destination

renderReport :: PerfResult -> Text
renderReport result =
  Text.unlines
    ( ["# Runtime performance validation", "", "Performance: " <> result.performanceVerdict, "Coverage: " <> result.coverage <> " (whole change: " <> result.changeCoverage <> ")", "Infrastructure: " <> result.infrastructureState, "Purpose: " <> result.purpose, "Gating eligible: " <> Text.pack (show result.gatingEligible), "Exit: " <> Text.pack (show result.exitCode), "", "| Profile | Verdict | Valid pairs | Infrastructure |", "| --- | --- | --- | --- |"]
        <> ["| " <> item.profile <> " | " <> item.performanceVerdict <> " | " <> Text.pack (show item.validPairs) <> "/" <> Text.pack (show item.requiredPairs) <> " | " <> item.infrastructureState <> " |" | item <- result.configurations]
        <> concatMap effects result.configurations
        <> ["", "Runtime blockers: " <> if null result.blockingRuntimeFailures then "none" else Text.intercalate "; " result.blockingRuntimeFailures]
        <> ["", "## Scope", ""]
        <> ["- Excluded " <> scenario <> ": " <> reason | (scenario, reason) <- result.exclusions]
        <> ["- Uncovered: " <> scenario | scenario <- result.uncoveredScenarios]
        <> ["", "## Timings", "", "| Stage | Seconds |", "| --- | ---: |"]
        <> ["| " <> stage <> " | " <> number seconds <> " |" | (stage, seconds) <- Map.toAscList result.stageSeconds]
        <> ["", "Full report retrieval time is recorded in the linked publication receipt.", "", "## Evidence", ""]
        <> ["- [" <> reference.kind <> "](" <> reference.uri <> ")" <> maybe "" (\digest -> " (`" <> digest <> "`)") reference.sha256 | reference <- result.artifactReferences]
    )
  where
    number amount = Text.pack (printf "%.4g" (amount :: Double))
    effects configuration = case configuration.comparison of
      Nothing -> []
      Just document -> case parseEither metrics document of
        Left problem -> ["", "Retained metric report could not be rendered: " <> Text.pack problem]
        Right (rows, reasons) -> ["", "## " <> configuration.profile, "", "Intervals and thresholds below are retained comparison results.", "", "| Metric | Unit | Ratio estimate [low, high] | Delta estimate [low, high] | Relative limit | Absolute floor | Status | Explanation |", "| --- | --- | --- | --- | ---: | ---: | --- | --- |"] <> rows <> ["", "Comparison reasons: " <> if null reasons then "none" else Text.intercalate "; " reasons]
    metrics = withObject "Comparison" \document -> do
      values <- document .: "metrics"
      reasons <- document .:? "reasons" .!= []
      rows <- traverse metric (Map.toAscList (values :: Map.Map Text Value))
      pure (rows, reasons)
    metric (name, document) =
      withObject
        "MetricComparison"
        ( \fields -> do
            unit <- fields .: "unit"
            ratio <- fields .: "ratio" >>= interval
            delta <- fields .: "delta" >>= interval
            limit <- fields .: "relativeLimit"
            floorValue <- fields .: "absoluteFloor"
            status <- fields .: "status"
            message <- fields .: "message"
            pure ("| " <> name <> " | " <> unit <> " | " <> ratio <> " | " <> delta <> " | " <> number limit <> " | " <> number floorValue <> " | " <> status <> " | " <> Text.replace "\n" " " (Text.replace "|" "\\|" message) <> " |")
        )
        document
    interval = withObject "Interval" \fields -> do
      low <- fields .: "low"
      estimate <- fields .: "estimate"
      high <- fields .: "high"
      pure (number estimate <> " [" <> number low <> ", " <> number high <> "]")

openStore :: Bucket -> IO (ObjectStore, Text)
openStore bucket = do
  selected <- lookupEnv "KENSHOU_CELL_STORE"
  case selected of
    Just setting -> do
      path <- maybe (ioError (userError "KENSHOU_CELL_STORE must be file:<dir> or unset for GCS")) (pure . Text.unpack) (Text.stripPrefix "file:" (Text.pack setting))
      when (null path) (ioError (userError "file-backed cell store directory is empty"))
      absolute <- makeAbsolute path
      store <- newFileStore absolute
      pure (store, "file:" <> Text.pack absolute)
    Nothing -> do
      tokens <- newTokenProvider bucket
      store <- newGcsStore tokens
      pure (store, "gcs")

readJSON :: (FromJSON a) => FilePath -> IO a
readJSON path = eitherDecodeFileStrict' path >>= either (ioError . userError . ((path <> ": ") <>)) pure

decode :: (FromJSON a) => Value -> Either Text a
decode document = case fromJSON document of Error problem -> Left (Text.pack problem); Success result -> Right result

require :: Either Text a -> IO a
require = either (ioError . userError . Text.unpack) pure

input :: Either Text a -> IO a
input = either (throwIO . SessionFailure 2) pure

exit :: Int -> ExitCode
exit 0 = ExitSuccess
exit code = ExitFailure code

guarded :: IO ExitCode -> IO ExitCode
guarded action = catch executeFailure (\(SessionFailure code problem) -> TextIO.hPutStrLn stderr problem >> pure (exit code))
  where
    executeFailure = do
      result <- try @IOException action
      case result of
        Right code -> pure code
        Left problem -> do
          TextIO.hPutStrLn stderr (Text.pack (displayException problem))
          pure (ExitFailure 4)