packages feed

kenshou-cli-0.1.0.0: src/Kenshou/Cli/Command/Cell.hs

module Kenshou.Cli.Command.Cell (cellCommand, startCellInstances) where

import Control.Concurrent (threadDelay)
import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)
import Data.Aeson (FromJSON, Value (..), eitherDecode, eitherDecodeFileStrict', encode, object, (.=))
import Data.Aeson qualified as Aeson
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.ByteString.Lazy.Char8 qualified as LazyByteString
import Data.Foldable (traverse_)
import Data.List (find, nub, sort)
import Data.List.NonEmpty (NonEmpty (..))
import Data.List.NonEmpty qualified as NonEmpty
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe, isNothing)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as TextIO
import Data.Time (diffUTCTime, getCurrentTime)
import Kenshou.Cli.Command.Overhead qualified as LocalOverhead
import Kenshou.Core.Bundle (lookupScenario, registryRoutingRules)
import Kenshou.Core.Cli (CliCommand (..), CliEnv (..), CliGroup (..))
import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))
import Kenshou.Core.Id (RunId, mkSeed, newRunId, parseRunId, parseScenarioId, renderRunId, unSeed)
import Kenshou.Core.RunSpec (EnvironmentSpec (..), RunSpec (..), SpecPlacement (..))
import Kenshou.Core.Scenario (Scenario (..))
import Kenshou.Diagnose.Leak (analyseRunDirectory, defaultLeakSpec)
import Kenshou.Measure.Compare (CompareError (..), Comparison (..), compareRuns, verdictExitCode)
import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..), compatibilityInputs, varyingValue)
import Kenshou.Measure.Compare.Ordering (Arm (..), PairedOrdering (..))
import Kenshou.Measure.Compare.Policy (Policy (..), decodePolicy)
import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))
import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))
import Kenshou.Plan.Policy (PlanPolicy (..), defaultPlanPolicy)
import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (PlanInputs), PlannedRun (..), RunPlan (..), TrialInfo (..))
import Kenshou.Plan.Selector (parseSelector)
import Kenshou.Remote.Cell.Control (CellSnapshot (..), readCellSnapshot)
import Kenshou.Remote.Cell.Debug (DebugCommand (..), runCellDebug)
import Kenshou.Remote.Cell.Docs (CachePolicy (..), CellBuckets (..), CellDescriptor (..), CellManifest (..), CellNodes (..), CellOutcome (..), CellStatus (..), Limits (..), Rejected (..), Submission (..))
import Kenshou.Remote.Cell.Exec (cellExec)
import Kenshou.Remote.Cell.Fetch (FetchError (..), fetchCellRun, verifyCellRun)
import Kenshou.Remote.Cell.Index (deriveCellRunIndex, writeCellRunIndex)
import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), Quarantine (..), acquireLease, leaseSnapshot, reattachLease, releaseLease, resizeLease, validCellName, withHeartbeat)
import Kenshou.Remote.Cell.Parity (ParityOptions (..), ParityReport (..), compareForParity)
import Kenshou.Remote.Cell.Prepare (Granularity (..), OtlpSink (..), PrepareOptions (..))
import Kenshou.Remote.Cell.RouteJson (RouteDocuments (..), routeWorkJson)
import Kenshou.Remote.Cell.RouteRules (CellCapabilities (..), descriptorDigest, loadCellCapabilities)
import Kenshou.Remote.Cell.Session (SessionTransition (..))
import Kenshou.Remote.Cell.Session.Build (BuildOptions (..), BuiltSession (..), buildSession)
import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)
import Kenshou.Remote.Cell.Session.Resume (resumeHeldSession, resumeObservedSlices)
import Kenshou.Remote.Cell.Session.Runner (runPlannedSlices)
import Kenshou.Remote.Cell.Submit (PublishOutcome (Submitted), publishSubmission)
import Kenshou.Remote.Cell.Watch (WatchEvent (..), WatchTerminal (..), watchCellRun)
import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)
import Kenshou.Remote.Pair (PairRequest (..), PairState (..), SliceResult (..), judgePairs, pairPlan)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))
import Kenshou.Remote.Payload.Publisher (PublishError (..), PublishOptions (..), publishPayload)
import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..))
import Kenshou.Remote.Store.File (newFileStore)
import Kenshou.Remote.Store.Gcs (newGcsStore, newTokenProvider)
import Kenshou.Telemetry.Overhead (OverheadHooks (..), OverheadPlan (..), OverheadReport (..), OverheadRequest (..), OverheadState (..), UsageError (..), analyseOverhead, executeOverhead, loadOverheadState, overheadVerdictExitCode, planOverhead)
import Kenshou.Telemetry.Overhead.Policy (OverheadPolicy)
import Options.Applicative
import System.Directory (createDirectoryIfMissing, createDirectoryLink, doesDirectoryExist, doesPathExist, makeAbsolute, renameFile)
import System.Environment (getExecutablePath, lookupEnv)
import System.Exit (ExitCode (..))
import System.FilePath (takeDirectory, (</>))
import System.IO (hFlush, stderr, stdout)
import System.IO.Temp (withSystemTempDirectory)
import System.Process (readProcessWithExitCode)

data CellLocation = CellLocation !Text !(Maybe Text)

data LeaseCli = LeaseCli !CellLocation !Text !(Maybe Text) !Int !Int !Bool !Bool !Bool

data RouteCli = RouteCli ![Text] !(Maybe Text) !FilePath !(Maybe FilePath) !(Maybe FilePath) !FilePath !Bool !Bool

data SubmitSettings = SubmitSettings ![String] !FilePath !FilePath !Granularity !CachePolicy ![Text] !Bool !Bool !Bool !(Maybe FilePath) !OtlpSink !(Maybe Text) !Bool

data SubmitCli = SubmitCli !CellLocation !Text !SubmitSettings !Bool

data RunCli = RunCli !CellLocation !SubmitSettings !Int !Int !Bool !Bool

data ProbeCli = ProbeCli !CellLocation !FilePath !(Maybe FilePath)

data PairCli = PairCli !CellLocation !FilePath !FilePath !FilePath !(Maybe Int) !PairedOrdering !FilePath ![Text] !Int !Bool !FilePath

data PairObservation = PairObservation !PlannedRun !FilePath !Bool

data OverheadCli = OverheadCli !CellLocation !FilePath !OtlpSink ![Text] !Bool !LocalOverhead.OverheadOptions

data PayloadPublishCli = PayloadPublishCli !Text !Text !(Maybe Text) !FilePath !FilePath !Bool

data CellAction
  = Fetch !Text !Text !FilePath
  | Verify !FilePath
  | Parity !FilePath !FilePath !(Maybe FilePath) ![Text]
  | Status !CellLocation !Bool
  | LeaseCell !LeaseCli
  | Release !CellLocation !Text
  | Watch !CellLocation !Text
  | Debug !CellLocation !DebugCommand
  | Route !RouteCli
  | Submit !SubmitCli
  | Run !RunCli
  | Probe !ProbeCli
  | Pair !PairCli
  | Overhead !OverheadCli
  | Resume !FilePath !(Maybe Text) !Bool
  | PublishPayload !PayloadPublishCli
  | ShowPayload !FilePath
  | Exec !FilePath !FilePath

cellCommand :: CliCommand
cellCommand = CliCommand "cell" "Run and inspect leased verification cells" Execution False (runCell <$> cellParser)

cellParser :: Parser CellAction
cellParser =
  hsubparser $
    command "fetch" (info (fetchParser <**> helper) (progDesc "Fetch and verify one sealed cell run"))
      <> command "verify" (info (verifyParser <**> helper) (progDesc "Verify a fetched tree or a sealed GCS run"))
      <> command "parity" (info (parityParser <**> helper) (progDesc "Compare local and cell verdicts with named placement differences"))
      <> command "status" (info (statusParser <**> helper) (progDesc "Inspect cell descriptor and lease state"))
      <> command "lease" (info (leaseParser <**> helper) (progDesc "Acquire a cell lease"))
      <> command "release" (info (releaseParser <**> helper) (progDesc "Release an owned cell lease"))
      <> command "watch" (info (watchParser <**> helper) (progDesc "Follow a cell run to its terminal status"))
      <> command "debug" (info (debugParser <**> helper) (progDesc "Inspect a cell role through the owner IAP script"))
      <> command "route" (info (routeParser <**> helper) (progDesc "Split a plan across compatible cells"))
      <> command "submit" (info (submitParser <**> helper) (progDesc "Submit a prepared plan under an existing lease"))
      <> command "run" (info (runParser <**> helper) (progDesc "Lease, submit, verify and release a cell session"))
      <> command "probe" (info (probeParser <**> helper) (progDesc "Run a cell environment probe and cache capabilities"))
      <> command "pair" (info (pairParser <**> helper) (progDesc "Measure interleaved payload arms under one cell lease"))
      <> command "overhead" (info (overheadCellParser <**> helper) (progDesc "Measure telemetry overhead under one cell lease"))
      <> command "resume" (info (resumeParser <**> helper) (progDesc "Continue a saved cell session"))
      <> command "payload" (info (payloadParser <**> helper) (progDesc "Build and publish a checked cell payload"))
      <> (command "exec" (info (cellExecParser <**> helper) (progDesc "Execute prepared work on a cell driver")) <> internal)

payloadParser :: Parser CellAction
payloadParser =
  hsubparser $
    command "publish" (info (publishPayloadParser <**> helper) (progDesc "Publish a Nix closure after checking its runtime cohort"))
      <> command "show" (info (ShowPayload <$> strArgument (metavar "FILE") <**> helper) (progDesc "Show a payload descriptor after checking its bundle exists"))

publishPayloadParser :: Parser CellAction
publishPayloadParser =
  PublishPayload
    <$> ( PayloadPublishCli
            <$> strOption (long "cohort" <> metavar "released|head" <> help "Pinned runtime cohort")
            <*> strOption (long "variant" <> metavar "default|info-table|profiled" <> value "default" <> showDefault <> help "Executable build variant")
            <*> optional (strOption (long "control-bucket" <> metavar "BUCKET" <> help "Payload control bucket"))
            <*> strOption (long "out" <> metavar "FILE" <> help "Payload descriptor JSON output")
            <*> strOption (long "root" <> metavar "DIR" <> value "." <> showDefault <> help "Flake and cohort descriptor root")
            <*> switch (long "allow-dirty" <> help "Permit a payload from a dirty worktree")
        )

cellLocationParser :: Parser CellLocation
cellLocationParser =
  CellLocation
    <$> strOption (long "cell" <> metavar "NAME" <> help "Cell name")
    <*> optional (strOption (long "control-bucket" <> metavar "BUCKET" <> help "Cell control bucket"))

statusParser :: Parser CellAction
statusParser = Status <$> cellLocationParser <*> switch (long "json" <> help "Print JSON")

leaseParser :: Parser CellAction
leaseParser =
  LeaseCell
    <$> ( LeaseCli
            <$> cellLocationParser
            <*> strOption (long "purpose" <> metavar "TEXT" <> help "Purpose recorded in the lease")
            <*> optional (strOption (long "owner" <> metavar "TEXT" <> help "Lease owner; defaults to USER"))
            <*> option auto (long "ttl" <> metavar "SECONDS" <> value 120 <> showDefault <> help "Lease lifetime between renewals")
            <*> option auto (long "wait" <> metavar "SECONDS" <> value 0 <> showDefault <> help "Wait for a busy lease")
            <*> switch (long "start" <> help "Start stopped cell instances before acquiring a lease")
            <*> switch (long "hold" <> help "Renew in the foreground until interrupted")
            <*> switch (long "json" <> help "Print the lease document as JSON")
        )

releaseParser :: Parser CellAction
releaseParser = Release <$> cellLocationParser <*> strOption (long "lease-id" <> metavar "UUID" <> help "Expected lease identifier")

watchParser :: Parser CellAction
watchParser = Watch <$> cellLocationParser <*> strArgument (metavar "CELL_RUN_ID")

debugParser :: Parser CellAction
debugParser =
  Debug
    <$> cellLocationParser
    <*> hsubparser
      ( command "ssh" (info (DebugSsh <$> strArgument (metavar "ROLE") <*> many (strArgument (metavar "COMMAND")) <**> helper) (progDesc "Open a role shell or run a remote command"))
          <> command "journal" (info (pure DebugJournal <**> helper) (progDesc "Read the driver agent journal"))
          <> command "tunnel" (info (DebugTunnel <$> strArgument (metavar "ROLE") <*> argument auto (metavar "REMOTE_PORT") <*> argument auto (metavar "LOCAL_PORT") <**> helper) (progDesc "Forward a local port through IAP SSH until interrupted"))
      )

routeParser :: Parser CellAction
routeParser =
  Route
    <$> ( RouteCli
            <$> some (strOption (long "cell" <> metavar "NAME" <> help "Candidate cell in preference order; repeat to add cells"))
            <*> optional (strOption (long "control-bucket" <> metavar "BUCKET" <> help "Cell control bucket"))
            <*> strOption (long "plan" <> metavar "FILE" <> help "Run plan JSON, or - for stdin")
            <*> optional (strOption (long "payload" <> metavar "FILE" <> help "Payload descriptor for cohort preparation"))
            <*> optional (strOption (long "routing-rules" <> metavar "FILE" <> help "Replace the built-in routing policy with JSON, or - for stdin"))
            <*> strOption (long "out" <> metavar "DIR" <> help "Directory for routed plans and refusal report")
            <*> switch (long "coerce-durable" <> help "Use durable PostgreSQL when the scenario supports it")
            <*> switch (long "ephemeral-on-driver" <> help "Keep server-control correctness work on the cell driver")
        )

submitParser :: Parser CellAction
submitParser =
  Submit
    <$> ( SubmitCli
            <$> cellLocationParser
            <*> strOption (long "lease-id" <> metavar "UUID" <> help "Active lease identifier")
            <*> submitSettingsParser (switch (long "dry-run" <> help "Print a planned session without writing or submitting"))
            <*> pure False
        )

runParser :: Parser CellAction
runParser =
  Run
    <$> ( RunCli
            <$> cellLocationParser
            <*> submitSettingsParser (pure False)
            <*> option auto (long "ttl" <> metavar "SECONDS" <> value 120 <> showDefault <> help "Held lease lifetime between renewals")
            <*> option auto (long "wait" <> metavar "SECONDS" <> value 0 <> showDefault <> help "Wait for a busy cell")
            <*> switch (long "start" <> help "Start stopped cell instances before acquiring a lease")
            <*> switch (long "detach" <> help "Submit one slice and leave its budgeted lease to run after this process exits")
        )

probeParser :: Parser CellAction
probeParser =
  Probe
    <$> ( ProbeCli
            <$> cellLocationParser
            <*> strOption (long "payload" <> metavar "FILE" <> help "Published payload descriptor with the cell-environment scenario")
            <*> optional (strOption (long "out" <> metavar "FILE" <> help "Capability cache JSON; defaults to the configured cache directory"))
        )

pairParser :: Parser CellAction
pairParser =
  Pair
    <$> ( PairCli
            <$> cellLocationParser
            <*> strOption (long "candidate" <> metavar "FILE" <> help "Candidate payload descriptor")
            <*> strOption (long "baseline" <> metavar "FILE" <> help "Baseline payload descriptor")
            <*> strOption (long "plan" <> metavar "FILE" <> help "Benchmark run plan JSON")
            <*> optional (option auto (long "pairs" <> metavar "N" <> help "Requested valid pairs; defaults to the policy minimum"))
            <*> option (eitherReader parsePairOrdering) (long "ordering" <> metavar "abba|baab" <> value ABBA <> showDefaultWith (const "abba") <> help "Order baseline and candidate in adjacent pairs")
            <*> strOption (long "policy" <> metavar "FILE" <> value "policies/default.json" <> showDefault <> help "Comparison policy JSON")
            <*> many (strOption (long "pg-setting" <> metavar "KEY=VALUE" <> help "PostgreSQL reset setting; repeat as needed"))
            <*> option auto (long "max-replacements" <> metavar "N" <> value (-1) <> help "Additional pairs after invalid trials; defaults to requested pairs")
            <*> switch (long "start" <> help "Start stopped cell instances before acquiring a lease")
            <*> strOption (long "out" <> metavar "DIR" <> help "New directory for paired cell sessions and comparison")
        )

parsePairOrdering :: String -> Either String PairedOrdering
parsePairOrdering "abba" = Right ABBA
parsePairOrdering "baab" = Right BAAB
parsePairOrdering _ = Left "expected abba or baab"

overheadCellParser :: Parser CellAction
overheadCellParser =
  Overhead
    <$> ( OverheadCli
            <$> cellLocationParser
            <*> strOption (long "payload" <> metavar "FILE" <> help "Payload descriptor for every telemetry arm")
            <*> option (eitherReader parseOtlpSink) (long "otlp-sink" <> metavar "null|file" <> value NullSink <> showDefaultWith (const "null") <> help "Cell OTLP sink")
            <*> many (strOption (long "pg-setting" <> metavar "KEY=VALUE" <> help "PostgreSQL reset setting; repeat as needed"))
            <*> switch (long "start" <> help "Start stopped cell instances before acquiring a lease")
            <*> LocalOverhead.overheadParser
        )

submitSettingsParser :: Parser Bool -> Parser SubmitSettings
submitSettingsParser dryFlag =
  SubmitSettings
    <$> some (strOption (long "payload" <> metavar "[LABEL=]FILE" <> help "Payload descriptor; repeat for trial arms"))
    <*> strOption (long "plan" <> metavar "FILE" <> help "Run plan JSON, or - for stdin")
    <*> strOption (long "out" <> metavar "DIR" <> help "New session directory")
    <*> option (eitherReader parseGranularity) (long "granularity" <> metavar "auto|plan|run" <> value GranularityAuto <> showDefaultWith (const "auto") <> help "Submission slice boundary")
    <*> option (eitherReader parseCachePolicy) (long "cache-policy" <> metavar "cold|warm" <> value Cold <> showDefaultWith (const "cold") <> help "Cell cache reset policy")
    <*> many (strOption (long "pg-setting" <> metavar "KEY=VALUE" <> help "PostgreSQL reset setting; repeat as needed"))
    <*> switch (long "skip-incompatible" <> help "Record incompatible runs and submit compatible ones")
    <*> switch (long "coerce-durable" <> help "Use durable PostgreSQL when the scenario supports it")
    <*> switch (long "ephemeral-on-driver" <> help "Keep server-control correctness work on the cell driver")
    <*> optional (strOption (long "routing-rules" <> metavar "FILE" <> help "Replace the built-in routing policy JSON"))
    <*> option (eitherReader parseOtlpSink) (long "otlp-sink" <> metavar "null|file" <> value NullSink <> showDefaultWith (const "null") <> help "Cell OTLP sink")
    <*> optional (strOption (long "rts" <> metavar "OPTS" <> help "Runtime system options passed to the payload"))
    <*> dryFlag

parseGranularity :: String -> Either String Granularity
parseGranularity "auto" = Right GranularityAuto
parseGranularity "plan" = Right GranularityPlan
parseGranularity "run" = Right GranularityRun
parseGranularity _ = Left "expected auto, plan or run"

parseCachePolicy :: String -> Either String CachePolicy
parseCachePolicy "cold" = Right Cold
parseCachePolicy "warm" = Right Warm
parseCachePolicy _ = Left "expected cold or warm"

parseOtlpSink :: String -> Either String OtlpSink
parseOtlpSink "null" = Right NullSink
parseOtlpSink "file" = Right FileSink
parseOtlpSink _ = Left "expected null or file"

resumeParser :: Parser CellAction
resumeParser =
  Resume
    <$> strOption (long "session" <> metavar "DIR" <> help "Directory containing session.json")
    <*> optional (strOption (long "lease-id" <> metavar "UUID" <> help "Use this active lease instead of the journal lease"))
    <*> switch (long "start" <> help "Start stopped cell instances before resuming")

cellExecParser :: Parser CellAction
cellExecParser = Exec <$> strArgument (metavar "WORK_FILE") <*> strArgument (metavar "OUT_DIR")

fetchParser :: Parser CellAction
fetchParser =
  Fetch
    <$> strOption (long "results-bucket" <> metavar "BUCKET" <> help "Results bucket containing the sealed run")
    <*> strArgument (metavar "CELL_RUN_ID")
    <*> strOption (long "out" <> metavar "DIR" <> help "Directory for the fetched cell run")

verifyParser :: Parser CellAction
verifyParser = Verify <$> strArgument (metavar "DIR_OR_GS_URI")

parityParser :: Parser CellAction
parityParser =
  Parity
    <$> strOption (long "local" <> metavar "DIR" <> help "Local run directory")
    <*> strOption (long "cell" <> metavar "DIR" <> help "Fetched nested cell run directory")
    <*> optional (strOption (long "out" <> metavar "FILE" <> help "Write the parity report to FILE instead of stdout"))
    <*> many (strOption (long "volatile" <> metavar "PATH" <> help "Allow a named volatile verdict field; repeat as needed"))

runCell :: CellAction -> CliEnv -> IO ExitCode
runCell selected cli = case selected of
  PublishPayload (PayloadPublishCli cohort variant selectedBucket output root allowDirty) -> do
    configured <- lookupEnv "KENSHOU_CELL_CONTROL_BUCKET"
    let bucketName = fromMaybe (maybe "tan-nb-exp-cells-control" Text.pack configured) selectedBucket
    case validateBucket bucketName of
      Left problem -> usage problem
      Right bucket -> guardIO do
        store <- openStore bucket
        published <- publishPayload store (PublishOptions root cohort variant bucket output allowDirty)
        case published of
          Left (PublishInvalidSelection problem) -> usage problem
          Left (PublishCohortMismatch problem) -> failVerification problem
          Left PublishDirtyWorktree -> unavailable "worktree is dirty; pass --allow-dirty to publish it"
          Left problem -> unavailable (Text.pack (show problem))
          Right _ -> TextIO.putStrLn (Text.pack output) >> pure ExitSuccess
  ShowPayload path -> guardIO do
    decoded <- eitherDecodeFileStrict' path
    case decoded of
      Left problem -> failVerification (Text.pack problem)
      Right (descriptor :: PayloadDescriptor) -> case parsePayloadUri descriptor.cell.bundle.uri of
        Left problem -> failVerification problem
        Right (bucket, bundleObject) -> do
          store <- openStore bucket
          observed <- store.statObject bucket bundleObject
          case observed of
            Nothing -> failVerification "payload bundle does not exist"
            Just meta
              | meta.size /= descriptor.cell.bundle.bytes -> failVerification "payload bundle size differs from descriptor"
              | otherwise -> LazyByteString.putStrLn (encode descriptor) >> pure ExitSuccess
  Fetch bucket identifier outDir -> case (validateBucket bucket, parseRunId identifier) of
    (Left problem, _) -> usage problem
    (_, Left problem) -> usage problem
    (Right resultsBucket, Right cellRun) -> guardIO $ do
      store <- openStore resultsBucket
      fetched <- fetchCellRun store resultsBucket cellRun outDir
      case fetched of
        Left problem -> fetchFailure problem
        Right tree -> do
          indexed <- deriveCellRunIndex resultsBucket Nothing tree
          case indexed of
            Left problems -> failVerification (Text.pack (show (NonEmpty.toList problems)))
            Right index -> do
              path <- writeCellRunIndex tree index
              TextIO.putStrLn (Text.pack path)
              pure ExitSuccess
  Verify location -> case Text.stripPrefix "gs://" (Text.pack location) of
    Nothing -> guardIO $ do
      let nested = location </> "tree"
      isRunDirectory <- doesDirectoryExist nested
      exists <- doesDirectoryExist location
      if exists then verifyTree (if isRunDirectory then nested else location) else unavailable "verification directory does not exist"
    Just suffix -> case parseResultsUri suffix of
      Left problem -> usage problem
      Right (bucket, identifier) -> guardIO $
        withSystemTempDirectory "kenshou-cell-verify" \temporary -> do
          store <- openStore bucket
          fetched <- fetchCellRun store bucket identifier temporary
          case fetched of
            Left problem -> fetchFailure problem
            Right tree -> verifyTree tree
  Parity localDir cellDir output volatile
    | any (\path -> not ("result.summaries.verdicts." `Text.isPrefixOf` path || "verdicts." `Text.isPrefixOf` path)) volatile -> usage "--volatile must name a field under result.summaries.verdicts or verdicts"
    | otherwise -> guardIO do
        report <- compareForParity (ParityOptions volatile) localDir cellDir
        case output of
          Nothing -> LazyByteString.putStrLn (encode report)
          Just path -> do
            createDirectoryIfMissing True (takeDirectory path)
            LazyByteString.writeFile path (encode report)
            TextIO.putStrLn (Text.pack path)
        pure (if null report.unexpected then ExitSuccess else ExitFailure 1)
  Status location asJson -> withControl location \_ _ snapshot -> do
    if asJson
      then LazyByteString.putStrLn (encode (object ["descriptor" .= snapshot.descriptor, "lease" .= snapshot.lease, "quarantine" .= snapshot.quarantine]))
      else do
        let descriptor = snapshot.descriptor
        TextIO.putStrLn ("cell " <> descriptor.name <> " project=" <> descriptor.project <> " zone=" <> descriptor.zone <> " pg=" <> Text.pack (show descriptor.postgresMajor))
        TextIO.putStrLn ("lease " <> maybe "none" (renderRunId . (.leaseId)) snapshot.lease)
        TextIO.putStrLn ("quarantine " <> maybe "none" (.reason) snapshot.quarantine)
    pure ExitSuccess
  Debug location debugCommand -> withControl location \_ _ snapshot -> do
    result <- runCellDebug snapshot.descriptor debugCommand
    either unavailable pure result
  LeaseCell (LeaseCli location purpose owner ttl waitSeconds startInstances hold asJson)
    | ttl <= 0 || waitSeconds < 0 || Text.null purpose || maybe False Text.null owner -> usage "lease purpose and TTL must be positive; wait cannot be negative"
    | otherwise -> withControl location \store ref observed -> do
        if startInstances then startCellInstances observed.descriptor else pure ()
        defaultOwner <- Text.pack . fromMaybe "kenshou" <$> lookupEnv "USER"
        let request = LeaseRequest (fromMaybe defaultOwner owner) purpose ttl
        start <- getCurrentTime
        let acquire = do
              outcome <- acquireLease store ref request
              case outcome of
                Acquired handle -> do
                  record <- leaseSnapshot handle
                  if asJson
                    then LazyByteString.putStrLn (encode record)
                    else TextIO.putStrLn ("lease " <> renderRunId record.leaseId <> " ttl=" <> Text.pack (show record.ttlSeconds))
                  if hold
                    then withHeartbeat store ref handle (holdLease) `finally` (do _ <- releaseLease store ref handle; pure ())
                    else pure ExitSuccess
                Busy current -> do
                  now <- getCurrentTime
                  if diffUTCTime now start < fromIntegral waitSeconds
                    then threadDelay 2000000 >> acquire
                    else unavailable ("cell is busy under lease " <> renderRunId current.leaseId)
                Quarantined record -> unavailable ("cell is quarantined: " <> record.reason)
        acquire
  Release location leaseText -> case parseRunId leaseText of
    Left problem -> usage problem
    Right expected -> withControl location \store ref _ -> do
      reattached <- reattachLease store ref expected
      case reattached of
        Nothing -> unavailable "cell has no matching lease"
        Just handle -> do
          released <- releaseLease store ref handle
          if released then TextIO.putStrLn ("released " <> leaseText) >> pure ExitSuccess else unavailable "cell lease changed before release"
  Watch location runText -> case parseRunId runText of
    Left problem -> usage problem
    Right identifier -> withControl location \store ref _ -> do
      terminal <- watchCellRun store ref identifier emitWatchEvent
      case terminal of
        RunRejected rejected -> do
          TextIO.hPutStrLn stderr ("rejected: " <> rejected.reason)
          pure (if rejected.reason `elem` ["lease-mismatch", "payload-digest-mismatch"] then ExitFailure 4 else ExitFailure 2)
        RunSealed status -> do
          TextIO.hPutStrLn stderr ("sealed: " <> Text.pack (show status.outcome))
          pure (if status.outcome == Just Completed then ExitSuccess else ExitFailure 4)
  Route (RouteCli names bucket planFile payloadFile rulesFile outDir coerce ephemeral)
    | length (nub names) /= length names || not (all validCellName names) -> usage "cell names must be valid and unique"
    | planFile == "-" && rulesFile == Just "-" -> usage "only one JSON document may be read from stdin"
    | payloadFile == Just "-" -> usage "payload must name a file"
    | otherwise -> case NonEmpty.nonEmpty names of
        Nothing -> usage "at least one cell is required"
        Just (first :| rest) -> withControl (CellLocation first bucket) \store ref observed -> do
          allowed <- maybe ["tan-nb-exp"] (map Text.strip . Text.splitOn "," . Text.pack) <$> lookupEnv "KENSHOU_GCP_ALLOWED_PROJECTS"
          remaining <- traverse (\name -> readCellSnapshot store (CellRef name ref.controlBucket) allowed) rest
          case sequence remaining of
            Left problem -> unavailable problem
            Right snapshots -> do
              let descriptors = observed.descriptor : fmap (.descriptor) snapshots
              capabilityResult <- loadCapabilitiesFor descriptors
              planResult <- readJsonDocument planFile
              rulesResult <- maybe (pure (Right (registryRoutingRules cli.registry))) readJsonDocument rulesFile
              payloadResult <- traverse eitherDecodeFileStrict' payloadFile
              case (capabilityResult, planResult, rulesResult, sequence payloadResult) of
                (Left problem, _, _, _) -> usage problem
                (_, Left failure, _, _) -> usage (Text.pack failure)
                (_, _, Left failure, _) -> usage (Text.pack failure)
                (_, _, _, Left failure) -> usage (Text.pack failure)
                (Right caches, Right plan, Right rules, Right payload) ->
                  case NonEmpty.nonEmpty (zip descriptors caches) >>= \cells -> Just (routeWorkJson cli.registry cells rules (Map.singleton "default" <$> payload) (PrepareOptions coerce ephemeral [] Cold) plan) of
                    Nothing -> usage "at least one cell is required"
                    Just (Left problem) -> usage problem
                    Just (Right documents) -> guardIO do
                      exists <- doesPathExist outDir
                      if exists
                        then usage "route output path already exists; choose a new directory"
                        else do
                          createDirectoryIfMissing True outDir
                          traverse_ (\(name, planDocument) -> LazyByteString.writeFile (outDir </> "plan." <> Text.unpack name <> ".json") (encode planDocument)) (Map.toAscList documents.perCell)
                          traverse_ (\planDocument -> LazyByteString.writeFile (outDir </> "plan.local.json") (encode planDocument)) documents.local
                          LazyByteString.writeFile (outDir </> "unroutable.json") (encode documents.report)
                          TextIO.hPutStrLn stderr ("routed plans: " <> Text.pack (show (Map.size documents.perCell)) <> "; see " <> Text.pack (outDir </> "unroutable.json"))
                          pure (if documents.routeComplete then ExitSuccess else ExitFailure 2)
  Submit (SubmitCli location leaseText (SubmitSettings payloadFiles planFile outDir granularity cache settingTexts skip coerce ephemeral rulesFile sink rts dryRun) detached)
    | planFile == "-" && rulesFile == Just "-" -> usage "only one JSON document may be read from stdin"
    | otherwise -> case (parseRunId leaseText, traverse parsePgSetting settingTexts) of
        (Left problem, _) -> usage problem
        (_, Left problem) -> usage problem
        (Right leaseId, Right settings) -> withControl location \store ref observed -> do
          handle <- reattachLease store ref leaseId
          case handle of
            Nothing -> unavailable "cell has no matching lease"
            Just active -> do
              payloadResult <- loadPayloads payloadFiles
              rulesResult <- maybe (pure (Right (registryRoutingRules cli.registry))) readJsonDocument rulesFile
              case (payloadResult, rulesResult) of
                (Left problem, _) -> usage problem
                (_, Left failure) -> usage (Text.pack failure)
                (Right payloads, Right rules) -> do
                  capabilityResult <- loadCapabilitiesFor [observed.descriptor]
                  case capabilityResult of
                    Left problem -> usage problem
                    Right [cachedCapabilities] -> do
                      bytes <- if planFile == "-" then LazyByteString.getContents else LazyByteString.readFile planFile
                      selectedStore <- lookupEnv "KENSHOU_CELL_STORE"
                      let options = BuildOptions (PrepareOptions coerce ephemeral settings cache) granularity skip sink rts (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0"
                          storeLabel = Text.pack (fromMaybe "gs://" selectedStore)
                      built <- buildSession cli.registry observed.descriptor cachedCapabilities rules payloads options storeLabel leaseId bytes
                      case built of
                        Left problem -> usage problem
                        Right session
                          | dryRun -> LazyByteString.putStrLn (encode session.journal) >> pure ExitSuccess
                          | detached && length session.journal.slices /= 1 -> usage "detached sessions require exactly one slice; use --granularity plan for a compatible plan"
                          | otherwise -> guardIO do
                              exists <- doesPathExist outDir
                              if exists
                                then usage "session output path already exists; choose a new directory"
                                else do
                                  if detached
                                    then do
                                      slice <- singleSlice session.journal
                                      let budget = toInteger slice.submission.limits.wallClockSeconds + 600
                                      if budget > toInteger (maxBound :: Int)
                                        then ioError (userError "detached lease budget exceeds supported TTL")
                                        else do
                                          lease <- leaseSnapshot active
                                          enlarged <- resizeLease store ref active (max lease.ttlSeconds (fromInteger budget))
                                          if enlarged then pure () else ioError (userError "cell lease changed before detached submission")
                                    else pure ()
                                  traverse_
                                    ( \(relative, content) -> do
                                        let destination = outDir </> relative
                                        createDirectoryIfMissing True (takeDirectory destination)
                                        LazyByteString.writeFile destination content
                                    )
                                    session.workFiles
                                  if detached
                                    then do
                                      slice <- singleSlice session.journal
                                      let journal = session.journal {leaseMode = Detached}
                                          journalPath = outDir </> "session.json"
                                      writeSessionJournal journalPath journal
                                      work <- LazyByteString.readFile (outDir </> slice.workPath)
                                      published <- publishSubmission store ref active slice.submission work
                                      case published of
                                        Submitted -> do
                                          now <- getCurrentTime
                                          checkpoint <- either (ioError . userError . Text.unpack) pure (applyTransition now slice.cellRun SubmissionPublished journal)
                                          writeSessionJournal journalPath checkpoint
                                          TextIO.hPutStrLn stderr ("detached session " <> renderRunId checkpoint.sessionId <> " submitted; collect with cell resume --session " <> Text.pack outDir)
                                          pure ExitSuccess
                                        other -> unavailable ("detached submission failed: " <> Text.pack (show other))
                                    else do
                                      result <- withHeartbeat store ref active \_ -> runPlannedSlices store ref (Bucket observed.descriptor.buckets.results) active (outDir </> "session.json") session.journal emitWatchEvent
                                      case result of
                                        Left failure -> unavailable (Text.pack (show failure))
                                        Right finished -> do
                                          TextIO.hPutStrLn stderr ("session " <> renderRunId finished.sessionId <> " verified; journal " <> Text.pack (outDir </> "session.json"))
                                          pure (sessionExitCode finished)
                    Right _ -> unavailable "cell capability cache count differs from descriptor count"
  Run (RunCli location settings ttl waitSeconds startInstances detached)
    | ttl <= 0 || waitSeconds < 0 -> usage "lease TTL must be positive and wait cannot be negative"
    | otherwise -> withControl location \store ref observed -> do
        if startInstances then startCellInstances observed.descriptor else pure ()
        defaultOwner <- Text.pack . fromMaybe "kenshou" <$> lookupEnv "USER"
        start <- getCurrentTime
        let request = LeaseRequest defaultOwner "kenshou cell run" ttl
            acquire = do
              outcome <- acquireLease store ref request
              case outcome of
                Acquired handle -> do
                  lease <- leaseSnapshot handle
                  let execute = runCell (Submit (SubmitCli location (renderRunId lease.leaseId) settings detached)) cli
                      releaseCurrent = do
                        current <- reattachLease store ref lease.leaseId
                        traverse_ (\active -> do _ <- releaseLease store ref active; pure ()) current
                      releaseUnlessPublished = do
                        let SubmitSettings _ _ outDir _ _ _ _ _ _ _ _ _ _ = settings
                            journalPath = outDir </> "session.json"
                        present <- doesPathExist journalPath
                        if not present
                          then releaseCurrent
                          else do
                            decoded <- readSessionJournal journalPath
                            case decoded of
                              Left _ -> pure ()
                              Right journal | journal.leaseId /= lease.leaseId || journal.cell /= ref.cellName -> releaseCurrent
                              Right journal -> do
                                markers <- traverse (\slice -> store.statObject ref.controlBucket (ObjectName ("cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/submission.json"))) journal.slices
                                if all isNothing markers then releaseCurrent else pure ()
                  execute `finally` (if detached then releaseUnlessPublished else releaseCurrent)
                Busy current -> do
                  now <- getCurrentTime
                  if diffUTCTime now start < fromIntegral waitSeconds
                    then threadDelay 2000000 >> acquire
                    else unavailable ("cell is busy under lease " <> renderRunId current.leaseId)
                Quarantined record -> unavailable ("cell is quarantined: " <> record.reason)
        acquire
  Probe request -> runProbe cli request
  Pair request -> runPair cli request
  Overhead request -> runCellOverhead cli request
  Resume sessionDir selectedLease startInstances -> guardIO do
    let journalPath = sessionDir </> "session.json"
    present <- doesPathExist journalPath
    if not present
      then unavailable "session.json is missing"
      else do
        decoded <- readSessionJournal journalPath
        case decoded of
          Left problem -> usage problem
          Right journal -> do
            selectedStore <- Text.pack . fromMaybe "gs://" <$> lookupEnv "KENSHOU_CELL_STORE"
            if selectedStore /= journal.store
              then usage "selected cell store differs from the session journal"
              else case traverse parseRunId selectedLease of
                Left problem -> usage problem
                Right requested -> withControl (CellLocation journal.cell (Just journal.controlBucket)) \store ref observed -> do
                  if observed.descriptor.buckets.results /= journal.resultsBucket
                    then usage "cell results bucket differs from the session journal"
                    else do
                      if startInstances then startCellInstances observed.descriptor else pure ()
                      let expected = fromMaybe journal.leaseId requested
                      existing <- reattachLease store ref expected
                      case existing of
                        Just _ | journal.leaseMode == Detached -> continueDetached store ref journalPath journal
                        Just handle -> continueSession store ref journalPath journal handle `finally` (do _ <- releaseLease store ref handle; pure ())
                        Nothing | requested /= Nothing -> unavailable "cell has no matching selected lease"
                        Nothing | journal.leaseMode == Detached -> continueDetached store ref journalPath journal
                        Nothing -> do
                          owner <- Text.pack . fromMaybe "kenshou" <$> lookupEnv "USER"
                          acquired <- acquireLease store ref (LeaseRequest owner ("resume " <> renderRunId journal.sessionId) 120)
                          case acquired of
                            Busy current -> unavailable ("cell is busy under lease " <> renderRunId current.leaseId)
                            Quarantined record -> unavailable ("cell is quarantined: " <> record.reason)
                            Acquired handle -> continueSession store ref journalPath journal handle `finally` (do _ <- releaseLease store ref handle; pure ())
  Exec workFile outDir -> cellExec cli.registry workFile outDir

continueSession :: ObjectStore -> CellRef -> FilePath -> SessionJournal -> LeaseHandle -> IO ExitCode
continueSession store ref journalPath journal handle = do
  resumed <- withHeartbeat store ref handle \_ -> resumeHeldSession store ref (Bucket journal.resultsBucket) handle journalPath emitWatchEvent
  case resumed of
    Left failure -> unavailable (Text.pack (show failure))
    Right finished -> do
      TextIO.hPutStrLn stderr ("session " <> renderRunId finished.sessionId <> " verified; journal " <> Text.pack journalPath)
      pure (sessionExitCode finished)

continueDetached :: ObjectStore -> CellRef -> FilePath -> SessionJournal -> IO ExitCode
continueDetached store ref journalPath journal = do
  observed <- resumeObservedSlices store ref (Bucket journal.resultsBucket) journalPath emitWatchEvent
  case observed of
    Left failure -> unavailable (Text.pack (show failure))
    Right finished ->
      if not (null finished.slices) && all (\slice -> slice.state `elem` [SliceRejected, SliceVerified]) finished.slices
        then do
          current <- reattachLease store ref journal.leaseId
          traverse_ (\active -> do _ <- releaseLease store ref active; pure ()) current
          TextIO.hPutStrLn stderr ("detached session " <> renderRunId finished.sessionId <> " collected; journal " <> Text.pack journalPath)
          pure (sessionExitCode finished)
        else unavailable "detached session has not reached a terminal cell status; resume later"

singleSlice :: SessionJournal -> IO SliceJournal
singleSlice journal = case journal.slices of
  [slice] -> pure slice
  _ -> ioError (userError "detached session must contain exactly one slice")

sessionExitCode :: SessionJournal -> ExitCode
sessionExitCode journal
  | any ((/= SliceVerified) . (.state)) journal.slices = ExitFailure 4
  | any ((/= Just Completed) . (.cellOutcome)) journal.slices = ExitFailure 4
  | any ((== Just 4) . (.entryExitCode)) journal.slices = ExitFailure 4
  | any ((== Just 3) . (.entryExitCode)) journal.slices = ExitFailure 3
  | any ((== Just 1) . (.entryExitCode)) journal.slices = ExitFailure 1
  | all ((== Just 0) . (.entryExitCode)) journal.slices = ExitSuccess
  | otherwise = ExitFailure 4

runProbe :: CliEnv -> ProbeCli -> IO ExitCode
runProbe cli (ProbeCli location payload output) = withControl location \_ _ observed -> do
  cacheDirectory <- fromMaybe ".dev/cells" <$> lookupEnv "KENSHOU_CELL_CAPABILITIES_DIR"
  identifier <- newRunId
  let sessionDirectory = cacheDirectory </> "probes" </> Text.unpack (renderRunId identifier)
      planFile = sessionDirectory <> ".plan.json"
      cacheFile = fromMaybe (cacheDirectory </> Text.unpack observed.descriptor.name <> ".capabilities.json") output
      settings = SubmitSettings [payload] planFile sessionDirectory GranularityRun Cold [] False False False Nothing NullSink Nothing False
      planArguments =
        [ "plan",
          "--all",
          "--select",
          "selftest/remote/correctness/cell-environment",
          "--placement",
          "cell",
          "--seed",
          "7",
          "--dim",
          "pg.durability=durable",
          "--dim",
          "pg.version=" <> show observed.descriptor.postgresMajor,
          "--out",
          planFile
        ]
  createDirectoryIfMissing True (takeDirectory planFile)
  executable <- getExecutablePath
  (planned, _, planErrors) <- readProcessWithExitCode executable planArguments ""
  if planned /= ExitSuccess
    then usage ("cell probe could not plan its environment scenario: " <> Text.pack planErrors)
    else do
      result <- runCell (Run (RunCli location settings 120 0 False False)) cli
      if result /= ExitSuccess
        then pure result
        else do
          saved <- readSessionJournal (sessionDirectory </> "session.json")
          case saved of
            Right journal -> case journal.slices of
              [slice] | slice.state == SliceVerified -> case slice.runIds of
                [nested] -> do
                  let runResult = takeDirectory sessionDirectory </> Text.unpack (renderRunId slice.cellRun) </> "tree" </> "output" </> Text.unpack (renderRunId nested) </> "run-result.json"
                  document <- eitherDecodeFileStrict' runResult
                  case document >>= extractProbeCapabilities slice.cellRun of
                    Left problem -> failVerification ("cell probe result has no valid capabilities: " <> Text.pack problem)
                    Right capabilities -> do
                      now <- getCurrentTime
                      let cache = CellCapabilities observed.descriptor.name (descriptorDigest observed.descriptor) slice.cellRun now capabilities
                          temporary = cacheFile <> "." <> Text.unpack (renderRunId identifier) <> ".tmp"
                      createDirectoryIfMissing True (takeDirectory cacheFile)
                      LazyByteString.writeFile temporary (encode cache)
                      renameFile temporary cacheFile
                      TextIO.putStrLn (Text.pack cacheFile)
                      pure ExitSuccess
                _ -> failVerification "cell probe verified a slice with an unexpected nested run count"
              _ -> failVerification "cell probe did not verify exactly one slice"
            Left problem -> failVerification ("cell probe session is invalid: " <> problem)

extractProbeCapabilities :: RunId -> Value -> Either String (Map.Map Text Value)
extractProbeCapabilities expectedCellRun (Object root) = do
  schema <- member "schema" root
  if schema == String "kenshou.run-result/v1" then pure () else Left "unexpected run-result schema"
  outcome <- member "outcome" root
  if outcome == String "passed" then pure () else Left "probe run did not pass"
  fingerprint <- member "fingerprint" root
  cell <- asObject "fingerprint.cell" =<< member "cell" =<< asObject "fingerprint" fingerprint
  cellRun <- member "cellRun" cell
  if cellRun == Aeson.toJSON expectedCellRun then pure () else Left "probe cell run identity differs from the verified slice"
  capabilities <- member "capabilities" cell
  case Aeson.fromJSON capabilities of
    Aeson.Error problem -> Left problem
    Aeson.Success values
      | all primitive (Map.elems values) && all (`Map.member` values) required -> Right values
      | otherwise -> Left "probe capabilities are missing or contain a non-primitive value"
  where
    primitive (Bool _) = True
    primitive (String _) = True
    primitive (Number _) = True
    primitive _ = False
    required = ["postgres.major", "postgres.pg_partman", "postgres.second-server", "postgres.control-hook", "broker", "otlp.null", "otlp.file", "fault-hook"]
    member :: Text -> KeyMap.KeyMap Value -> Either String Value
    member name fields = maybe (Left ("missing " <> Text.unpack name)) Right (KeyMap.lookup (Key.fromText name) fields)
    asObject :: String -> Value -> Either String (KeyMap.KeyMap Value)
    asObject _ (Object fields) = Right fields
    asObject label _ = Left (label <> " is not an object")
extractProbeCapabilities _ _ = Left "run result is not an object"

runPair :: CliEnv -> PairCli -> IO ExitCode
runPair cli (PairCli location candidateFile baselineFile planFile selectedPairs ordering policyFile settingTexts replacementOption startInstances outDir) = guardIO do
  settings <- case traverse parsePgSetting settingTexts of
    Left problem -> pure (Left problem)
    Right parsed -> pure (Right parsed)
  policyBytes <- ByteString.readFile policyFile
  sourceDocument <- eitherDecodeFileStrict' planFile :: IO (Either String Value)
  payloadResult <- loadPayloads ["baseline=" <> baselineFile, "candidate=" <> candidateFile]
  case (settings, decodePolicy policyBytes, sourceDocument >>= either (Left . Text.unpack) Right . decodeWorkPlan, payloadResult) of
    (Left problem, _, _, _) -> usage problem
    (_, Left problem, _, _) -> usage problem
    (_, _, Left problem, _) -> usage (Text.pack problem)
    (_, _, _, Left problem) -> usage problem
    (Right selectedPgSettings, Right policy, Right source, Right payloads) -> do
      let requested = fromMaybe policy.minimumPairs selectedPairs
          replacements = if replacementOption < 0 then requested else replacementOption
          request = PairRequest "candidate" "baseline" requested ordering (unSeed source.policy.seed) replacements
          pgSettings = selectedPgSettings <> [(name, assigned) | (name, assigned) <- [("checkpoint_timeout", "30min"), ("max_wal_size", "16GB")], name `notElem` fmap fst selectedPgSettings]
      case pairPlan request source of
        Left problem -> usage problem
        Right schedule -> do
          let groups = nub [trial.group | entry <- schedule.runs, Just trial <- [entry.trial]]
          if length groups /= 1
            then usage "cell pair currently requires one benchmark configuration per plan"
            else withControl location \store ref observed -> do
              exists <- doesPathExist outDir
              if exists
                then usage "pair output path already exists; choose a new directory"
                else do
                  if startInstances then startCellInstances observed.descriptor else pure ()
                  defaultOwner <- Text.pack . fromMaybe "kenshou" <$> lookupEnv "USER"
                  acquired <- acquireLease store ref (LeaseRequest defaultOwner "kenshou cell pair" 120)
                  case acquired of
                    Busy lease -> unavailable ("cell is busy under lease " <> renderRunId lease.leaseId)
                    Quarantined record -> unavailable ("cell is quarantined: " <> record.reason)
                    Acquired handle -> do
                      lease <- leaseSnapshot handle
                      let releaseCurrent = do
                            current <- reattachLease store ref lease.leaseId
                            traverse_ (\active -> do _ <- releaseLease store ref active; pure ()) current
                      pairUnderLease cli location lease.leaseId baselineFile candidateFile request schedule policy payloads pgSettings outDir `finally` releaseCurrent

runCellOverhead :: CliEnv -> OverheadCli -> IO ExitCode
runCellOverhead cli (OverheadCli location payloadFile sink settingTexts startInstances options) = guardIO do
  let requestedScenario =
        parseScenarioId options.scenario >>= \identifier ->
          maybe (Left ("unknown scenario " <> options.scenario)) Right (lookupScenario cli.registry identifier)
      input = LocalOverhead.parseInputs options
      settings = traverse parsePgSetting settingTexts
  selectedPolicy <- LocalOverhead.loadPolicy options.policy
  payloadResult <- loadPayloads [payloadFile]
  case (requestedScenario, input, settings, selectedPolicy, payloadResult) of
    (Left problem, _, _, _, _) -> usage problem
    (_, Left problem, _, _, _) -> usage problem
    (_, _, Left problem, _, _) -> usage problem
    (_, _, _, Left problem, _) -> usage problem
    (_, _, _, _, Left problem) -> usage problem
    (Right scenario, Right (factors, fixedKnobs, fixedDimensions), Right pgSettings, Right policy, Right payloads) ->
      case Map.lookup "default" payloads of
        Nothing -> usage "cell overhead requires one default payload"
        Just payload -> do
          resumed <- if options.resume || options.analyseOnly then LocalOverhead.findResumeDirectory options.out scenario.id else pure Nothing
          case (options.resume || options.analyseOnly, resumed) of
            (True, Nothing) -> usage "no matching overhead state was found under --out"
            (_, Just directory) -> do
              loaded <- loadOverheadState directory
              case loaded of
                Left problem -> usage ("invalid overhead state: " <> Text.pack problem)
                Right state -> continue directory state.plan (Just state) payload policy pgSettings
            (_, Nothing) -> do
              identifier <- renderRunId <$> newRunId
              let directory = options.out </> ("overhead-" <> Text.unpack identifier)
                  request = OverheadRequest identifier scenario.id factors options.mode options.control options.trials fixedKnobs fixedDimensions options.seed options.settleSeconds options.retries
              case planOverhead request scenario of
                Left (UsageError problem) -> usage problem
                Right plan -> continue directory plan Nothing payload policy pgSettings
  where
    continue directory plan existing payload policy pgSettings = do
      configuredBucket <- lookupEnv "KENSHOU_CELL_CONTROL_BUCKET"
      let CellLocation name selectedBucket = location
          bucketName = fromMaybe (maybe "tan-nb-exp-cells-control" Text.pack configuredBucket) selectedBucket
          configuration = object ["schema" .= ("kenshou.cell-overhead/v1" :: Text), "cell" .= name, "controlBucket" .= bucketName, "payload" .= payload, "otlpSink" .= show sink, "pgSettings" .= pgSettings]
          configPath = directory </> "cell-overhead.json"
      case existing of
        Nothing -> do
          present <- doesPathExist directory
          if present
            then usage "overhead output path already exists"
            else do
              createDirectoryIfMissing True directory
              LazyByteString.writeFile configPath (encode configuration)
              execute directory plan payload policy pgSettings
        Just state -> do
          saved <- eitherDecodeFileStrict' configPath :: IO (Either String Value)
          if saved /= Right configuration
            then usage "cell overhead resume arguments differ from the saved cell and payload configuration"
            else
              if options.analyseOnly
                then analyse directory plan state policy
                else execute directory plan payload policy pgSettings

    execute directory plan payload policy pgSettings = withControl location \store ref observed -> do
      if startInstances then startCellInstances observed.descriptor else pure ()
      owner <- Text.pack . fromMaybe "kenshou" <$> lookupEnv "USER"
      acquired <- acquireLease store ref (LeaseRequest owner "kenshou cell overhead" 120)
      case acquired of
        Busy current -> unavailable ("cell is busy under lease " <> renderRunId current.leaseId)
        Quarantined record -> unavailable ("cell is quarantined: " <> record.reason)
        Acquired handle -> do
          lease <- leaseSnapshot handle
          let releaseCurrent = do
                current <- reattachLease store ref lease.leaseId
                traverse_ (\active -> do _ <- releaseLease store ref active; pure ()) current
              hooks = overheadHooks plan (runCellOverheadChild cli location lease.leaseId payloadFile payload sink pgSettings directory)
          withHeartbeat
            store
            ref
            handle
            ( \_ -> do
                state <- executeOverhead hooks plan directory
                analyseOverheadResult options.json directory plan state policy hooks
            )
            `finally` releaseCurrent

    analyse directory plan state policy = analyseOverheadResult options.json directory plan state policy (overheadHooks plan (\_ _ -> pure (ExitFailure 4)))

overheadHooks :: OverheadPlan -> (RunSpec -> FilePath -> IO ExitCode) -> OverheadHooks
overheadHooks plan runChild =
  OverheadHooks runChild compareCellOverhead (Just leakCheck)
  where
    leakCheck directory = fmap (either (const Nothing) (Just . Aeson.toJSON)) (analyseRunDirectory directory defaultLeakSpec plan.seed)

-- On a cell, the resolver binds the chosen collector address only for an
-- sdk-otlp tracing arm. That address is a consequence of the declared tracing
-- change, so name its knob when the two arms actually have different values.
compareCellOverhead :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison)
compareCellOverhead policy axes baselines candidates = do
  derivedEndpoint <- case (baselines, candidates) of
    (baseline : _, candidate : _)
      | VaryDimension "telemetry.tracing" `elem` NonEmpty.toList axes -> do
          left <- readInputs baseline
          right <- readInputs candidate
          pure case (left, right) of
            (Just a, Just b) ->
              case (tracing a, tracing b, endpoint a, endpoint b) of
                (Just (String "sdk-otlp"), Just _, Just leftEndpoint, Just rightEndpoint) -> leftEndpoint /= rightEndpoint
                (Just _, Just (String "sdk-otlp"), Just leftEndpoint, Just rightEndpoint) -> leftEndpoint /= rightEndpoint
                _ -> False
            _ -> False
    _ -> pure False
  let selectedAxes = if derivedEndpoint then axes <> (VaryKnob "otel.endpoint" :| []) else axes
  compareRuns policy selectedAxes baselines candidates
  where
    tracing = varyingValue (VaryDimension "telemetry.tracing")
    endpoint = varyingValue (VaryKnob "otel.endpoint")
    readInputs directory = do
      decoded <- eitherDecodeFileStrict' (directory </> "run-result.json") :: IO (Either String Value)
      pure (either (const Nothing) compatibilityInputs decoded)

analyseOverheadResult :: Bool -> FilePath -> OverheadPlan -> OverheadState -> OverheadPolicy -> OverheadHooks -> IO ExitCode
analyseOverheadResult asJson directory plan state policy hooks = do
  report <- analyseOverhead hooks policy plan state directory
  if asJson
    then LazyByteString.putStrLn (encode report)
    else TextIO.putStrLn ("verdict " <> LocalOverhead.verdictText report.verdict <> "  " <> Text.pack directory)
  pure case overheadVerdictExitCode report.verdict of 0 -> ExitSuccess; code -> ExitFailure code

runCellOverheadChild :: CliEnv -> CellLocation -> RunId -> FilePath -> PayloadDescriptor -> OtlpSink -> [(Text, Text)] -> FilePath -> RunSpec -> FilePath -> IO ExitCode
runCellOverheadChild cli location leaseId payloadFile payload sink pgSettings directory rawSpec runsRoot = case rawSpec.runId of
  Nothing -> pure (ExitFailure 4)
  Just runId -> do
    planId <- newRunId
    now <- getCurrentTime
    selector <- either (ioError . userError . Text.unpack) pure (parseSelector "**")
    seed <- either (ioError . userError . Text.unpack) pure (mkSeed 1)
    let reason = Reason (Change (ComponentRef (ComponentId "cell-overhead") Nothing) Named "telemetry overhead slot") [] selector 0
        spec = rawSpec {environment = rawSpec.environment {Kenshou.Core.RunSpec.placement = RunOnCell}}
        context = PlanContext Nothing "cell-overhead" payload.cohort payload.cohortIdentity.identityPlanHash.unPlanHash (PlanInputs (object ["overhead" .= True])) [] []
        plan = RunPlan planId now context (defaultPlanPolicy seed) [PlannedRun 0 runId 10 (reason :| []) Nothing spec] [] 10
        planPath = directory </> "cell-sessions" </> Text.unpack (renderRunId runId) <> ".plan.json"
        sessionDir = directory </> "cell-sessions" </> Text.unpack (renderRunId runId)
        settings = SubmitSettings [payloadFile] planPath sessionDir GranularityRun Cold (fmap (\(name, assigned) -> name <> "=" <> assigned) pgSettings) False True False Nothing sink Nothing False
    createDirectoryIfMissing True (takeDirectory planPath)
    LazyByteString.writeFile planPath (encode plan)
    result <- runCell (Submit (SubmitCli location (renderRunId leaseId) settings False)) cli
    journal <- readSessionJournal (sessionDir </> "session.json")
    case journal of
      Right saved -> case saved.slices of
        [slice] | slice.state == SliceVerified && slice.runIds == [runId] -> do
          let target = takeDirectory sessionDir </> Text.unpack (renderRunId slice.cellRun) </> "tree" </> "output" </> Text.unpack (renderRunId runId)
              link = runsRoot </> Text.unpack (renderRunId runId)
          available <- doesDirectoryExist target
          if available
            then do
              createDirectoryIfMissing True runsRoot
              absolute <- makeAbsolute target
              createDirectoryLink absolute link
              pure result
            else pure (ExitFailure 4)
        _ -> pure (ExitFailure 4)
      Left _ -> pure (ExitFailure 4)

pairUnderLease :: CliEnv -> CellLocation -> RunId -> FilePath -> FilePath -> PairRequest -> RunPlan -> Policy -> Map.Map Text PayloadDescriptor -> [(Text, Text)] -> FilePath -> IO ExitCode
pairUnderLease cli location leaseId baselineFile candidateFile request schedule policy payloads pgSettings outDir = do
  createDirectoryIfMissing True outDir
  LazyByteString.writeFile (outDir </> "pair-plan.json") (encode schedule)
  let requested = request.pairs
      limit = requested + request.maxReplacements
      batchFor predicate = filter (\entry -> maybe False predicate entry.trial) schedule.runs
      initial = batchFor (\trial -> trial.index < requested)
      runRound :: Int -> [PlannedRun] -> IO (Either Text [PairObservation])
      runRound roundNumber entries = do
        let planPath = outDir </> ("round-" <> show roundNumber <> ".plan.json")
            sessionDir = outDir </> ("round-" <> show roundNumber)
            plan = schedule {runs = entries, estimateMinutes = sum (fmap (.estimateMinutes) entries)}
            settings = SubmitSettings ["baseline=" <> baselineFile, "candidate=" <> candidateFile] planPath sessionDir GranularityRun Cold (fmap (\(name, assigned) -> name <> "=" <> assigned) pgSettings) False False False Nothing NullSink Nothing False
        LazyByteString.writeFile planPath (encode plan)
        result <- runCell (Submit (SubmitCli location (renderRunId leaseId) settings False)) cli
        journal <- readSessionJournal (sessionDir </> "session.json")
        case journal of
          Left problem -> pure (Left ("pair submission did not produce a readable journal (" <> Text.pack (show result) <> "): " <> problem))
          Right verified -> case find ((== SliceRejected) . (.state)) verified.slices of
            Just rejected -> pure (Left ("pair submission rejected: " <> fromMaybe "unknown reason" rejected.rejectionReason))
            Nothing -> case find ((/= SliceVerified) . (.state)) verified.slices of
              Just incomplete -> pure (Left ("pair submission is incomplete at " <> renderRunId incomplete.cellRun <> "; resume " <> Text.pack sessionDir))
              Nothing -> pure (collectPairObservations outDir entries verified)
      continue roundNumber nextIndex observed = do
        let valid = validPairIndexes observed
        if length valid >= request.pairs || nextIndex >= limit
          then finishPair policy payloads outDir request observed valid
          else do
            let entries = batchFor (\trial -> trial.index == nextIndex)
            next <- runRound roundNumber entries
            case next of
              Left problem -> unavailable problem
              Right results -> continue (roundNumber + 1) (nextIndex + 1) (observed <> results)
  initialResult <- runRound 0 initial
  case initialResult of
    Left problem -> unavailable problem
    Right observed -> continue 1 requested observed

collectPairObservations :: FilePath -> [PlannedRun] -> SessionJournal -> Either Text [PairObservation]
collectPairObservations outDir entries journal = traverse collect entries
  where
    collect entry = case find (elem entry.runId . (.runIds)) journal.slices of
      Nothing -> Left ("pair journal omits planned run " <> renderRunId entry.runId)
      Just slice ->
        let path = outDir </> Text.unpack (renderRunId slice.cellRun) </> "tree" </> "output" </> Text.unpack (renderRunId entry.runId)
            passed = slice.state == SliceVerified && slice.cellOutcome == Just Completed && slice.entryExitCode == Just 0
         in Right (PairObservation entry path passed)

validPairIndexes :: [PairObservation] -> [Int]
validPairIndexes observations = sort [index | ((_, index), PairValid) <- judgePairs (fmap asSlice observations)]
  where
    asSlice (PairObservation entry _ passed) = case entry.trial of
      Just trial -> SliceResult trial.group trial.index (if trial.arm == "baseline" then Baseline else Candidate) passed
      Nothing -> error "paired plan lost trial membership"

finishPair :: Policy -> Map.Map Text PayloadDescriptor -> FilePath -> PairRequest -> [PairObservation] -> [Int] -> IO ExitCode
finishPair policy payloads outDir request observations valid =
  if null valid
    then unavailable "all paired trials failed; no comparison can be computed"
    else do
      let chosen = take request.pairs valid
          pathFor index arm = case find (matches index arm) observations of
            Just (PairObservation _ path _) -> path
            Nothing -> error "valid pair has no run path"
          matches index arm (PairObservation entry _ _) = case entry.trial of
            Just trial -> trial.index == index && trial.arm == arm
            Nothing -> False
          baselines = fmap (`pathFor` "baseline") chosen
          candidates = fmap (`pathFor` "candidate") chosen
          varying = case (Map.lookup "baseline" payloads, Map.lookup "candidate" payloads) of
            (Just baseline, Just candidate) | baseline.cohortIdentity.identityPlanHash /= candidate.cohortIdentity.identityPlanHash -> VaryCohort
            _ -> VaryControl
      compared <- compareRuns policy (varying :| []) baselines candidates
      case compared of
        Left (CompareError problem) -> usage ("cell pair comparison failed: " <> problem)
        Right comparison -> do
          LazyByteString.writeFile (outDir </> "comparison.json") (encode comparison <> "\n")
          TextIO.putStrLn ("comparison " <> Text.pack (show comparison.verdict) <> "; valid pairs " <> Text.pack (show (length chosen)) <> "/" <> Text.pack (show request.pairs))
          pure case verdictExitCode comparison.verdict of 0 -> ExitSuccess; code -> ExitFailure code

parsePgSetting :: Text -> Either Text (Text, Text)
parsePgSetting setting = case Text.breakOn "=" setting of
  (name, assignment) | not (Text.null name) && Text.length assignment > 1 -> Right (name, Text.drop 1 assignment)
  _ -> Left "--pg-setting must be KEY=VALUE"

loadPayloads :: [String] -> IO (Either Text (Map.Map Text PayloadDescriptor))
loadPayloads paths = do
  loaded <- traverse load paths
  pure do
    entries <- sequence loaded
    let labels = fmap fst entries
    if length (nub labels) == length labels
      then Right (Map.fromList entries)
      else Left "payload labels must be unique"
  where
    load input = case break (== '=') input of
      (label, '=' : path) | not (null label) && not (null path) && path /= "-" -> readPayload (Text.pack label) path
      (_, '=' : _) -> pure (Left "--payload must be [LABEL=]FILE with a nonempty file path")
      (_, _) | input == "-" -> pure (Left "payload must name a file")
      _ -> readPayload "default" input
    readPayload label path = do
      result <- eitherDecodeFileStrict' path
      pure (either (Left . Text.pack) (\descriptor -> Right (label, descriptor)) result)

loadCapabilitiesFor :: [CellDescriptor] -> IO (Either Text [Maybe CellCapabilities])
loadCapabilitiesFor descriptors = do
  directory <- fromMaybe ".dev/cells" <$> lookupEnv "KENSHOU_CELL_CAPABILITIES_DIR"
  sequence <$> traverse (loadCellCapabilities directory) descriptors

readJsonDocument :: (FromJSON value) => FilePath -> IO (Either String value)
readJsonDocument "-" = eitherDecode <$> LazyByteString.getContents
readJsonDocument path = eitherDecodeFileStrict' path

withControl :: CellLocation -> (ObjectStore -> CellRef -> CellSnapshot -> IO ExitCode) -> IO ExitCode
withControl (CellLocation name selectedBucket) operation = do
  configured <- lookupEnv "KENSHOU_CELL_CONTROL_BUCKET"
  let bucketName = fromMaybe (maybe "tan-nb-exp-cells-control" Text.pack configured) selectedBucket
  case validateBucket bucketName of
    Left problem -> usage problem
    Right bucket
      | not (validCellName name) -> usage "invalid cell name"
      | otherwise -> guardIO do
          store <- openStore bucket
          allowed <- maybe ["tan-nb-exp"] (map Text.strip . Text.splitOn "," . Text.pack) <$> lookupEnv "KENSHOU_GCP_ALLOWED_PROJECTS"
          let ref = CellRef name bucket
          snapshot <- readCellSnapshot store ref allowed
          case snapshot of
            Left problem -> unavailable problem
            Right observed -> operation store ref observed

startCellInstances :: CellDescriptor -> IO ()
startCellInstances descriptor = do
  let names = descriptor.instances.postgres : descriptor.instances.drivers <> [descriptor.instances.monitoring] <> maybe [] pure descriptor.instances.broker
      location = ["--project=" <> Text.unpack descriptor.project, "--zone=" <> Text.unpack descriptor.zone]
  states <-
    traverse
      ( \name -> do
          (code, output, failure) <- readProcessWithExitCode "gcloud" (["compute", "instances", "describe", Text.unpack name] <> location <> ["--format=value(status)"]) ""
          if code == ExitSuccess
            then pure (name, Text.strip (Text.pack output))
            else ioError (userError ("cannot inspect cell instance " <> Text.unpack name <> ": " <> failure))
      )
      names
  case [(name, state) | (name, state) <- states, state `notElem` ["RUNNING", "TERMINATED"]] of
    (name, state) : _ -> ioError (userError ("cell instance " <> Text.unpack name <> " is " <> Text.unpack state))
    [] -> do
      let stopped = [Text.unpack name | (name, "TERMINATED") <- states]
      if null stopped
        then pure ()
        else do
          (code, _, failure) <- readProcessWithExitCode "gcloud" (["compute", "instances", "start"] <> stopped <> location <> ["--quiet"]) ""
          if code == ExitSuccess
            then TextIO.hPutStrLn stderr ("started " <> Text.pack (show (length stopped)) <> " instances for cell " <> descriptor.name)
            else ioError (userError ("could not start cell " <> Text.unpack descriptor.name <> ": " <> failure))

holdLease :: IO Bool -> IO ExitCode
holdLease stillHeld = do
  held <- stillHeld
  if held then threadDelay 1000000 >> holdLease stillHeld else unavailable "lease was lost"

emitWatchEvent :: WatchEvent -> IO ()
emitWatchEvent event = case event of
  PhaseChanged phase -> TextIO.hPutStrLn stderr ("phase: " <> Text.pack (show phase))
  StdoutChunk bytes -> LazyByteString.hPut stdout bytes >> hFlush stdout
  StderrChunk bytes -> LazyByteString.hPut stderr bytes >> hFlush stderr

verifyTree :: FilePath -> IO ExitCode
verifyTree tree = do
  verified <- verifyCellRun tree
  case verified of
    Left problems -> failVerification (Text.pack (show (NonEmpty.toList problems)))
    Right manifest -> TextIO.putStrLn ("verified " <> renderRunId manifest.runId) >> pure ExitSuccess

openStore :: Bucket -> IO ObjectStore
openStore bucket = do
  selected <- lookupEnv "KENSHOU_CELL_STORE"
  case selected of
    Nothing -> newTokenProvider bucket >>= newGcsStore
    Just setting -> case Text.stripPrefix "file:" (Text.pack setting) of
      Just root | not (Text.null root) -> newFileStore (Text.unpack root)
      _ -> ioError (userError "KENSHOU_CELL_STORE must be file:<dir> or unset for GCS")

parseResultsUri :: Text -> Either Text (Bucket, RunId)
parseResultsUri suffix = case Text.splitOn "/" suffix of
  [bucket, "runs", identifier] -> do
    checked <- validateBucket bucket
    parsed <- parseRunId identifier
    pure (checked, parsed)
  _ -> Left "expected gs://BUCKET/runs/CELL_RUN_ID"

parsePayloadUri :: Text -> Either Text (Bucket, ObjectName)
parsePayloadUri uri = case Text.stripPrefix "gs://" uri of
  Nothing -> Left "payload bundle URI must use gs://"
  Just suffix -> case Text.breakOn "/" suffix of
    (bucketName, path) | not (Text.null path) -> do
      bucket <- validateBucket bucketName
      pure (bucket, ObjectName (Text.drop 1 path))
    _ -> Left "payload bundle URI must name an object"

validateBucket :: Text -> Either Text Bucket
validateBucket bucket
  | Text.null bucket || Text.any (\character -> not (character `elem` (['a' .. 'z'] <> ['A' .. 'Z'] <> ['0' .. '9'] <> "-_."))) bucket = Left "invalid results bucket"
  | otherwise = Right (Bucket bucket)

guardIO :: IO ExitCode -> IO ExitCode
guardIO operation = do
  result <- try operation :: IO (Either SomeException ExitCode)
  case result of
    Left failure | Just async <- (fromException failure :: Maybe SomeAsyncException) -> throwIO async
    Left failure -> unavailable (Text.pack (displayException failure))
    Right code -> pure code

fetchFailure :: FetchError -> IO ExitCode
fetchFailure problem = case problem of
  Unsealed -> unavailable "cell run is not sealed"
  ObjectChanged _ -> unavailable (Text.pack (show problem))
  _ -> failVerification (Text.pack (show problem))

unavailable :: Text -> IO ExitCode
unavailable problem = TextIO.hPutStrLn stderr ("kenshou cell: " <> problem) >> pure (ExitFailure 4)

failVerification :: Text -> IO ExitCode
failVerification problem = TextIO.hPutStrLn stderr ("kenshou cell: " <> problem) >> pure (ExitFailure 1)

usage :: Text -> IO ExitCode
usage problem = TextIO.hPutStrLn stderr ("kenshou cell: " <> problem) >> pure (ExitFailure 2)