packages feed

kenshou-cli-0.1.0.0: test/Main.hs

module Main (main) where

import Control.Concurrent (forkIO, killThread, threadDelay)
import Control.Exception (bracket, finally)
import Control.Monad (forM_)
import Data.Aeson (Value, object, (.=))
import Data.Aeson qualified as Aeson
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Foldable (toList)
import Data.List (find)
import Data.List qualified as List
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text
import Data.Time (addUTCTime, getCurrentTime)
import Kenshou.Cli (Program (..), runWithArgsWith, selfTestAssembly)
import Kenshou.Cli.Cohort (resolveDefaultCohortIdentity)
import Kenshou.Cli.Version (appVersionWithGit)
import Kenshou.Core.Canonical (sha256Hex)
import Kenshou.Core.Cohort (CohortIdentity (..), CohortName (..))
import Kenshou.Core.Id (newRunId, renderRunId)
import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..))
import Kenshou.Core.Outcome qualified as Outcome
import Kenshou.Perf.Artifacts (sealPerfDirectory)
import Kenshou.Perf.Plan qualified as Perf
import Kenshou.Perf.Prepared qualified as Prepared
import Kenshou.Remote.Cell.Docs (Artifact (..), CellBuckets (..), CellDescriptor (..), CellManifest (..), CellOutcome (..), CellPhase (..), CellRunResult (..), CellStatus (..), Limits (..), LogChunks (..), ManifestPayload (..), Rejected (..), Submission (..), WorkObject (..))
import Kenshou.Remote.Cell.Lease (Lease (..))
import Kenshou.Remote.Cell.RouteRules (descriptorDigest)
import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), readSessionJournal, writeSessionJournal)
import Kenshou.Remote.Cell.Submit (workObjectFor)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))
import Kenshou.Remote.Payload qualified as Payload
import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))
import Kenshou.Remote.Store.File (newFileStore)
import Kenshou.Telemetry.Overhead (OverheadState (..), SlotRun (..), loadOverheadState)
import PerfSessionSpec qualified
import System.Directory (Permissions (..), copyFile, createDirectoryIfMissing, doesFileExist, doesPathExist, getPermissions, listDirectory, makeAbsolute, setPermissions, withCurrentDirectory)
import System.Environment (lookupEnv, setEnv, unsetEnv)
import System.Exit (ExitCode (..))
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import System.Timeout (timeout)
import Test.Hspec (describe, expectationFailure, hspec, it, shouldBe, shouldReturn, shouldSatisfy)

main :: IO ()
main = hspec do
  PerfSessionSpec.spec
  describe "performance planning CLI" do
    it "exposes calibration publication and rejects malformed batches before storage effects" $ withSystemTempDirectory "kenshou-calibration-malformed" \directory -> do
      let input = directory </> "definition.json"
      Aeson.encodeFile input Aeson.Null
      runWithArgs ["perf", "calibration", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["perf", "calibration", "declare", "--definition", input, "--uri", "gs://fixture-evidence/calibration/definition.json"] `shouldReturnCode` ExitFailure 2
    it "publishes a calibration definition create-only and retrieves its identical bytes" $ withSystemTempDirectory "kenshou-calibration-publication" \directory -> do
      raw <- withPerfRoot (Aeson.eitherDecodeFileStrict' "kenshou-core/test/golden/perf-calibration-definition.synthetic.json" >>= either fail pure)
      now <- getCurrentTime
      let input = directory </> "definition.json"
          storeRoot = directory </> "store"
          document = replacePerfField "expiresAt" (Aeson.toJSON (addUTCTime 3600 now)) raw
          uri = "gs://fixture-evidence/calibration/definition.json"
      Aeson.encodeFile input document
      store <- newFileStore storeRoot
      withCellStore storeRoot do
        runWithArgs ["perf", "calibration", "declare", "--definition", input, "--uri", uri] `shouldReturnCode` ExitSuccess
        bytes <- LazyByteString.readFile input
        retained <- store.getObject (Bucket "fixture-evidence") (ObjectName "calibration/definition.json")
        fmap fst retained `shouldBe` Just bytes
        runWithArgs ["perf", "calibration", "declare", "--definition", input, "--uri", uri] `shouldReturnCode` ExitSuccess
        Aeson.encodeFile input (replacePerfField "injectionMechanism" (Aeson.String "changed after declaration") document)
        runWithArgs ["perf", "calibration", "declare", "--definition", input, "--uri", uri] `shouldReturnCode` ExitFailure 4
        store.getObject (Bucket "fixture-evidence") (ObjectName "calibration/definition.json") >>= \current -> fmap fst current `shouldBe` Just bytes
    it "exposes persistent execution, status, resume and evidence replay" do
      forM_ ["run", "status", "resume", "verify"] \name -> do
        runWithArgs ["perf", name, "--help"] `shouldReturnCode` ExitSuccess
        runWithArgs ["perf", name] `shouldReturnCode` ExitFailure 2
    it "rejects malformed prepared input before creating a session or contacting a cell" $ withSystemTempDirectory "kenshou-perf-run-malformed" \directory -> do
      let input = directory </> "prepared.json"
          output = directory </> "session"
      Aeson.encodeFile input Aeson.Null
      runWithArgs ["perf", "run", "--prepared", input, "--out", output, "--evidence-base-uri", "gs://fixture-evidence/performance"] `shouldReturnCode` ExitFailure 2
      doesPathExist output `shouldReturn` False
    it "reads retained status without cloud authentication or execution" $ withSystemTempDirectory "kenshou-perf-status" \directory -> do
      withPerfRoot (copyFile "fixtures/consumer-baseline/kenshou-remote/test/golden/perf-session.fixture.json" (directory </> "session.json"))
      runWithArgs ["perf", "status", "--session", directory] `shouldReturnCode` ExitSuccess
    it "publishes and replays incomplete scope without leasing, and detects storage tampering" $ withSystemTempDirectory "kenshou-perf-incomplete" \directory -> do
      let storeRoot = directory </> "store"
          preparation = directory </> "preparation"
          output = directory </> "session"
          bytes = "synthetic payload; this control executes no runtime work"
          digest = Text.drop 7 (sha256Hex (LazyByteString.toStrict bytes))
          bundle = Bundle ("gs://control/payloads/sha256/" <> digest <> ".nar.zst") digest (LazyByteString.length bytes)
          changePayload arm = case Aeson.fromJSON arm.payload of
            Aeson.Error problem -> fail problem
            Aeson.Success payload -> pure arm {Prepared.payload = Aeson.toJSON (payload {Payload.cell = payload.cell {Payload.bundle = bundle}} :: PayloadDescriptor)}
      original <- withPerfRoot (Aeson.eitherDecodeFileStrict' "fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= either fail pure)
      baseline <- changePayload (original :: Prepared.PerfPrepared).baseline
      candidate <- changePayload original.candidate
      let prepared = original {Prepared.plan = original.plan {Perf.selected = [], Perf.coverage = "incomplete", Perf.estimateSeconds = 0}, Prepared.baseline = baseline, Prepared.candidate = candidate}
      createDirectoryIfMissing True preparation
      Aeson.encodeFile (preparation </> "prepared.json") prepared
      sealPerfDirectory preparation >>= either (fail . Text.unpack) pure
      descriptor <- withPerfRoot (LazyByteString.readFile "kenshou-remote/test/golden/cell/cell.descriptor.v1.json")
      store <- newFileStore storeRoot
      _ <- store.putObject (Bucket "control") (ObjectName ("payloads/sha256/" <> digest <> ".nar.zst")) "application/octet-stream" DoesNotExist bytes
      _ <- store.putObject (Bucket "tan-nb-exp-cells-control") (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor
      withCellStore storeRoot $ withCapabilityDir (directory </> "capabilities") do
        runWithArgs ["perf", "run", "--prepared", preparation </> "prepared.json", "--out", output, "--evidence-base-uri", "gs://fixture-evidence/performance"] `shouldReturnCode` ExitFailure 3
        store.getObject (Bucket "tan-nb-exp-cells-control") (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing
        runWithArgs ["perf", "verify", "--session", output] `shouldReturnCode` ExitFailure 3
        runWithArgs ["perf", "verify", "--session", output, "--offline"] `shouldReturnCode` ExitFailure 3
        originalResult <- LazyByteString.readFile (output </> "result.json")
        runWithArgs ["perf", "resume", "--session", output] `shouldReturnCode` ExitFailure 3
        LazyByteString.readFile (output </> "result.json") `shouldReturn` originalResult
        published <- store.listObjects (Bucket "fixture-evidence") "performance/sessions/"
        let reports = [name | (name, _) <- published, "/report.md" `Text.isSuffixOf` name.unObjectName]
        length reports `shouldBe` 1
        forM_ reports \name -> do
          _ <- store.putObject (Bucket "fixture-evidence") name "text/markdown" NoPrecondition "tampered"
          pure ()
        runWithArgs ["perf", "verify", "--session", output] `shouldReturnCode` ExitFailure 4
    it "exposes the planning command and requires request/output arguments" do
      runWithArgs ["perf", "plan", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["perf", "plan"] `shouldReturnCode` ExitFailure 2
    it "exposes exact-revision preparation and rejects malformed requests without building" do
      runWithArgs ["perf", "prepare", "--help"] `shouldReturnCode` ExitSuccess
      withSystemTempDirectory "kenshou-perf-prepare-malformed" \directory -> do
        let input = directory </> "request.json"
            output = directory </> "preparation"
        Aeson.encodeFile input Aeson.Null
        runWithArgs ["perf", "prepare", "--request", input, "--out", output] `shouldReturnCode` ExitFailure 2
        doesPathExist output `shouldReturn` False
    it "rejects malformed requests before creating an output directory" do
      withSystemTempDirectory "kenshou-perf-malformed" \directory -> do
        let input = directory </> "request.json"
            output = directory </> "plan"
        Aeson.encodeFile input Aeson.Null
        runWithArgs ["perf", "plan", "--request", input, "--out", output] `shouldReturnCode` ExitFailure 2
        doesPathExist output `shouldReturn` False
    it "rejects target components from a second repository before fetching" do
      withPerfRoot $ withSystemTempDirectory "kenshou-perf-targets" \directory -> do
        raw <- Aeson.eitherDecodeFileStrict' "fixtures/consumer-baseline/kenshou-core/test/golden/perf-request.json" >>= either fail pure
        let input = directory </> "request.json"
            output = directory </> "plan"
        targets <- foreignTargets raw
        Aeson.encodeFile input (replacePerfField "targetComponents" (Aeson.toJSON targets) raw)
        withConsumerBaseline ["perf", "plan", "--request", input, "--out", output] `shouldReturnCode` ExitFailure 2
        doesPathExist output `shouldReturn` False
    it "rejects unknown profiles before fetching immutable sources" do
      withPerfRoot $ withSystemTempDirectory "kenshou-perf-profiles" \directory -> do
        raw <- Aeson.eitherDecodeFileStrict' "fixtures/consumer-baseline/kenshou-core/test/golden/perf-request.json" >>= either fail pure
        let input = directory </> "request.json"
            output = directory </> "plan"
        Aeson.encodeFile input (replacePerfField "profiles" (Aeson.toJSON (["missing"] :: [Text])) raw)
        withConsumerBaseline ["perf", "plan", "--request", input, "--out", output] `shouldReturnCode` ExitFailure 2
        doesPathExist output `shouldReturn` False
    it "refuses to overwrite an existing planning directory" do
      withSystemTempDirectory "kenshou-perf-existing" \directory -> do
        runWithArgs ["perf", "plan", "--request", "missing", "--out", directory] `shouldReturnCode` ExitFailure 2
  describe "CLI exit contract" do
    it "resolves the wrapped cohort identity outside a source checkout" $
      withSystemTempDirectory "kenshou-wrapped-cohort" \root -> do
        fixture <- makeAbsolute "../kenshou-core/test/fixtures/cohort-identity.golden.json"
        withEnvVariable "KENSHOU_COHORT_IDENTITY" fixture $
          withCurrentDirectory root do
            resolved <- resolveDefaultCohortIdentity
            case resolved of
              Left problem -> expectationFailure (show problem)
              Right identity -> identity.identityCohort `shouldBe` CohortName "released"

    it "returns success for help" do
      runWithArgs ["--help"] `shouldReturnCode` ExitSuccess

    it "rejects a pinned knob absent from every selected scenario" $
      withSystemTempDirectory "kenshou-plan-pin" \root -> do
        let output = root </> "run-plan.json"
        runWithArgs
          [ "plan",
            "--all",
            "--select",
            "selftest/kernel/correctness/always-pass",
            "--set",
            "diagnose.major-gc-interval-ms=5000",
            "--out",
            output
          ]
          `shouldReturnCode` ExitFailure 2
        doesFileExist output `shouldReturn` False

    it "returns success for command-specific help" do
      runWithArgs ["record", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["attest", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["history", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["run", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "fetch", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "verify", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "status", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "lease", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "release", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "watch", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "route", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "submit", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "run", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "probe", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "pair", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "overhead", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "resume", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "payload", "publish", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["cell", "payload", "show", "--help"] `shouldReturnCode` ExitSuccess
      runWithArgs ["help", "cells"] `shouldReturnCode` ExitSuccess

    it "shows a payload only when its bundle exists at the recorded size" $
      withSystemTempDirectory "kenshou-cell-payload" \root -> do
        store <- newFileStore root
        let descriptor = "../kenshou-remote/test/golden/payload.json"
            bundle = ObjectName "payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst"
            showPayload = ["cell", "payload", "show", descriptor]
        withCellStore root do
          runWithArgs showPayload `shouldReturnCode` ExitFailure 1
          _ <- store.putObject (Bucket "control") bundle "application/octet-stream" DoesNotExist "xx"
          runWithArgs showPayload `shouldReturnCode` ExitFailure 1
          _ <- store.putObject (Bucket "control") bundle "application/octet-stream" NoPrecondition "x"
          runWithArgs showPayload `shouldReturnCode` ExitSuccess

    it "rejects malformed cell result identifiers and URIs" do
      runWithArgs ["cell", "fetch", "--results-bucket", "test-results", "not-a-run-id", "--out", "test-output"] `shouldReturnCode` ExitFailure 2
      runWithArgs ["cell", "verify", "gs://test-results/runs/not-a-run-id"] `shouldReturnCode` ExitFailure 2

    it "distinguishes an unavailable cell tree from invalid evidence" do
      runWithArgs ["cell", "verify", "test/fixtures/does-not-exist"] `shouldReturnCode` ExitFailure 4
      runWithArgs ["cell", "verify", leakingRun] `shouldReturnCode` ExitFailure 1

    it "acquires, reports and releases a lease through the file-backed cell CLI" $
      withSystemTempDirectory "kenshou-cell-cli" \root -> do
        store <- newFileStore root
        descriptor <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        let control = Bucket "tan-nb-exp-cells-control"
            leaseObject = ObjectName "cells/alpha/lease.json"
            common = ["--cell", "alpha", "--control-bucket", "tan-nb-exp-cells-control"]
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor
        withCellStore root do
          runWithArgs (["cell", "status"] <> common <> ["--json"]) `shouldReturnCode` ExitSuccess
          runWithArgs (["cell", "lease"] <> common <> ["--purpose", "cli-test"]) `shouldReturnCode` ExitSuccess
          stored <- store.getObject control leaseObject
          record <- case stored of
            Just (bytes, _) -> case Aeson.eitherDecode bytes of
              Right lease -> pure (lease :: Lease)
              Left problem -> expectationFailure problem >> error "unreachable"
            Nothing -> expectationFailure "lease was not published" >> error "unreachable"
          runWithArgs (["cell", "status"] <> common) `shouldReturnCode` ExitSuccess
          runWithArgs (["cell", "release"] <> common <> ["--lease-id", Text.unpack (renderRunId record.leaseId)]) `shouldReturnCode` ExitSuccess
          store.statObject control leaseObject `shouldReturn` Nothing

    it "routes a public plan through file-backed cell descriptors" $
      withSystemTempDirectory "kenshou-cell-route" \root -> do
        store <- newFileStore root
        descriptor <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptor :: Either String CellDescriptor)
        let control = Bucket "tan-nb-exp-cells-control"
            outDir = root </> "routed"
            common = ["cell", "route", "--cell", "beta", "--cell", "alpha", "--control-bucket", "tan-nb-exp-cells-control", "--plan", "../kenshou-core/test/golden/run-plan.minimal.json"]
        _ <- store.putObject control (ObjectName "cells/beta/descriptor.json") "application/json" DoesNotExist (Aeson.encode (observed {name = "beta", postgresMajor = 17}))
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor
        withCellStore root do
          runWithArgs (common <> ["--out", outDir, "--coerce-durable"]) `shouldReturnCode` ExitSuccess
          doesFileExist (outDir </> "plan.alpha.json") `shouldReturn` True
          doesFileExist (outDir </> "plan.beta.json") `shouldReturn` False
          report <- Aeson.eitherDecodeFileStrict' (outDir </> "unroutable.json") >>= either fail pure
          case report of
            Aeson.Object fields -> KeyMap.lookup "schema" fields `shouldBe` Just (Aeson.String "kenshou.cell-route/v1")
            _ -> expectationFailure "expected route report"
          runWithArgs (common <> ["--out", outDir, "--coerce-durable"]) `shouldReturnCode` ExitFailure 2
          runWithArgs (common <> ["--out", root </> "refused"]) `shouldReturnCode` ExitFailure 2
          doesFileExist (root </> "refused" </> "unroutable.json") `shouldReturn` True

    it "uses a matching capability cache and ignores one from an old descriptor" $
      withSystemTempDirectory "kenshou-cell-capability-route" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        descriptor <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        probeId <- newRunId
        now <- getCurrentTime
        let control = Bucket "tan-nb-exp-cells-control"
            directory = root </> "capabilities"
            rulesPath = root </> "rules.json"
            cachePath = directory </> "alpha.capabilities.json"
            command outDir = ["cell", "route", "--cell", "alpha", "--control-bucket", "tan-nb-exp-cells-control", "--plan", "../kenshou-core/test/golden/run-plan.minimal.json", "--routing-rules", rulesPath, "--out", outDir, "--coerce-durable"]
            cache digest = object ["schema" .= ("kenshou.cell-capabilities/v1" :: Text), "cell" .= ("alpha" :: Text), "descriptorSha256" .= digest, "probedBy" .= renderRunId probeId, "probedAt" .= now, "capabilities" .= object ["postgres.pg_partman" .= False]]
            rule = object ["when" .= object ["scenario" .= ("selftest/kernel/**" :: Text)], "requires" .= ("postgres.pg_partman" :: Text), "why" .= ("requires extension" :: Text)]
        createDirectoryIfMissing True directory
        LazyByteString.writeFile rulesPath (Aeson.encode [rule])
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptorBytes
        withCellStore root $ withCapabilityDir directory do
          LazyByteString.writeFile cachePath (Aeson.encode (cache (descriptorDigest descriptor)))
          runWithArgs (command (root </> "denied")) `shouldReturnCode` ExitFailure 2
          doesFileExist (root </> "denied" </> "plan.alpha.json") `shouldReturn` False
          LazyByteString.writeFile cachePath (Aeson.encode (cache (Text.replicate 64 "0")))
          runWithArgs (command (root </> "unprobed")) `shouldReturnCode` ExitSuccess
          doesFileExist (root </> "unprobed" </> "plan.alpha.json") `shouldReturn` True

    it "prepares a dry-run submission under an existing file-backed lease" $
      withSystemTempDirectory "kenshou-cell-submit" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        let control = Bucket "control"
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            location = ["--cell", "alpha", "--control-bucket", "control"]
            outDir = root </> "session"
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        withCellStore root do
          runWithArgs (["cell", "lease"] <> location <> ["--purpose", "submit-test"]) `shouldReturnCode` ExitSuccess
          stored <- store.getObject control (ObjectName "cells/alpha/lease.json")
          lease <- case stored of
            Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String Lease)
            Nothing -> expectationFailure "lease was not published" >> error "unreachable"
          let common = ["cell", "submit"] <> location <> ["--lease-id", Text.unpack (renderRunId lease.leaseId), "--payload", "../kenshou-remote/test/golden/payload.json", "--plan", "../kenshou-core/test/golden/run-plan.minimal.json", "--out", outDir, "--dry-run"]
          runWithArgs (common <> ["--coerce-durable"]) `shouldReturnCode` ExitSuccess
          doesPathExist outDir `shouldReturn` False
          runWithArgs common `shouldReturnCode` ExitFailure 2
          template <- Aeson.eitherDecodeFileStrict' "../kenshou-remote/test/golden/cell-session.json" >>= either fail pure
          let complete = (template :: SessionJournal) {store = Text.pack ("file:" <> root), controlBucket = "control", resultsBucket = descriptor.buckets.results, leaseId = lease.leaseId, slices = []}
          createDirectoryIfMissing True outDir
          writeSessionJournal (outDir </> "session.json") complete
          runWithArgs ["cell", "resume", "--session", outDir] `shouldReturnCode` ExitSuccess

    it "releases a one-command lease when plan preparation fails" $
      withSystemTempDirectory "kenshou-cell-run" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        let control = Bucket "control"
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            outDir = root </> "run-session"
            command = ["cell", "run", "--cell", "alpha", "--control-bucket", "control", "--payload", "../kenshou-remote/test/golden/payload.json", "--plan", "../kenshou-core/test/golden/run-plan.minimal.json", "--out", outDir]
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        withCellStore root do
          runWithArgs command `shouldReturnCode` ExitFailure 2
          store.statObject control (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing
          doesPathExist outDir `shouldReturn` False

    it "checkpoints a rejected one-command submission and releases its lease" $
      withSystemTempDirectory "kenshou-cell-run-rejected" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        let control = Bucket "control"
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            outDir = root </> "run-session"
            command = ["cell", "run", "--cell", "alpha", "--control-bucket", "control", "--payload", "../kenshou-remote/test/golden/payload.json", "--plan", "../kenshou-core/test/golden/run-plan.minimal.json", "--out", outDir, "--coerce-durable"]
            awaitMarker = do
              objects <- store.listObjects control "cells/alpha/submissions/"
              case find (Text.isSuffixOf "/submission.json" . (.unObjectName) . fst) objects of
                Nothing -> threadDelay 10000 >> awaitMarker
                Just (name, _) -> do
                  stored <- store.getObject control name
                  case stored of
                    Nothing -> expectationFailure "submission marker disappeared"
                    Just (bytes, _) -> do
                      submission <- either fail pure (Aeson.eitherDecode bytes :: Either String Submission)
                      now <- getCurrentTime
                      let prefix = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/rejected.json"
                      _ <- store.putObject control (ObjectName prefix) "application/json" DoesNotExist (Aeson.encode (Rejected submission.runId "fixture-rejected" now))
                      pure ()
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        worker <- forkIO awaitMarker
        ( withCellStore root do
            result <- timeout 15000000 (runWithArgs command)
            result `shouldBe` Just (ExitFailure 4)
            store.statObject control (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing
            journal <- readSessionJournal (outDir </> "session.json") >>= either (fail . Text.unpack) pure
            fmap (.state) journal.slices `shouldBe` [SliceRejected]
          )
          `finally` killThread worker

    it "fetches and verifies a sealed one-command submission" $
      withSystemTempDirectory "kenshou-cell-run-sealed" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        let control = Bucket "control"
            results = Bucket observed.buckets.results
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            outDir = root </> "run-session"
            command = ["cell", "run", "--cell", "alpha", "--control-bucket", "control", "--payload", "../kenshou-remote/test/golden/payload.json", "--plan", "../kenshou-core/test/golden/run-plan.minimal.json", "--out", outDir, "--coerce-durable"]
            awaitMarker = do
              objects <- store.listObjects control "cells/alpha/submissions/"
              case find (Text.isSuffixOf "/submission.json" . (.unObjectName) . fst) objects of
                Nothing -> threadDelay 10000 >> awaitMarker
                Just (name, _) -> do
                  marker <- store.getObject control name
                  submission <- case marker of
                    Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String Submission)
                    Nothing -> fail "submission marker disappeared"
                  journal <- readSessionJournal (outDir </> "session.json") >>= either (fail . Text.unpack) pure
                  nestedId <- case journal.slices of
                    [slice] -> case slice.runIds of
                      [identifier] -> pure identifier
                      _ -> fail "expected one nested run"
                    _ -> fail "expected one cell slice"
                  let prefix = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/"
                  work <- store.getObject control (ObjectName (prefix <> "work"))
                  workBytes <- case work of
                    Just (bytes, _) -> pure bytes
                    Nothing -> fail "submission work disappeared"
                  now <- getCurrentTime
                  let payloadDigest = submission.payload.bundle.sha256
                      nestedBytes = Aeson.encode (object ["schema" .= ("kenshou.run-result/v1" :: Text), "runId" .= nestedId, "scenario" .= ("selftest/kernel/correctness/postgres-roundtrip" :: Text), "outcome" .= Outcome.Passed, "fingerprint" .= object ["cell" .= object ["payload" .= object ["bundleSha256" .= payloadDigest]]]])
                      nestedInfo = workObjectFor "application/json" nestedBytes
                      nestedManifest = Manifest nestedId now [ManifestFile "run-result.json" ("sha256:" <> nestedInfo.sha256) (fromIntegral nestedInfo.bytes) "application/json"]
                      files =
                        [ ("submission/work", workBytes),
                          ("submission/submission.json", Aeson.encode submission),
                          ("cell/result.json", Aeson.encode (CellRunResult submission.runId "alpha" submission.leaseId 1 Completed (Just 0) Nothing [])),
                          ("output/" <> renderRunId nestedId <> "/run-result.json", nestedBytes),
                          ("output/" <> renderRunId nestedId <> "/manifest.json", Aeson.encode nestedManifest)
                        ]
                      artifactFor (path, bytes) = let info = workObjectFor "application/json" bytes in Artifact path info.sha256 info.bytes "application/json"
                      manifest = CellManifest submission.runId "alpha" submission.leaseId 1 now "0.1.0" (ManifestPayload payloadDigest submission.payload.storePath) Completed 3600 (map artifactFor files)
                      manifestBytes = Aeson.encode manifest
                      manifestDigest = (workObjectFor "application/json" manifestBytes).sha256
                  mapM_
                    ( \(path, bytes) -> do
                        _ <- store.putObject results (ObjectName ("runs/" <> renderRunId submission.runId <> "/" <> path)) "application/json" DoesNotExist bytes
                        pure ()
                    )
                    files
                  _ <- store.putObject results (ObjectName ("runs/" <> renderRunId submission.runId <> "/manifest.json")) "application/json" DoesNotExist manifestBytes
                  _ <- store.putObject control (ObjectName (prefix <> "status.json")) "application/json" DoesNotExist (Aeson.encode (CellStatus submission.runId Sealed (Just 1) now Nothing (LogChunks 0 0) (Just Completed) (Just manifestDigest) (Just [])))
                  pure ()
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        worker <- forkIO awaitMarker
        ( withCellStore root do
            result <- timeout 15000000 (runWithArgs command)
            result `shouldBe` Just ExitSuccess
            store.statObject control (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing
            journal <- readSessionJournal (outDir </> "session.json") >>= either (fail . Text.unpack) pure
            fmap (.state) journal.slices `shouldBe` [SliceVerified]
            case journal.slices of
              [slice] -> doesFileExist (root </> Text.unpack (renderRunId slice.cellRun) </> "cell-run.json") `shouldReturn` True
              _ -> expectationFailure "expected one verified cell slice"
          )
          `finally` killThread worker

    it "runs overhead slots behind one cell lease and links their verified run directories" $
      withSystemTempDirectory "kenshou-cell-overhead" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        let control = Bucket "control"
            results = Bucket observed.buckets.results
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            outRoot = root </> "overhead-output"
            command =
              [ "cell",
                "overhead",
                "selftest/telemetry/benchmark/arms-on-synthetic-service",
                "--cell",
                "alpha",
                "--control-bucket",
                "control",
                "--payload",
                "../kenshou-remote/test/golden/payload.json",
                "--arms",
                "tracing=off,noop",
                "--trials",
                "3",
                "--settle-seconds",
                "0",
                "--retries",
                "0",
                "--policy",
                "../policies/telemetry-overhead.json",
                "--out",
                outRoot
              ]
            awaitMarkers seen = do
              objects <- store.listObjects control "cells/alpha/submissions/"
              case find (\(name, _) -> Text.isSuffixOf "/submission.json" name.unObjectName && name `Set.notMember` seen) objects of
                Nothing -> threadDelay 10000 >> awaitMarkers seen
                Just (name, _) -> do
                  marker <- store.getObject control name
                  submission <- case marker of
                    Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String Submission)
                    Nothing -> fail "overhead submission marker disappeared"
                  let prefix = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/"
                  work <- store.getObject control (ObjectName (prefix <> "work"))
                  workBytes <- case work of
                    Just (bytes, _) -> pure bytes
                    Nothing -> fail "overhead work disappeared"
                  workValue <- either fail pure (Aeson.eitherDecode workBytes :: Either String Value)
                  nestedId <- case workValue of
                    Aeson.Object fields -> case KeyMap.lookup "runs" fields of
                      Just (Aeson.Array entries) -> case toList entries of
                        [Aeson.Object entry] -> case KeyMap.lookup "runId" entry of
                          Just value -> case Aeson.fromJSON value of Aeson.Success identifier -> pure identifier; Aeson.Error problem -> fail problem
                          Nothing -> fail "overhead work has no nested run ID"
                        _ -> fail "overhead work must contain one valid run entry"
                      _ -> fail "overhead work has no runs"
                    _ -> fail "overhead work is invalid"
                  now <- getCurrentTime
                  let sequenceNumber = Set.size seen + 1
                      payloadDigest = submission.payload.bundle.sha256
                      nestedBytes = Aeson.encode (object ["schema" .= ("kenshou.run-result/v1" :: Text), "runId" .= nestedId, "scenario" .= ("selftest/telemetry/benchmark/arms-on-synthetic-service" :: Text), "outcome" .= Outcome.Passed, "fingerprint" .= object ["cell" .= object ["payload" .= object ["bundleSha256" .= payloadDigest]]]])
                      nestedInfo = workObjectFor "application/json" nestedBytes
                      nestedManifest = Manifest nestedId now [ManifestFile "run-result.json" ("sha256:" <> nestedInfo.sha256) (fromIntegral nestedInfo.bytes) "application/json"]
                      files =
                        [ ("submission/work", workBytes),
                          ("submission/submission.json", Aeson.encode submission),
                          ("cell/result.json", Aeson.encode (CellRunResult submission.runId "alpha" submission.leaseId sequenceNumber Completed (Just 0) Nothing [])),
                          ("output/" <> renderRunId nestedId <> "/run-result.json", nestedBytes),
                          ("output/" <> renderRunId nestedId <> "/manifest.json", Aeson.encode nestedManifest)
                        ]
                      artifactFor (path, bytes) = let info = workObjectFor "application/json" bytes in Artifact path info.sha256 info.bytes "application/json"
                      manifest = CellManifest submission.runId "alpha" submission.leaseId sequenceNumber now "0.1.0" (ManifestPayload payloadDigest submission.payload.storePath) Completed 3600 (map artifactFor files)
                      manifestBytes = Aeson.encode manifest
                      manifestDigest = (workObjectFor "application/json" manifestBytes).sha256
                  mapM_ (\(path, bytes) -> do _ <- store.putObject results (ObjectName ("runs/" <> renderRunId submission.runId <> "/" <> path)) "application/json" DoesNotExist bytes; pure ()) files
                  _ <- store.putObject results (ObjectName ("runs/" <> renderRunId submission.runId <> "/manifest.json")) "application/json" DoesNotExist manifestBytes
                  _ <- store.putObject control (ObjectName (prefix <> "status.json")) "application/json" DoesNotExist (Aeson.encode (CellStatus submission.runId Sealed (Just sequenceNumber) now Nothing (LogChunks 0 0) (Just Completed) (Just manifestDigest) (Just [])))
                  if sequenceNumber < 6 then awaitMarkers (Set.insert name seen) else pure ()
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        worker <- forkIO (awaitMarkers Set.empty)
        ( withCellStore root do
            result <- timeout 30000000 (runWithArgs command)
            result `shouldBe` Just (ExitFailure 4)
            store.statObject control (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing
            [invocation] <- listDirectory outRoot
            state <- loadOverheadState (outRoot </> invocation) >>= either fail pure
            length state.slots `shouldBe` 6
            all (.complete) state.slots `shouldBe` True
            observations <-
              traverse
                ( \slot -> do
                    runId <- case reverse slot.runIds of identifier : _ -> pure identifier; [] -> fail "overhead slot has no run"
                    doesFileExist (outRoot </> invocation </> "runs" </> Text.unpack (renderRunId runId) </> "manifest.json") `shouldReturn` True
                    journal <- readSessionJournal (outRoot </> invocation </> "cell-sessions" </> Text.unpack (renderRunId runId) </> "session.json") >>= either (fail . Text.unpack) pure
                    slice <- case journal.slices of [one] -> pure one; _ -> fail "overhead child must have one slice"
                    sealed <- store.getObject results (ObjectName ("runs/" <> renderRunId slice.cellRun <> "/manifest.json"))
                    manifest <- case sealed of
                      Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String CellManifest)
                      Nothing -> fail "overhead child has no cell manifest"
                    pure (journal.leaseId, manifest.leaseSequence)
                )
                state.slots
            length (Set.fromList (fmap fst observations)) `shouldBe` 1
            List.sort (fmap snd observations) `shouldBe` [1 .. 6]
            resumed <- timeout 15000000 (runWithArgs (command <> ["--resume"]))
            resumed `shouldBe` Just (ExitFailure 4)
            markers <- store.listObjects control "cells/alpha/submissions/"
            length (filter (Text.isSuffixOf "/submission.json" . (.unObjectName) . fst) markers) `shouldBe` 6
          )
          `finally` killThread worker

    it "leaves a detached single-slice lease alive and collects its rejection later" $
      withSystemTempDirectory "kenshou-cell-run-detached" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        let control = Bucket "control"
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            outDir = root </> "run-session"
            leaseObject = ObjectName "cells/alpha/lease.json"
            command = ["cell", "run", "--cell", "alpha", "--control-bucket", "control", "--payload", "../kenshou-remote/test/golden/payload.json", "--plan", "../kenshou-core/test/golden/run-plan.minimal.json", "--out", outDir, "--coerce-durable", "--detach"]
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        withCellStore root do
          runWithArgs command `shouldReturnCode` ExitSuccess
          journal <- readSessionJournal (outDir </> "session.json") >>= either (fail . Text.unpack) pure
          journal.leaseMode `shouldBe` Detached
          fmap (.state) journal.slices `shouldBe` [SliceSubmitted]
          leaseStored <- store.getObject control leaseObject
          lease <- case leaseStored of
            Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String Lease)
            Nothing -> expectationFailure "detached lease was released" >> error "unreachable"
          slice <- case journal.slices of
            [entry] -> pure entry
            _ -> expectationFailure "expected one detached slice" >> error "unreachable"
          lease.ttlSeconds `shouldSatisfy` (>= fromIntegral slice.submission.limits.wallClockSeconds + 600)
          runWithArgs ["cell", "resume", "--session", outDir] `shouldReturnCode` ExitFailure 4
          store.statObject control leaseObject >>= (`shouldSatisfy` (maybe False (const True)))
          now <- getCurrentTime
          let rejectedObject = ObjectName ("cells/alpha/submissions/" <> renderRunId slice.cellRun <> "/rejected.json")
          _ <- store.putObject control rejectedObject "application/json" DoesNotExist (Aeson.encode (Rejected slice.cellRun "fixture-rejected" now))
          runWithArgs ["cell", "resume", "--session", outDir] `shouldReturnCode` ExitFailure 4
          store.statObject control leaseObject `shouldReturn` Nothing
          finished <- readSessionJournal (outDir </> "session.json") >>= either (fail . Text.unpack) pure
          fmap (.state) finished.slices `shouldBe` [SliceRejected]
          runWithArgs command `shouldReturnCode` ExitFailure 2
          store.statObject control leaseObject `shouldReturn` Nothing

    it "starts stopped instances and releases a reattached held lease after a terminal resume" $
      withSystemTempDirectory "kenshou-cell-resume-release" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        template <- Aeson.eitherDecodeFileStrict' "../kenshou-remote/test/golden/cell-session.json" >>= either fail pure
        let control = Bucket "control"
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            outDir = root </> "session"
            leaseObject = ObjectName "cells/alpha/lease.json"
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        withCellStore root do
          runWithArgs ["cell", "lease", "--cell", "alpha", "--control-bucket", "control", "--purpose", "resume-test"] `shouldReturnCode` ExitSuccess
          stored <- store.getObject control leaseObject
          lease <- case stored of
            Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String Lease)
            Nothing -> fail "fixture lease missing"
          let journal = (template :: SessionJournal) {store = Text.pack ("file:" <> root), controlBucket = "control", resultsBucket = descriptor.buckets.results, leaseId = lease.leaseId, leaseMode = Held, slices = map (\slice -> slice {state = SliceRejected, rejectionReason = Just "fixture-rejected", cellOutcome = Nothing, entryExitCode = Nothing, manifestSha256 = Nothing, fetchedPath = Nothing}) template.slices}
          createDirectoryIfMissing True outDir
          writeSessionJournal (outDir </> "session.json") journal
          let binDir = root </> "bin"
              fakeGcloud = binDir </> "gcloud"
              started = root </> "instances-started"
          createDirectoryIfMissing True binDir
          writeFile fakeGcloud "#!/bin/sh\nif [ \"$3\" = describe ]; then echo TERMINATED; else printf started > \"$KENSHOU_TEST_STARTED\"; fi\n"
          permissions <- getPermissions fakeGcloud
          setPermissions fakeGcloud permissions {executable = True}
          oldPath <- lookupEnv "PATH"
          withEnvVariable "PATH" (binDir <> ":" <> maybe "" id oldPath) $
            withEnvVariable "KENSHOU_TEST_STARTED" started $
              runWithArgs ["cell", "resume", "--start", "--session", outDir] `shouldReturnCode` ExitFailure 4
          doesFileExist started `shouldReturn` True
          store.statObject control leaseObject `shouldReturn` Nothing
          runWithArgs ["cell", "resume", "--start", "--help"] `shouldReturnCode` ExitSuccess

    it "rebinds a submitted marker that a prior held lease never accepted" $
      withSystemTempDirectory "kenshou-cell-resume-unaccepted" \root -> do
        store <- newFileStore root
        descriptorBytes <- LazyByteString.readFile "../kenshou-remote/test/golden/cell/cell.descriptor.v1.json"
        observed <- either fail pure (Aeson.eitherDecode descriptorBytes :: Either String CellDescriptor)
        template <- Aeson.eitherDecodeFileStrict' "../kenshou-remote/test/golden/cell-session.json" >>= either fail pure
        let control = Bucket "control"
            descriptor = observed {buckets = observed.buckets {control = "control"}}
            outDir = root </> "session"
            leaseObject = ObjectName "cells/alpha/lease.json"
        _ <- store.putObject control (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist (Aeson.encode descriptor)
        withCellStore root do
          runWithArgs ["cell", "lease", "--cell", "alpha", "--control-bucket", "control", "--purpose", "old-session"] `shouldReturnCode` ExitSuccess
          oldStored <- store.getObject control leaseObject
          oldLease <- case oldStored of
            Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String Lease)
            Nothing -> expectationFailure "old lease missing" >> error "unreachable"
          fixtureSlice <- case (template :: SessionJournal).slices of
            [entry] -> pure entry
            _ -> expectationFailure "expected one fixture slice" >> error "unreachable"
          let fixtureSubmission = fixtureSlice.submission
              fixturePayload = fixtureSubmission.payload
              fixtureBundle = fixturePayload.bundle
              payload = fixturePayload {bundle = fixtureBundle {uri = "gs://control/payloads/sha256/" <> fixtureBundle.sha256 <> ".nar.zst"}}
              oldSubmission = fixtureSubmission {leaseId = oldLease.leaseId, payload = payload, work = workObjectFor "application/json" "{}"}
              oldSlice = fixtureSlice {submission = oldSubmission, state = SliceSubmitted}
              journal = template {store = Text.pack ("file:" <> root), controlBucket = "control", resultsBucket = descriptor.buckets.results, leaseId = oldLease.leaseId, slices = [oldSlice]}
              oldMarker = ObjectName ("cells/alpha/submissions/" <> renderRunId oldSlice.cellRun <> "/submission.json")
              awaitRetry = do
                objects <- store.listObjects control "cells/alpha/submissions/"
                case find (\(name, _) -> name /= oldMarker && Text.isSuffixOf "/submission.json" name.unObjectName) objects of
                  Nothing -> threadDelay 10000 >> awaitRetry
                  Just (name, _) -> do
                    stored <- store.getObject control name
                    submission <- case stored of
                      Just (bytes, _) -> either fail pure (Aeson.eitherDecode bytes :: Either String Submission)
                      Nothing -> fail "retry marker disappeared"
                    now <- getCurrentTime
                    let rejected = ObjectName ("cells/alpha/submissions/" <> renderRunId submission.runId <> "/rejected.json")
                    _ <- store.putObject control rejected "application/json" DoesNotExist (Aeson.encode (Rejected submission.runId "fixture-rejected" now))
                    pure ()
          createDirectoryIfMissing True (outDir </> "slice-0")
          LazyByteString.writeFile (outDir </> "slice-0/work.json") "{}"
          writeSessionJournal (outDir </> "session.json") journal
          _ <- store.putObject control oldMarker "application/json" DoesNotExist (Aeson.encode oldSubmission)
          runWithArgs ["cell", "release", "--cell", "alpha", "--control-bucket", "control", "--lease-id", Text.unpack (renderRunId oldLease.leaseId)] `shouldReturnCode` ExitSuccess
          worker <- forkIO awaitRetry
          ( do
              result <- timeout 15000000 (runWithArgs ["cell", "resume", "--session", outDir])
              result `shouldBe` Just (ExitFailure 4)
              finished <- readSessionJournal (outDir </> "session.json") >>= either (fail . Text.unpack) pure
              fmap (.state) finished.slices `shouldBe` [SliceRejected]
              fmap (.cellRun) finished.slices `shouldSatisfy` (/= [oldSlice.cellRun])
              store.statObject control leaseObject `shouldReturn` Nothing
            )
            `finally` killThread worker

    it "reads scenario history from a bundle" do
      runWithArgs ["history", "--bundle", "../fixtures/consumer-baseline/docs/verification", "--bundle-uri", "mori://shinzui/kenshou/okf/verification", "--scenario", "selftest/kernel/correctness/always-pass", "--json"] `shouldReturnCode` ExitSuccess
      runWithArgs ["history", "--bundle", "../fixtures/consumer-baseline/docs/verification", "--scenario", "selftest/kernel/correctness/always-pass", "--json"] `shouldReturnCode` ExitFailure 2

    it "requires one record source" do
      runWithArgs ["record", "--purpose", "release"] `shouldReturnCode` ExitFailure 2
      runWithArgs ["record", "a-run", "--comparison", "comparison.json", "--purpose", "release"] `shouldReturnCode` ExitFailure 2

    it "rejects a human anomaly exception without an interactive terminal" do
      runWithArgs ["attest", "invalid", "--project", "fixture", "--accept-anomaly", "--authority", "human:fixture", "--reason", "fixture"] `shouldReturnCode` ExitFailure 2

    it "returns 2 for an unknown subcommand" do
      runWithArgs ["cohort", "bogus"] `shouldReturnCode` ExitFailure 2

    it "returns 1 when offline leak diagnosis finds growth" do
      runWithArgs ["diagnose", "leak", leakingRun] `shouldReturnCode` ExitFailure 1

    it "returns 1 when an offline stall diagnosis exists" do
      runWithArgs ["diagnose", "stall", stalledRun] `shouldReturnCode` ExitFailure 1

    it "returns 0 when no offline stall diagnosis exists" do
      runWithArgs ["diagnose", "stall", leakingRun] `shouldReturnCode` ExitSuccess

    it "returns 3 when leak evidence is insufficient" do
      runWithArgs ["diagnose", "leak", stalledRun] `shouldReturnCode` ExitFailure 3

    it "returns 4 when diagnosis input is unavailable" do
      runWithArgs ["diagnose", "leak", "test/fixtures/does-not-exist"] `shouldReturnCode` ExitFailure 4

    it "returns 2 for invalid diagnose syntax" do
      runWithArgs ["diagnose", "leak", "--bogus"] `shouldReturnCode` ExitFailure 2

    it "does not mutate a sealed run manifest" do
      before <- Text.readFile (leakingRun <> "/manifest.json")
      _ <- runWithArgs ["diagnose", "leak", leakingRun]
      after <- Text.readFile (leakingRun <> "/manifest.json")
      after `shouldBe` before

  describe "version" do
    it "includes the cabal package version and a short revision" do
      appVersionWithGit `shouldSatisfy` Text.isPrefixOf "kenshou v0.1.0.0 ("
      Text.dropAround (`elem` ['(', ')']) (Text.takeWhileEnd (/= ' ') appVersionWithGit)
        `shouldSatisfy` (\revision -> revision == "dirty" || Text.length revision == 7)

runWithArgs :: [String] -> IO ExitCode
runWithArgs = runWithArgsWith selfTestAssembly

-- Runs from the consumer-baseline bridge with the consumer's component graph, for the
-- perf planning checks whose request names the consumer's packages. Called from the
-- repository root.
withConsumerBaseline :: [String] -> IO ExitCode
withConsumerBaseline arguments = do
  graph <- ByteString.readFile "fixtures/consumer-baseline/kenshou-core/data/components.json"
  withCurrentDirectory "fixtures/consumer-baseline" (runWithArgsWith selfTestAssembly {componentGraph = graph} arguments)

shouldReturnCode :: IO ExitCode -> ExitCode -> IO ()
shouldReturnCode action expected = action >>= (`shouldBe` expected)

leakingRun :: FilePath
leakingRun = "../kenshou-diagnose/test/fixtures/run-leaking"

stalledRun :: FilePath
stalledRun = "../kenshou-diagnose/test/fixtures/run-stalled"

withCellStore :: FilePath -> IO value -> IO value
withCellStore root operation = bracket (lookupEnv "KENSHOU_CELL_STORE") (restore "KENSHOU_CELL_STORE") \_ ->
  bracket (lookupEnv "KENSHOU_GCP_ALLOWED_PROJECTS") (restore "KENSHOU_GCP_ALLOWED_PROJECTS") \_ -> do
    setEnv "KENSHOU_CELL_STORE" ("file:" <> root)
    setEnv "KENSHOU_GCP_ALLOWED_PROJECTS" "tan-nb-exp"
    operation
  where
    restore name Nothing = unsetEnv name
    restore name (Just prior) = setEnv name prior

withCapabilityDir :: FilePath -> IO value -> IO value
withCapabilityDir directory operation = bracket (lookupEnv "KENSHOU_CELL_CAPABILITIES_DIR") restore \_ -> do
  setEnv "KENSHOU_CELL_CAPABILITIES_DIR" directory
  operation
  where
    restore Nothing = unsetEnv "KENSHOU_CELL_CAPABILITIES_DIR"
    restore (Just prior) = setEnv "KENSHOU_CELL_CAPABILITIES_DIR" prior

withEnvVariable :: String -> String -> IO value -> IO value
withEnvVariable name selected operation = bracket (lookupEnv name) restore \_ -> do
  setEnv name selected
  operation
  where
    restore Nothing = unsetEnv name
    restore (Just prior) = setEnv name prior

withPerfRoot :: IO a -> IO a
withPerfRoot operation = do
  exists <- doesFileExist "cabal.project"
  withCurrentDirectory (if exists then "." else "..") operation

replacePerfField :: Key.Key -> Value -> Value -> Value
replacePerfField key value (Aeson.Object fields) = Aeson.Object (KeyMap.insert key value fields)
replacePerfField _ _ value = value

-- The request's own target components plus one component of its cohort template
-- that belongs to a different project, read from the consumer-baseline bridge.
foreignTargets :: Value -> IO [Text]
foreignTargets request = do
  let field name (Aeson.Object fields) = KeyMap.lookup name fields
      field _ _ = Nothing
      texts value = case value of Just (Aeson.Array items) -> [text | Aeson.String text <- toList items]; _ -> []
      owned = texts (field "targetComponents" request)
  template <- case field "cohortTemplate" request of
    Just (Aeson.String path) -> Aeson.eitherDecodeFileStrict' ("fixtures/consumer-baseline/" <> Text.unpack path) >>= either fail pure
    _ -> fail "the request names no cohort template"
  let components = case field "components" template of Just (Aeson.Array items) -> toList items; _ -> []
  case [name | component <- components, Just (Aeson.String name) <- [field "id" component], field "moriUri" component /= field "project" request] of
    other : _ -> pure (owned <> [other])
    [] -> fail "the cohort template has no component from another project"