packages feed

kenshou-remote-0.1.0.0: test/PrepareSpec.hs

module PrepareSpec (spec) where

import Data.Aeson (Value (..), eitherDecode, eitherDecodeFileStrict', encode, toJSON)
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Foldable (toList)
import Data.List.NonEmpty (NonEmpty (..))
import Data.List.NonEmpty qualified as NonEmpty
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import Kenshou.Core.Bundle (LayerBundle (..), lookupScenario, mkBundle, mkRegistry, registryRoutingRules)
import Kenshou.Core.Bundle qualified as Bundle
import Kenshou.Core.Dimension (PgDurability (..), PgVersion (..), noDimensions, postgresDimensions)
import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))
import Kenshou.Core.Id (Kind (..), mkSeed, newRunId, parseScenarioId, renderKind, selftestLayer)
import Kenshou.Core.Knob (RawKnob (..), mkKnobName)
import Kenshou.Core.Resource (CellBinding (..), ProvisionedResource (..), ResourceProvider (..), mkResourceName)
import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), minimalRunSpec)
import Kenshou.Core.RunSpec qualified as RunSpec
import Kenshou.Core.Scenario (Scenario (..))
import Kenshou.Core.Selftest qualified as Selftest
import Kenshou.Core.Selftest.Schema (selftestMigrationSet)
import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))
import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))
import Kenshou.Plan.Policy (defaultPlanPolicy)
import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (PlannedRun), RunPlan (..))
import Kenshou.Plan.Selector (parseSelector)
import Kenshou.Remote.Cell.Docs (CachePolicy (..), CellBuckets (..), CellDescriptor (..), CellImages (..), CollectOptions (..), Limits (..), PgReset (..), ResetBlock (..), Submission (..))
import Kenshou.Remote.Cell.Lease (CellRef (..))
import Kenshou.Remote.Cell.Prepare (Granularity (..), OtlpSink (..), PrepareOptions (..), Prepared (..), PreparedRun (..), RejectReason (..), Routed (..), Slice (..), SubmissionInputs (..), prepareForCell, routePlan, slicePlan, sliceRuns, submissionFor)
import Kenshou.Remote.Cell.Prepare qualified as Prepare
import Kenshou.Remote.Cell.RouteJson (RouteDocuments (..), routeWorkJson)
import Kenshou.Remote.Cell.RouteRules (CellCapabilities (..), RoutingRule (..), RuleCondition (..), descriptorDigest, loadCellCapabilities, matchingRules)
import Kenshou.Remote.Cell.Session.Build (BuildOptions (..), BuiltSession (..), buildSession)
import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..))
import Kenshou.Remote.Cell.Submit (workObjectFor)
import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan, renderPreparedWork)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))
import Kenshou.Remote.Store (Bucket (..))
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec

spec :: Spec
spec = describe "cell run slicing" do
  it "invalidates machine profiles when cell images change and preserves equivalent hardware profiles" do
    (_, descriptor, _) <- preparationFixture
    let original = Prepare.cellMachineProfile descriptor
    Prepare.cellMachineProfile descriptor {name = "same-shape-other-cell"} `shouldBe` original
    Prepare.cellMachineProfile descriptor {images = descriptor.images {driver = "different-driver-image"}} `shouldNotBe` original
    Prepare.cellMachineProfile descriptor {images = descriptor.images {postgres = "different-postgres-image"}} `shouldNotBe` original

  it "prepares durable PostgreSQL work and records reset settings and payload cohort" do
    (plan, descriptor, payload) <- preparationFixture
    let registry = either (error . show) id (mkRegistry [Selftest.bundle])
        payloads = Map.singleton "default" payload
        options = PrepareOptions False False [] Cold
        rejected = prepareForCell registry descriptor Nothing [] payloads options plan
        accepted = prepareForCell registry descriptor Nothing [] payloads (options {coerceDurable = True}) plan
    fmap snd rejected.rejected `shouldBe` [DurabilityNotDurable]
    case accepted.accepted of
      [run] -> do
        run.spec.environment.postgres `shouldBe` Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))
        case run.reset.postgres of
          Just reset -> do
            Map.lookup "fsync" reset.settings `shouldBe` Just "on"
            Map.lookup "synchronous_commit" reset.settings `shouldBe` Just "on"
          Nothing -> expectationFailure "expected PostgreSQL reset"
        run.spec.cohortExpectation `shouldSatisfy` (/= Nothing)
        accepted.warnings `shouldSatisfy` (not . null)
      _ -> expectationFailure "expected one prepared run"
    fmap snd (prepareForCell registry (descriptor {postgresMajor = 17}) Nothing [] payloads (options {coerceDurable = True}) plan).rejected `shouldBe` [PgVersionMismatch 18 17]
    fmap snd (prepareForCell registry descriptor Nothing [] payloads (options {coerceDurable = True, pgSettings = [("fsync", "off")]}) plan).rejected `shouldBe` [ConflictingPostgresSetting "fsync"]

  it "keeps image-owned preloaded libraries out of PostgreSQL reset settings" do
    (plan, descriptor, payload) <- preparationFixture
    let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])
        scenarioId = case plan.runs of
          [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario
          _ -> error "expected one planned run"
        original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)
        requirement = maybe (error "missing PostgreSQL requirement") id original.requires.postgres
        required = PostgresRequirement requirement.schemas [("shared_preload_libraries", "'pg_stat_statements'")] requirement.needsServerControl
        withPreload = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements (Just required) original.requires.extraPostgres original.requires.resources) original.knownDefect original.run
        registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [withPreload] []) {migrations = [selftestMigrationSet]}])
        prepared = prepareForCell registry descriptor Nothing [] (Map.singleton "default" payload) (PrepareOptions True False [] Cold) plan
    case prepared.accepted of
      [run] -> case run.reset.postgres of
        Just reset -> Map.member "shared_preload_libraries" reset.settings `shouldBe` False
        Nothing -> expectationFailure "expected PostgreSQL reset"
      _ -> expectationFailure "expected one prepared run"

  it "keeps the owner PostgreSQL reset path for a worker with no database requirement" do
    (plan, descriptor, payload) <- preparationFixture
    worker <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseScenarioId "selftest/kernel/concurrency/worker-echo")
    let registry = either (error . show) id (mkRegistry [Selftest.bundle])
        payloads = Map.singleton "default" payload
        workerPlan = case plan.runs of
          [PlannedRun ordinal identifier estimate reasons trial _] ->
            let base = minimalRunSpec worker
                workerSpec = RunSpec (Just identifier) worker base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels
             in plan {runs = [PlannedRun ordinal identifier estimate reasons trial workerSpec]}
          _ -> error "expected one planned run"
    case (prepareForCell registry descriptor Nothing [] payloads (PrepareOptions False False [] Warm) workerPlan).accepted of
      [preparedRun] -> do
        preparedRun.spec.environment.postgres `shouldBe` Nothing
        preparedRun.reset.postgres `shouldBe` Just (PgReset descriptor.postgresMajor [] Map.empty)
      _ -> expectationFailure "expected one worker run"

  it "rejects a resource the cell cannot bind or does not report" do
    (plan, descriptor, payload) <- preparationFixture
    now <- getCurrentTime
    let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])
        scenarioId = case plan.runs of
          [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario
          _ -> error "expected one planned run"
        original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)
        objectStore = either (error . Text.unpack) id (mkResourceName "object-store")
        needsStore = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements original.requires.postgres original.requires.extraPostgres [objectStore]) original.knownDefect original.run
        provider binding = ResourceProvider objectStore "Object store" (Right . maybe Null id) (\_ value action -> action (ProvisionedResource value Null)) binding
        registryWith binding = either (error . show) id (mkRegistry [(mkBundle selftestLayer [needsStore] []) {migrations = [selftestMigrationSet], Bundle.resources = [provider binding]}])
        payloads = Map.singleton "default" payload
        options = PrepareOptions True False [] Cold
        binding = Just (CellBinding "object-store" Nothing)
        reporting = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "object-store" (Bool True))
    fmap snd (prepareForCell (registryWith Nothing) descriptor Nothing [] payloads options plan).rejected `shouldBe` [ResourceUnavailable objectStore]
    fmap snd (prepareForCell (registryWith binding) descriptor Nothing [] payloads options plan).rejected `shouldBe` [ResourceUnavailable objectStore]
    length (prepareForCell (registryWith binding) descriptor (Just reporting) [] payloads options plan).accepted `shouldBe` 1

  it "keeps server-control work off the cell server unless the driver is explicitly selected" do
    (plan, descriptor, payload) <- preparationFixture
    let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])
        scenarioId = case plan.runs of
          [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario
          _ -> error "expected one planned run"
        original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)
        requirement = maybe (error "missing PostgreSQL requirement") id original.requires.postgres
        controlled = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements (Just (requirement {needsServerControl = True})) original.requires.extraPostgres original.requires.resources) original.knownDefect original.run
        registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [controlled] []) {migrations = [selftestMigrationSet]}])
        payloads = Map.singleton "default" payload
        options = PrepareOptions False False [] Cold
    fmap snd (prepareForCell registry descriptor Nothing [] payloads options plan).rejected `shouldBe` [NeedsServerControl "primary"]
    case (prepareForCell registry descriptor Nothing [] payloads (options {ephemeralOnDriver = True}) plan).accepted of
      [run] -> do
        run.spec.environment.postgres `shouldBe` Just (PostgresEphemeral [])
        run.reset.postgres `shouldBe` Just (PgReset descriptor.postgresMajor [] Map.empty)
      _ -> expectationFailure "expected one driver-local prepared run"
    let routed = routePlan registry ((descriptor, Nothing) :| []) [] payloads options plan
    fmap (.planId) routed.local `shouldBe` Just plan.planId
    Map.null routed.perCell `shouldBe` True

  it "routes PostgreSQL majors to matching cells while preserving plan order and run IDs" do
    (plan, descriptor18, payload) <- preparationFixture
    secondId <- newRunId
    let original = case plan.runs of
          [PlannedRun _ _ _ originalReasons originalTrial originalSpec] -> (originalReasons, originalTrial, originalSpec)
          _ -> error "expected one planned run"
        (reasons, trial, firstSpec) = original
        secondSpec = RunSpec (Just secondId) firstSpec.scenario firstSpec.scenarioRevision firstSpec.knobs [("pg.version", "17")] firstSpec.seed firstSpec.phases firstSpec.timeoutSeconds firstSpec.environment firstSpec.cohortExpectation firstSpec.comparison firstSpec.labels
        second = PlannedRun 1 secondId 1 reasons trial secondSpec
        twoRunPlan = RunPlan plan.planId plan.createdAt plan.context plan.policy (plan.runs <> [second]) plan.skipped 2
        descriptor17 = descriptor18 {name = "beta", postgresMajor = 17}
        baseRegistry = either (error . show) id (mkRegistry [Selftest.bundle])
        scenario = maybe (error "missing selftest") id (lookupScenario baseRegistry firstSpec.scenario)
        bothMajors = Scenario scenario.id scenario.revision scenario.summary scenario.tier scenario.placement scenario.knobs (postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) noDimensions) scenario.phases (EnvRequirements (Just (PostgresRequirement [] [] False)) [] []) scenario.knownDefect scenario.run
        registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [bothMajors] []) {migrations = [selftestMigrationSet]}])
        routed = routePlan registry ((descriptor17, Nothing) :| [(descriptor18, Nothing)]) [] (Map.singleton "default" payload) (PrepareOptions True False [] Cold) twoRunPlan
        runIds selected = fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) selected.runs
    fmap runIds (Map.lookup "alpha" routed.perCell) `shouldBe` Just [case plan.runs of [PlannedRun _ identifier _ _ _ _] -> identifier; _ -> error "expected one planned run"]
    fmap runIds (Map.lookup "beta" routed.perCell) `shouldBe` Just [secondId]
    routed.local `shouldBe` Nothing
    routed.unroutable `shouldBe` []

  it "routes a public JSON plan while preserving its provenance and reporting refusals" do
    (plan, descriptor, payload) <- preparationFixture
    let registry = either (error . show) id (mkRegistry [Selftest.bundle])
        input = toJSON plan
        route = routeWorkJson registry ((descriptor, Nothing) :| []) [] (Just (Map.singleton "default" payload)) (PrepareOptions True False [] Cold) input
    documents <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure route
    documents.routeComplete `shouldBe` True
    documents.local `shouldBe` Nothing
    case (input, Map.lookup descriptor.name documents.perCell) of
      (Object source, Just (Object selected)) -> do
        KeyMap.lookup "planId" selected `shouldBe` KeyMap.lookup "planId" source
        KeyMap.lookup "createdAt" selected `shouldBe` KeyMap.lookup "createdAt" source
        KeyMap.lookup "policy" selected `shouldBe` KeyMap.lookup "policy" source
        KeyMap.lookup "changes" selected `shouldBe` KeyMap.lookup "changes" source
        KeyMap.lookup "runs" selected `shouldNotBe` KeyMap.lookup "runs" source
      _ -> expectationFailure "expected a routed plan object"
    case input of
      Object source -> do
        let originalRun = case KeyMap.lookup "runs" source of
              Just (Array runs) -> case toList runs of
                [run] -> run
                _ -> error "expected one run"
              _ -> error "missing runs"
            duplicate = Object (KeyMap.insert "runs" (toJSON [originalRun, originalRun]) source)
        routeWorkJson registry ((descriptor, Nothing) :| []) [] (Just (Map.singleton "default" payload)) (PrepareOptions True False [] Cold) duplicate `shouldSatisfy` isLeft
      _ -> expectationFailure "expected a plan object"
    withoutPayload <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (routeWorkJson registry ((descriptor, Nothing) :| []) [] Nothing (PrepareOptions True False [] Cold) input)
    withoutPayload.routeComplete `shouldBe` True
    case Map.lookup descriptor.name withoutPayload.perCell of
      Just (Object selected) -> case KeyMap.lookup "runs" selected of
        Just (Array rows) -> case toList rows of
          [Object row] -> case KeyMap.lookup "spec" row of
            Just (Object selectedSpec) -> KeyMap.lookup "cohortExpectation" selectedSpec `shouldBe` Just Null
            _ -> expectationFailure "expected a selected spec"
          _ -> expectationFailure "expected one selected run"
        _ -> expectationFailure "expected runs"
      _ -> expectationFailure "expected a routed plan"

  it "renders prepared slices from the original plan with reset settings intact" do
    (plan, descriptor, payload) <- preparationFixture
    let input = toJSON plan
        registry = either (error . show) id (mkRegistry [Selftest.bundle])
        options = PrepareOptions True False [] Cold
    decoded <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (decodeWorkPlan input)
    decoded.planId `shouldBe` plan.planId
    let selectedWork = prepareForCell registry descriptor Nothing [] (Map.singleton "default" payload) options decoded
    case selectedWork.accepted of
      [selected] -> do
        rendered <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (renderPreparedWork input [selected])
        case (input, rendered) of
          (Object original, Object slice) -> do
            KeyMap.lookup "policy" slice `shouldBe` KeyMap.lookup "policy" original
            KeyMap.lookup "changes" slice `shouldBe` KeyMap.lookup "changes" original
            KeyMap.lookup "runs" slice `shouldNotBe` KeyMap.lookup "runs" original
          _ -> expectationFailure "expected a rendered slice"
        renderPreparedWork input [selected {ordinal = 99}] `shouldSatisfy` isLeft
      _ -> expectationFailure "expected a prepared run"

  it "builds a durable session and refuses incompatible work before publication" do
    (plan, descriptor, payload) <- preparationFixture
    lease <- newRunId
    let registry = either (error . show) id (mkRegistry [Selftest.bundle])
        payloads = Map.singleton "default" payload
        matchingDescriptor = descriptor {buckets = descriptor.buckets {control = "control"}}
        options coerce = BuildOptions (PrepareOptions coerce False [] Cold) GranularityAuto False NullSink Nothing (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0"
        source = encode plan
    rejected <- buildSession registry matchingDescriptor Nothing [] payloads (options False) "file:fixture" lease source
    rejected `shouldSatisfy` isLeft
    built <- buildSession registry matchingDescriptor Nothing [] payloads (options True) "file:fixture" lease source
    case built of
      Left problem -> expectationFailure (Text.unpack problem)
      Right session -> do
        session.journal.leaseId `shouldBe` lease
        length session.journal.slices `shouldBe` 1
        length session.workFiles `shouldBe` 1
        case (session.journal.slices, session.workFiles) of
          ([slice], [(_, bytes)]) -> do
            slice.runIds `shouldBe` fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) plan.runs
            case eitherDecode bytes of
              Right (Object rendered) -> KeyMap.lookup "policy" rendered `shouldBe` case toJSON plan of Object original -> KeyMap.lookup "policy" original; _ -> Nothing
              _ -> expectationFailure "expected a work plan"
          _ -> expectationFailure "expected one slice and work file"

  it "uses cached denials and warns when a matching capability has not been probed" do
    (plan, descriptor, payload) <- preparationFixture
    now <- getCurrentTime
    selector <- either (error . Text.unpack) pure (parseSelector "selftest/kernel/**")
    let registry = either (error . show) id (mkRegistry [Selftest.bundle])
        rule = RoutingRule (ScenarioMatches selector) "postgres.pg_partman" "requires extension"
        options = PrepareOptions True False [] Cold
        payloads = Map.singleton "default" payload
        unprobed = prepareForCell registry descriptor Nothing [rule] payloads options plan
        deniedCache = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "postgres.pg_partman" (Bool False))
        denied = prepareForCell registry descriptor (Just deniedCache) [rule] payloads options plan
        stale = prepareForCell registry (descriptor {postgresMajor = 17}) (Just deniedCache) [rule] payloads options plan
    length unprobed.accepted `shouldBe` 1
    unprobed.warnings `shouldSatisfy` any (Text.isInfixOf "kenshou cell probe")
    fmap snd denied.rejected `shouldBe` [MissingCapability "postgres.pg_partman"]
    fmap snd stale.rejected `shouldBe` [PgVersionMismatch 18 17]

  it "reads the routing rules registered bundles contribute" do
    scenario <- either (error . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/always-pass")
    knob <- either (error . Text.unpack) pure (mkKnobName "synthetic.queue-kind")
    let rule = RoutingRule (KnobEquals "synthetic.queue-kind" "partitioned") "postgres.pg_partman" "Partitioned queues need pg_partman on the cell PostgreSQL server."
        registry = either (error . show) id (mkRegistry [Selftest.bundle, (mkBundle selftestLayer [] []) {routingRules = [rule]}])
        base = minimalRunSpec scenario
        withKnob = RunSpec base.runId base.scenario base.scenarioRevision [(knob, RawText "partitioned")] base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels
    registryRoutingRules registry `shouldBe` [rule]
    fmap (.requires) (matchingRules (registryRoutingRules registry) withKnob) `shouldBe` ["postgres.pg_partman"]
    matchingRules (registryRoutingRules registry) base `shouldBe` []

  it "round-trips a probed capability cache" do
    cache <- eitherDecodeFileStrict' "test/golden/cell-capabilities.json" >>= either fail pure
    eitherDecode (encode cache) `shouldBe` Right (cache :: CellCapabilities)

  it "loads only a valid cache for the current cell descriptor" $
    withSystemTempDirectory "kenshou-capabilities" \directory -> do
      (plan, descriptor, _) <- preparationFixture
      now <- getCurrentTime
      let path = directory </> Text.unpack descriptor.name <> ".capabilities.json"
          cache = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "postgres.pg_partman" (Bool False))
      loadCellCapabilities directory descriptor `shouldReturn` Right Nothing
      LazyByteString.writeFile path (encode cache)
      loadCellCapabilities directory descriptor `shouldReturn` Right (Just cache)
      loadCellCapabilities directory (descriptor {postgresMajor = 17}) `shouldReturn` Right Nothing
      LazyByteString.writeFile path "invalid JSON"
      malformed <- loadCellCapabilities directory descriptor
      malformed `shouldSatisfy` isLeft

  it "shares adjacent correctness resets and isolates benchmark and soak runs" do
    first <- prepared 0 Correctness "default" cold
    second <- prepared 1 Concurrency "default" cold
    benchmark <- prepared 2 Benchmark "default" cold
    third <- prepared 3 Correctness "default" cold
    soak <- prepared 4 Soak "default" cold
    fourth <- prepared 5 Correctness "default" cold
    slices <- expectSlices (sliceRuns GranularityAuto [first, second, benchmark, third, soak, fourth])
    fmap (NonEmpty.length . (.entries)) slices `shouldBe` [2, 1, 1, 1, 1]
    fmap (.index) slices `shouldBe` [0, 1, 2, 3, 4]
    fmap (.wallClockSeconds) slices `shouldBe` [320, 310, 310, 910, 310]

  it "leaves bounded time for full-soak drain and diagnosis without extending benchmark limits" do
    soak <- prepared 0 Soak "default" cold
    benchmark <- prepared 1 Benchmark "default" cold
    slices <- expectSlices (sliceRuns GranularityRun [soak {Prepare.timeoutSeconds = 14400}, benchmark {Prepare.timeoutSeconds = 600}])
    fmap (.wallClockSeconds) slices `shouldBe` [15300, 900]

  it "budgets finalization for each soak in a grouped slice and rejects overflow" do
    first <- prepared 0 Soak "default" cold
    second <- prepared 1 Soak "default" cold
    slices <- expectSlices (sliceRuns GranularityPlan [first {Prepare.timeoutSeconds = 14400}, second {Prepare.timeoutSeconds = 14400}])
    fmap (.wallClockSeconds) slices `shouldBe` [30300]
    sliceRuns GranularityRun [first {Prepare.timeoutSeconds = maxBound}] `shouldSatisfy` isLeft

  it "starts a new automatic slice when the payload or reset changes" do
    first <- prepared 0 Correctness "default" cold
    second <- prepared 1 Correctness "candidate" cold
    third <- prepared 2 Correctness "candidate" warm
    fourth <- prepared 3 Correctness "candidate" warm
    slices <- expectSlices (sliceRuns GranularityAuto [first, second, third, fourth])
    fmap (NonEmpty.length . (.entries)) slices `shouldBe` [1, 1, 2]
    fmap (.payloadLabel) slices `shouldBe` ["default", "candidate", "candidate"]

  it "requires a homogeneous plan slice and refuses duplicate identities" do
    first <- prepared 0 Correctness "default" cold
    second <- prepared 1 Correctness "default" cold
    runSlices <- expectSlices (sliceRuns GranularityRun [first, second])
    fmap (NonEmpty.length . (.entries)) runSlices `shouldBe` [1, 1]
    planSlices <- expectSlices (sliceRuns GranularityPlan [first, second])
    fmap (NonEmpty.length . (.entries)) planSlices `shouldBe` [2]
    sliceRuns GranularityPlan [first, PreparedRun second.ordinal second.runId second.spec second.payloadLabel warm second.timeoutSeconds] `shouldBe` Left "plan granularity requires one payload and reset for every run"
    sliceRuns GranularityRun [first, first {ordinal = 1}] `shouldSatisfy` isLeft

  it "keeps the original plan metadata while replacing only selected run specs" do
    first <- prepared 0 Correctness "default" cold
    second <- prepared 1 Correctness "default" cold
    planId <- newRunId
    now <- getCurrentTime
    selector <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseSelector "selftest/**")
    seed <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (mkSeed 1)
    let reason = Reason (Change (ComponentRef (ComponentId "remote") Nothing) Everything "fixture") [] selector 0
        planned run = PlannedRun run.ordinal run.runId 1 (reason :| []) Nothing run.spec
        planContext = PlanContext Nothing "graph" "released" "plan-hash" (PlanInputs Null) [] []
        original = RunPlan planId now planContext (defaultPlanPolicy seed) [planned first, planned second] [] 2
        changed = PreparedRun first.ordinal first.runId (first.spec {RunSpec.timeoutSeconds = Just 42}) first.payloadLabel first.reset first.timeoutSeconds
    slices <- expectSlices (sliceRuns GranularityAuto [changed, second])
    case slices of
      [slice] -> do
        sliced <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (slicePlan original slice)
        sliced.planId `shouldBe` original.planId
        fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) sliced.runs `shouldBe` [first.runId, second.runId]
        fmap (\(PlannedRun _ _ _ _ _ runSpec) -> runSpec.timeoutSeconds) sliced.runs `shouldBe` [Just 42, Nothing]
        sliced.estimateMinutes `shouldBe` 2
        slicePlan (original {runs = [planned first]}) slice `shouldSatisfy` isLeft
      _ -> expectationFailure "expected one slice"

  it "builds a cell submission with payload transport, limits and session labels" do
    planned <- prepared 0 Correctness "default" cold
    slices <- expectSlices (sliceRuns GranularityRun [planned])
    slice <- case slices of
      [value] -> pure value
      _ -> expectationFailure "expected one slice" >> error "unreachable"
    bytes <- LazyByteString.readFile "test/golden/payload.json"
    payload <- either (\problem -> expectationFailure problem >> error "unreachable") pure (eitherDecode bytes :: Either String PayloadDescriptor)
    cellRun <- newRunId
    lease <- newRunId
    session <- newRunId
    plan <- newRunId
    let inputs = SubmissionInputs cellRun lease session plan FileSink (Just "-N2 -T") (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0" Nothing
        ref = CellRef "alpha" (Bucket "control")
        work = workObjectFor "application/json" "{}"
    submission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref inputs payload slice work)
    submission.runId `shouldBe` cellRun
    submission.payload `shouldBe` payload.cell
    submission.limits.wallClockSeconds `shouldBe` 310
    Map.lookup "KENSHOU_PAYLOAD_BUNDLE_SHA256" submission.env `shouldBe` Just payload.cell.bundle.sha256
    Map.lookup "KENSHOU_OTLP_SINK" submission.env `shouldBe` Just "file"
    fmap (.traces) submission.collect `shouldBe` Just True
    fmap (.profiles) submission.collect `shouldBe` Just False
    Map.lookup "GHCRTS" submission.env `shouldBe` Just "-N2 -T"
    Map.lookup "payload" submission.labels `shouldBe` Just "default"
    defaultSubmission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref (inputs {otlpSink = NullSink, rtsOptions = Nothing}) payload slice work)
    Map.lookup "KENSHOU_OTLP_SINK" defaultSubmission.env `shouldBe` Just "null"
    defaultSubmission.collect `shouldBe` Nothing
    Map.lookup "GHCRTS" defaultSubmission.env `shouldBe` Nothing
    profileSubmission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref (inputs {otlpSink = NullSink, rtsOptions = Just "-p"}) payload slice work)
    profileSubmission.collect `shouldBe` Just (CollectOptions False True)
    eitherDecode (encode profileSubmission) `shouldBe` Right profileSubmission
    submissionFor (CellRef "alpha" (Bucket "other")) inputs payload slice work `shouldBe` Left "payload bundle is outside the cell control bucket"
    submissionFor ref inputs payload slice (workObjectFor "text/plain" "{}") `shouldBe` Left "cell work must be a nonempty JSON run plan"

prepared :: Int -> Kind -> Text -> ResetBlock -> IO PreparedRun
prepared ordinal kind label reset = do
  identifier <- newRunId
  scenario <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseScenarioId ("selftest/remote/" <> renderKind kind <> "/slice"))
  pure (PreparedRun ordinal identifier (minimalRunSpec scenario) label reset 10)

expectSlices :: Either Text [Slice] -> IO [Slice]
expectSlices = either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure

isLeft :: Either left right -> Bool
isLeft (Left _) = True
isLeft _ = False

cold :: ResetBlock
cold = ResetBlock Cold Nothing Nothing Map.empty

warm :: ResetBlock
warm = ResetBlock Warm Nothing Nothing Map.empty

preparationFixture :: IO (RunPlan, CellDescriptor, PayloadDescriptor)
preparationFixture = do
  planId <- newRunId
  runId <- newRunId
  now <- getCurrentTime
  selector <- either (error . Text.unpack) pure (parseSelector "selftest/kernel/**")
  scenario <- either (error . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/postgres-roundtrip")
  seed <- either (error . Text.unpack) pure (mkSeed 7)
  descriptor <- eitherDecodeFileStrict' "test/golden/cell/cell.descriptor.v1.json" >>= either fail pure
  payload <- eitherDecodeFileStrict' "test/golden/payload.json" >>= either fail pure
  let reason = Reason (Change (ComponentRef (ComponentId "kernel") Nothing) Named "cell prepare") [] selector 0
      base = minimalRunSpec scenario
      runSpec = RunSpec (Just runId) scenario base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels
      run = PlannedRun 0 runId 1 (reason :| []) Nothing runSpec
      planContext = PlanContext Nothing "fixture" "released" "sha256:fixture" (PlanInputs Null) [] []
  pure (RunPlan planId now planContext (defaultPlanPolicy seed) [run] [] 1, descriptor, payload)