packages feed

kenshou-remote (empty) → 0.1.0.0

raw patch · 105 files changed

+12010/−0 lines, 105 filesdep +aesondep +basedep +base16-bytestring

Dependencies added: aeson, base, base16-bytestring, bytestring, containers, cryptohash-sha256, directory, filepath, hspec, http-client, http-client-tls, http-types, kenshou-core, kenshou-measure, kenshou-remote, network, process, temporary, text, time, unix

Files

+ CHANGELOG.md view
@@ -0,0 +1,12 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of the leased verification cell client, payload publishing, and file and GCS object stores.+- Cells bind named resources through a `resources` block; `Kenshou.Remote.Cell.Legacy` adapts cells that still describe a broker.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+   list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+   this list of conditions and the following disclaimer in the documentation+   and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+   may be used to endorse or promote products derived from this software without+   specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ kenshou-remote.cabal view
@@ -0,0 +1,155 @@+cabal-version: 3.0+name: kenshou-remote+version: 0.1.0.0+synopsis: Leased verification cell client and storage protocol+description: Leases verification cells, publishes immutable run payloads, submits and recovers cell runs, and stores evidence objects in files or Google Cloud Storage.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+  test/golden/*.json+  test/golden/cell/*.json+  test/golden/cell/SOURCE++common warnings+  ghc-options:+    -Wall+    -Wcompat+    -Widentities+    -Wincomplete-record-updates+    -Wincomplete-uni-patterns+    -Wpartial-fields+    -Wredundant-constraints++  default-language: GHC2024+  default-extensions:+    BlockArguments+    DeriveAnyClass+    DeriveGeneric+    DuplicateRecordFields+    ImportQualifiedPost+    LambdaCase+    NoFieldSelectors+    OverloadedRecordDot+    OverloadedStrings++library+  import: warnings+  hs-source-dirs: src+  exposed-modules:+    Kenshou.Remote.Cell.Control+    Kenshou.Remote.Cell.Debug+    Kenshou.Remote.Cell.Docs+    Kenshou.Remote.Cell.Exec+    Kenshou.Remote.Cell.Fetch+    Kenshou.Remote.Cell.Health+    Kenshou.Remote.Cell.Index+    Kenshou.Remote.Cell.Lease+    Kenshou.Remote.Cell.Legacy+    Kenshou.Remote.Cell.Parity+    Kenshou.Remote.Cell.Prepare+    Kenshou.Remote.Cell.RouteJson+    Kenshou.Remote.Cell.RouteRules+    Kenshou.Remote.Cell.Session+    Kenshou.Remote.Cell.Session.Build+    Kenshou.Remote.Cell.Session.Journal+    Kenshou.Remote.Cell.Session.Rebind+    Kenshou.Remote.Cell.Session.Resume+    Kenshou.Remote.Cell.Session.Runner+    Kenshou.Remote.Cell.Submit+    Kenshou.Remote.Cell.Watch+    Kenshou.Remote.Cell.WorkJson+    Kenshou.Remote.Pair+    Kenshou.Remote.Payload+    Kenshou.Remote.Payload.Nix+    Kenshou.Remote.Payload.Publish+    Kenshou.Remote.Payload.Publisher+    Kenshou.Remote.Perf.Attempt+    Kenshou.Remote.Perf.Cell+    Kenshou.Remote.Perf.Cleanup+    Kenshou.Remote.Perf.Conditions+    Kenshou.Remote.Perf.Journal+    Kenshou.Remote.Perf.Lock+    Kenshou.Remote.Perf.Publication+    Kenshou.Remote.Perf.Runner+    Kenshou.Remote.Selftest+    Kenshou.Remote.Store+    Kenshou.Remote.Store.File+    Kenshou.Remote.Store.Gcs++  build-depends:+    aeson >=2.2 && <2.3,+    base >=4.21 && <4.22,+    base16-bytestring >=1.0 && <1.1,+    bytestring >=0.12 && <0.13,+    containers >=0.7 && <0.8,+    cryptohash-sha256 >=0.11 && <0.12,+    directory >=1.3 && <1.4,+    filepath >=1.5 && <1.6,+    http-client >=0.7 && <0.8,+    http-client-tls >=0.4 && <0.5,+    http-types >=0.12 && <0.13,+    kenshou-core ^>=0.1.0.0,+    kenshou-measure ^>=0.1.0.0,+    network >=3.2 && <3.3,+    process >=1.6 && <1.7,+    text >=2.1 && <2.2,+    time >=1.14 && <1.15,+    unix >=2.8 && <2.9,++test-suite kenshou-remote-test+  import: warnings+  type: exitcode-stdio-1.0+  hs-source-dirs: test+  main-is: Main.hs+  other-modules:+    DebugSpec+    DocsSpec+    ExecSpec+    FetchSpec+    GcsSpec+    HealthSpec+    LeaseSpec+    PairSpec+    ParitySpec+    PayloadSpec+    PerfAttemptSpec+    PerfCellSpec+    PerfCleanupSpec+    PerfConditionsSpec+    PerfJournalSpec+    PerfLockSpec+    PerfPublicationSpec+    PerfRunnerSpec+    PrepareSpec+    SessionSpec+    SubmitSpec+    WatchSpec++  build-depends:+    aeson,+    base >=4.21 && <4.22,+    bytestring,+    containers >=0.7 && <0.8,+    directory >=1.3 && <1.4,+    filepath,+    hspec >=2.11 && <2.12,+    kenshou-core ^>=0.1.0.0,+    kenshou-measure ^>=0.1.0.0,+    kenshou-remote,+    network >=3.2 && <3.3,+    process >=1.6 && <1.7,+    temporary,+    text,+    time >=1.14 && <1.15,++source-repository head+  type: git+  location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Remote/Cell/Control.hs view
@@ -0,0 +1,55 @@+module Kenshou.Remote.Cell.Control+  ( CellSnapshot (..),+    readCellSnapshot,+  )+where++import Data.Aeson (FromJSON, eitherDecode)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Remote.Cell.Docs (CellBuckets (..), CellDescriptor (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), Quarantine (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..))++data CellSnapshot = CellSnapshot+  { descriptor :: !CellDescriptor,+    lease :: !(Maybe Lease),+    quarantine :: !(Maybe Quarantine)+  }+  deriving stock (Eq, Show)++-- Read the same control objects as the cell owner. A descriptor is mandatory+-- before any client action so a bucket or project mismatch cannot be ignored.+readCellSnapshot :: ObjectStore -> CellRef -> [Text] -> IO (Either Text CellSnapshot)+readCellSnapshot store ref allowedProjects = do+  descriptor <- readDocument store ref "descriptor.json"+  case descriptor of+    Left problem -> pure (Left problem)+    Right Nothing -> pure (Left "cell descriptor is missing")+    Right (Just current)+      | current.name /= ref.cellName -> pure (Left "cell descriptor names another cell")+      | current.buckets.control /= ref.controlBucket.unBucket -> pure (Left "cell descriptor control bucket differs from selected bucket")+      | current.project `notElem` allowedProjects -> pure (Left ("cell project is outside KENSHOU_GCP_ALLOWED_PROJECTS: " <> current.project))+      | otherwise -> do+          lease <- readDocument store ref "lease.json"+          quarantine <- readDocument store ref "quarantine.json"+          pure do+            observedLease <- lease+            observedQuarantine <- quarantine+            case observedLease of+              Just record | record.cell /= ref.cellName -> Left "cell lease names another cell"+              _ -> pure ()+            case observedQuarantine of+              Just record | record.cell /= ref.cellName -> Left "cell quarantine names another cell"+              _ -> pure ()+            pure (CellSnapshot current observedLease observedQuarantine)++readDocument :: (FromJSON document) => ObjectStore -> CellRef -> Text -> IO (Either Text (Maybe document))+readDocument store ref name = do+  let object = ObjectName ("cells/" <> ref.cellName <> "/" <> name)+  stored <- store.getObject ref.controlBucket object+  pure case stored of+    Nothing -> Right Nothing+    Just (bytes, _) -> case eitherDecode bytes of+      Left failure -> Left ("invalid " <> name <> ": " <> Text.pack failure)+      Right document -> Right (Just document)
+ src/Kenshou/Remote/Cell/Debug.hs view
@@ -0,0 +1,102 @@+module Kenshou.Remote.Cell.Debug+  ( DebugCommand (..),+    resolveDebugRole,+    remoteCommand,+    runCellDebug,+  )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Remote.Cell.Docs (CellDescriptor (..), CellNodes (..))+import System.Directory (doesFileExist)+import System.Environment (getEnvironment, lookupEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO (hPutStrLn, stderr)+import System.Process (CreateProcess (..), StdStream (..), createProcess, proc, readProcessWithExitCode, waitForProcess)+import Text.Read (readMaybe)++data DebugCommand+  = DebugSsh !Text ![String]+  | DebugJournal+  | DebugTunnel !Text !Int !Int+  deriving stock (Eq, Show)++resolveDebugRole :: CellDescriptor -> Text -> Either Text Text+resolveDebugRole descriptor role = case role of+  "postgres" -> Right descriptor.instances.postgres+  "monitoring" -> Right descriptor.instances.monitoring+  "driver" -> driver 0+  _ -> case Text.stripPrefix "driver-" role >>= readMaybe . Text.unpack of+    Just index -> driver index+    Nothing -> Left "role must be postgres, monitoring, driver, or driver-N"+  where+    driver index = case drop index descriptor.instances.drivers of+      name : _ | index >= 0 -> Right name+      _ -> Left "cell descriptor has no driver at that index"++remoteCommand :: [String] -> String+remoteCommand = unwords . map quote+  where+    quote value = "'" <> concatMap escape value <> "'"+    escape '\'' = "'\\''"+    escape character = [character]++runCellDebug :: CellDescriptor -> DebugCommand -> IO (Either Text ExitCode)+runCellDebug descriptor selected = do+  scriptResult <- locateScript+  case scriptResult of+    Left problem -> pure (Left problem)+    Right script -> case arguments of+      Left problem -> pure (Left problem)+      Right args -> do+        environment <- getEnvironment+        let overrides =+              [ ("CLOUDSDK_CORE_PROJECT", Text.unpack descriptor.project),+                ("ZONE", Text.unpack descriptor.zone)+              ]+            inherited = filter (\(key, _) -> key `notElem` map fst overrides) environment+        hPutStrLn stderr ("cell debug: " <> unwords (script : args))+        (_, _, _, handle) <-+          createProcess+            (proc script args)+              { env = Just (overrides <> inherited),+                std_in = Inherit,+                std_out = Inherit,+                std_err = Inherit+              }+        Right <$> waitForProcess handle+  where+    arguments = case selected of+      DebugSsh role command -> do+        instanceName <- resolveDebugRole descriptor role+        pure (["ssh", Text.unpack instanceName, "--"] <> [remoteCommand command | not (null command)])+      DebugJournal -> do+        instanceName <- resolveDebugRole descriptor "driver"+        pure ["ssh", Text.unpack instanceName, "--", remoteCommand ["sudo", "journalctl", "-u", "cell-agent", "-n", "100", "--no-pager"]]+      DebugTunnel role remotePort localPort -> do+        instanceName <- resolveDebugRole descriptor role+        if validPort remotePort && validPort localPort+          then Right ["ssh-tunnel", Text.unpack instanceName, show remotePort, show localPort]+          else Left "tunnel ports must be between 1 and 65535"+    validPort port = port >= 1 && port <= 65535++locateScript :: IO (Either Text FilePath)+locateScript = do+  configured <- lookupEnv "KENSHOU_LTI_DIR"+  root <- case configured of+    Just value | not (null value) -> pure (Right value)+    _ -> do+      (code, output, failure) <- readProcessWithExitCode "mori" ["path", "shinzui/load-testing-infra"] ""+      pure case code of+        ExitSuccess -> case lines output of+          [path] -> Right path+          _ -> Left "mori did not return one load-testing-infra path"+        _ -> Left ("mori could not locate load-testing-infra: " <> Text.pack failure)+  case root of+    Left problem -> pure (Left problem)+    Right directory -> do+      let script = directory </> "scripts" </> "iap-ssh.sh"+      exists <- doesFileExist script+      pure if exists then Right script else Left ("cell debug needs the owner IAP script at " <> Text.pack script)
+ src/Kenshou/Remote/Cell/Docs.hs view
@@ -0,0 +1,725 @@+module Kenshou.Remote.Cell.Docs+  ( CellPhase (..),+    CellOutcome (..),+    CellNodes (..),+    CellImages (..),+    CellBuckets (..),+    CellDescriptor (..),+    CellPostgres (..),+    CellBroker (..),+    OtlpEndpoint (..),+    OtlpSinks (..),+    CellDriver (..),+    CellEnvironment (..),+    WorkObject (..),+    PgDatabase (..),+    PgReset (..),+    BrokerReset (..),+    CachePolicy (..),+    ResetBlock (..),+    Limits (..),+    Requirements (..),+    CollectOptions (..),+    Submission (..),+    LogChunks (..),+    CellStatus (..),+    Rejected (..),+    CellRunResult (..),+    ManifestPayload (..),+    Artifact (..),+    CellManifest (..),+  )+where++import Control.Monad (unless)+import Data.Aeson (FromJSON (..), Object, ToJSON (..), Value (..), object, withObject, withText, (.!=), (.:), (.:?), (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser)+import Data.Int (Int64)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Core.Resource.Types (ResourceName)+import Kenshou.Remote.Payload (CellPayload)+import Text.Read (readMaybe)++data CellPhase = Accepted | Resetting | Fetching | Running | Publishing | Sealed+  deriving stock (Eq, Show)++data CellOutcome = Completed | InfrastructureFailure | Cancelled | TimedOut+  deriving stock (Eq, Show)++data CellNodes = CellNodes+  { postgres :: !Text,+    drivers :: ![Text],+    monitoring :: !Text,+    -- | The optional broker role, present on cells that run one.+    broker :: !(Maybe Text)+  }+  deriving stock (Eq, Show)++data CellImages = CellImages+  { driver :: !Text,+    postgres :: !Text,+    monitoring :: !Text,+    broker :: !Text+  }+  deriving stock (Eq, Show)++data CellBuckets = CellBuckets+  { control :: !Text,+    results :: !Text+  }+  deriving stock (Eq, Show)++data CellDescriptor = CellDescriptor+  { name :: !Text,+    project :: !Text,+    zone :: !Text,+    postgresMajor :: !Int,+    instances :: !CellNodes,+    addresses :: !CellNodes,+    shape :: !Value,+    images :: !CellImages,+    buckets :: !CellBuckets+  }+  deriving stock (Eq, Show)++data CellPostgres = CellPostgres+  { major :: !Int,+    host :: !Text,+    port :: !Int,+    database :: !Text,+    user :: !Text,+    connectionString :: !Text+  }+  deriving stock (Eq, Show)++data CellBroker = CellBroker+  { bootstrapServers :: !Text,+    adminUrl :: !Text,+    implementation :: !Text,+    version :: !Text+  }+  deriving stock (Eq, Show)++data OtlpEndpoint = OtlpEndpoint+  { grpc :: !Text,+    http :: !Text+  }+  deriving stock (Eq, Show)++data OtlpSinks = OtlpSinks+  { nullEndpoint :: !OtlpEndpoint,+    fileEndpoint :: !OtlpEndpoint+  }+  deriving stock (Eq, Show)++data CellDriver = CellDriver+  { index :: !Int,+    host :: !Text+  }+  deriving stock (Eq, Show)++data CellEnvironment = CellEnvironment+  { cell :: !Text,+    runId :: !RunId,+    leaseId :: !RunId,+    postgres :: !CellPostgres,+    broker :: !(Maybe CellBroker),+    otlp :: !(Maybe OtlpSinks),+    victoriaMetricsUrl :: !Text,+    drivers :: ![CellDriver],+    clockSkewBoundMicros :: !(Maybe Int),+    faultHook :: !(Maybe Text),+    -- | Bound values of named resources, keyed by resource name. A kenshou extension+    -- of @cell.environment/v1@, written only when non-empty.+    resources :: !(Map ResourceName Value)+  }+  deriving stock (Eq, Show)++data WorkObject = WorkObject+  { sha256 :: !Text,+    bytes :: !Int64,+    mediaType :: !Text+  }+  deriving stock (Eq, Show)++data PgDatabase = PgDatabase+  { name :: !Text,+    owner :: !Text,+    template :: !Text+  }+  deriving stock (Eq, Show)++data PgReset = PgReset+  { major :: !Int,+    databases :: ![PgDatabase],+    settings :: !(Map Text Text)+  }+  deriving stock (Eq, Show)++newtype BrokerReset = BrokerReset {wipe :: Bool}+  deriving stock (Eq, Show)++data CachePolicy = Cold | Warm+  deriving stock (Eq, Show)++data ResetBlock = ResetBlock+  { cachePolicy :: !CachePolicy,+    postgres :: !(Maybe PgReset),+    broker :: !(Maybe BrokerReset),+    -- | Per-resource reset requests, keyed by resource name. A kenshou extension of+    -- @cell.submission/v1@, written only when non-empty.+    resources :: !(Map ResourceName Value)+  }+  deriving stock (Eq, Show)++data Limits = Limits+  { wallClockSeconds :: !Int64,+    memoryMaxBytes :: !Int64,+    outputMaxBytes :: !Int64+  }+  deriving stock (Eq, Show)++data Requirements = Requirements+  { minAgentVersion :: !Text,+    capabilities :: !(Maybe [Text])+  }+  deriving stock (Eq, Show)++data CollectOptions = CollectOptions+  { traces :: !Bool,+    profiles :: !Bool+  }+  deriving stock (Eq, Show)++data Submission = Submission+  { runId :: !RunId,+    leaseId :: !RunId,+    payload :: !CellPayload,+    work :: !WorkObject,+    env :: !(Map Text Text),+    reset :: !ResetBlock,+    limits :: !Limits,+    requires :: !Requirements,+    collect :: !(Maybe CollectOptions),+    labels :: !(Map Text Text)+  }+  deriving stock (Eq, Show)++data LogChunks = LogChunks+  { stdout :: !Int,+    stderr :: !Int+  }+  deriving stock (Eq, Show)++data CellStatus = CellStatus+  { runId :: !RunId,+    phase :: !CellPhase,+    leaseSequence :: !(Maybe Int),+    updatedAt :: !UTCTime,+    phaseStartedAt :: !(Maybe UTCTime),+    logChunks :: !LogChunks,+    outcome :: !(Maybe CellOutcome),+    manifestSha256 :: !(Maybe Text),+    reasons :: !(Maybe [Text])+  }+  deriving stock (Eq, Show)++data Rejected = Rejected+  { runId :: !RunId,+    reason :: !Text,+    at :: !UTCTime+  }+  deriving stock (Eq, Show)++data CellRunResult = CellRunResult+  { runId :: !RunId,+    cell :: !Text,+    leaseId :: !RunId,+    leaseSequence :: !Int,+    outcome :: !CellOutcome,+    entryExitCode :: !(Maybe Int),+    entrySignal :: !(Maybe Int),+    reasons :: ![Text]+  }+  deriving stock (Eq, Show)++data ManifestPayload = ManifestPayload+  { sha256 :: !Text,+    storePath :: !Text+  }+  deriving stock (Eq, Show)++data Artifact = Artifact+  { path :: !Text,+    sha256 :: !Text,+    bytes :: !Int64,+    mediaType :: !Text+  }+  deriving stock (Eq, Show)++data CellManifest = CellManifest+  { runId :: !RunId,+    cell :: !Text,+    leaseId :: !RunId,+    leaseSequence :: !Int,+    sealedAt :: !UTCTime,+    agentVersion :: !Text,+    payload :: !ManifestPayload,+    outcome :: !CellOutcome,+    retentionSeconds :: !Int64,+    artifacts :: ![Artifact]+  }+  deriving stock (Eq, Show)++instance ToJSON CellPhase where+  toJSON = toJSON . renderPhase++instance FromJSON CellPhase where+  parseJSON = withText "cell phase" \case+    "accepted" -> pure Accepted+    "resetting" -> pure Resetting+    "fetching" -> pure Fetching+    "running" -> pure Running+    "publishing" -> pure Publishing+    "sealed" -> pure Sealed+    _ -> fail "unknown cell phase"++renderPhase :: CellPhase -> Text+renderPhase Accepted = "accepted"+renderPhase Resetting = "resetting"+renderPhase Fetching = "fetching"+renderPhase Running = "running"+renderPhase Publishing = "publishing"+renderPhase Sealed = "sealed"++instance ToJSON CellOutcome where+  toJSON = toJSON . renderOutcome++instance FromJSON CellOutcome where+  parseJSON = withText "cell outcome" \case+    "completed" -> pure Completed+    "infrastructure-failure" -> pure InfrastructureFailure+    "cancelled" -> pure Cancelled+    "timed-out" -> pure TimedOut+    _ -> fail "unknown cell outcome"++renderOutcome :: CellOutcome -> Text+renderOutcome Completed = "completed"+renderOutcome InfrastructureFailure = "infrastructure-failure"+renderOutcome Cancelled = "cancelled"+renderOutcome TimedOut = "timed-out"++instance ToJSON CellNodes where+  toJSON nodes = object (["postgres" .= nodes.postgres, "drivers" .= nodes.drivers, "monitoring" .= nodes.monitoring] <> maybe [] (\name -> ["broker" .= name]) nodes.broker)++instance FromJSON CellNodes where+  parseJSON = withObject "cell nodes" \value -> do+    nodes <- CellNodes <$> value .: "postgres" <*> value .: "drivers" <*> value .: "monitoring" <*> value .:? "broker"+    unless (not (Text.null nodes.postgres) && not (null nodes.drivers) && all (not . Text.null) nodes.drivers && not (Text.null nodes.monitoring) && maybe True (not . Text.null) nodes.broker) (fail "invalid cell nodes")+    pure nodes++instance ToJSON CellImages where+  toJSON images = object ["driver" .= images.driver, "postgres" .= images.postgres, "monitoring" .= images.monitoring, "broker" .= images.broker]++instance FromJSON CellImages where+  parseJSON = withObject "cell images" \value -> do+    images <- CellImages <$> value .: "driver" <*> value .: "postgres" <*> value .: "monitoring" <*> value .: "broker"+    unless (all (not . Text.null) [images.driver, images.postgres, images.monitoring]) (fail "invalid cell images")+    pure images++instance ToJSON CellBuckets where+  toJSON buckets = object ["control" .= buckets.control, "results" .= buckets.results]++instance FromJSON CellBuckets where+  parseJSON = withObject "cell buckets" \value -> do+    buckets <- CellBuckets <$> value .: "control" <*> value .: "results"+    unless (not (Text.null buckets.control) && not (Text.null buckets.results)) (fail "invalid cell buckets")+    pure buckets++instance ToJSON CellDescriptor where+  toJSON descriptor =+    object+      [ "schema" .= ("cell.descriptor/v1" :: Text),+        "name" .= descriptor.name,+        "project" .= descriptor.project,+        "zone" .= descriptor.zone,+        "postgresMajor" .= descriptor.postgresMajor,+        "instances" .= descriptor.instances,+        "addresses" .= descriptor.addresses,+        "shape" .= descriptor.shape,+        "images" .= descriptor.images,+        "buckets" .= descriptor.buckets,+        "agentProtocol" .= ("cell.protocol/v1" :: Text)+      ]++instance FromJSON CellDescriptor where+  parseJSON = withObject "cell descriptor" \value -> do+    expectSchema "cell.descriptor/v1" value+    protocol <- value .: "agentProtocol"+    unless (protocol == ("cell.protocol/v1" :: Text)) (fail "unsupported cell agent protocol")+    descriptor <- CellDescriptor <$> value .: "name" <*> value .: "project" <*> value .: "zone" <*> value .: "postgresMajor" <*> value .: "instances" <*> value .: "addresses" <*> value .: "shape" <*> value .: "images" <*> value .: "buckets"+    unless (validName descriptor.name && Text.length descriptor.name <= 41 && not (Text.null descriptor.project) && not (Text.null descriptor.zone) && descriptor.postgresMajor `elem` [17, 18] && all validIpv4 (descriptor.addresses.postgres : descriptor.addresses.monitoring : descriptor.addresses.drivers) && validShape descriptor.shape) (fail "invalid cell descriptor")+    pure descriptor++instance ToJSON CellPostgres where+  toJSON postgres = object ["major" .= postgres.major, "host" .= postgres.host, "port" .= postgres.port, "database" .= postgres.database, "user" .= postgres.user, "connectionString" .= postgres.connectionString]++instance FromJSON CellPostgres where+  parseJSON = withObject "cell PostgreSQL" \value -> do+    postgres <- CellPostgres <$> value .: "major" <*> value .: "host" <*> value .: "port" <*> value .: "database" <*> value .: "user" <*> value .: "connectionString"+    unless (postgres.major `elem` [17, 18] && validIpv4 postgres.host && postgres.port >= 1 && postgres.port <= 65535 && all (not . Text.null) [postgres.database, postgres.user, postgres.connectionString]) (fail "invalid cell PostgreSQL endpoint")+    pure postgres++instance ToJSON CellBroker where+  toJSON broker = object ["bootstrapServers" .= broker.bootstrapServers, "adminUrl" .= broker.adminUrl, "implementation" .= broker.implementation, "version" .= broker.version]++instance FromJSON CellBroker where+  parseJSON = withObject "cell broker" \value -> do+    broker <- CellBroker <$> value .: "bootstrapServers" <*> value .: "adminUrl" <*> value .: "implementation" <*> value .: "version"+    unless (all (not . Text.null) [broker.bootstrapServers, broker.implementation, broker.version] && validUri broker.adminUrl) (fail "invalid cell broker")+    pure broker++instance ToJSON OtlpEndpoint where+  toJSON endpoint = object ["grpc" .= endpoint.grpc, "http" .= endpoint.http]++instance FromJSON OtlpEndpoint where+  parseJSON = withObject "cell OTLP endpoint" \value -> do+    endpoint <- OtlpEndpoint <$> value .: "grpc" <*> value .: "http"+    unless (validUri endpoint.grpc && validUri endpoint.http) (fail "invalid cell OTLP endpoint")+    pure endpoint++instance ToJSON OtlpSinks where+  toJSON sinks = object ["null" .= sinks.nullEndpoint, "file" .= sinks.fileEndpoint]++instance FromJSON OtlpSinks where+  parseJSON = withObject "cell OTLP sinks" \value -> OtlpSinks <$> value .: "null" <*> value .: "file"++instance ToJSON CellDriver where+  toJSON driver = object ["index" .= driver.index, "host" .= driver.host]++instance FromJSON CellDriver where+  parseJSON = withObject "cell driver" \value -> do+    driver <- CellDriver <$> value .: "index" <*> value .: "host"+    unless (driver.index >= 0 && validIpv4 driver.host) (fail "invalid cell driver")+    pure driver++instance ToJSON CellEnvironment where+  toJSON environment =+    object $+      [ "schema" .= ("cell.environment/v1" :: Text),+        "cell" .= environment.cell,+        "runId" .= environment.runId,+        "leaseId" .= environment.leaseId,+        "postgres" .= environment.postgres,+        "broker" .= environment.broker,+        "otlp" .= environment.otlp,+        "metrics" .= object ["victoriaMetricsUrl" .= environment.victoriaMetricsUrl],+        "drivers" .= environment.drivers+      ]+        <> catMaybes+          [ ("clock" .=) . object . pure . ("skewBoundMicros" .=) <$> environment.clockSkewBoundMicros,+            ("faultHook" .=) <$> environment.faultHook+          ]+        <> ["resources" .= environment.resources | not (Map.null environment.resources)]++instance FromJSON CellEnvironment where+  parseJSON = withObject "cell environment" \value -> do+    expectSchema "cell.environment/v1" value+    metrics <- value .: "metrics"+    clock <- value .:? "clock"+    url <- metrics .: "victoriaMetricsUrl"+    skew <- traverse (.: "skewBoundMicros") clock+    environment <- CellEnvironment <$> value .: "cell" <*> value .: "runId" <*> value .: "leaseId" <*> value .: "postgres" <*> value .: "broker" <*> value .: "otlp" <*> pure url <*> value .: "drivers" <*> pure skew <*> value .:? "faultHook" <*> value .:? "resources" .!= Map.empty+    unless (validName environment.cell && validUri environment.victoriaMetricsUrl && not (null environment.drivers) && maybe True (>= 0) environment.clockSkewBoundMicros && maybe True (not . Text.null) environment.faultHook) (fail "invalid cell environment")+    pure environment++instance ToJSON WorkObject where+  toJSON work = object ["sha256" .= work.sha256, "bytes" .= work.bytes, "mediaType" .= work.mediaType]++instance FromJSON WorkObject where+  parseJSON = withObject "cell work object" \value -> do+    work <- WorkObject <$> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"+    unless (validDigest work.sha256 && work.bytes >= 0 && not (Text.null work.mediaType)) (fail "invalid cell work object")+    pure work++instance ToJSON PgDatabase where+  toJSON database = object ["name" .= database.name, "owner" .= database.owner, "template" .= database.template]++instance FromJSON PgDatabase where+  parseJSON = withObject "cell PostgreSQL database" \value -> do+    database <- PgDatabase <$> value .: "name" <*> value .: "owner" <*> value .: "template"+    unless (all validPgName [database.name, database.owner, database.template]) (fail "invalid cell PostgreSQL database")+    pure database++instance ToJSON PgReset where+  toJSON reset = object ["major" .= reset.major, "databases" .= reset.databases, "settings" .= reset.settings]++instance FromJSON PgReset where+  parseJSON = withObject "cell PostgreSQL reset" \value -> do+    reset <- PgReset <$> value .: "major" <*> value .: "databases" <*> value .: "settings"+    unless (reset.major `elem` [17, 18] && unique reset.databases && validStringMap reset.settings) (fail "invalid cell PostgreSQL reset")+    pure reset++instance ToJSON BrokerReset where+  toJSON reset = object ["wipe" .= reset.wipe]++instance FromJSON BrokerReset where+  parseJSON = withObject "cell broker reset" \value -> BrokerReset <$> value .: "wipe"++instance ToJSON CachePolicy where+  toJSON Cold = toJSON ("cold" :: Text)+  toJSON Warm = toJSON ("warm" :: Text)++instance FromJSON CachePolicy where+  parseJSON = withText "cell cache policy" \case+    "cold" -> pure Cold+    "warm" -> pure Warm+    _ -> fail "unknown cell cache policy"++instance ToJSON ResetBlock where+  toJSON reset =+    object $+      ["cachePolicy" .= reset.cachePolicy]+        <> catMaybes+          [ ("postgres" .=) <$> reset.postgres,+            ("broker" .=) <$> reset.broker+          ]+        <> ["resources" .= reset.resources | not (Map.null reset.resources)]++instance FromJSON ResetBlock where+  parseJSON = withObject "cell reset" \value -> ResetBlock <$> value .: "cachePolicy" <*> value .:? "postgres" <*> value .:? "broker" <*> value .:? "resources" .!= Map.empty++instance ToJSON Limits where+  toJSON limits = object ["wallClockSeconds" .= limits.wallClockSeconds, "memoryMaxBytes" .= limits.memoryMaxBytes, "outputMaxBytes" .= limits.outputMaxBytes]++instance FromJSON Limits where+  parseJSON = withObject "cell limits" \value -> do+    limits <- Limits <$> value .: "wallClockSeconds" <*> value .: "memoryMaxBytes" <*> value .: "outputMaxBytes"+    unless (limits.wallClockSeconds > 0 && limits.memoryMaxBytes > 0 && limits.outputMaxBytes > 0) (fail "invalid cell limits")+    pure limits++instance ToJSON Requirements where+  toJSON requirements =+    object $+      [ "protocol" .= ("cell.protocol/v1" :: Text),+        "minAgentVersion" .= requirements.minAgentVersion+      ]+        <> catMaybes [("capabilities" .=) <$> requirements.capabilities]++instance FromJSON Requirements where+  parseJSON = withObject "cell requirements" \value -> do+    protocol <- value .: "protocol"+    unless (protocol == ("cell.protocol/v1" :: Text)) (fail "unsupported cell protocol")+    requirements <- Requirements <$> value .: "minAgentVersion" <*> value .:? "capabilities"+    unless (validVersion requirements.minAgentVersion && maybe True (\items -> unique items && all (not . Text.null) items) requirements.capabilities) (fail "invalid cell requirements")+    pure requirements++instance ToJSON Submission where+  toJSON submission =+    object+      ( [ "schema" .= ("cell.submission/v1" :: Text),+          "runId" .= submission.runId,+          "leaseId" .= submission.leaseId,+          "payload" .= submission.payload,+          "work" .= submission.work,+          "env" .= submission.env,+          "reset" .= submission.reset,+          "limits" .= submission.limits,+          "requires" .= submission.requires,+          "labels" .= submission.labels+        ]+          <> maybe [] (\options -> ["collect" .= object (["traces" .= options.traces] <> ["profiles" .= True | options.profiles])]) submission.collect+      )++instance FromJSON Submission where+  parseJSON = withObject "cell submission" \value -> do+    expectSchema "cell.submission/v1" value+    submission <- Submission <$> value .: "runId" <*> value .: "leaseId" <*> value .: "payload" <*> value .: "work" <*> value .: "env" <*> value .: "reset" <*> value .: "limits" <*> value .: "requires" <*> (value .:? "collect" >>= traverse parseCollect) <*> value .: "labels"+    unless (validStringMap submission.env && validStringMap submission.labels) (fail "invalid cell submission map keys")+    pure submission+    where+      parseCollect = withObject "cell collection options" \options -> CollectOptions <$> options .: "traces" <*> (options .:? "profiles" .!= False)++instance ToJSON LogChunks where+  toJSON chunks = object ["stdout" .= chunks.stdout, "stderr" .= chunks.stderr]++instance FromJSON LogChunks where+  parseJSON = withObject "cell log chunks" \value -> do+    chunks <- LogChunks <$> value .: "stdout" <*> value .: "stderr"+    unless (chunks.stdout >= 0 && chunks.stderr >= 0) (fail "negative cell log chunk count")+    pure chunks++instance ToJSON CellStatus where+  toJSON status =+    object $+      [ "schema" .= ("cell.status/v1" :: Text),+        "runId" .= status.runId,+        "phase" .= status.phase,+        "updatedAt" .= status.updatedAt,+        "logChunks" .= status.logChunks+      ]+        <> catMaybes+          [ ("leaseSequence" .=) <$> status.leaseSequence,+            ("phaseStartedAt" .=) <$> status.phaseStartedAt,+            ("outcome" .=) <$> status.outcome,+            ("manifestSha256" .=) <$> status.manifestSha256,+            ("reasons" .=) <$> status.reasons+          ]++instance FromJSON CellStatus where+  parseJSON = withObject "cell status" \value -> do+    expectSchema "cell.status/v1" value+    status <- CellStatus <$> value .: "runId" <*> value .: "phase" <*> value .:? "leaseSequence" <*> value .: "updatedAt" <*> value .:? "phaseStartedAt" <*> value .: "logChunks" <*> value .:? "outcome" <*> value .:? "manifestSha256" <*> value .:? "reasons"+    unless (maybe True (>= 0) status.leaseSequence && maybe True validDigest status.manifestSha256 && maybe True (all (not . Text.null)) status.reasons) (fail "invalid cell status fields")+    unless (if status.phase == Sealed then status.outcome /= Nothing && status.manifestSha256 /= Nothing else status.outcome == Nothing && status.manifestSha256 == Nothing) (fail "cell status terminal fields disagree with phase")+    pure status++instance ToJSON Rejected where+  toJSON rejected = object ["schema" .= ("cell.rejected/v1" :: Text), "runId" .= rejected.runId, "reason" .= rejected.reason, "at" .= rejected.at]++instance FromJSON Rejected where+  parseJSON = withObject "cell rejection" \value -> do+    expectSchema "cell.rejected/v1" value+    rejected <- Rejected <$> value .: "runId" <*> value .: "reason" <*> value .: "at"+    unless (not (Text.null rejected.reason)) (fail "empty cell rejection reason")+    pure rejected++instance ToJSON CellRunResult where+  toJSON result =+    object+      [ "schema" .= ("cell.run-result/v1" :: Text),+        "runId" .= result.runId,+        "cell" .= result.cell,+        "leaseId" .= result.leaseId,+        "leaseSequence" .= result.leaseSequence,+        "outcome" .= result.outcome,+        "entryExitCode" .= result.entryExitCode,+        "entrySignal" .= result.entrySignal,+        "reasons" .= result.reasons+      ]++instance FromJSON CellRunResult where+  parseJSON = withObject "cell run result" \value -> do+    expectSchema "cell.run-result/v1" value+    result <- CellRunResult <$> value .: "runId" <*> value .: "cell" <*> value .: "leaseId" <*> value .: "leaseSequence" <*> value .: "outcome" <*> value .: "entryExitCode" <*> value .: "entrySignal" <*> value .: "reasons"+    unless (validName result.cell && result.leaseSequence >= 0 && all (not . Text.null) result.reasons) (fail "invalid cell run result")+    pure result++instance ToJSON ManifestPayload where+  toJSON payload = object ["kind" .= ("nix-nar-bundle" :: Text), "sha256" .= payload.sha256, "storePath" .= payload.storePath]++instance FromJSON ManifestPayload where+  parseJSON = withObject "manifest payload" \value -> do+    kind <- value .: "kind"+    unless (kind == ("nix-nar-bundle" :: Text)) (fail "unsupported manifest payload kind")+    payload <- ManifestPayload <$> value .: "sha256" <*> value .: "storePath"+    unless (validDigest payload.sha256 && validStorePath payload.storePath) (fail "invalid manifest payload")+    pure payload++instance ToJSON Artifact where+  toJSON artifact = object ["path" .= artifact.path, "sha256" .= artifact.sha256, "bytes" .= artifact.bytes, "mediaType" .= artifact.mediaType]++instance FromJSON Artifact where+  parseJSON = withObject "cell artifact" \value -> do+    artifact <- Artifact <$> value .: "path" <*> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"+    unless (validArtifactPath artifact.path && validDigest artifact.sha256 && artifact.bytes >= 0 && not (Text.null artifact.mediaType)) (fail "invalid cell artifact")+    pure artifact++instance ToJSON CellManifest where+  toJSON manifest =+    object+      [ "schema" .= ("cell.artifact-manifest/v1" :: Text),+        "layout" .= (1 :: Int),+        "runId" .= manifest.runId,+        "cell" .= manifest.cell,+        "leaseId" .= manifest.leaseId,+        "leaseSequence" .= manifest.leaseSequence,+        "sealedAt" .= manifest.sealedAt,+        "agentVersion" .= manifest.agentVersion,+        "payload" .= manifest.payload,+        "outcome" .= manifest.outcome,+        "retention" .= object ["policy" .= ("bucket-retention" :: Text), "retentionSeconds" .= manifest.retentionSeconds],+        "artifacts" .= manifest.artifacts+      ]++instance FromJSON CellManifest where+  parseJSON = withObject "cell manifest" \value -> do+    expectSchema "cell.artifact-manifest/v1" value+    layout <- value .: "layout"+    unless (layout == (1 :: Int)) (fail "unsupported cell manifest layout")+    retention <- value .: "retention"+    policy <- retention .: "policy"+    unless (policy == ("bucket-retention" :: Text)) (fail "unsupported cell retention policy")+    seconds <- retention .: "retentionSeconds"+    manifest <- CellManifest <$> value .: "runId" <*> value .: "cell" <*> value .: "leaseId" <*> value .: "leaseSequence" <*> value .: "sealedAt" <*> value .: "agentVersion" <*> value .: "payload" <*> value .: "outcome" <*> pure seconds <*> value .: "artifacts"+    unless (validName manifest.cell && manifest.leaseSequence >= 0 && manifest.retentionSeconds >= 0 && validVersion manifest.agentVersion && not (null manifest.artifacts) && all ((/= "manifest.json") . (.path)) manifest.artifacts && Set.size (Set.fromList (map (.path) manifest.artifacts)) == length manifest.artifacts) (fail "invalid cell manifest")+    pure manifest++expectSchema :: Text -> Object -> Parser ()+expectSchema expected value = do+  schema <- value .: "schema"+  unless (schema == expected) (fail "unsupported cell document schema")++validName :: Text -> Bool+validName value = Text.length value >= 3 && Text.length value <= 63 && Text.head value `elem` (['a' .. 'z'] <> ['0' .. '9']) && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'] <> "-")) value++validShape :: Value -> Bool+validShape (Object fields) = all (`KeyMap.member` fields) ["postgres", "driver", "monitoring", "scheduling"]+validShape _ = False++validIpv4 :: Text -> Bool+validIpv4 value = case traverse (readMaybe . Text.unpack) (Text.splitOn "." value) :: Maybe [Int] of+  Just [first, second, third, fourth] -> all (\piece -> piece >= 0 && piece <= 255) [first, second, third, fourth]+  _ -> False++validUri :: Text -> Bool+validUri value = case Text.breakOn ":" value of+  (scheme, rest) ->+    not (Text.null scheme)+      && Text.length rest > 1+      && Text.head scheme `elem` (['a' .. 'z'] <> ['A' .. 'Z'])+      && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['A' .. 'Z'] <> ['0' .. '9'] <> "+.-")) scheme++validDigest :: Text -> Bool+validDigest value = Text.length value == 64 && Text.all (\character -> character `elem` (['0' .. '9'] <> ['a' .. 'f'])) value++validVersion :: Text -> Bool+validVersion value = case Text.splitOn "." value of+  [major, minor, patch] -> all (\piece -> not (Text.null piece) && Text.all (\character -> character `elem` ['0' .. '9']) piece) [major, minor, patch]+  _ -> False++validStorePath :: Text -> Bool+validStorePath value = case Text.stripPrefix "/nix/store/" value of+  Just suffix -> let (digest, name) = Text.breakOn "-" suffix in Text.length digest == 32 && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'])) digest && Text.length name > 1 && not (Text.any (== '/') name)+  Nothing -> False++validArtifactPath :: Text -> Bool+validArtifactPath value =+  not (Text.null value)+    && not (Text.isPrefixOf "/" value)+    && not (Text.any (`elem` ['\\', '\0']) value)+    && all (\part -> not (Text.null part) && part /= "." && part /= "..") (Text.splitOn "/" value)++validPgName :: Text -> Bool+validPgName value = case Text.uncons value of+  Just (first, rest) -> Text.length value <= 63 && first `elem` ['a' .. 'z'] && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'] <> "_")) rest+  Nothing -> False++validStringMap :: Map Text Text -> Bool+validStringMap = all validKey . Map.keys+  where+    validKey value = case Text.uncons value of+      Just (first, rest) -> first `elem` (['A' .. 'Z'] <> ['a' .. 'z'] <> "_") && Text.all (\character -> character `elem` (['A' .. 'Z'] <> ['a' .. 'z'] <> ['0' .. '9'] <> "_.-")) rest+      Nothing -> False++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Cell/Exec.hs view
@@ -0,0 +1,351 @@+module Kenshou.Remote.Cell.Exec (resolveOnCell, resolveWorkJson, cellExec) where++import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)+import Control.Monad (when)+import Data.Aeson (Value (..), encode, object, (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (toList)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as TextIO+import Kenshou.Core.Bundle (Registry, lookupScenario)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), TracingArm (..), resolveDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), parseRunId, renderScenarioId)+import Kenshou.Core.Knob (KnobSpec (..), RawKnob (..), mkKnobName)+import Kenshou.Core.Resource.Types (legacyKafkaName, renderResourceName)+import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Scenario (..))+import Kenshou.Plan.Execute (ExecuteOptions (..), executePlan)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Plan.Summary (summaryExitCode)+import Kenshou.Remote.Cell.Docs (CellBroker (..), CellEnvironment (..), CellPostgres (..), OtlpEndpoint (..), OtlpSinks (..))+import Kenshou.Remote.Cell.Health (withCellHealthFile)+import Kenshou.Remote.Cell.Legacy (brokerAsResource)+import Kenshou.Remote.Cell.Prepare (OtlpSink (..))+import System.Directory (Permissions (..), createDirectoryIfMissing, doesFileExist, getPermissions)+import System.Environment (lookupEnv, setEnv, unsetEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO (stderr)+import System.Process (readProcessWithExitCode)++-- Bind the generic environment in a prepared work plan to endpoints available+-- on this cell. Run IDs and plan metadata are preserved; no run is silently+-- dropped here.+resolveOnCell :: Registry -> CellEnvironment -> OtlpSink -> RunPlan -> Either Text RunPlan+resolveOnCell registry environment sink plan = do+  runs <- traverse resolveRun plan.runs+  pure (RunPlan plan.planId plan.createdAt plan.context plan.policy runs plan.skipped plan.estimateMinutes)+  where+    resolveRun entry = do+      resolved <- resolveSpec registry environment sink entry.spec+      pure (PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial resolved)++-- The executor accepts a narrow run-plan JSON document rather than decoding+-- every planning provenance type. Rewrite just the spec member of each entry.+resolveWorkJson :: Registry -> CellEnvironment -> OtlpSink -> Value -> Either Text Value+resolveWorkJson registry environment sink work = fst <$> resolveWorkDocument registry environment sink work++resolveWorkDocument :: Registry -> CellEnvironment -> OtlpSink -> Value -> Either Text (Value, [RunSpec])+resolveWorkDocument registry environment sink work = case work of+  Object fields -> case (KeyMap.lookup "schema" fields, KeyMap.lookup "runs" fields) of+    (Just (String "kenshou.run-plan/v1"), Just (Array runs)) -> do+      resolved <- traverse resolveEntry runs+      pure (Object (KeyMap.insert "runs" (Array (fmap fst resolved)) fields), toList (fmap snd resolved))+    _ -> Left "cell work is not a kenshou.run-plan/v1 document"+  _ -> Left "cell work is not a JSON object"+  where+    resolveEntry (Object entry) = do+      identifier <- case KeyMap.lookup "runId" entry of+        Just value -> (decodeValue "planned run ID" value :: Either Text RunId)+        Nothing -> Left "cell work entry has no run ID"+      spec <- case KeyMap.lookup "spec" entry of+        Just value -> (decodeValue "planned run spec" value :: Either Text RunSpec)+        Nothing -> Left "cell work entry has no run spec"+      if maybe True (== identifier) spec.runId+        then pure ()+        else Left "cell work entry and spec have different run IDs"+      resolved <- resolveSpec registry environment sink spec+      pure (Object (KeyMap.insert "spec" (Aeson.toJSON resolved) entry), resolved)+    resolveEntry _ = Left "cell work run is not an object"++    decodeValue :: (Aeson.FromJSON document) => Text -> Value -> Either Text document+    decodeValue label value = case Aeson.fromJSON value of+      Aeson.Error failure -> Left (label <> ": " <> Text.pack failure)+      Aeson.Success document -> Right document++resolveSpec :: Registry -> CellEnvironment -> OtlpSink -> RunSpec -> Either Text RunSpec+resolveSpec registry environment sink spec = do+  scenario <- maybe (Left ("unknown cell scenario: " <> renderScenarioId spec.scenario)) Right (lookupScenario registry spec.scenario)+  if scenario.placement == PlaceLocal+    then Left ("scenario is local-only: " <> renderScenarioId scenario.id)+    else pure ()+  dimensions <- either (Left . Text.intercalate "; " . NonEmpty.toList) Right (resolveDimensions scenario.dimensions spec.dimensions)+  postgres <- bindPostgres scenario dimensions+  extras <- bindExtras scenario postgres+  resources <- Map.fromList <$> traverse (bindResource scenario) scenario.requires.resources+  telemetryKnobs <- bindOtlp scenario dimensions+  knobs <- bindRemoteScenario telemetryKnobs+  let prior = spec.environment+      cellEnvironment = EnvironmentSpec RunOnCell prior.machineProfile postgres extras resources prior.telemetry+  pure (RunSpec spec.runId spec.scenario spec.scenarioRevision knobs spec.dimensions spec.seed spec.phases spec.timeoutSeconds cellEnvironment spec.cohortExpectation spec.comparison spec.labels)+  where+    bindPostgres scenario dimensions = case scenario.requires.postgres of+      Nothing -> pure spec.environment.postgres+      Just requirement+        | requirement.needsServerControl ->+            if scenario.id.kind `elem` [Correctness, Concurrency]+              then case spec.environment.postgres of+                Just ephemeral@(PostgresEphemeral _) -> pure (Just ephemeral)+                _ -> Left ("scenario needs driver-local PostgreSQL server control: " <> renderScenarioId scenario.id)+              else Left ("benchmark or soak cannot control the cell PostgreSQL server: " <> renderScenarioId scenario.id)+        | otherwise -> do+            if dimensions.pgDurability /= Just PgDurable+              then Left ("cell run requires pg.durability=durable: " <> renderScenarioId scenario.id)+              else pure ()+            let requested = case dimensions.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> 0+            if requested /= environment.postgres.major+              then Left ("cell PostgreSQL major differs from pg.version for " <> renderScenarioId scenario.id)+              else pure (Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL")))++    bindExtras scenario primary =+      if case primary of Just (PostgresEphemeral _) -> True; _ -> False+        then Map.fromList <$> traverse driverExtra scenario.requires.extraPostgres+        else do+          let requirements = scenario.requires.extraPostgres+              variableNames = fmap (variableName . fst) requirements+          if any (.needsServerControl) (fmap snd requirements)+            then Left ("extra PostgreSQL server needs driver-local control: " <> renderScenarioId scenario.id)+            else pure ()+          if length variableNames /= length (unique variableNames)+            then Left "extra PostgreSQL names collide as environment variables"+            else pure (Map.fromList [(name, PostgresExternal (ConnFromEnv ("KENSHOU_CELL_PG_URL_" <> variableName name))) | (name, _) <- requirements])+      where+        driverExtra (name, _) = case Map.lookup name spec.environment.extraPostgres of+          Nothing -> pure (name, PostgresEphemeral [])+          Just local@(PostgresEphemeral _) -> pure (name, local)+          Just (PostgresExternal _) -> Left ("driver-local PostgreSQL requires every extra server to be ephemeral: " <> name)++    -- The cell environment binds a resource by name; the legacy broker fields still+    -- bind the resource they always described.+    bindResource scenario name = case (Map.lookup name environment.resources, environment.broker) of+      (Just value, _) -> pure (name, value)+      (Nothing, Just broker) | name == legacyKafkaName -> (name,) <$> brokerAsResource broker+      _ -> Left ("cell has no resource " <> renderResourceName name <> " for " <> renderScenarioId scenario.id)++    bindOtlp scenario dimensions+      | dimensions.tracing /= Just TracingSdkOtlp = pure spec.knobs+      | otherwise = do+          name <- mkKnobName "otel.endpoint"+          if name `notElem` fmap (.name) scenario.knobs+            then Left ("tracing scenario lacks otel.endpoint: " <> renderScenarioId scenario.id)+            else pure ()+          sinks <- maybe (Left "cell has no OTLP sink") Right environment.otlp+          let endpoint = case sink of NullSink -> sinks.nullEndpoint.http; FileSink -> sinks.fileEndpoint.http+          if Text.null endpoint+            then Left "cell OTLP sink has no HTTP endpoint"+            else pure ((name, RawText endpoint) : filter ((/= name) . fst) spec.knobs)++    bindRemoteScenario knobs+      | renderScenarioId spec.scenario /= "selftest/remote/correctness/cell-environment" = pure knobs+      | otherwise = do+          otlpName <- mkKnobName "remote.otlp-endpoint"+          let selectedOtlp = case environment.otlp of+                Nothing -> ""+                Just sinks -> case sink of NullSink -> sinks.nullEndpoint.http; FileSink -> sinks.fileEndpoint.http+          pure (assignWhenBlank otlpName selectedOtlp knobs)++    assignWhenBlank name value knobs+      | Text.null value || maybe False (not . blank) (lookup name knobs) = knobs+      | otherwise = assign name value knobs+    blank (RawText value) = Text.null value+    blank (RawJson (String value)) = Text.null value+    blank _ = False+    assign name value knobs = (name, RawText value) : filter ((/= name) . fst) knobs++variableName :: Text -> Text+variableName = Text.map (\character -> if character == '-' then '_' else character) . Text.toUpper++unique :: (Eq value) => [value] -> [value]+unique [] = []+unique (first : rest) = first : unique (filter (/= first) rest)++-- The generic cell agent invokes this command on the first driver with the+-- submitted work and an environment document. Every child run receives the+-- resolved plan, the cell context and the same payload identity variables.+cellExec :: Registry -> FilePath -> FilePath -> IO ExitCode+cellExec registry workFile outDir = do+  result <- try (executeOnDriver registry workFile outDir) :: IO (Either SomeException ExitCode)+  case result of+    Left failure | Just async <- (fromException failure :: Maybe SomeAsyncException) -> throwIO async+    Left failure -> adapterFailure outDir (Text.pack (displayException failure))+    Right code -> pure code++executeOnDriver :: Registry -> FilePath -> FilePath -> IO ExitCode+executeOnDriver registry workFile outDir = do+  driverIndex <- integerEnv "CELL_DRIVER_INDEX" 0+  driverCount <- integerEnv "CELL_DRIVER_COUNT" 1+  when (driverIndex >= driverCount) (ioError (userError "CELL_DRIVER_INDEX exceeds CELL_DRIVER_COUNT"))+  let cellDirectory = outDir </> "kenshou-cell"+  createDirectoryIfMissing True cellDirectory+  if driverIndex /= 0+    then do+      LazyByteString.writeFile (outDir </> "idle-driver.json") (encode (object ["schema" .= ("kenshou.cell-idle-driver/v1" :: Text), "index" .= driverIndex, "count" .= driverCount]))+      pure ExitSuccess+    else do+      environmentPath <- requiredEnv "CELL_ENV_FILE"+      environment <- Aeson.eitherDecodeFileStrict' environmentPath >>= either (ioError . userError . ("invalid CELL_ENV_FILE: " <>)) pure+      cellRun <- requiredEnv "CELL_RUN_ID" >>= either (ioError . userError . Text.unpack) pure . parseRunId . Text.pack+      when (cellRun /= environment.runId) (ioError (userError "CELL_RUN_ID differs from the cell environment"))+      scratch <- requiredEnv "CELL_SCRATCH_DIR"+      sink <-+        requiredEnv "KENSHOU_OTLP_SINK" >>= \case+          "null" -> pure NullSink+          "file" -> pure FileSink+          _ -> ioError (userError "KENSHOU_OTLP_SINK must be null or file")+      identity <- requiredEnv "KENSHOU_COHORT_IDENTITY"+      present <- doesFileExist identity+      when (not present) (ioError (userError "payload cohort identity file is missing"))+      payload <- requiredPayloadIdentity+      work <- Aeson.eitherDecodeFileStrict' workFile >>= either (ioError . userError . ("invalid cell work: " <>)) pure+      (resolved, specs) <- either (ioError . userError . Text.unpack) pure (resolveWorkDocument registry environment sink work)+      let planPath = cellDirectory </> "plan.resolved.json"+          contextPath = cellDirectory </> "context.json"+          noticePath = cellDirectory </> "health-notices.jsonl"+          cachePath = scratch </> "cache"+          tempPath = scratch </> "tmp"+      createDirectoryIfMissing True cachePath+      createDirectoryIfMissing True tempPath+      LazyByteString.writeFile planPath (encode resolved)+      LazyByteString.writeFile noticePath ""+      setEnv "KENSHOU_CELL_PG_URL" (Text.unpack environment.postgres.connectionString)+      mapM_ (setExtraPostgres environment.postgres.connectionString) specs+      setEnv "KENSHOU_CELL_FINGERPRINT" contextPath+      setEnv "KENSHOU_HEALTH_NOTICES" noticePath+      setEnv "XDG_CACHE_HOME" cachePath+      setEnv "TMPDIR" tempPath+      faultEnabled <- case environment.faultHook of+        Nothing -> unsetEnv "KENSHOU_CELL_FAULT_HOOK" >> pure False+        Just hook -> do+          available <- doesFileExist (Text.unpack hook)+          executableHook <- if available then (.executable) <$> getPermissions (Text.unpack hook) else pure False+          if executableHook then setEnv "KENSHOU_CELL_FAULT_HOOK" (Text.unpack hook) else unsetEnv "KENSHOU_CELL_FAULT_HOOK"+          pure executableHook+      pgPartman <- probePgPartman environment.postgres.connectionString+      LazyByteString.writeFile contextPath (encode (cellContext environment driverIndex driverCount sink faultEnabled pgPartman payload))+      if driverCount > 1+        then setEnv "KENSHOU_CLOCK_SKEW_BOUND_MICROS" (show (maybe 50000 id environment.clockSkewBoundMicros))+        else unsetEnv "KENSHOU_CLOCK_SKEW_BOUND_MICROS"+      healthFile <- lookupEnv "CELL_HEALTH_FILE"+      let runPlan = executePlan (ExecuteOptions planPath outDir False False Nothing Nothing)+          withHealth = withCellHealthFile healthFile noticePath cellRun runPlan+      summary <- if faultEnabled then healAll environment.faultHook >> withHealth `finally` healAll environment.faultHook else withHealth+      pure (summaryExitCode summary)++healAll :: Maybe Text -> IO ()+healAll Nothing = pure ()+healAll (Just hook) = do+  (code, _, errors) <- readProcessWithExitCode (Text.unpack hook) ["heal-all"] ""+  when (code /= ExitSuccess) (ioError (userError ("cell fault hook heal-all failed: " <> errors)))++setExtraPostgres :: Text -> RunSpec -> IO ()+setExtraPostgres connection spec = mapM_ setOne (Map.elems spec.environment.extraPostgres)+  where+    setOne (PostgresExternal (ConnFromEnv name)) | "KENSHOU_CELL_PG_URL_" `Text.isPrefixOf` name = setEnv (Text.unpack name) (Text.unpack connection)+    setOne _ = pure ()++probePgPartman :: Text -> IO (Maybe Text)+probePgPartman connection = do+  (code, output, errors) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", "SELECT COALESCE((SELECT default_version FROM pg_available_extensions WHERE name = 'pg_partman'), '')"] ""+  case code of+    ExitSuccess -> pure case Text.strip (Text.pack output) of+      "" -> Nothing+      version -> Just version+    _ -> ioError (userError ("PostgreSQL extension probe failed: " <> errors))++cellContext :: CellEnvironment -> Int -> Int -> OtlpSink -> Bool -> Maybe Text -> Map.Map Text Text -> Value+cellContext environment driverIndex driverCount sink faultEnabled pgPartman payload =+  object+    [ "schema" .= ("kenshou.cell-context/v1" :: Text),+      "cell" .= environment.cell,+      "cellRun" .= environment.runId,+      "leaseId" .= environment.leaseId,+      "driver" .= object ["index" .= driverIndex, "count" .= driverCount],+      "postgres" .= object ["major" .= environment.postgres.major, "host" .= environment.postgres.host, "port" .= environment.postgres.port, "placement" .= ("cell-server" :: Text), "extraPostgres" .= ("shared-server" :: Text), "extensions" .= object ["pg_partman" .= object ["available" .= maybe False (const True) pgPartman, "version" .= pgPartman]]],+      "broker" .= environment.broker,+      "capabilities"+        .= object+          [ "postgres.major" .= Text.pack (show environment.postgres.major),+            "postgres.pg_partman" .= maybe False (const True) pgPartman,+            "postgres.second-server" .= False,+            "postgres.control-hook" .= False,+            "broker" .= maybe "none" (\broker -> broker.implementation <> " " <> broker.version) environment.broker,+            "otlp.null" .= maybe False (not . Text.null . (.nullEndpoint.http)) environment.otlp,+            "otlp.file" .= maybe False (not . Text.null . (.fileEndpoint.http)) environment.otlp,+            "fault-hook" .= faultEnabled+          ],+      "otlpSink" .= (case sink of NullSink -> "null" :: Text; FileSink -> "file"),+      "faultHook" .= faultEnabled,+      "clock" .= object ["skewBoundMicros" .= environment.clockSkewBoundMicros],+      "payload"+        .= object+          [ "bundleSha256" .= field "KENSHOU_PAYLOAD_BUNDLE_SHA256",+            "storePath" .= field "KENSHOU_PAYLOAD_STORE_PATH",+            "narHash" .= field "KENSHOU_PAYLOAD_NAR_HASH",+            "cohort" .= field "KENSHOU_PAYLOAD_COHORT",+            "harness" .= object ["revision" .= field "KENSHOU_HARNESS_REVISION", "dirty" .= (field "KENSHOU_HARNESS_DIRTY" == "true")]+          ]+    ]+  where+    field name = Map.findWithDefault "" name payload++requiredPayloadIdentity :: IO (Map.Map Text Text)+requiredPayloadIdentity = do+  values <- Map.fromList <$> traverse load (filter (/= "KENSHOU_COHORT_IDENTITY") payloadIdentityNames)+  when (Map.lookup "KENSHOU_HARNESS_DIRTY" values `notElem` [Just "true", Just "false"]) (ioError (userError "KENSHOU_HARNESS_DIRTY must be true or false"))+  pure values+  where+    load name = (name,) . Text.pack <$> requiredEnv (Text.unpack name)++requiredEnv :: String -> IO String+requiredEnv name =+  lookupEnv name >>= \case+    Just value | not (null value) -> pure value+    _ -> ioError (userError (name <> " is required by the cell payload"))++integerEnv :: String -> Int -> IO Int+integerEnv name fallback =+  lookupEnv name >>= \case+    Nothing -> pure fallback+    Just raw -> case reads raw of+      [(value, "")] | value >= 0 -> pure value+      _ -> ioError (userError (name <> " must be a non-negative integer"))++adapterFailure :: FilePath -> Text -> IO ExitCode+adapterFailure outDir detail = do+  let directory = outDir </> "kenshou-cell"+      reason :: Text+      reason+        | "CELL_ENV_FILE" `Text.isInfixOf` detail = "cell-environment-missing"+        | any (`Text.isInfixOf` detail) payloadIdentityNames = "payload-identity-missing"+        | otherwise = "cell-exec-failed"+  createDirectoryIfMissing True directory+  LazyByteString.writeFile (directory </> "adapter-error.json") (encode (object ["schema" .= ("kenshou.cell-adapter-error/v1" :: Text), "reason" .= reason, "detail" .= detail]))+  TextIO.hPutStrLn stderr ("kenshou cell exec: " <> detail)+  pure (ExitFailure 4)++payloadIdentityNames :: [Text]+payloadIdentityNames =+  [ "KENSHOU_COHORT_IDENTITY",+    "KENSHOU_HARNESS_REVISION",+    "KENSHOU_HARNESS_DIRTY",+    "KENSHOU_PAYLOAD_BUNDLE_SHA256",+    "KENSHOU_PAYLOAD_STORE_PATH",+    "KENSHOU_PAYLOAD_NAR_HASH",+    "KENSHOU_PAYLOAD_COHORT"+  ]
+ src/Kenshou/Remote/Cell/Fetch.hs view
@@ -0,0 +1,387 @@+module Kenshou.Remote.Cell.Fetch+  ( FetchError (..),+    VerifyProblem (..),+    fetchCellRun,+    verifyCellTree,+    verifyCellRun,+    verifyCellRunWithStatus,+    effectiveOutcome,+  )+where++import Control.Exception (bracket)+import Control.Monad (forM)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (FromJSON, Value (..), eitherDecode)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List (sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, listToMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Kenshou.Core.Id (RunId, parseRunId, renderRunId)+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..), verifyManifest)+import Kenshou.Core.Outcome qualified as Outcome+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellPhase (..), CellRunResult (..), CellStatus (..), ManifestPayload (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Docs qualified as Cell+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket, ObjectMeta (..), ObjectName (..), ObjectStore (..))+import System.Directory (createDirectoryIfMissing, doesDirectoryExist, doesFileExist, listDirectory, pathIsSymbolicLink, removeFile, renameFile)+import System.FilePath (takeDirectory, (</>))+import System.IO (IOMode (ReadMode), hClose, openBinaryTempFile, withBinaryFile)++data VerifyProblem+  = InvalidManifest !Text+  | Missing !Text+  | Extra !Text+  | DigestMismatch !Text+  | SizeMismatch !Text+  | UnsafeEntry !Text+  | PayloadMismatch !Text+  | ManifestsDisagree !Text+  | NestedManifestProblem !Text+  | RunResultMismatch !Text+  | StatusDigestMismatch+  | StatusMismatch !Text+  deriving stock (Eq, Show)++data FetchError+  = Unsealed+  | ManifestProblem !Text+  | ObjectSetMismatch ![Text] ![Text] -- missing, extra+  | ObjectChanged !Text+  | ObjectCorrupt !Text+  | VerificationFailed !(NonEmpty VerifyProblem)+  deriving stock (Eq, Show)++effectiveOutcome :: Cell.CellOutcome -> Bool -> Outcome.Outcome -> Outcome.Outcome+effectiveOutcome Cell.Completed _ recorded = recorded+effectiveOutcome Cell.InfrastructureFailure _ _ = Outcome.InfrastructureFailure+effectiveOutcome Cell.Cancelled complete recorded = if complete then recorded else Outcome.Errored+effectiveOutcome Cell.TimedOut complete recorded = if complete then recorded else Outcome.Errored++fetchCellRun :: ObjectStore -> Bucket -> RunId -> FilePath -> IO (Either FetchError FilePath)+fetchCellRun store bucket identifier outDir = do+  let prefix = "runs/" <> renderRunId identifier <> "/"+      manifestObject = ObjectName (prefix <> "manifest.json")+      tree = outDir </> Text.unpack (renderRunId identifier) </> "tree"+  sealed <- store.getObject bucket manifestObject+  case sealed of+    Nothing -> pure (Left Unsealed)+    Just (manifestBytes, manifestMeta) -> case eitherDecode manifestBytes :: Either String CellManifest of+      Left failure -> pure (Left (ManifestProblem (Text.pack failure)))+      Right manifest+        | manifest.runId /= identifier -> pure (Left (ManifestProblem "manifest run ID differs from result prefix"))+        | otherwise -> do+            listed <- store.listObjects bucket prefix+            let actual = Set.fromList [object.unObjectName | (object, _) <- listed]+                expected = Set.fromList ((prefix <> "manifest.json") : [prefix <> artifact.path | artifact <- manifest.artifacts])+                absent = Set.toList (expected Set.\\ actual)+                unexpected = Set.toList (actual Set.\\ expected)+            if not (null absent && null unexpected)+              then pure (Left (ObjectSetMismatch absent unexpected))+              else case lookup manifestObject listed of+                Nothing -> pure (Left Unsealed)+                Just listedManifest | listedManifest.generation /= manifestMeta.generation -> pure (Left (ObjectChanged "manifest.json"))+                Just _ -> do+                  createDirectoryIfMissing True tree+                  rootSymbolic <- pathIsSymbolicLink tree+                  if rootSymbolic+                    then pure (Left (VerificationFailed (UnsafeEntry "tree" :| [])))+                    else do+                      (_, unsafe) <- listTree tree+                      case unsafe of+                        first : rest -> pure (Left (VerificationFailed (UnsafeEntry first :| map UnsafeEntry rest)))+                        [] -> do+                          writeVerified tree "manifest.json" manifestBytes+                          fetched <- traverse (fetchArtifact store bucket prefix tree listed) manifest.artifacts+                          case firstFailure fetched of+                            Just failure -> pure (Left failure)+                            Nothing -> do+                              verified <- verifyCellTree tree+                              pure case verified of+                                Left problems -> Left (VerificationFailed problems)+                                Right _ -> Right tree++fetchArtifact :: ObjectStore -> Bucket -> Text -> FilePath -> [(ObjectName, ObjectMeta)] -> Artifact -> IO (Maybe FetchError)+fetchArtifact store bucket prefix tree listed artifact = do+  let name = ObjectName (prefix <> artifact.path)+      target = tree </> Text.unpack artifact.path+  case lookup name listed of+    Nothing -> pure (Just (ObjectChanged artifact.path))+    Just meta+      | meta.size /= artifact.bytes -> pure (Just (ObjectCorrupt artifact.path))+      | otherwise -> do+          existing <- doesFileExist target+          valid <- if existing then matches target artifact else pure False+          if valid+            then pure Nothing+            else withTemporary target \temporary -> do+              fetched <- store.downloadTo bucket name temporary+              case fetched of+                Nothing -> pure (Just (ObjectChanged artifact.path))+                Just got+                  | got.generation /= meta.generation -> pure (Just (ObjectChanged artifact.path))+                  | otherwise -> do+                      sound <- matches temporary artifact+                      if sound+                        then renameFile temporary target >> pure Nothing+                        else pure (Just (ObjectCorrupt artifact.path))++verifyCellTree :: FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)+verifyCellTree tree = do+  let manifestPath = tree </> "manifest.json"+  rootExists <- doesDirectoryExist tree+  if not rootExists+    then pure (Left (Missing "manifest.json" :| []))+    else do+      rootSymbolic <- pathIsSymbolicLink tree+      if rootSymbolic+        then pure (Left (UnsafeEntry "tree" :| []))+        else do+          exists <- doesFileExist manifestPath+          if not exists+            then pure (Left (Missing "manifest.json" :| []))+            else do+              symbolic <- pathIsSymbolicLink manifestPath+              if symbolic+                then pure (Left (UnsafeEntry "manifest.json" :| []))+                else do+                  bytes <- LazyByteString.readFile manifestPath+                  case eitherDecode bytes :: Either String CellManifest of+                    Left failure -> pure (Left (InvalidManifest (Text.pack failure) :| []))+                    Right manifest -> do+                      (files, unsafe) <- listTree tree+                      let actual = Set.fromList files+                          expected = Set.fromList ("manifest.json" : map (.path) manifest.artifacts)+                          missing = map Missing (Set.toList (expected Set.\\ actual))+                          extra = map Extra (Set.toList (actual Set.\\ expected))+                      damaged <- fmap concat $ forM manifest.artifacts \artifact -> do+                        if Set.member artifact.path actual+                          then do+                            (digest, size) <- digestFile (tree </> Text.unpack artifact.path)+                            pure ([DigestMismatch artifact.path | digest /= artifact.sha256] <> [SizeMismatch artifact.path | size /= artifact.bytes])+                          else pure []+                      let problems = map UnsafeEntry unsafe <> missing <> extra <> damaged+                      pure case problems of+                        [] -> Right manifest+                        first : rest -> Left (first :| rest)++-- This is the evidence gate after the cell-level size and digest verification.+-- Validate nested paths before calling the kernel verifier, whose standalone+-- entry point assumes its manifest came from a trusted local run directory.+verifyCellRun :: FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)+verifyCellRun tree = do+  verified <- verifyCellTree tree+  case verified of+    Left problems -> pure (Left problems)+    Right manifest -> do+      let byPath = Map.fromList [(artifact.path, artifact) | artifact <- manifest.artifacts]+      submission <- readRequired tree byPath "submission/submission.json" :: IO (Either VerifyProblem Submission)+      result <- readRequired tree byPath "cell/result.json" :: IO (Either VerifyProblem CellRunResult)+      case (submission, result) of+        (Right submitted, Right cellResult) -> do+          let identityProblems =+                [PayloadMismatch "submission run, lease or payload differs from cell manifest" | submitted.runId /= manifest.runId || submitted.leaseId /= manifest.leaseId || submitted.payload.bundle.sha256 /= manifest.payload.sha256 || submitted.payload.storePath /= manifest.payload.storePath]+                  <> [RunResultMismatch "cell result differs from cell manifest" | cellResult.runId /= manifest.runId || cellResult.cell /= manifest.cell || cellResult.leaseId /= manifest.leaseId || cellResult.leaseSequence /= manifest.leaseSequence || cellResult.outcome /= manifest.outcome]+          workProblems <- checkWork tree byPath submitted+          planned <- readPlannedRuns tree+          let nested = nestedManifestPaths manifest.artifacts+              nestedIds = Set.fromList (map fst nested)+              (planProblems, expectedIds) = case planned of+                Left problem -> ([problem], Set.empty)+                Right identifiers -> ([], identifiers)+              absentNested =+                [ NestedManifestProblem ("output/" <> run <> "/manifest.json: missing")+                | manifest.outcome == Cell.Completed,+                  run <- Set.toList (expectedIds Set.\\ nestedIds)+                ]+              unexpectedNested =+                [NestedManifestProblem ("output/" <> run <> "/manifest.json: run was not planned") | run <- Set.toList (nestedIds Set.\\ expectedIds)]+          nestedProblems <- concat <$> traverse (checkNested tree byPath manifest.payload.sha256) nested+          let problems = identityProblems <> workProblems <> planProblems <> absentNested <> unexpectedNested <> nestedProblems+          pure case problems of+            [] -> Right manifest+            first : rest -> Left (first :| rest)+        _ -> pure (Left (firstProblems submission result))++verifyCellRunWithStatus :: CellStatus -> FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)+verifyCellRunWithStatus status tree = do+  verified <- verifyCellRun tree+  case verified of+    Left problems -> pure (Left problems)+    Right manifest -> do+      (digest, _) <- digestFile (tree </> "manifest.json")+      let problems =+            [StatusMismatch "status is not sealed or names another cell run" | status.phase /= Sealed || status.runId /= manifest.runId]+              <> [StatusMismatch "status sequence or outcome differs from cell manifest" | status.leaseSequence /= Just manifest.leaseSequence || status.outcome /= Just manifest.outcome]+              <> [StatusDigestMismatch | status.manifestSha256 /= Just digest]+      pure case problems of+        [] -> Right manifest+        first : rest -> Left (first :| rest)++readRequired :: (FromJSON document) => FilePath -> Map.Map Text Artifact -> Text -> IO (Either VerifyProblem document)+readRequired tree byPath relative = case Map.lookup relative byPath of+  Nothing -> pure (Left (Missing relative))+  Just _ -> do+    bytes <- LazyByteString.readFile (tree </> Text.unpack relative)+    pure case eitherDecode bytes of+      Left failure -> Left (InvalidManifest (relative <> ": " <> Text.pack failure))+      Right document -> Right document++firstProblems :: Either VerifyProblem left -> Either VerifyProblem right -> NonEmpty VerifyProblem+firstProblems left right = case catMaybes [either Just (const Nothing) left, either Just (const Nothing) right] of+  first : rest -> first :| rest+  [] -> InvalidManifest "missing required cell document" :| []++checkWork :: FilePath -> Map.Map Text Artifact -> Submission -> IO [VerifyProblem]+checkWork tree byPath submitted = case Map.lookup "submission/work" byPath of+  Nothing -> pure [Missing "submission/work"]+  Just artifact -> do+    (digest, size) <- digestFile (tree </> "submission" </> "work")+    pure [PayloadMismatch "submission work differs from its declared digest or size" | digest /= submitted.work.sha256 || size /= submitted.work.bytes || artifact.sha256 /= submitted.work.sha256 || artifact.bytes /= submitted.work.bytes]++readPlannedRuns :: FilePath -> IO (Either VerifyProblem (Set.Set Text))+readPlannedRuns tree = do+  bytes <- LazyByteString.readFile (tree </> "submission" </> "work")+  pure case eitherDecode bytes :: Either String Value of+    Left failure -> Left (InvalidManifest ("submission/work: " <> Text.pack failure))+    Right value -> case decodeWorkPlan value of+      Left failure -> Left (InvalidManifest ("submission/work: " <> failure))+      Right plan -> Right (Set.fromList (map (renderRunId . (.runId)) plan.runs))++nestedManifestPaths :: [Artifact] -> [(Text, Text)]+nestedManifestPaths artifacts =+  [(run, artifact.path) | artifact <- artifacts, ["output", run, "manifest.json"] <- [Text.splitOn "/" artifact.path], Right _ <- [parseRunId run]]++checkNested :: FilePath -> Map.Map Text Artifact -> Text -> (Text, Text) -> IO [VerifyProblem]+checkNested tree byPath bundleDigest (run, path) = do+  decoded <- readRequired tree byPath path :: IO (Either VerifyProblem Manifest)+  case decoded of+    Left problem -> pure [problem]+    Right nested+      | renderRunId nested.runId /= run -> pure [NestedManifestProblem (path <> ": run ID differs from directory")]+      | otherwise -> do+          let relativeFiles = map (Text.pack . (.path)) nested.files+          if any (not . safeRelative) relativeFiles || length relativeFiles /= Set.size (Set.fromList relativeFiles)+            then pure [NestedManifestProblem (path <> ": unsafe or duplicate nested path")]+            else do+              let prefix = "output/" <> run <> "/"+                  expected = sort (path : map (prefix <>) relativeFiles)+                  actual = sort [name | name <- Map.keys byPath, prefix `Text.isPrefixOf` name]+                  setProblem = [ManifestsDisagree (path <> ": nested file set differs from cell manifest") | expected /= actual]+                  metadataProblems =+                    [ ManifestsDisagree (full <> ": digest or size differs between manifests")+                    | file <- nested.files,+                      let full = prefix <> Text.pack file.path,+                      Just artifact <- [Map.lookup full byPath],+                      Just artifact.sha256 /= Text.stripPrefix "sha256:" file.sha256 || fromIntegral artifact.bytes /= file.bytes+                    ]+              checked <- verifyManifest (tree </> "output" </> Text.unpack run)+              let kernelProblems = case checked of+                    Left problems -> [NestedManifestProblem (path <> ": " <> Text.pack (show problems))]+                    Right () -> []+              fingerprintProblems <- checkFingerprint tree prefix run bundleDigest+              pure (setProblem <> metadataProblems <> kernelProblems <> fingerprintProblems)++checkFingerprint :: FilePath -> Text -> Text -> Text -> IO [VerifyProblem]+checkFingerprint tree prefix run expected = do+  let relative = prefix <> "run-result.json"+  present <- doesFileExist (tree </> Text.unpack relative)+  if not present+    then pure [RunResultMismatch (relative <> ": missing")]+    else do+      bytes <- LazyByteString.readFile (tree </> Text.unpack relative)+      pure case eitherDecode bytes :: Either String Value of+        Left failure -> [RunResultMismatch (relative <> ": " <> Text.pack failure)]+        Right value ->+          [RunResultMismatch (relative <> ": run ID differs from directory") | resultRunId value /= Just run]+            <> [RunResultMismatch (relative <> ": payload fingerprint differs from submission") | fingerprintDigest value /= Just expected]++resultRunId :: Value -> Maybe Text+resultRunId (Object result) = do+  String identifier <- KeyMap.lookup "runId" result+  pure identifier+resultRunId _ = Nothing++fingerprintDigest :: Value -> Maybe Text+fingerprintDigest (Object result) = do+  Object fingerprint <- KeyMap.lookup "fingerprint" result+  Object cell <- KeyMap.lookup "cell" fingerprint+  Object payload <- KeyMap.lookup "payload" cell+  String digest <- KeyMap.lookup "bundleSha256" payload+  pure digest+fingerprintDigest _ = Nothing++safeRelative :: Text -> Bool+safeRelative value =+  not (Text.null value)+    && not (Text.isPrefixOf "/" value)+    && not (Text.any (`elem` ['\\', '\0']) value)+    && all (\part -> not (Text.null part) && part /= "." && part /= "..") (Text.splitOn "/" value)++listTree :: FilePath -> IO ([Text], [Text])+listTree root = go ""+  where+    go relative = do+      let directory = if Text.null relative then root else root </> Text.unpack relative+      names <- listDirectory directory+      parts <- forM names \name -> do+        let path = if Text.null relative then Text.pack name else relative <> "/" <> Text.pack name+            full = root </> Text.unpack path+        symbolic <- pathIsSymbolicLink full+        if symbolic+          then pure ([], [path])+          else do+            directoryEntry <- doesDirectoryExist full+            if directoryEntry+              then go path+              else do+                regular <- doesFileExist full+                pure (if regular then [path] else [], if regular then [] else [path])+      pure (concatMap fst parts, concatMap snd parts)++matches :: FilePath -> Artifact -> IO Bool+matches path artifact = do+  (digest, size) <- digestFile path+  pure (digest == artifact.sha256 && size == artifact.bytes)++digestFile :: FilePath -> IO (Text, Int64)+digestFile path = withBinaryFile path ReadMode (go SHA256.init 0)+  where+    go context size handle = do+      chunk <- ByteString.hGetSome handle 65536+      if ByteString.null chunk+        then pure (TextEncoding.decodeUtf8 (Base16.encode (SHA256.finalize context)), size)+        else do+          let nextContext = SHA256.update context chunk+              nextSize = size + fromIntegral (ByteString.length chunk)+          nextContext `seq` nextSize `seq` go nextContext nextSize handle++writeVerified :: FilePath -> FilePath -> LazyByteString.ByteString -> IO ()+writeVerified root relative bytes = do+  let target = root </> relative+  withTemporary target \temporary -> LazyByteString.writeFile temporary bytes >> renameFile temporary target++withTemporary :: FilePath -> (FilePath -> IO value) -> IO value+withTemporary target action = do+  createDirectoryIfMissing True (takeDirectory target)+  bracket (openBinaryTempFile (takeDirectory target) ".kenshou-fetch-") cleanup \(temporary, handle) -> do+    hClose handle+    action temporary+  where+    cleanup (temporary, _) = do+      exists <- doesFileExist temporary+      if exists then removeFile temporary else pure ()++firstFailure :: [Maybe failure] -> Maybe failure+firstFailure = listToMaybe . catMaybes
+ src/Kenshou/Remote/Cell/Health.hs view
@@ -0,0 +1,103 @@+module Kenshou.Remote.Cell.Health (mapCellHealth, withCellHealthFile) where++import Control.Concurrent (forkIO, killThread, threadDelay)+import Control.Exception (IOException, catch, finally, mask_)+import Data.Aeson (Value, eitherDecodeStrict', encode, object, withObject, (.:), (.:?), (.=))+import Data.Aeson.Types (Parser, parseEither)+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as TextIO+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId)+import System.Directory (doesFileExist)+import System.IO (stderr)++-- The cell owner emits a health document for each observation window. Keep+-- only tripped gates, because an ok gate is not a run health notice.+mapCellHealth :: RunId -> Value -> Either Text [Value]+mapCellHealth expected document = either (Left . Text.pack) Right (parseEither parseDocument document)+  where+    parseDocument = withObject "cell health" \fields -> do+      schema <- fields .: "schema"+      if schema == ("cell.health/v1" :: Text) then pure () else fail "unsupported cell health schema"+      runId <- fields .: "runId"+      if runId == expected then pure () else fail "cell health run ID differs from the active cell run"+      gates <- fields .: "gates" :: Parser [Value]+      concat <$> traverse parseGate gates++    parseGate = withObject "cell health gate" \gate -> do+      name <- gate .: "name" :: Parser Text+      machine <- gate .: "machine" :: Parser Text+      status <- gate .: "status" :: Parser Text+      at <- gate .: "at" :: Parser UTCTime+      detail <- gate .:? "detail" :: Parser (Maybe Text)+      if name `elem` ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"] && not (Text.null machine)+        then pure ()+        else fail "unknown cell health gate or empty machine"+      case status of+        "ok" -> pure []+        "tripped" ->+          pure+            [ object+                [ "schema" .= ("kenshou.health-notice/v1" :: Text),+                  "source" .= ("cell-health:" <> name <> ":" <> machine),+                  "severity" .= (if name == "background-load" then "soft" else "hard" :: Text),+                  "at" .= at,+                  "detail" .= maybe ("cell health gate tripped: " <> name) (\message -> if Text.null message then "cell health gate tripped: " <> name else message) detail+                ]+            ]+        _ -> fail "unknown cell health gate status"++-- Re-read the rolling file because its producer may replace it atomically or+-- append JSON lines. Deduplicate notices across repeated health snapshots.+withCellHealthFile :: Maybe FilePath -> FilePath -> RunId -> IO result -> IO result+withCellHealthFile Nothing _ _ action = action+withCellHealthFile (Just source) destination runId action = do+  initial <- safePoll Set.empty+  latest <- newIORef initial+  thread <- forkIO (loop latest initial)+  action `finally` do+    killThread thread+    readIORef latest >>= safePoll >>= writeIORef latest+  where+    loop latest seen = do+      threadDelay 1000000+      next <- mask_ do+        updated <- safePoll seen+        writeIORef latest updated+        pure updated+      loop latest next++    safePoll seen =+      poll seen `catch` \(failure :: IOException) -> do+        TextIO.hPutStrLn stderr ("kenshou cell health: " <> Text.pack (show failure))+        pure seen++    poll seen = do+      exists <- doesFileExist source+      if not exists+        then pure seen+        else do+          bytes <- ByteString.readFile source+          foldLines seen (completeLines bytes)++    foldLines seen [] = pure seen+    foldLines seen (line : rest) = do+      case eitherDecodeStrict' line >>= firstText . mapCellHealth runId of+        Left problem -> TextIO.hPutStrLn stderr ("kenshou cell health: " <> Text.pack problem) >> foldLines seen rest+        Right notices -> do+          let new = filter (\notice -> encode notice `Set.notMember` seen) notices+          mapM_ (LazyByteString.appendFile destination . (<> "\n") . encode) new+          foldLines (foldr (Set.insert . encode) seen new) rest++    firstText = either (Left . Text.unpack) Right++completeLines :: ByteString.ByteString -> [ByteString.ByteString]+completeLines bytes | ByteString.null bytes = []+completeLines bytes = case ByteString.split 10 bytes of+  [] -> []+  parts -> filter (not . ByteString.null) (if ByteString.last bytes == 10 then parts else init parts)
+ src/Kenshou/Remote/Cell/Index.hs view
@@ -0,0 +1,195 @@+module Kenshou.Remote.Cell.Index+  ( CellManifestLink (..),+    EvidenceLink (..),+    RunLink (..),+    CellRunIndex (..),+    deriveCellRunIndex,+    writeCellRunIndex,+  )+where++import Control.Exception (bracket)+import Control.Monad (forM)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (FromJSON (..), ToJSON (..), Value, eitherDecode, encode, object, withObject, (.:), (.:?), (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Time (UTCTime, getCurrentTime)+import Kenshou.Core.Id (RunId, parseRunId, renderRunId)+import Kenshou.Core.Outcome (Outcome)+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellRunResult (..), CellStatus)+import Kenshou.Remote.Cell.Fetch (VerifyProblem (..), effectiveOutcome, verifyCellRun, verifyCellRunWithStatus)+import Kenshou.Remote.Store (Bucket (..))+import System.Directory (doesFileExist, removeFile, renameFile)+import System.FilePath (takeDirectory, (</>))+import System.IO (hClose, openBinaryTempFile)++data CellManifestLink = CellManifestLink+  { path :: !Text,+    sha256 :: !Text,+    bytes :: !Int64+  }+  deriving stock (Eq, Show)++data EvidenceLink = EvidenceLink+  { kind :: !Text,+    path :: !Text,+    sha256 :: !Text,+    bytes :: !Int64,+    mediaType :: !Text+  }+  deriving stock (Eq, Show)++data RunLink = RunLink+  { runId :: !RunId,+    path :: !Text,+    scenario :: !Text,+    manifestSha256 :: !Text,+    recordedOutcome :: !Outcome,+    effectiveOutcome :: !Outcome,+    overrideReason :: !(Maybe Text)+  }+  deriving stock (Eq, Show)++data CellRunIndex = CellRunIndex+  { cellRun :: !RunId,+    cell :: !Text,+    leaseId :: !RunId,+    leaseSequence :: !Int,+    resultsBaseUri :: !Text,+    dataBaseUri :: !Text,+    cellManifest :: !CellManifestLink,+    cellOutcome :: !CellOutcome,+    reasons :: ![Text],+    entryExitCode :: !(Maybe Int),+    evidence :: ![EvidenceLink],+    runs :: ![RunLink],+    verifiedAt :: !UTCTime+  }+  deriving stock (Eq, Show)++instance ToJSON CellManifestLink where+  toJSON link = object ["path" .= link.path, "sha256" .= link.sha256, "bytes" .= link.bytes]++instance FromJSON CellManifestLink where+  parseJSON = withObject "cell manifest link" \value -> CellManifestLink <$> value .: "path" <*> value .: "sha256" <*> value .: "bytes"++instance ToJSON EvidenceLink where+  toJSON link = object ["kind" .= link.kind, "path" .= link.path, "sha256" .= link.sha256, "bytes" .= link.bytes, "mediaType" .= link.mediaType]++instance FromJSON EvidenceLink where+  parseJSON = withObject "cell evidence link" \value -> EvidenceLink <$> value .: "kind" <*> value .: "path" <*> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"++instance ToJSON RunLink where+  toJSON run = object ["runId" .= run.runId, "path" .= run.path, "scenario" .= run.scenario, "manifestSha256" .= run.manifestSha256, "recordedOutcome" .= run.recordedOutcome, "effectiveOutcome" .= run.effectiveOutcome, "overrideReason" .= run.overrideReason]++instance FromJSON RunLink where+  parseJSON = withObject "cell nested run" \value -> RunLink <$> value .: "runId" <*> value .: "path" <*> value .: "scenario" <*> value .: "manifestSha256" <*> value .: "recordedOutcome" <*> value .: "effectiveOutcome" <*> value .:? "overrideReason"++instance ToJSON CellRunIndex where+  toJSON index =+    object+      [ "schema" .= ("kenshou.cell-run/v1" :: Text),+        "cellRun" .= index.cellRun,+        "cell" .= index.cell,+        "leaseId" .= index.leaseId,+        "leaseSequence" .= index.leaseSequence,+        "resultsBaseUri" .= index.resultsBaseUri,+        "dataBaseUri" .= index.dataBaseUri,+        "cellManifest" .= index.cellManifest,+        "cellOutcome" .= index.cellOutcome,+        "reasons" .= index.reasons,+        "entryExitCode" .= index.entryExitCode,+        "evidence" .= index.evidence,+        "runs" .= index.runs,+        "verifiedAt" .= index.verifiedAt+      ]++instance FromJSON CellRunIndex where+  parseJSON = withObject "cell run index" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.cell-run/v1" :: Text) then fail "unsupported cell run index" else pure ()+    CellRunIndex <$> value .: "cellRun" <*> value .: "cell" <*> value .: "leaseId" <*> value .: "leaseSequence" <*> value .: "resultsBaseUri" <*> value .: "dataBaseUri" <*> value .: "cellManifest" <*> value .: "cellOutcome" <*> value .: "reasons" <*> value .:? "entryExitCode" <*> value .: "evidence" <*> value .: "runs" <*> value .: "verifiedAt"++deriveCellRunIndex :: Bucket -> Maybe CellStatus -> FilePath -> IO (Either (NonEmpty VerifyProblem) CellRunIndex)+deriveCellRunIndex bucket status tree = do+  verified <- case status of+    Nothing -> verifyCellRun tree+    Just sealed -> verifyCellRunWithStatus sealed tree+  case verified of+    Left problems -> pure (Left problems)+    Right manifest -> do+      resultBytes <- LazyByteString.readFile (tree </> "cell" </> "result.json")+      case eitherDecode resultBytes :: Either String CellRunResult of+        Left failure -> pure (Left (InvalidManifest ("cell/result.json: " <> Text.pack failure) :| []))+        Right cellResult -> do+          manifestBytes <- LazyByteString.readFile (tree </> "manifest.json")+          let base = "gs://" <> bucket.unBucket <> "/runs/" <> renderRunId manifest.runId+              linked = CellManifestLink "manifest.json" (digest manifestBytes) (LazyByteString.length manifestBytes)+              evidence = [EvidenceLink kind artifact.path artifact.sha256 artifact.bytes artifact.mediaType | artifact <- manifest.artifacts, Just kind <- [evidenceKind artifact.path]]+          indexed <- forM (nestedRuns manifest.artifacts) \(identifier, nestedPath) -> do+            bytes <- LazyByteString.readFile (tree </> Text.unpack nestedPath </> "run-result.json")+            nestedManifest <- LazyByteString.readFile (tree </> Text.unpack nestedPath </> "manifest.json")+            pure case runFields bytes of+              Left failure -> Left (RunResultMismatch (nestedPath <> ": " <> failure))+              Right (scenario, recorded) ->+                let effective = effectiveOutcome manifest.outcome True recorded+                    override = if manifest.outcome == InfrastructureFailure || effective /= recorded then Just (reasonFor cellResult.reasons) else Nothing+                 in Right (RunLink identifier nestedPath scenario (digest nestedManifest) recorded effective override)+          case sequence indexed of+            Left problem -> pure (Left (problem :| []))+            Right runs -> do+              now <- getCurrentTime+              pure (Right (CellRunIndex manifest.runId manifest.cell manifest.leaseId manifest.leaseSequence base (base <> "/output") linked manifest.outcome cellResult.reasons cellResult.entryExitCode evidence runs now))++writeCellRunIndex :: FilePath -> CellRunIndex -> IO FilePath+writeCellRunIndex tree index = do+  let destination = takeDirectory tree </> "cell-run.json"+  bracket (openBinaryTempFile (takeDirectory tree) ".kenshou-cell-run-") cleanup \(temporary, handle) -> do+    hClose handle+    LazyByteString.writeFile temporary (encode index)+    renameFile temporary destination+    pure destination+  where+    cleanup (temporary, _) = do+      exists <- doesFileExist temporary+      if exists then removeFile temporary else pure ()++nestedRuns :: [Artifact] -> [(RunId, Text)]+nestedRuns artifacts =+  [ (identifier, "output/" <> renderRunId identifier)+  | artifact <- artifacts,+    ["output", rendered, "manifest.json"] <- [Text.splitOn "/" artifact.path],+    Right identifier <- [parseRunId rendered]+  ]++runFields :: LazyByteString.ByteString -> Either Text (Text, Outcome)+runFields bytes = case eitherDecode bytes :: Either String Value of+  Left failure -> Left (Text.pack failure)+  Right (Aeson.Object fields) -> case (KeyMap.lookup "scenario" fields, KeyMap.lookup "outcome" fields) of+    (Just (Aeson.String scenario), Just verdict) -> case Aeson.fromJSON verdict of+      Aeson.Success outcome -> Right (scenario, outcome)+      Aeson.Error failure -> Left (Text.pack failure)+    _ -> Left "scenario or outcome is missing"+  Right _ -> Left "run result is not an object"++evidenceKind :: Text -> Maybe Text+evidenceKind "cell/fingerprint.json" = Just "cell-fingerprint"+evidenceKind "cell/health.json" = Just "cell-health"+evidenceKind "cell/reset-evidence.json" = Just "cell-reset-evidence"+evidenceKind "metrics/export.jsonl.zst" = Just "cell-metrics-export"+evidenceKind _ = Nothing++reasonFor :: [Text] -> Text+reasonFor [] = "cell-infrastructure-failure"+reasonFor reasons = Text.intercalate "; " reasons++digest :: LazyByteString.ByteString -> Text+digest = TextEncoding.decodeUtf8 . Base16.encode . SHA256.hashlazy
+ src/Kenshou/Remote/Cell/Lease.hs view
@@ -0,0 +1,332 @@+module Kenshou.Remote.Cell.Lease+  ( CellRef (..),+    LeaseId,+    Lease (..),+    Quarantine (..),+    LeaseRequest (..),+    LeaseHandle,+    AcquireOutcome (..),+    acquireLease,+    acquireLeaseWithId,+    renewLease,+    resizeLease,+    releaseLease,+    reattachLease,+    reattachActiveLease,+    requestCancel,+    leaseSnapshot,+    leaseHeld,+    withHeartbeat,+    validCellName,+  )+where++import Control.Concurrent (forkIO, killThread, threadDelay)+import Control.Concurrent.MVar (MVar, modifyMVar, modifyMVar_, newMVar, readMVar)+import Control.Exception (SomeAsyncException, SomeException, bracket, displayException, fromException, throwIO, try)+import Control.Monad (unless)+import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecode, encode, object, withObject, (.:), (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, getCurrentTime)+import Kenshou.Core.Id (RunId, newRunId)+import Kenshou.Remote.Store (Bucket, ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import System.IO (hPutStrLn, stderr)++data CellRef = CellRef+  { cellName :: !Text,+    controlBucket :: !Bucket+  }+  deriving stock (Eq, Show)++type LeaseId = RunId++data Lease = Lease+  { leaseId :: !LeaseId,+    cell :: !Text,+    owner :: !Text,+    purpose :: !Text,+    ttlSeconds :: !Int,+    acquiredAt :: !UTCTime,+    heartbeatAt :: !UTCTime,+    cancelRequested :: !Bool,+    runsStarted :: !Int+  }+  deriving stock (Eq, Show)++data Quarantine = Quarantine+  { cell :: !Text,+    reason :: !Text,+    runId :: !(Maybe Text),+    at :: !UTCTime+  }+  deriving stock (Eq, Show)++data LeaseRequest = LeaseRequest+  { owner :: !Text,+    purpose :: !Text,+    ttlSeconds :: !Int+  }+  deriving stock (Eq, Show)++data LeaseHandle = LeaseHandle+  { state :: !(MVar (Lease, Int64, Bool))+  }++data AcquireOutcome = Acquired !LeaseHandle | Busy !Lease | Quarantined !Quarantine++instance ToJSON Lease where+  toJSON lease =+    object+      [ "schema" .= ("cell.lease/v1" :: Text),+        "leaseId" .= lease.leaseId,+        "cell" .= lease.cell,+        "owner" .= lease.owner,+        "purpose" .= lease.purpose,+        "ttlSeconds" .= lease.ttlSeconds,+        "acquiredAt" .= lease.acquiredAt,+        "heartbeatAt" .= lease.heartbeatAt,+        "cancelRequested" .= lease.cancelRequested,+        "runsStarted" .= lease.runsStarted+      ]++instance FromJSON Lease where+  parseJSON = withObject "cell lease" \value -> do+    schema <- value .: "schema"+    unless (schema == ("cell.lease/v1" :: Text)) (fail "unsupported cell lease schema")+    lease <- Lease <$> value .: "leaseId" <*> value .: "cell" <*> value .: "owner" <*> value .: "purpose" <*> value .: "ttlSeconds" <*> value .: "acquiredAt" <*> value .: "heartbeatAt" <*> value .: "cancelRequested" <*> value .: "runsStarted"+    unless (validCellName lease.cell && not (Text.null lease.owner) && not (Text.null lease.purpose) && lease.ttlSeconds > 0 && lease.runsStarted >= 0) (fail "invalid cell lease")+    pure lease++instance ToJSON Quarantine where+  toJSON quarantine =+    object+      [ "schema" .= ("cell.quarantine/v1" :: Text),+        "cell" .= quarantine.cell,+        "reason" .= quarantine.reason,+        "runId" .= quarantine.runId,+        "at" .= quarantine.at+      ]++instance FromJSON Quarantine where+  parseJSON = withObject "cell quarantine" \value -> do+    schema <- value .: "schema"+    unless (schema == ("cell.quarantine/v1" :: Text)) (fail "unsupported cell quarantine schema")+    quarantine <- Quarantine <$> value .: "cell" <*> value .: "reason" <*> value .: "runId" <*> value .: "at"+    unless (validCellName quarantine.cell && not (Text.null quarantine.reason)) (fail "invalid cell quarantine")+    pure quarantine++acquireLease :: ObjectStore -> CellRef -> LeaseRequest -> IO AcquireOutcome+acquireLease store ref request = newRunId >>= acquireLeaseWithId store ref request++-- The caller may durably retain this identity before issuing a remote claim.+-- Reconciliation never renews or revives an expired/cancelled claim.+acquireLeaseWithId :: ObjectStore -> CellRef -> LeaseRequest -> LeaseId -> IO AcquireOutcome+acquireLeaseWithId store ref request proposedId = do+  unless (validCellName ref.cellName && not (Text.null request.owner) && not (Text.null request.purpose) && request.ttlSeconds > 0) (ioError (userError "invalid lease request"))+  current <- readCurrent store ref+  now <- store.serverTime+  case current of+    Just (lease, meta)+      | lease.leaseId == proposedId ->+          if not lease.cancelRequested && addUTCTime (fromIntegral lease.ttlSeconds + 30) meta.updated >= now+            then accept lease meta+            else pure (Busy lease)+    _ -> acquire proposedId (0 :: Int)+  where+    acquire identifier attempts = do+      unless (attempts < maxCasAttempts) (ioError (userError "lease claim contention exceeded retry limit"))+      now <- store.serverTime+      let proposed = Lease identifier ref.cellName request.owner request.purpose request.ttlSeconds now now False 0+      result <- store.putObject ref.controlBucket (leaseName ref) "application/json" DoesNotExist (encode proposed)+      case result of+        Written meta -> accept proposed meta+        PreconditionFailed -> do+          observed <- readCurrent store ref+          case observed of+            Nothing -> acquire identifier (attempts + 1)+            Just (current, meta) -> do+              clock <- store.serverTime+              if addUTCTime (fromIntegral current.ttlSeconds + 30) meta.updated < clock+                then do+                  replaced <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs meta.generation) (encode proposed)+                  case replaced of+                    Written next -> accept proposed next+                    PreconditionFailed -> acquire identifier (attempts + 1)+                else pure (Busy current)+    accept lease meta = do+      quarantine <- readQuarantine store ref+      case quarantine of+        Nothing -> Acquired <$> newHandle lease meta.generation+        Just record -> do+          released <- store.deleteObject ref.controlBucket (leaseName ref) (GenerationIs meta.generation)+          unless released (ioError (userError "quarantined cell lease could not be released"))+          pure (Quarantined record)++renewLease :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool+renewLease store ref handle = modifyMVar handle.state \(lease, generation, held) ->+  if not held || lease.cancelRequested+    then pure ((lease, generation, False), False)+    else renewCurrent lease generation (0 :: Int)+  where+    renewCurrent lease generation attempts = do+      unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))+      now <- store.serverTime+      let renewed = lease {heartbeatAt = now}+      result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs generation) (encode renewed)+      case result of+        Written meta -> pure ((renewed, meta.generation, True), True)+        PreconditionFailed -> do+          current <- readCurrent store ref+          case current of+            Just (observed, meta)+              | attempts + 1 < maxCasAttempts,+                observed.leaseId == lease.leaseId,+                not observed.cancelRequested,+                addUTCTime (fromIntegral observed.ttlSeconds + 30) meta.updated >= now ->+                  renewCurrent observed meta.generation (attempts + 1)+            _ -> pure ((lease, generation, False), False)++-- A detached submission needs a lease that outlives its wall-clock budget.+-- Fence the TTL change before publishing its marker.+resizeLease :: ObjectStore -> CellRef -> LeaseHandle -> Int -> IO Bool+resizeLease store ref handle ttl = do+  unless (ttl > 0) (ioError (userError "lease TTL must be positive"))+  modifyMVar handle.state \(lease, generation, held) ->+    if not held || lease.cancelRequested+      then pure ((lease, generation, False), False)+      else do+        unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))+        now <- store.serverTime+        let resized = lease {ttlSeconds = ttl, heartbeatAt = now}+        result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs generation) (encode resized)+        case result of+          Written meta -> pure ((resized, meta.generation, True), True)+          PreconditionFailed -> pure ((lease, generation, False), False)++releaseLease :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool+releaseLease store ref handle = modifyMVar handle.state \(lease, generation, held) ->+  if not held+    then pure ((lease, generation, False), False)+    else do+      unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))+      released <- store.deleteObject ref.controlBucket (leaseName ref) (GenerationIs generation)+      pure ((lease, generation, False), released)++reattachLease :: ObjectStore -> CellRef -> LeaseId -> IO (Maybe LeaseHandle)+reattachLease store ref identifier = do+  current <- readCurrent store ref+  case current of+    Just (lease, meta) | lease.leaseId == identifier -> Just <$> newHandle lease meta.generation+    _ -> pure Nothing++-- Performance recovery must treat an expired or cancelled lease as lost, even+-- when its old object still exists. The unfenced reattachment above remains+-- available to cancellation/release code that needs its original generation.+reattachActiveLease :: ObjectStore -> CellRef -> LeaseId -> IO (Maybe LeaseHandle)+reattachActiveLease store ref identifier = do+  current <- readCurrent store ref+  now <- store.serverTime+  case current of+    Just (lease, meta)+      | lease.leaseId == identifier,+        not lease.cancelRequested,+        addUTCTime (fromIntegral lease.ttlSeconds + 30) meta.updated >= now ->+          Just <$> newHandle lease meta.generation+    _ -> pure Nothing++requestCancel :: ObjectStore -> CellRef -> Maybe LeaseId -> Bool -> IO Bool+requestCancel store ref expected force = do+  unless (force || isJust expected) (ioError (userError "cancelling another owner's lease requires force"))+  attempt (0 :: Int)+  where+    attempt count = do+      unless (count < maxCasAttempts) (ioError (userError "lease cancellation contention exceeded retry limit"))+      current <- readCurrent store ref+      case current of+        Nothing -> pure False+        Just (lease, meta)+          | not force && Just lease.leaseId /= expected -> pure False+          | lease.cancelRequested -> pure True+          | otherwise -> do+              result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs meta.generation) (encode (lease {cancelRequested = True}))+              case result of+                Written _ -> pure True+                PreconditionFailed -> attempt (count + 1)++leaseHeld :: LeaseHandle -> IO Bool+leaseHeld handle = (\(_, _, held) -> held) <$> readMVar handle.state++leaseSnapshot :: LeaseHandle -> IO Lease+leaseSnapshot handle = (\(lease, _, _) -> lease) <$> readMVar handle.state++withHeartbeat :: ObjectStore -> CellRef -> LeaseHandle -> (IO Bool -> IO value) -> IO value+withHeartbeat store ref handle action = bracket start killThread (const (action (leaseHeld handle)))+  where+    start = forkIO loop+    loop = do+      (lease, _, held) <- readMVar handle.state+      if held+        then do+          threadDelay (max 100000 (lease.ttlSeconds * 1000000 `div` 3))+          renew+        else pure ()+    renew = do+      result <- try (renewLease store ref handle) :: IO (Either SomeException Bool)+      case result of+        Right True -> loop+        Right False -> markLost+        Left failure | Just async <- (fromException failure :: Maybe SomeAsyncException) -> throwIO async+        Left failure -> do+          hPutStrLn stderr ("cell lease heartbeat renewal failed; retrying before TTL: " <> displayException failure)+          current <- leaseSnapshot handle+          now <- getCurrentTime+          if now <= addUTCTime (fromIntegral current.ttlSeconds + 30) current.heartbeatAt+            then threadDelay 1000000 >> renew+            else markLost+    markLost = modifyMVar_ handle.state \(current, generation, _) -> pure (current, generation, False)++newHandle :: Lease -> Int64 -> IO LeaseHandle+newHandle lease generation = LeaseHandle <$> newMVar (lease, generation, True)++readCurrent :: ObjectStore -> CellRef -> IO (Maybe (Lease, ObjectMeta))+readCurrent store ref = do+  stored <- store.getObject ref.controlBucket (leaseName ref)+  case stored of+    Nothing -> pure Nothing+    Just (bytes, meta) -> do+      lease <- decodeDocument "cell lease" bytes+      unless (lease.cell == ref.cellName) (ioError (userError "cell lease names another cell"))+      pure (Just (lease, meta))++readQuarantine :: ObjectStore -> CellRef -> IO (Maybe Quarantine)+readQuarantine store ref = do+  stored <- store.getObject ref.controlBucket (quarantineName ref)+  case stored of+    Nothing -> pure Nothing+    Just (bytes, _) -> do+      record <- decodeDocument "cell quarantine" bytes+      unless (record.cell == ref.cellName) (ioError (userError "cell quarantine names another cell"))+      pure (Just record)++decodeDocument :: (FromJSON value) => String -> LazyByteString.ByteString -> IO value+decodeDocument label bytes = either (ioError . userError . ((label <> ": ") <>)) pure (eitherDecode bytes)++leaseName :: CellRef -> ObjectName+leaseName ref = ObjectName ("cells/" <> ref.cellName <> "/lease.json")++quarantineName :: CellRef -> ObjectName+quarantineName ref = ObjectName ("cells/" <> ref.cellName <> "/quarantine.json")++validCellName :: Text -> Bool+validCellName value =+  Text.length value >= 3+    && Text.length value <= 63+    && Text.head value `elem` (['a' .. 'z'] <> ['0' .. '9'])+    && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'] <> "-")) value++maxCasAttempts :: Int+maxCasAttempts = 32
+ src/Kenshou/Remote/Cell/Legacy.hs view
@@ -0,0 +1,41 @@+-- | The one place that maps the cell protocol's dedicated broker fields onto the+-- named resource that documents once carried as a top-level field. The cell agent+-- in mori://shinzui/load-testing-infra still describes a broker with+-- @CellEnvironment.broker@ and resets it with @reset.broker@; this module binds and+-- requests it through the generic resource path. Delete it when load-testing-infra+-- publishes @resources@ in the cell environment and accepts @reset.resources@ in+-- the submission.+module Kenshou.Remote.Cell.Legacy+  ( brokerAsResource,+    legacyBrokerReset,+    legacyBrokerAvailable,+  )+where++import Data.Aeson (Value, object, (.=))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Resource.Types (legacyKafkaName)+import Kenshou.Remote.Cell.Docs (BrokerReset (..), CellBroker (..), CellDescriptor (..), CellImages (..), ResetBlock (..))++-- | The resource value a cell broker binds to, exactly as the cell client has+-- always passed it to scenarios.+brokerAsResource :: CellBroker -> Either Text Value+brokerAsResource broker+  | null addresses = Left "cell broker has no bootstrap servers"+  | otherwise = Right (object ["backend" .= ("external" :: Text), "brokers" .= addresses, "lanes" .= (0 :: Int)])+  where+    addresses = filter (not . Text.null) (fmap Text.strip (Text.splitOn "," broker.bootstrapServers))++-- | Move a reset of that resource into the dedicated broker wipe. The entry leaves+-- @reset.resources@, because the cell agent decodes submissions strictly and would+-- reject the extension; the submission keeps the exact shape it always had.+legacyBrokerReset :: ResetBlock -> ResetBlock+legacyBrokerReset reset+  | Map.member legacyKafkaName reset.resources = reset {broker = Just (BrokerReset True), resources = Map.delete legacyKafkaName reset.resources}+  | otherwise = reset++-- | Whether the cell's descriptor names a broker image.+legacyBrokerAvailable :: CellDescriptor -> Bool+legacyBrokerAvailable descriptor = not (Text.null descriptor.images.broker)
+ src/Kenshou/Remote/Cell/Parity.hs view
@@ -0,0 +1,189 @@+module Kenshou.Remote.Cell.Parity+  ( ParityOptions (..),+    ParityDifference (..),+    ParityReport (..),+    compareForParity,+  )+where++import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (ToJSON (..), Value (..), eitherDecodeStrict', object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.Foldable (toList)+import Data.List (isPrefixOf, sort)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import System.Directory (doesDirectoryExist, listDirectory)+import System.FilePath (takeExtension, (</>))++newtype ParityOptions = ParityOptions {volatilePaths :: [Text]}+  deriving stock (Eq, Show)++data ParityDifference = ParityDifference+  { path :: !Text,+    localValue :: !(Maybe Value),+    cellValue :: !(Maybe Value)+  }+  deriving stock (Eq, Show)++instance ToJSON ParityDifference where+  toJSON difference =+    object+      [ "path" .= difference.path,+        "local" .= difference.localValue,+        "cell" .= difference.cellValue+      ]++data ParityReport = ParityReport+  { schema :: !Text,+    localRunResultSha256 :: !Text,+    cellRunResultSha256 :: !Text,+    equal :: ![Text],+    intentional :: ![ParityDifference],+    unexpected :: ![ParityDifference]+  }+  deriving stock (Eq, Show)++instance ToJSON ParityReport where+  toJSON report =+    object+      [ "schema" .= report.schema,+        "localRunResultSha256" .= report.localRunResultSha256,+        "cellRunResultSha256" .= report.cellRunResultSha256,+        "equal" .= report.equal,+        "intentional" .= report.intentional,+        "unexpected" .= report.unexpected+      ]++compareForParity :: ParityOptions -> FilePath -> FilePath -> IO ParityReport+compareForParity options localDir cellDir = do+  localBytes <- ByteString.readFile (localDir </> "run-result.json")+  cellBytes <- ByteString.readFile (cellDir </> "run-result.json")+  localResult <- decode "local run-result.json" localBytes+  cellResult <- decode "cell run-result.json" cellBytes+  localSpec <- readJson "local run-spec.json" (localDir </> "run-spec.json")+  cellSpec <- readJson "cell run-spec.json" (cellDir </> "run-spec.json")+  localVerdicts <- readVerdicts localDir+  cellVerdicts <- readVerdicts cellDir+  let comparisons =+        compareJson ["result"] localResult cellResult+          <> compareJson ["spec"] localSpec cellSpec+          <> compareVerdicts localVerdicts cellVerdicts+      allEqual = [renderPath path | (path, Just left, Just right) <- comparisons, left == right]+      differences = [ParityDifference (renderPath path) left right | (path, left, right) <- comparisons, left /= right]+      intentionalDiffs = filter (isIntentional options . (.path)) differences+      unexpectedDiffs = filter (not . isIntentional options . (.path)) differences+  pure+    ParityReport+      { schema = "kenshou.parity-report/v1",+        localRunResultSha256 = digest localBytes,+        cellRunResultSha256 = digest cellBytes,+        equal = allEqual,+        intentional = intentionalDiffs,+        unexpected = unexpectedDiffs+      }++decode :: Text -> ByteString.ByteString -> IO Value+decode label bytes =+  case eitherDecodeStrict' bytes of+    Left problem -> ioError (userError (Text.unpack label <> ": " <> problem))+    Right value -> pure value++readJson :: Text -> FilePath -> IO Value+readJson label path = ByteString.readFile path >>= decode label++readVerdicts :: FilePath -> IO (Map.Map Text Value)+readVerdicts runDir = do+  let verdictDir = runDir </> "verdicts"+  exists <- doesDirectoryExist verdictDir+  if not exists+    then pure Map.empty+    else do+      names <- sort . filter ((== ".json") . takeExtension) <$> listDirectory verdictDir+      Map.fromList+        <$> traverse+          ( \name -> do+              value <- readJson (Text.pack name) (verdictDir </> name)+              pure (Text.pack name, projectVerdict value)+          )+          names++projectVerdict :: Value -> Value+projectVerdict (Object value) =+  Object (KeyMap.filterWithKey (\key _ -> Key.toText key `elem` ["status", "counts"]) value)+projectVerdict value = value++compareVerdicts :: Map.Map Text Value -> Map.Map Text Value -> [([Text], Maybe Value, Maybe Value)]+compareVerdicts localVerdicts cellVerdicts =+  concatMap compareName (Set.toAscList (Map.keysSet localVerdicts `Set.union` Map.keysSet cellVerdicts))+  where+    compareName name =+      case (Map.lookup name localVerdicts, Map.lookup name cellVerdicts) of+        (Just localValue, Just cellValue) -> compareJson ["verdicts", name] localValue cellValue+        (left, right) -> [(["verdicts", name], left, right)]++compareJson :: [Text] -> Value -> Value -> [([Text], Maybe Value, Maybe Value)]+compareJson prefix localValue cellValue =+  [ (path, Map.lookup path localLeaves, Map.lookup path cellLeaves)+  | path <- Set.toAscList (Map.keysSet localLeaves `Set.union` Map.keysSet cellLeaves)+  ]+  where+    localLeaves = Map.fromList (flatten prefix localValue)+    cellLeaves = Map.fromList (flatten prefix cellValue)++flatten :: [Text] -> Value -> [([Text], Value)]+flatten prefix (Object value)+  | not (KeyMap.null value) = concatMap (\(key, child) -> flatten (prefix <> [Key.toText key]) child) (KeyMap.toList value)+flatten prefix (Array value)+  | not (null value) = concatMap (\(index, child) -> flatten (prefix <> [Text.pack (show index)]) child) (zip [0 :: Int ..] (toList value))+flatten prefix value = [(prefix, value)]++renderPath :: [Text] -> Text+renderPath = Text.intercalate "."++isIntentional :: ParityOptions -> Text -> Bool+isIntentional options rendered =+  natural || volatile+  where+    path = Text.splitOn "." rendered+    starts prefix = prefix `isPrefixOf` path+    natural =+      or+        [ starts ["result", "runId"],+          starts ["result", "timings"],+          starts ["result", "compatibility"],+          starts ["result", "invocation"],+          starts ["result", "spec", "sha256"],+          starts ["result", "spec", "path"],+          starts ["result", "cohort", "arch"],+          starts ["result", "cohort", "os"],+          starts ["result", "cohort", "resolver"],+          starts ["result", "cohort", "planHash"],+          starts ["result", "cohort", "cabalVersion"],+          starts ["result", "fingerprint", "cell"],+          starts ["result", "fingerprint", "host"],+          starts ["result", "fingerprint", "runtime"],+          starts ["result", "fingerprint", "machineProfile"],+          starts ["result", "fingerprint", "placement"],+          starts ["result", "fingerprint", "postgres"],+          starts ["result", "fingerprint", "kenshou", "executableSha256"],+          starts ["result", "fingerprint", "kenshou", "revision"],+          starts ["spec", "runId"],+          starts ["spec", "cohortExpectation"],+          starts ["spec", "environment", "placement"],+          starts ["spec", "environment", "machineProfile"],+          starts ["spec", "environment", "postgres"],+          path == ["spec", "labels"],+          starts ["spec", "labels", "postgresPlacement"]+        ]+    volatileScope = starts ["result", "summaries", "verdicts"] || starts ["verdicts"]+    volatile = volatileScope && any (\selected -> selected == rendered || (selected <> ".") `Text.isPrefixOf` rendered) options.volatilePaths++digest :: ByteString.ByteString -> Text+digest bytes = "sha256:" <> TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash bytes))
+ src/Kenshou/Remote/Cell/Prepare.hs view
@@ -0,0 +1,376 @@+module Kenshou.Remote.Cell.Prepare+  ( Granularity (..),+    RejectReason (..),+    PrepareOptions (..),+    Prepared (..),+    Routed (..),+    PreparedRun (..),+    Slice (..),+    OtlpSink (..),+    SubmissionInputs (..),+    sliceRuns,+    slicePlan,+    submissionFor,+    prepareForCell,+    routePlan,+    routePlanWithoutPayload,+    cellMachineProfile,+  )+where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe, isNothing)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (Registry, lookupResource, lookupScenario)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), resolveDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), renderRunId)+import Kenshou.Core.Resource (CellBinding (..), ResourceName, ResourceProvider (..))+import Kenshou.Core.Resource.Types (legacyKafkaName)+import Kenshou.Core.RunSpec (CohortExpectation (..), ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Scenario (..))+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Remote.Cell.Docs (BrokerReset (..), CachePolicy (..), CellDescriptor (..), CellImages (..), CollectOptions (..), Limits (..), PgReset (..), Requirements (..), ResetBlock (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Legacy (legacyBrokerAvailable, legacyBrokerReset)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities (..), RoutingRule (..), capabilityKnown, descriptorDigest, matchingRules)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))+import Kenshou.Remote.Store (Bucket (..))++cellMachineProfile :: CellDescriptor -> Text+cellMachineProfile descriptor =+  "gcp/" <> descriptor.zone <> "/shape-" <> Text.take 16 (Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor.shape)))) <> "/images-" <> Text.take 16 (Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor.images)))) <> "/pg" <> Text.pack (show descriptor.postgresMajor)++data Granularity = GranularityAuto | GranularityPlan | GranularityRun+  deriving stock (Eq, Show)++data RejectReason+  = PlacementLocalOnly+  | UnknownScenario+  | PgVersionMismatch Int Int+  | DurabilityNotDurable+  | NeedsServerControl Text+  | ResourceUnavailable ResourceName+  | PayloadLabelUnknown Text+  | ConflictingPostgresSetting Text+  | CollidingExtraPostgresName Text+  | RunIdMismatch+  | InvalidTimeout+  | MissingCapability Text+  | UnsupportedDimensions Text+  deriving stock (Eq, Show)++data PrepareOptions = PrepareOptions+  { coerceDurable :: !Bool,+    ephemeralOnDriver :: !Bool,+    pgSettings :: ![(Text, Text)],+    cachePolicy :: !CachePolicy+  }+  deriving stock (Eq, Show)++data Prepared = Prepared+  { accepted :: ![PreparedRun],+    rejected :: ![(PlannedRun, RejectReason)],+    warnings :: ![Text]+  }+  deriving stock (Eq, Show)++data Routed = Routed+  { perCell :: !(Map.Map Text RunPlan),+    local :: !(Maybe RunPlan),+    unroutable :: ![(PlannedRun, RejectReason)],+    warnings :: ![Text]+  }+  deriving stock (Eq, Show)++data PreparedRun = PreparedRun+  { ordinal :: !Int,+    runId :: !RunId,+    spec :: !RunSpec,+    payloadLabel :: !Text,+    reset :: !ResetBlock,+    timeoutSeconds :: !Int+  }+  deriving stock (Eq, Show)++data Slice = Slice+  { index :: !Int,+    payloadLabel :: !Text,+    reset :: !ResetBlock,+    entries :: !(NonEmpty PreparedRun),+    wallClockSeconds :: !Int+  }+  deriving stock (Eq, Show)++data OtlpSink = NullSink | FileSink+  deriving stock (Eq, Show)++data SubmissionInputs = SubmissionInputs+  { cellRun :: !RunId,+    leaseId :: !RunId,+    sessionId :: !RunId,+    planId :: !RunId,+    otlpSink :: !OtlpSink,+    rtsOptions :: !(Maybe Text),+    memoryMaxBytes :: !Int64,+    outputMaxBytes :: !Int64,+    minAgentVersion :: !Text,+    requiredCapabilities :: !(Maybe [Text])+  }+  deriving stock (Eq, Show)++-- Preflight a plan before it is sliced. Nothing is dropped silently: every+-- rejected entry remains paired with its original run identity.+prepareForCell :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Map.Map Text PayloadDescriptor -> PrepareOptions -> RunPlan -> Prepared+prepareForCell registry descriptor cachedCapabilities routingRules payloads options plan =+  prepareWithPayload registry descriptor cachedCapabilities routingRules (Just payloads) options plan++prepareWithPayload :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Maybe (Map.Map Text PayloadDescriptor) -> PrepareOptions -> RunPlan -> Prepared+prepareWithPayload registry descriptor cachedCapabilities routingRules payloads options plan =+  Prepared+    [prepared | Right (prepared, _) <- results]+    [(entry, reason) | (entry, Left reason) <- zip plan.runs results]+    (concat [notices | Right (_, notices) <- results])+  where+    results = fmap prepare plan.runs+    activeCapabilities = case cachedCapabilities of+      Just cache | cache.cell == descriptor.name && cache.descriptorSha256 == descriptorDigest descriptor -> Just cache+      _ -> Nothing+    profile = cellMachineProfile descriptor++    prepare entry = do+      if maybe True (== entry.runId) entry.spec.runId then pure () else Left RunIdMismatch+      scenario <- maybe (Left UnknownScenario) Right (lookupScenario registry entry.spec.scenario)+      if scenario.placement == PlaceLocal then Left PlacementLocalOnly else pure ()+      let label = maybe "default" (\trial -> if maybe False (Map.member trial.arm) payloads then trial.arm else "default") entry.trial+      payload <- traverse (\available -> maybe (Left (PayloadLabelUnknown label)) Right (Map.lookup label available)) payloads+      let requirements = scenario.requires+          controlled = case requirements.postgres of+            Just requirement | requirement.needsServerControl -> Just "primary"+            _ -> fst <$> find (\(_, requirement) -> requirement.needsServerControl) requirements.extraPostgres+          driverLocal = maybe False (const (options.ephemeralOnDriver && entry.spec.scenario.kind `elem` [Correctness, Concurrency])) controlled+      case controlled of+        Just name | not driverLocal -> Left (NeedsServerControl name)+        _ -> pure ()+      bindings <- traverse cellBinding requirements.resources+      let rules = matchingRules routingRules entry.spec+          denied = find (\rule -> maybe False (\cache -> capabilityKnown cache rule.requires == Just False) activeCapabilities) rules+      case denied of+        Just rule -> Left (MissingCapability rule.requires)+        Nothing -> pure ()+      let originalSpec = entry.spec+          dimensions =+            if options.coerceDurable && not driverLocal && controlled == Nothing && (requirements.postgres /= Nothing || not (null requirements.extraPostgres))+              then ("pg.durability", "durable") : filter ((/= "pg.durability") . fst) originalSpec.dimensions+              else originalSpec.dimensions+      resolved <- either (Left . UnsupportedDimensions . Text.intercalate "; " . NonEmpty.toList) Right (resolveDimensions scenario.dimensions dimensions)+      let needsPostgres = requirements.postgres /= Nothing || not (null requirements.extraPostgres)+      if needsPostgres && not driverLocal && resolved.pgDurability /= Just PgDurable then Left DurabilityNotDurable else pure ()+      let requestedMajor = case resolved.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> descriptor.postgresMajor+      if needsPostgres && not driverLocal && requestedMajor /= descriptor.postgresMajor+        then Left (PgVersionMismatch requestedMajor descriptor.postgresMajor)+        else pure ()+      let extraVariables = fmap (variableName . fst) requirements.extraPostgres+      case find (\name -> length (filter (== name) extraVariables) > 1) extraVariables of+        Just name -> Left (CollidingExtraPostgresName name)+        Nothing -> pure ()+      settings <- if needsPostgres && not driverLocal then combinedSettings scenario originalSpec else Right Map.empty+      let prior = originalSpec.environment+          primary =+            if requirements.postgres == Nothing+              then Nothing+              else+                if driverLocal+                  then Just (case prior.postgres of Just ephemeral@(PostgresEphemeral _) -> ephemeral; _ -> PostgresEphemeral [])+                  else Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))+          extras =+            if driverLocal+              then Map.fromList [(name, case Map.lookup name prior.extraPostgres of Just ephemeral@(PostgresEphemeral _) -> ephemeral; _ -> PostgresEphemeral []) | (name, _) <- requirements.extraPostgres]+              else Map.fromList [(name, PostgresExternal (ConnFromEnv ("KENSHOU_CELL_PG_URL_" <> variableName name))) | (name, _) <- requirements.extraPostgres]+          env = EnvironmentSpec RunOnCell (Just profile) primary extras prior.resources prior.telemetry+          expectation = maybe originalSpec.cohortExpectation (\selected -> Just (CohortExpectation (Just selected.cohort) selected.cohortIdentity.identityPlanHash.unPlanHash)) payload+          labels = Map.insert "postgresPlacement" (if driverLocal then "driver-ephemeral" else "cell-server") originalSpec.labels+          spec = RunSpec (Just entry.runId) originalSpec.scenario originalSpec.scenarioRevision originalSpec.knobs dimensions originalSpec.seed originalSpec.phases originalSpec.timeoutSeconds env expectation originalSpec.comparison labels+          -- The released cell agent currently requires a PostgreSQL role reset+          -- even when the scenario itself has no database requirement.+          reset = legacyBrokerReset (ResetBlock options.cachePolicy (Just (PgReset descriptor.postgresMajor [] settings)) Nothing (Map.fromList [(name, value) | (name, binding) <- bindings, Just value <- [binding.reset]]))+          timeout = fromMaybe (max 60 (entry.estimateMinutes * 60)) originalSpec.timeoutSeconds+          notices =+            ["coerced pg.durability=durable for " <> renderRunId entry.runId | dimensions /= originalSpec.dimensions]+              <> ["capability " <> rule.requires <> " is unprobed for " <> renderRunId entry.runId <> "; run kenshou cell probe" | rule <- rules, maybe True (\cache -> capabilityKnown cache rule.requires == Nothing) activeCapabilities]+              <> ["payload not selected for " <> renderRunId entry.runId <> "; cohort expectation will be set at submission" | isNothing payloads]+      if timeout <= 0 then Left InvalidTimeout else Right (PreparedRun entry.ordinal entry.runId spec label reset timeout, notices)+    -- A required resource is available on the cell when its provider can bind it+    -- there and the cell reports the binding's capability; a cell broker image+    -- still counts for the resource the legacy broker fields describe.+    cellBinding name = do+      binding <- maybe (Left (ResourceUnavailable name)) Right (lookupResource registry name >>= (.cell))+      let reported = maybe Nothing (\cache -> capabilityKnown cache binding.capability) activeCapabilities+          available = reported == Just True || (name == legacyKafkaName && reported /= Just False && legacyBrokerAvailable descriptor)+      if available then Right (name, binding) else Left (ResourceUnavailable name)++    combinedSettings scenario spec = foldl add (Right Map.empty) sources+      where+        -- The cell image owns preloaded libraries. The external PostgreSQL+        -- adapter verifies required libraries against the running server.+        required = filter ((/= "shared_preload_libraries") . fst) (maybe [] (.settings) scenario.requires.postgres <> concatMap ((.settings) . snd) scenario.requires.extraPostgres)+        primary = case spec.environment.postgres of Just (PostgresEphemeral values) -> values; _ -> []+        extras = concatMap (\value -> case value of PostgresEphemeral values -> values; _ -> []) (Map.elems spec.environment.extraPostgres)+        sources = [("fsync", "on"), ("synchronous_commit", "on"), ("full_page_writes", "on")] <> required <> primary <> extras <> options.pgSettings+        add previous (name, value) = do+          settings <- previous+          case Map.lookup name settings of+            Nothing -> Right (Map.insert name value settings)+            Just existing | existing == value -> Right settings+            _ -> Left (ConflictingPostgresSetting name)++variableName :: Text -> Text+variableName = Text.map (\character -> if character == '-' then '_' else character) . Text.toUpper++-- Prefer the caller's cell order. A rejected run is either retained for local+-- execution or named as unroutable; its original identity never disappears.+routePlan :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Map.Map Text PayloadDescriptor -> PrepareOptions -> RunPlan -> Routed+routePlan registry cells routingRules payloads options plan = routePlanWith registry cells routingRules (Just payloads) options plan++routePlanWithoutPayload :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> PrepareOptions -> RunPlan -> Routed+routePlanWithoutPayload registry cells routingRules options plan = routePlanWith registry cells routingRules Nothing options plan++routePlanWith :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Maybe (Map.Map Text PayloadDescriptor) -> PrepareOptions -> RunPlan -> Routed+routePlanWith registry cells routingRules payloads options plan =+  Routed (Map.map makePlan assigned) (if null localRuns then Nothing else Just (makePlan localRuns)) rejectedRuns notices+  where+    choices = fmap routeEntry plan.runs+    assigned = foldl addCell Map.empty [(name, selected) | Right (name, selected, _) <- choices]+    localRuns = [entry | Left (entry, reason) <- choices, localReason reason]+    rejectedRuns = [(entry, reason) | Left (entry, reason) <- choices, not (localReason reason)]+    notices = concat [messages | Right (_, _, messages) <- choices]++    routeEntry entry = choose (NonEmpty.toList cells) Nothing+      where+        choose [] firstFailure = Left (entry, fromMaybe UnknownScenario firstFailure)+        choose ((descriptor, capabilities) : rest) firstFailure =+          let one = RunPlan plan.planId plan.createdAt plan.context plan.policy [entry] [] entry.estimateMinutes+              prepared = prepareWithPayload registry descriptor capabilities routingRules payloads options one+           in case prepared.accepted of+                [accepted] -> Right (descriptor.name, preparedEntry accepted, prepared.warnings)+                _ -> case prepared.rejected of+                  [(_, reason)] -> choose rest (Just (fromMaybe reason firstFailure))+                  _ -> choose rest (Just (fromMaybe UnknownScenario firstFailure))+        preparedEntry accepted = PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial accepted.spec++    addCell grouped (name, entry) = Map.insertWith (flip (<>)) name [entry] grouped+    makePlan entries = RunPlan plan.planId plan.createdAt plan.context plan.policy entries plan.skipped (sum (fmap (.estimateMinutes) entries))+    localReason PlacementLocalOnly = True+    localReason (NeedsServerControl _) = True+    localReason _ = False++sliceRuns :: Granularity -> [PreparedRun] -> Either Text [Slice]+sliceRuns granularity prepared = do+  if all valid prepared && unique (fmap (.runId) prepared) && unique (fmap (.ordinal) prepared)+    then pure ()+    else Left "prepared runs have an empty payload, invalid timeout, duplicate identity or invalid ordinal"+  groups <- case granularity of+    GranularityRun -> pure (fmap (:| []) prepared)+    GranularityAuto -> pure (autoGroups prepared)+    GranularityPlan -> case NonEmpty.nonEmpty prepared of+      Nothing -> pure []+      Just group+        | all (compatible (NonEmpty.head group)) (NonEmpty.tail group) -> pure [group]+        | otherwise -> Left "plan granularity requires one payload and reset for every run"+  traverse makeSlice (zip [0 ..] groups)+  where+    valid run = run.ordinal >= 0 && run.timeoutSeconds > 0 && not (Text.null run.payloadLabel) && maybe True (== run.runId) run.spec.runId+    makeSlice (index, group) =+      let runBudget :: PreparedRun -> Integer+          runBudget run = toInteger run.timeoutSeconds + if run.spec.scenario.kind == Soak then 600 else 0+          -- Soaks drain delayed work and compute resource verdicts after load ends.+          -- Finding 49 records a full inbox run exhausting the old shared margin.+          total = 300 + sum (fmap runBudget (NonEmpty.toList group))+       in if total > toInteger (maxBound :: Int)+            then Left "cell slice wall-clock limit is too large"+            else Right (Slice index (NonEmpty.head group).payloadLabel (NonEmpty.head group).reset group (fromInteger total))++slicePlan :: RunPlan -> Slice -> Either Text RunPlan+slicePlan plan slice = do+  selected <- traverse select (NonEmpty.toList slice.entries)+  pure (RunPlan plan.planId plan.createdAt plan.context plan.policy selected plan.skipped (sum (fmap (.estimateMinutes) selected)))+  where+    select prepared = case find ((== prepared.runId) . (.runId)) plan.runs of+      Nothing -> Left "slice contains a run absent from its plan"+      Just original+        | original.ordinal /= prepared.ordinal -> Left "slice ordinal differs from its plan"+        | original.spec.scenario /= prepared.spec.scenario -> Left "slice scenario differs from its plan"+        | otherwise -> Right (PlannedRun original.ordinal original.runId original.estimateMinutes original.reasons original.trial prepared.spec)++submissionFor :: CellRef -> SubmissionInputs -> PayloadDescriptor -> Slice -> WorkObject -> Either Text Submission+submissionFor ref inputs descriptor slice work = do+  case eitherDecode (encode descriptor) :: Either String PayloadDescriptor of+    Left failure -> Left ("invalid payload descriptor: " <> Text.pack failure)+    Right _ -> pure ()+  case eitherDecode (encode work) :: Either String WorkObject of+    Left failure -> Left ("invalid work object: " <> Text.pack failure)+    Right _ -> pure ()+  if work.mediaType /= "application/json" || work.bytes <= 0+    then Left "cell work must be a nonempty JSON run plan"+    else pure ()+  let bundle = descriptor.cell.bundle+      expectedPrefix = "gs://" <> ref.controlBucket.unBucket <> "/payloads/sha256/"+  if not (expectedPrefix `Text.isPrefixOf` bundle.uri)+    then Left "payload bundle is outside the cell control bucket"+    else pure ()+  if inputs.memoryMaxBytes <= 0 || inputs.outputMaxBytes <= 0 || Text.null inputs.minAgentVersion || maybe False Text.null inputs.rtsOptions+    then Left "invalid cell submission limits or options"+    else pure ()+  let env =+        Map.fromList+          ( [ ("KENSHOU_PAYLOAD_BUNDLE_SHA256", bundle.sha256),+              ("KENSHOU_PAYLOAD_STORE_PATH", descriptor.cell.storePath),+              ("KENSHOU_PAYLOAD_NAR_HASH", descriptor.cell.narHash),+              ("KENSHOU_PAYLOAD_COHORT", descriptor.cohort),+              ("KENSHOU_OTLP_SINK", case inputs.otlpSink of NullSink -> "null"; FileSink -> "file")+            ]+              <> maybe [] (\options -> [("GHCRTS", options)]) inputs.rtsOptions+          )+      labels =+        Map.fromList+          [ ("session", renderRunId inputs.sessionId),+            ("plan", renderRunId inputs.planId),+            ("slice", Text.pack (show slice.index)),+            ("payload", slice.payloadLabel)+          ]+      profileCollection = maybe False (any profileFlag . Text.words) inputs.rtsOptions+      traceCollection = inputs.otlpSink == FileSink+      collection = if profileCollection || traceCollection then Just (CollectOptions traceCollection profileCollection) else Nothing+      submission = Submission inputs.cellRun inputs.leaseId descriptor.cell work env slice.reset (Limits (fromIntegral slice.wallClockSeconds) inputs.memoryMaxBytes inputs.outputMaxBytes) (Requirements inputs.minAgentVersion inputs.requiredCapabilities) collection labels+  case eitherDecode (encode submission) :: Either String Submission of+    Left failure -> Left ("invalid cell submission: " <> Text.pack failure)+    Right _ -> Right submission+  where+    profileFlag flag = any (`Text.isPrefixOf` flag) ["-p", "-h", "-l"]++autoGroups :: [PreparedRun] -> [NonEmpty PreparedRun]+autoGroups [] = []+autoGroups (first : rest)+  | isolated first = (first :| []) : autoGroups rest+  | otherwise =+      let (same, later) = span (\candidate -> not (isolated candidate) && compatible first candidate) rest+       in (first :| same) : autoGroups later++compatible :: PreparedRun -> PreparedRun -> Bool+compatible first second = first.payloadLabel == second.payloadLabel && first.reset == second.reset++isolated :: PreparedRun -> Bool+isolated run = run.spec.scenario.kind `elem` [Benchmark, Soak]++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Cell/RouteJson.hs view
@@ -0,0 +1,123 @@+module Kenshou.Remote.Cell.RouteJson (RouteDocuments (..), routeWorkJson) where++import Data.Aeson (FromJSON, Value (..), object, (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time.Clock.POSIX (posixSecondsToUTCTime)+import Kenshou.Core.Bundle (Registry)+import Kenshou.Core.Id (RunId, mkSeed)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Docs (CellDescriptor)+import Kenshou.Remote.Cell.Prepare (PrepareOptions, Routed (..), routePlan, routePlanWithoutPayload)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities, RoutingRule)+import Kenshou.Remote.Payload (PayloadDescriptor)++data RouteDocuments = RouteDocuments+  { perCell :: !(Map Text Value),+    local :: !(Maybe Value),+    report :: !Value,+    routeComplete :: !Bool+  }+  deriving stock (Eq, Show)++-- Route the public JSON plan without decoding and re-encoding its provenance+-- types. The typed plan below carries only the fields used by preparation; the+-- output copies each original entry and replaces its prepared spec.+routeWorkJson :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Maybe (Map Text PayloadDescriptor) -> PrepareOptions -> Value -> Either Text RouteDocuments+routeWorkJson registry cells rules payloads options work = do+  fields <- case work of Object value -> Right value; _ -> Left "run plan is not a JSON object"+  case KeyMap.lookup "schema" fields of+    Just (String "kenshou.run-plan/v1") -> pure ()+    _ -> Left "run plan has an unsupported schema"+  planId <- required "planId" fields+  rawRuns <- case KeyMap.lookup "runs" fields of+    Just (Array entries) -> Right (toList entries)+    _ -> Left "run plan has no runs array"+  entries <- traverse parseEntry rawRuns+  let identifiers = fmap (\(entry, _) -> entry.runId) entries+      ordinals = fmap (\(entry, _) -> entry.ordinal) entries+  if unique identifiers && unique ordinals then pure () else Left "run plan contains duplicate run IDs or ordinals"+  selector <- parseSelector "**"+  seed <- mkSeed 1+  let placeholder = Reason (Change (ComponentRef (ComponentId "cell-route") Nothing) Named "internal routing placeholder") [] selector 0+      planned = fmap (toPlanned placeholder . fst) entries+      context = PlanContext Nothing "" "" "" (PlanInputs Null) [] []+      typed = RunPlan planId (posixSecondsToUTCTime 0) context (defaultPlanPolicy seed) planned [] (sum (fmap (.estimateMinutes) planned))+      routed = case payloads of+        Just selected -> routePlan registry cells rules selected options typed+        Nothing -> routePlanWithoutPayload registry cells rules options typed+      originals = Map.fromList [(entry.runId, value) | (entry, value) <- entries]+  cellDocs <- traverse (renderPlan fields originals True) routed.perCell+  localDoc <- traverse (renderPlan fields originals False) routed.local+  let routeReport =+        object+          [ "schema" .= ("kenshou.cell-route/v1" :: Text),+            "planId" .= planId,+            "perCell" .= Map.map (fmap (.runId) . (.runs)) routed.perCell,+            "localRunIds" .= maybe [] (fmap (.runId) . (.runs)) routed.local,+            "unroutable" .= [object ["runId" .= entry.runId, "reason" .= Text.pack (show reason)] | (entry, reason) <- routed.unroutable],+            "warnings" .= routed.warnings+          ]+  pure (RouteDocuments cellDocs localDoc routeReport (null routed.unroutable))+  where+    required :: (FromJSON document) => Text -> Aeson.Object -> Either Text document+    required name fields = case KeyMap.lookup (Key.fromText name) fields of+      Nothing -> Left ("run plan is missing " <> name)+      Just value -> decode name value++    decode :: (FromJSON document) => Text -> Value -> Either Text document+    decode label value = case Aeson.fromJSON value of+      Aeson.Error failure -> Left (label <> ": " <> Text.pack failure)+      Aeson.Success result -> Right result++    parseEntry value = case value of+      Object fields -> do+        ordinal <- required "ordinal" fields+        runId <- required "runId" fields+        estimate <- required "estimateMinutes" fields+        spec <- required "spec" fields+        trial <- case KeyMap.lookup "trial" fields of+          Nothing -> Right Nothing+          Just Null -> Right Nothing+          Just (Object trialFields) -> Just <$> (TrialInfo <$> required "group" trialFields <*> required "arm" trialFields <*> required "index" trialFields <*> required "of" trialFields)+          _ -> Left "planned run trial is not an object"+        if ordinal < 0 || estimate <= 0 || maybe False (/= runId) spec.runId+          then Left "planned run has an invalid ordinal, estimate or spec run ID"+          else Right (RouteEntry ordinal runId estimate trial spec, value)+      _ -> Left "planned run is not an object"++    toPlanned placeholder entry = PlannedRun entry.ordinal entry.runId entry.estimateMinutes (placeholder :| []) entry.trial entry.spec++    renderPlan fields originals prepared plan = do+      rows <- traverse (renderEntry originals prepared) plan.runs+      let changed = KeyMap.insert "runs" (Aeson.toJSON rows) (KeyMap.insert "estimateMinutes" (Aeson.toJSON plan.estimateMinutes) fields)+      pure (Object changed)++    renderEntry originals prepared entry = case Map.lookup entry.runId originals of+      Just (Object original) ->+        Right (Object (if prepared then KeyMap.insert "spec" (Aeson.toJSON entry.spec) original else original))+      _ -> Left "routed run is absent from the input plan"++data RouteEntry = RouteEntry+  { ordinal :: !Int,+    runId :: !RunId,+    estimateMinutes :: !Int,+    trial :: !(Maybe TrialInfo),+    spec :: !RunSpec+  }++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Cell/RouteRules.hs view
@@ -0,0 +1,101 @@+module Kenshou.Remote.Cell.RouteRules+  ( CellCapabilities (..),+    RuleCondition (..),+    RoutingRule (..),+    defaultRoutingRules,+    matchingRules,+    capabilityKnown,+    descriptorDigest,+    loadCellCapabilities,+  )+where++import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), eitherDecodeFileStrict', encode, object, withObject, (.:), (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Contribution (RoutingRule (..), RuleCondition (..))+import Kenshou.Core.Id (RunId)+import Kenshou.Core.Knob (RawKnob (..), mkKnobName)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Plan.Selector (Selector, matches, parseSelector)+import Kenshou.Remote.Cell.Docs (CellDescriptor (..))+import System.Directory (doesFileExist)+import System.FilePath ((</>))++data CellCapabilities = CellCapabilities+  { cell :: !Text,+    descriptorSha256 :: !Text,+    probedBy :: !RunId,+    probedAt :: !UTCTime,+    capabilities :: !(Map Text Value)+  }+  deriving stock (Eq, Show)++-- | Superseded by 'Kenshou.Core.Bundle.registryRoutingRules': suites contribute+-- their routing rules on their bundles. Kept, empty, so existing imports compile.+defaultRoutingRules :: [RoutingRule]+defaultRoutingRules = []++instance FromJSON CellCapabilities where+  parseJSON = withObject "cell capabilities" \fields -> do+    schema <- fields .: "schema"+    if schema == ("kenshou.cell-capabilities/v1" :: Text) then pure () else fail "unsupported cell capabilities schema"+    document <- CellCapabilities <$> fields .: "cell" <*> fields .: "descriptorSha256" <*> fields .: "probedBy" <*> fields .: "probedAt" <*> fields .: "capabilities"+    if Text.null document.cell || Text.length document.descriptorSha256 /= 64 || not (Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) document.descriptorSha256)+      then fail "invalid cell capabilities identity"+      else pure document++instance ToJSON CellCapabilities where+  toJSON cache =+    object+      [ "schema" .= ("kenshou.cell-capabilities/v1" :: Text),+        "cell" .= cache.cell,+        "descriptorSha256" .= cache.descriptorSha256,+        "probedBy" .= cache.probedBy,+        "probedAt" .= cache.probedAt,+        "capabilities" .= cache.capabilities+      ]++matchingRules :: [RoutingRule] -> RunSpec -> [RoutingRule]+matchingRules rules spec = filter (matchesRule spec) rules+  where+    matchesRule run rule = case rule.condition of+      ScenarioMatches selector -> matches selector run.scenario+      KnobEquals name expected -> case mkKnobName name of+        Left _ -> False+        Right key -> case lookup key run.knobs of+          Just (RawText actual) -> actual == expected+          Just (RawJson (String actual)) -> actual == expected+          _ -> False++capabilityKnown :: CellCapabilities -> Text -> Maybe Bool+capabilityKnown cache name = case Map.lookup name cache.capabilities of+  Just (Bool available) -> Just available+  _ -> Nothing++-- The cache uses the canonical codec form, so object member order in the+-- downloaded descriptor cannot invalidate an otherwise identical cell shape.+descriptorDigest :: CellDescriptor -> Text+descriptorDigest = Text.drop 7 . sha256Hex . LazyByteString.toStrict . encode++-- A cache belongs to one descriptor revision. Missing and stale files leave+-- routing in its explicit unprobed state; malformed files must be repaired.+loadCellCapabilities :: FilePath -> CellDescriptor -> IO (Either Text (Maybe CellCapabilities))+loadCellCapabilities directory descriptor = do+  let path = directory </> Text.unpack descriptor.name <> ".capabilities.json"+  present <- doesFileExist path+  if not present+    then pure (Right Nothing)+    else do+      decoded <- eitherDecodeFileStrict' path+      pure case decoded of+        Left problem -> Left ("invalid capability cache " <> Text.pack path <> ": " <> Text.pack problem)+        Right cache+          | cache.cell == descriptor.name && cache.descriptorSha256 == descriptorDigest descriptor -> Right (Just cache)+          | otherwise -> Right Nothing
+ src/Kenshou/Remote/Cell/Session.hs view
@@ -0,0 +1,75 @@+module Kenshou.Remote.Cell.Session+  ( SessionError (..),+    SessionTransition (..),+    runSubmission,+    runSubmissionWithTransitions,+  )+where++import Control.Concurrent (threadDelay)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty)+import Data.Text (Text)+import Kenshou.Remote.Cell.Docs (CellStatus, Rejected (..), Submission (..))+import Kenshou.Remote.Cell.Fetch (FetchError, VerifyProblem, fetchCellRun)+import Kenshou.Remote.Cell.Index (CellRunIndex, deriveCellRunIndex, writeCellRunIndex)+import Kenshou.Remote.Cell.Lease (CellRef, LeaseHandle, withHeartbeat)+import Kenshou.Remote.Cell.Submit (PublishOutcome (..), publishSubmission)+import Kenshou.Remote.Cell.Watch (WatchEvent, WatchSnapshot (..), WatchTerminal (..), newWatchCursor, pollCellRun)+import Kenshou.Remote.Store (Bucket, ObjectStore)++data SessionError+  = LeaseLost+  | PublicationFailed !PublishOutcome+  | SubmissionRejected !Text+  | FetchFailed !FetchError+  | VerificationFailed !(NonEmpty VerifyProblem)+  deriving stock (Eq, Show)++data SessionTransition+  = SubmissionPublished+  | SubmissionSealed !CellStatus+  | SubmissionRejectedByCell !Rejected+  | ResultsFetched !FilePath+  | ResultsVerified !CellRunIndex+  deriving stock (Eq, Show)++-- One already prepared slice under an existing held lease. Its caller owns the+-- session document and advances its state after each returned transition.+runSubmission :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> Submission -> LazyByteString.ByteString -> FilePath -> (WatchEvent -> IO ()) -> IO (Either SessionError CellRunIndex)+runSubmission store ref resultsBucket handle submission workBytes outDir emit =+  runSubmissionWithTransitions store ref resultsBucket handle submission workBytes outDir emit (const (pure ()))++runSubmissionWithTransitions :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> Submission -> LazyByteString.ByteString -> FilePath -> (WatchEvent -> IO ()) -> (SessionTransition -> IO ()) -> IO (Either SessionError CellRunIndex)+runSubmissionWithTransitions store ref resultsBucket handle submission workBytes outDir emit checkpoint =+  withHeartbeat store ref handle \stillHeld -> do+    published <- publishSubmission store ref handle submission workBytes+    case published of+      Submitted -> checkpoint SubmissionPublished >> waitForSeal stillHeld newWatchCursor+      LostLease -> pure (Left LeaseLost)+      other -> pure (Left (PublicationFailed other))+  where+    waitForSeal stillHeld cursor = do+      held <- stillHeld+      if not held+        then pure (Left LeaseLost)+        else do+          snapshot <- pollCellRun store ref submission.runId cursor+          mapM_ emit snapshot.events+          case snapshot.terminal of+            Just (RunRejected rejected) -> checkpoint (SubmissionRejectedByCell rejected) >> pure (Left (SubmissionRejected rejected.reason))+            Just (RunSealed status) -> do+              checkpoint (SubmissionSealed status)+              fetched <- fetchCellRun store resultsBucket submission.runId outDir+              case fetched of+                Left failure -> pure (Left (FetchFailed failure))+                Right tree -> do+                  checkpoint (ResultsFetched tree)+                  indexed <- deriveCellRunIndex resultsBucket (Just status) tree+                  case indexed of+                    Left problems -> pure (Left (VerificationFailed problems))+                    Right index -> do+                      _ <- writeCellRunIndex tree index+                      checkpoint (ResultsVerified index)+                      pure (Right index)+            Nothing -> threadDelay 2000000 >> waitForSeal stillHeld snapshot.cursor
+ src/Kenshou/Remote/Cell/Session/Build.hs view
@@ -0,0 +1,118 @@+module Kenshou.Remote.Cell.Session.Build+  ( BuildOptions (..),+    BuiltSession (..),+    buildSession,+  )+where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Bundle (Registry)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Remote.Cell.Docs (CellBuckets (..), CellDescriptor (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Prepare (Granularity, OtlpSink, PrepareOptions, Prepared (..), PreparedRun (..), Slice (..), SubmissionInputs (..), prepareForCell, sliceRuns, submissionFor)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities, RoutingRule)+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), RejectedRun (..), SessionJournal (..), SliceJournal (..), SliceState (..))+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan, renderPreparedWork)+import Kenshou.Remote.Payload (PayloadDescriptor)+import Kenshou.Remote.Store (Bucket (..))++data BuildOptions = BuildOptions+  { preparation :: !PrepareOptions,+    granularity :: !Granularity,+    skipIncompatible :: !Bool,+    otlpSink :: !OtlpSink,+    rtsOptions :: !(Maybe Text),+    memoryMaxBytes :: !Int64,+    outputMaxBytes :: !Int64,+    minAgentVersion :: !Text+  }+  deriving stock (Eq, Show)++data BuiltSession = BuiltSession+  { journal :: !SessionJournal,+    workFiles :: ![(FilePath, LazyByteString.ByteString)]+  }+  deriving stock (Eq, Show)++-- Plan bytes are kept for their original digest, while each work file is+-- rendered from the original JSON so that provenance survives slicing.+buildSession :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Map Text PayloadDescriptor -> BuildOptions -> Text -> RunId -> LazyByteString.ByteString -> IO (Either Text BuiltSession)+buildSession registry descriptor capabilities rules payloads options storeLabel leaseId planBytes =+  case eitherDecode planBytes of+    Left failure -> pure (Left ("run plan: " <> Text.pack failure))+    Right document -> case decodeWorkPlan document of+      Left problem -> pure (Left problem)+      Right plan -> do+        let prepared = prepareForCell registry descriptor capabilities rules payloads options.preparation plan+            rejected = [RejectedRun entry.runId (Text.pack (show reason)) | (entry, reason) <- prepared.rejected]+        if not options.skipIncompatible && not (null rejected)+          then pure (Left ("cell cannot run " <> Text.intercalate ", " [renderRunId run.runId <> ": " <> run.reason | run <- rejected]))+          else case sliceRuns options.granularity prepared.accepted of+            Left problem -> pure (Left problem)+            Right [] -> pure (Left "no compatible runs remain in the plan")+            Right slices -> case traverse (buildWork document payloads) slices of+              Left problem -> pure (Left problem)+              Right work -> do+                sessionId <- newRunId+                cellRuns <- traverse (const newRunId) slices+                now <- getCurrentTime+                let ref = CellRef descriptor.name (Bucket descriptor.buckets.control)+                    planHash = (workObjectFor "application/json" planBytes).sha256+                    results = Bucket descriptor.buckets.results+                    mkSlice slice (payload, path, bytes) cellRun = do+                      let inputs =+                            SubmissionInputs+                              cellRun+                              leaseId+                              sessionId+                              plan.planId+                              options.otlpSink+                              options.rtsOptions+                              options.memoryMaxBytes+                              options.outputMaxBytes+                              options.minAgentVersion+                              Nothing+                      submission <- submissionFor ref inputs payload slice (workObjectFor "application/json" bytes)+                      pure+                        ( SliceJournal slice.index cellRun (fmap (.ordinal) (NonEmpty.toList slice.entries)) (fmap (.runId) (NonEmpty.toList slice.entries)) slice.reset submission path SlicePlanned Nothing Nothing Nothing Nothing Nothing,+                          (path, bytes)+                        )+                pure do+                  if options.memoryMaxBytes <= 0 || options.outputMaxBytes <= 0+                    then Left "cell memory and output limits must be positive"+                    else pure ()+                  assembled <- sequence (zipWith3 mkSlice slices work cellRuns)+                  let journal =+                        SessionJournal+                          sessionId+                          descriptor.name+                          storeLabel+                          (Just descriptor.project)+                          ref.controlBucket.unBucket+                          results.unBucket+                          leaseId+                          Held+                          payloads+                          planHash+                          rejected+                          (fmap fst assembled)+                          now+                          now+                  pure (BuiltSession journal (fmap snd assembled))+  where+    buildWork document available slice = do+      payload <- maybe (Left ("slice payload label is absent: " <> slice.payloadLabel)) Right (Map.lookup slice.payloadLabel available)+      rendered <- renderPreparedWork document (NonEmpty.toList slice.entries)+      let path = "slice-" <> show slice.index <> "/work.json"+      pure (payload, path, encode rendered)
+ src/Kenshou/Remote/Cell/Session/Journal.hs view
@@ -0,0 +1,232 @@+module Kenshou.Remote.Cell.Session.Journal+  ( LeaseMode (..),+    SliceState (..),+    RejectedRun (..),+    SliceJournal (..),+    SessionJournal (..),+    applyTransition,+    readSessionJournal,+    writeSessionJournal,+  )+where++import Control.Exception (bracket)+import Control.Monad (unless)+import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecode, encode, object, withObject, withText, (.:), (.:?), (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (find, sort)+import Data.Map.Strict (Map)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Remote.Cell.Docs (CellOutcome (..), CellStatus (..), Rejected (..), ResetBlock, Submission (..))+import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..), RunLink (..))+import Kenshou.Remote.Cell.Session (SessionTransition (..))+import Kenshou.Remote.Payload (PayloadDescriptor)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.FilePath (isAbsolute, splitDirectories, takeDirectory)+import System.IO (hClose, openBinaryTempFile)++data LeaseMode = Held | Detached+  deriving stock (Eq, Show)++data SliceState = SlicePlanned | SliceSubmitted | SliceSealed | SliceRejected | SliceFetched | SliceVerified+  deriving stock (Eq, Show)++data RejectedRun = RejectedRun+  { runId :: !RunId,+    reason :: !Text+  }+  deriving stock (Eq, Show)++data SliceJournal = SliceJournal+  { index :: !Int,+    cellRun :: !RunId,+    ordinals :: ![Int],+    runIds :: ![RunId],+    reset :: !ResetBlock,+    submission :: !Submission,+    workPath :: !FilePath,+    state :: !SliceState,+    cellOutcome :: !(Maybe CellOutcome),+    entryExitCode :: !(Maybe Int),+    manifestSha256 :: !(Maybe Text),+    rejectionReason :: !(Maybe Text),+    fetchedPath :: !(Maybe FilePath)+  }+  deriving stock (Eq, Show)++data SessionJournal = SessionJournal+  { sessionId :: !RunId,+    cell :: !Text,+    store :: !Text,+    project :: !(Maybe Text),+    controlBucket :: !Text,+    resultsBucket :: !Text,+    leaseId :: !RunId,+    leaseMode :: !LeaseMode,+    payloads :: !(Map Text PayloadDescriptor),+    planSha256 :: !Text,+    rejectedRuns :: ![RejectedRun],+    slices :: ![SliceJournal],+    createdAt :: !UTCTime,+    updatedAt :: !UTCTime+  }+  deriving stock (Eq, Show)++instance ToJSON LeaseMode where+  toJSON Held = toJSON ("held" :: Text)+  toJSON Detached = toJSON ("detached" :: Text)++instance FromJSON LeaseMode where+  parseJSON = withText "cell lease mode" \case+    "held" -> pure Held+    "detached" -> pure Detached+    _ -> fail "unknown cell lease mode"++instance ToJSON SliceState where+  toJSON SlicePlanned = toJSON ("planned" :: Text)+  toJSON SliceSubmitted = toJSON ("submitted" :: Text)+  toJSON SliceSealed = toJSON ("sealed" :: Text)+  toJSON SliceRejected = toJSON ("rejected" :: Text)+  toJSON SliceFetched = toJSON ("fetched" :: Text)+  toJSON SliceVerified = toJSON ("verified" :: Text)++instance FromJSON SliceState where+  parseJSON = withText "cell slice state" \case+    "planned" -> pure SlicePlanned+    "submitted" -> pure SliceSubmitted+    "sealed" -> pure SliceSealed+    "rejected" -> pure SliceRejected+    "fetched" -> pure SliceFetched+    "verified" -> pure SliceVerified+    _ -> fail "unknown cell slice state"++instance ToJSON RejectedRun where+  toJSON run = object ["runId" .= run.runId, "reason" .= run.reason]++instance FromJSON RejectedRun where+  parseJSON = withObject "rejected planned run" \value -> RejectedRun <$> value .: "runId" <*> value .: "reason"++instance ToJSON SliceJournal where+  toJSON slice =+    object+      [ "index" .= slice.index,+        "cellRun" .= slice.cellRun,+        "ordinals" .= slice.ordinals,+        "runIds" .= slice.runIds,+        "reset" .= slice.reset,+        "submission" .= slice.submission,+        "workPath" .= slice.workPath,+        "state" .= slice.state,+        "cellOutcome" .= slice.cellOutcome,+        "entryExitCode" .= slice.entryExitCode,+        "manifestSha256" .= slice.manifestSha256,+        "rejectionReason" .= slice.rejectionReason,+        "fetchedPath" .= slice.fetchedPath+      ]++instance FromJSON SliceJournal where+  parseJSON = withObject "cell slice journal" \value -> do+    slice <- SliceJournal <$> value .: "index" <*> value .: "cellRun" <*> value .: "ordinals" <*> value .: "runIds" <*> value .: "reset" <*> value .: "submission" <*> value .: "workPath" <*> value .: "state" <*> value .:? "cellOutcome" <*> value .:? "entryExitCode" <*> value .:? "manifestSha256" <*> value .:? "rejectionReason" <*> value .:? "fetchedPath"+    unless (slice.index >= 0 && slice.cellRun == slice.submission.runId && slice.reset == slice.submission.reset && not (null slice.ordinals) && not (null slice.runIds) && unique slice.ordinals && unique slice.runIds && validWorkPath slice.workPath && validSliceState slice) (fail "inconsistent cell slice journal")+    pure slice++instance ToJSON SessionJournal where+  toJSON journal =+    object+      [ "schema" .= ("kenshou.cell-session/v1" :: Text),+        "sessionId" .= journal.sessionId,+        "cell" .= journal.cell,+        "store" .= journal.store,+        "project" .= journal.project,+        "controlBucket" .= journal.controlBucket,+        "resultsBucket" .= journal.resultsBucket,+        "leaseId" .= journal.leaseId,+        "leaseMode" .= journal.leaseMode,+        "payloads" .= journal.payloads,+        "planSha256" .= journal.planSha256,+        "rejectedRuns" .= journal.rejectedRuns,+        "slices" .= journal.slices,+        "createdAt" .= journal.createdAt,+        "updatedAt" .= journal.updatedAt+      ]++instance FromJSON SessionJournal where+  parseJSON = withObject "cell session journal" \value -> do+    schema <- value .: "schema"+    unless (schema == ("kenshou.cell-session/v1" :: Text)) (fail "unsupported cell session journal")+    journal <- SessionJournal <$> value .: "sessionId" <*> value .: "cell" <*> value .: "store" <*> value .:? "project" <*> value .: "controlBucket" <*> value .: "resultsBucket" <*> value .: "leaseId" <*> value .: "leaseMode" <*> value .: "payloads" <*> value .: "planSha256" <*> value .: "rejectedRuns" <*> value .: "slices" <*> value .: "createdAt" <*> value .: "updatedAt"+    unless (unique (fmap (.index) journal.slices) && unique (fmap (.cellRun) journal.slices) && validDigest journal.planSha256 && journal.createdAt <= journal.updatedAt) (fail "inconsistent cell session journal")+    pure journal++applyTransition :: UTCTime -> RunId -> SessionTransition -> SessionJournal -> Either Text SessionJournal+applyTransition now identifier transition journal = do+  slice <- maybe (Left "cell run is not in session") Right (find ((== identifier) . (.cellRun)) journal.slices)+  changed <- advance slice transition+  pure journal {slices = fmap (\current -> if current.cellRun == identifier then changed else current) journal.slices, updatedAt = now}+  where+    advance slice SubmissionPublished = require SlicePlanned slice $ slice {state = SliceSubmitted}+    advance slice (SubmissionRejectedByCell rejected)+      | rejected.runId /= identifier = Left "rejection names another cell run"+      | otherwise = require SliceSubmitted slice $ slice {state = SliceRejected, rejectionReason = Just rejected.reason}+    advance slice (SubmissionSealed status)+      | status.runId /= identifier = Left "seal names another cell run"+      | otherwise = case (status.outcome, status.manifestSha256) of+          (Just outcome, Just digest) -> require SliceSubmitted slice $ slice {state = SliceSealed, cellOutcome = Just outcome, manifestSha256 = Just digest}+          _ -> Left "sealed status is missing outcome or manifest digest"+    advance slice (ResultsFetched path)+      | null path = Left "fetched tree path is empty"+      | otherwise = require SliceSealed slice $ slice {state = SliceFetched, fetchedPath = Just path}+    advance slice (ResultsVerified index)+      | index.cellRun /= identifier || index.leaseId /= slice.submission.leaseId = Left "verified index names another run or lease"+      | Just index.cellOutcome /= slice.cellOutcome || Just index.cellManifest.sha256 /= slice.manifestSha256 = Left "verified index disagrees with seal"+      | index.cellOutcome == Completed && sort (fmap (.runId) index.runs) /= sort slice.runIds = Left "verified index nested runs differ from planned slice"+      | any (\run -> run.runId `notElem` slice.runIds) index.runs = Left "verified index nested runs differ from planned slice"+      | otherwise = require SliceFetched slice $ slice {state = SliceVerified, entryExitCode = index.entryExitCode}+    require expected slice next+      | slice.state == expected = Right next+      | otherwise = Left ("invalid cell slice transition from " <> Text.pack (show slice.state))++readSessionJournal :: FilePath -> IO (Either Text SessionJournal)+readSessionJournal path = do+  bytes <- LazyByteString.readFile path+  pure case eitherDecode bytes of+    Left failure -> Left (Text.pack failure)+    Right journal -> Right journal++writeSessionJournal :: FilePath -> SessionJournal -> IO ()+writeSessionJournal path journal = do+  let directory = takeDirectory path+  createDirectoryIfMissing True directory+  bracket (openBinaryTempFile directory ".kenshou-session-") cleanup \(temporary, handle) -> do+    hClose handle+    LazyByteString.writeFile temporary (encode journal)+    renameFile temporary path+  where+    cleanup (temporary, _) = do+      exists <- doesFileExist temporary+      if exists then removeFile temporary else pure ()++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest++validWorkPath :: FilePath -> Bool+validWorkPath path = not (null path) && not (isAbsolute path) && all (`notElem` ["..", "."]) (splitDirectories path)++validSliceState :: SliceJournal -> Bool+validSliceState slice = case slice.state of+  SlicePlanned -> empty+  SliceSubmitted -> empty+  SliceSealed -> sealed && slice.fetchedPath == Nothing && slice.entryExitCode == Nothing+  SliceRejected -> slice.rejectionReason /= Nothing && slice.cellOutcome == Nothing && slice.manifestSha256 == Nothing && slice.fetchedPath == Nothing && slice.entryExitCode == Nothing+  SliceFetched -> sealed && slice.fetchedPath /= Nothing && slice.entryExitCode == Nothing+  SliceVerified -> sealed && slice.fetchedPath /= Nothing+  where+    empty = slice.cellOutcome == Nothing && slice.entryExitCode == Nothing && slice.manifestSha256 == Nothing && slice.rejectionReason == Nothing && slice.fetchedPath == Nothing+    sealed = slice.cellOutcome /= Nothing && maybe False validDigest slice.manifestSha256 && slice.rejectionReason == Nothing++validDigest :: Text -> Bool+validDigest digest = Text.length digest == 64 && Text.all (\character -> character `elem` ['0' .. '9'] || character `elem` ['a' .. 'f']) digest
+ src/Kenshou/Remote/Cell/Session/Rebind.hs view
@@ -0,0 +1,86 @@+module Kenshou.Remote.Cell.Session.Rebind+  ( RebindError (..),+    rebindPlannedSlices,+  )+where++import Control.Monad (forM)+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Submission (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseSnapshot, renewLease)+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..))++data RebindError+  = InvalidJournal !Text+  | LeaseUnavailable+  | UnresolvedSlice !RunId !SliceState+  | RemoteMarkerExists !RunId+  deriving stock (Eq, Show)++-- A planned entry may be rebound only when it has no remote claim marker.+-- A submitted entry may be rebound under a different, renewed lease when no+-- status, rejection or result object exists: the new lease fences the old+-- agent's start_run.+rebindPlannedSlices :: ObjectStore -> CellRef -> LeaseHandle -> FilePath -> IO (Either RebindError SessionJournal)+rebindPlannedSlices store ref handle journalPath = do+  decoded <- readSessionJournal journalPath+  case decoded of+    Left failure -> pure (Left (InvalidJournal failure))+    Right journal -> do+      lease <- leaseSnapshot handle+      held <- renewLease store ref handle+      if not held+        then pure (Left LeaseUnavailable)+        else+          if journal.cell /= ref.cellName || journal.controlBucket /= ref.controlBucket.unBucket || journal.leaseMode /= Held+            then pure (Left (InvalidJournal "session cell, control bucket or lease mode is incompatible with a held lease"))+            else case unresolved lease.leaseId journal.leaseId journal.slices of+              Just slice -> pure (Left (UnresolvedSlice slice.cellRun slice.state))+              Nothing -> do+                let planned = filter ((== SlicePlanned) . (.state)) journal.slices+                    pending = filter (\slice -> slice.state `elem` [SlicePlanned, SliceSubmitted]) journal.slices+                active <- firstSubmittedStatus (Bucket journal.resultsBucket) journal.slices+                case active of+                  Just slice -> pure (Left (UnresolvedSlice slice.cellRun slice.state))+                  Nothing -> do+                    present <- firstRemoteMarker planned+                    case present of+                      Just identifier -> pure (Left (RemoteMarkerExists identifier))+                      Nothing+                        | null pending -> pure (Right journal)+                        | otherwise -> do+                            rebound <- forM journal.slices \slice ->+                              if slice.state `elem` [SlicePlanned, SliceSubmitted]+                                then do+                                  identifier <- newRunId+                                  let previous = slice.submission+                                      submission = Submission identifier lease.leaseId previous.payload previous.work previous.env previous.reset previous.limits previous.requires previous.collect previous.labels+                                  pure (SliceJournal slice.index identifier slice.ordinals slice.runIds slice.reset submission slice.workPath SlicePlanned Nothing Nothing Nothing Nothing Nothing)+                                else pure slice+                            now <- getCurrentTime+                            let updated = journal {leaseId = lease.leaseId, slices = rebound, updatedAt = now}+                            writeSessionJournal journalPath updated+                            pure (Right updated)+  where+    unresolved _ _ [] = Nothing+    unresolved activeLease priorLease (slice : rest)+      | slice.state `elem` [SliceSealed, SliceFetched] || (slice.state == SliceSubmitted && activeLease == priorLease) = Just slice+      | otherwise = unresolved activeLease priorLease rest+    firstSubmittedStatus _ [] = pure Nothing+    firstSubmittedStatus resultsBucket (slice : rest)+      | slice.state /= SliceSubmitted = firstSubmittedStatus resultsBucket rest+      | otherwise = do+          let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/"+              resultPrefix = "runs/" <> renderRunId slice.cellRun <> "/"+          visible <- or <$> forM ["status.json", "rejected.json"] (fmap isJust . store.statObject ref.controlBucket . ObjectName . (prefix <>))+          results <- store.listObjects resultsBucket resultPrefix+          if visible || not (null results) then pure (Just slice) else firstSubmittedStatus resultsBucket rest+    firstRemoteMarker [] = pure Nothing+    firstRemoteMarker (slice : rest) = do+      let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/"+      found <- or <$> forM ["submission.json", "status.json", "rejected.json"] (fmap isJust . store.statObject ref.controlBucket . ObjectName . (prefix <>))+      if found then pure (Just slice.cellRun) else firstRemoteMarker rest
+ src/Kenshou/Remote/Cell/Session/Resume.hs view
@@ -0,0 +1,128 @@+module Kenshou.Remote.Cell.Session.Resume+  ( ResumeError (..),+    HeldResumeError (..),+    resumeObservedSlices,+    resumeHeldSession,+  )+where++import Data.Aeson (eitherDecode)+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.List.NonEmpty (NonEmpty)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Submission)+import Kenshou.Remote.Cell.Fetch (FetchError, VerifyProblem, fetchCellRun)+import Kenshou.Remote.Cell.Index (deriveCellRunIndex, writeCellRunIndex)+import Kenshou.Remote.Cell.Lease (CellRef (..), LeaseHandle)+import Kenshou.Remote.Cell.Session (SessionTransition (..))+import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Cell.Session.Rebind (RebindError, rebindPlannedSlices)+import Kenshou.Remote.Cell.Session.Runner (SessionRunError, runPendingSlices)+import Kenshou.Remote.Cell.Watch (WatchEvent, WatchSnapshot (..), WatchTerminal (..), newWatchCursor, pollCellRun)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..))+import System.FilePath (takeDirectory)++data ResumeError+  = InvalidJournal !Text+  | ResultFetchFailed !RunId !FetchError+  | ResultVerificationFailed !RunId !(NonEmpty VerifyProblem)+  deriving stock (Eq, Show)++data HeldResumeError+  = ObservationError !ResumeError+  | RebindingError !RebindError+  | ContinuationError !SessionRunError+  deriving stock (Eq, Show)++resumeHeldSession :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> (WatchEvent -> IO ()) -> IO (Either HeldResumeError SessionJournal)+resumeHeldSession store ref resultsBucket handle journalPath emit = do+  observed <- resumeObservedSlices store ref resultsBucket journalPath emit+  case observed of+    Left failure -> pure (Left (ObservationError failure))+    Right observed | not (null observed.slices) && all (\slice -> slice.state `elem` [SliceVerified, SliceRejected]) observed.slices -> pure (Right observed)+    Right _ -> do+      rebound <- rebindPlannedSlices store ref handle journalPath+      case rebound of+        Left failure -> pure (Left (RebindingError failure))+        Right _ -> do+          continued <- runPendingSlices store ref resultsBucket handle journalPath emit+          pure (either (Left . ContinuationError) Right continued)++-- Reconcile observations and collect already sealed results. Planned or still+-- running slices remain untouched for a later lease-aware resume step.+resumeObservedSlices :: ObjectStore -> CellRef -> Bucket -> FilePath -> (WatchEvent -> IO ()) -> IO (Either ResumeError SessionJournal)+resumeObservedSlices store ref resultsBucket journalPath emit = do+  decoded <- readSessionJournal journalPath+  case decoded of+    Left problem -> pure (Left (InvalidJournal problem))+    Right journal+      | journal.cell /= ref.cellName || journal.controlBucket /= ref.controlBucket.unBucket || journal.resultsBucket /= resultsBucket.unBucket -> pure (Left (InvalidJournal "session cell or buckets differ from the requested store"))+      | otherwise -> do+          current <- newIORef journal+          reconcile current journal.slices+  where+    outRoot = takeDirectory (takeDirectory journalPath)+    checkpoint current identifier transition = do+      before <- readIORef current+      now <- getCurrentTime+      changed <- either (ioError . userError . Text.unpack) pure (applyTransition now identifier transition before)+      writeSessionJournal journalPath changed+      writeIORef current changed+    reconcile current [] = Right <$> readIORef current+    reconcile current (slice : rest) = do+      outcome <- case slice.state of+        SlicePlanned -> do+          let name = ObjectName ("cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/submission.json")+          marker <- store.getObject ref.controlBucket name+          case marker of+            Nothing -> pure (Right ())+            Just (bytes, _) -> case eitherDecode bytes :: Either String Submission of+              Right submission | submission == slice.submission -> do+                checkpoint current slice.cellRun SubmissionPublished+                observeSubmitted current slice+              _ -> pure (Left (InvalidJournal "retained planned submission differs from the visible marker"))+        SliceRejected -> pure (Right ())+        SliceVerified -> pure (Right ())+        SliceSubmitted -> observeSubmitted current slice+        SliceSealed -> do+          status <- sealedStatus slice.cellRun+          collect current slice.cellRun status True+        SliceFetched -> do+          status <- sealedStatus slice.cellRun+          collect current slice.cellRun status False+      case outcome of+        Left failure -> pure (Left failure)+        Right () -> reconcile current rest+    observeSubmitted current slice = do+      observed <- observe slice.cellRun+      case observed of+        Nothing -> pure (Right ())+        Just (RunRejected rejected) -> checkpoint current slice.cellRun (SubmissionRejectedByCell rejected) >> pure (Right ())+        Just (RunSealed status) -> do+          checkpoint current slice.cellRun (SubmissionSealed status)+          collect current slice.cellRun (Just status) True+    observe identifier = do+      snapshot <- pollCellRun store ref identifier newWatchCursor+      mapM_ emit snapshot.events+      pure snapshot.terminal+    sealedStatus identifier = do+      terminal <- observe identifier+      pure case terminal of+        Just (RunSealed status) -> Just status+        _ -> Nothing+    collect current identifier status shouldCheckpointFetch = do+      fetched <- fetchCellRun store resultsBucket identifier outRoot+      case fetched of+        Left failure -> pure (Left (ResultFetchFailed identifier failure))+        Right tree -> do+          if shouldCheckpointFetch then checkpoint current identifier (ResultsFetched tree) else pure ()+          verified <- deriveCellRunIndex resultsBucket status tree+          case verified of+            Left problems -> pure (Left (ResultVerificationFailed identifier problems))+            Right index -> do+              _ <- writeCellRunIndex tree index+              checkpoint current identifier (ResultsVerified index)+              pure (Right ())
+ src/Kenshou/Remote/Cell/Session/Runner.hs view
@@ -0,0 +1,120 @@+module Kenshou.Remote.Cell.Session.Runner+  ( SessionRunError (..),+    runPlannedSlices,+    runPendingSlices,+  )+where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Remote.Cell.Docs (Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseSnapshot)+import Kenshou.Remote.Cell.Session (SessionError, SessionTransition, runSubmissionWithTransitions)+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.Watch (WatchEvent)+import Kenshou.Remote.Store (Bucket (..), ObjectStore)+import System.Directory (doesFileExist)+import System.FilePath (takeDirectory, (</>))++data SessionRunError+  = InvalidSession !Text+  | SessionFileExists !FilePath+  | SessionFileMissing !FilePath+  | UnresolvedSlice !RunId !SliceState+  | SliceExecutionFailed !RunId !SessionError+  deriving stock (Eq, Show)++-- Run a new, already prepared journal. The journal and work files are the+-- caller's durable inputs; this function never rewrites a prior session.+runPlannedSlices :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> SessionJournal -> (WatchEvent -> IO ()) -> IO (Either SessionRunError SessionJournal)+runPlannedSlices store ref resultsBucket handle journalPath journal emit = do+  exists <- doesFileExist journalPath+  if exists+    then pure (Left (SessionFileExists journalPath))+    else do+      lease <- leaseSnapshot handle+      case validateSession ref resultsBucket lease journal of+        Left problem -> pure (Left (InvalidSession problem))+        Right ()+          | any ((/= SlicePlanned) . (.state)) journal.slices -> pure (Left (InvalidSession "new session contains a slice that is not planned"))+          | otherwise -> do+              preflight <- checkWorkFiles (takeDirectory journalPath) journal.slices+              case preflight of+                Left problem -> pure (Left (InvalidSession problem))+                Right () -> do+                  writeSessionJournal journalPath journal+                  executeSlices store ref resultsBucket handle journalPath journal journal.slices emit++-- Continue a journal whose planned slices have already been rebound to this+-- lease. Verified and rejected slices are retained without execution.+runPendingSlices :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> (WatchEvent -> IO ()) -> IO (Either SessionRunError SessionJournal)+runPendingSlices store ref resultsBucket handle journalPath emit = do+  exists <- doesFileExist journalPath+  if not exists+    then pure (Left (SessionFileMissing journalPath))+    else do+      decoded <- readSessionJournal journalPath+      case decoded of+        Left problem -> pure (Left (InvalidSession problem))+        Right journal -> do+          lease <- leaseSnapshot handle+          case validateSession ref resultsBucket lease journal of+            Left problem -> pure (Left (InvalidSession problem))+            Right () -> case filter (\slice -> slice.state `notElem` [SlicePlanned, SliceRejected, SliceVerified]) journal.slices of+              slice : _ -> pure (Left (UnresolvedSlice slice.cellRun slice.state))+              [] -> do+                let planned = filter ((== SlicePlanned) . (.state)) journal.slices+                preflight <- checkWorkFiles (takeDirectory journalPath) planned+                case preflight of+                  Left problem -> pure (Left (InvalidSession problem))+                  Right () -> executeSlices store ref resultsBucket handle journalPath journal planned emit++validateSession :: CellRef -> Bucket -> Lease -> SessionJournal -> Either Text ()+validateSession ref resultsBucket lease journal = do+  case eitherDecode (encode journal) :: Either String SessionJournal of+    Left failure -> Left (Text.pack failure)+    Right _ -> pure ()+  if journal.cell == ref.cellName && journal.controlBucket == ref.controlBucket.unBucket && journal.resultsBucket == resultsBucket.unBucket && journal.leaseId == lease.leaseId && journal.leaseMode == Held+    then pure ()+    else Left "session cell, bucket, lease or mode differs from the active held lease"++checkWorkFiles :: FilePath -> [SliceJournal] -> IO (Either Text ())+checkWorkFiles _ [] = pure (Right ())+checkWorkFiles directory (slice : rest) = do+  let path = directory </> slice.workPath+  present <- doesFileExist path+  if not present+    then pure (Left ("missing prepared work file: " <> Text.pack slice.workPath))+    else do+      bytes <- LazyByteString.readFile path+      if workObjectFor slice.submission.work.mediaType bytes /= slice.submission.work+        then pure (Left ("prepared work digest or size differs: " <> Text.pack slice.workPath))+        else checkWorkFiles directory rest++executeSlices :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> SessionJournal -> [SliceJournal] -> (WatchEvent -> IO ()) -> IO (Either SessionRunError SessionJournal)+executeSlices store ref resultsBucket handle journalPath journal slices emit = do+  current <- newIORef journal+  go current slices+  where+    directory = takeDirectory journalPath+    outRoot = takeDirectory directory+    go current [] = Right <$> readIORef current+    go current (slice : rest) = do+      bytes <- LazyByteString.readFile (directory </> slice.workPath)+      let checkpoint :: SessionTransition -> IO ()+          checkpoint transition = do+            before <- readIORef current+            now <- getCurrentTime+            changed <- either (ioError . userError . Text.unpack) pure (applyTransition now slice.cellRun transition before)+            writeSessionJournal journalPath changed+            writeIORef current changed+      result <- runSubmissionWithTransitions store ref resultsBucket handle slice.submission bytes outRoot emit checkpoint+      case result of+        Left failure -> pure (Left (SliceExecutionFailed slice.cellRun failure))+        Right _ -> go current rest
+ src/Kenshou/Remote/Cell/Submit.hs view
@@ -0,0 +1,96 @@+module Kenshou.Remote.Cell.Submit+  ( PublishOutcome (..),+    workObjectFor,+    publishSubmission,+  )+where++import Control.Monad (unless)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text.Encoding qualified as TextEncoding+import Data.Time (addUTCTime)+import Kenshou.Core.Id (renderRunId)+import Kenshou.Remote.Cell.Docs (Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseHeld, leaseSnapshot)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))++data PublishOutcome+  = Submitted+  | LostLease+  | Quarantined+  | WorkAlreadyExists+  | SubmissionAlreadyExists+  deriving stock (Eq, Show)++workObjectFor :: Text -> LazyByteString.ByteString -> WorkObject+workObjectFor mediaType bytes =+  WorkObject+    { sha256 = TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash (LazyByteString.toStrict bytes))),+      bytes = LazyByteString.length bytes,+      mediaType = mediaType+    }++-- The agent increments runsStarted after it claims submission.json. The client+-- only publishes the opaque work object and then the visible submission marker.+publishSubmission :: ObjectStore -> CellRef -> LeaseHandle -> Submission -> LazyByteString.ByteString -> IO PublishOutcome+publishSubmission store ref handle submission workBytes = do+  expected <- leaseSnapshot handle+  unless (expected.cell == ref.cellName && submission.leaseId == expected.leaseId) (ioError (userError "submission lease does not match cell lease handle"))+  case eitherDecode (encode submission) :: Either String Submission of+    Left failure -> ioError (userError ("invalid cell submission: " <> failure))+    Right _ -> pure ()+  let bundle = submission.payload.bundle+      bundleUri = "gs://" <> ref.controlBucket.unBucket <> "/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"+  unless (bundle.uri == bundleUri) (ioError (userError "submission payload bundle is not in the cell control bucket"))+  unless (submission.work == workObjectFor submission.work.mediaType workBytes) (ioError (userError "submission work digest or size does not match its bytes"))+  active <- leaseActive store ref handle+  if not active+    then pure LostLease+    else do+      quarantined <- cellQuarantined store ref+      if quarantined+        then pure Quarantined+        else do+          let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId submission.runId <> "/"+          workResult <- store.putObject ref.controlBucket (ObjectName (prefix <> "work")) submission.work.mediaType DoesNotExist workBytes+          case workResult of+            PreconditionFailed -> pure WorkAlreadyExists+            Written _ -> do+              stillActive <- leaseActive store ref handle+              if not stillActive+                then pure LostLease+                else do+                  nowQuarantined <- cellQuarantined store ref+                  if nowQuarantined+                    then pure Quarantined+                    else do+                      marker <- store.putObject ref.controlBucket (ObjectName (prefix <> "submission.json")) "application/json" DoesNotExist (encode submission)+                      pure case marker of+                        Written _ -> Submitted+                        PreconditionFailed -> SubmissionAlreadyExists++cellQuarantined :: ObjectStore -> CellRef -> IO Bool+cellQuarantined store ref = do+  status <- store.statObject ref.controlBucket (ObjectName ("cells/" <> ref.cellName <> "/quarantine.json"))+  pure (case status of Nothing -> False; Just _ -> True)++leaseActive :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool+leaseActive store ref handle = do+  held <- leaseHeld handle+  if not held+    then pure False+    else do+      expected <- leaseSnapshot handle+      stored <- store.getObject ref.controlBucket (ObjectName ("cells/" <> ref.cellName <> "/lease.json"))+      case stored of+        Nothing -> pure False+        Just (body, meta) -> case eitherDecode body of+          Left failure -> ioError (userError ("invalid active cell lease: " <> failure))+          Right (current :: Lease) -> do+            now <- store.serverTime+            pure (current.cell == ref.cellName && current.leaseId == expected.leaseId && not current.cancelRequested && addUTCTime (fromIntegral current.ttlSeconds + 30) meta.updated >= now)
+ src/Kenshou/Remote/Cell/Watch.hs view
@@ -0,0 +1,99 @@+module Kenshou.Remote.Cell.Watch+  ( WatchCursor,+    WatchEvent (..),+    WatchTerminal (..),+    WatchSnapshot (..),+    newWatchCursor,+    pollCellRun,+    watchCellRun,+  )+where++import Control.Concurrent (threadDelay)+import Control.Monad (forM, unless)+import Data.Aeson (FromJSON, eitherDecode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (RunId, renderRunId)+import Kenshou.Remote.Cell.Docs (CellPhase (..), CellStatus (..), LogChunks (..), Rejected (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Store (ObjectName (..), ObjectStore (..))++data WatchCursor = WatchCursor+  { lastPhase :: !(Maybe CellPhase),+    stdoutNext :: !Int,+    stderrNext :: !Int+  }+  deriving stock (Eq, Show)++data WatchEvent+  = PhaseChanged !CellPhase+  | StdoutChunk !LazyByteString.ByteString+  | StderrChunk !LazyByteString.ByteString+  deriving stock (Eq, Show)++data WatchTerminal = RunSealed !CellStatus | RunRejected !Rejected+  deriving stock (Eq, Show)++data WatchSnapshot = WatchSnapshot+  { cursor :: !WatchCursor,+    events :: ![WatchEvent],+    terminal :: !(Maybe WatchTerminal)+  }+  deriving stock (Eq, Show)++newWatchCursor :: WatchCursor+newWatchCursor = WatchCursor Nothing 0 0++pollCellRun :: ObjectStore -> CellRef -> RunId -> WatchCursor -> IO WatchSnapshot+pollCellRun store ref identifier cursor = do+  let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId identifier <> "/"+  rejected <- readDocument store ref (ObjectName (prefix <> "rejected.json"))+  case rejected of+    Just record -> do+      unless (record.runId == identifier) (ioError (userError "cell rejection names another run"))+      pure (WatchSnapshot cursor [] (Just (RunRejected record)))+    Nothing -> do+      status <- readDocument store ref (ObjectName (prefix <> "status.json"))+      case status of+        Nothing -> pure (WatchSnapshot cursor [] Nothing)+        Just current -> do+          unless (current.runId == identifier) (ioError (userError "cell status names another run"))+          unless (current.logChunks.stdout >= cursor.stdoutNext && current.logChunks.stderr >= cursor.stderrNext) (ioError (userError "cell log chunk count moved backwards"))+          stdout <- readChunks "stdout" cursor.stdoutNext current.logChunks.stdout StdoutChunk prefix+          stderr <- readChunks "stderr" cursor.stderrNext current.logChunks.stderr StderrChunk prefix+          let changed = if cursor.lastPhase == Just current.phase then [] else [PhaseChanged current.phase]+              next = WatchCursor (Just current.phase) current.logChunks.stdout current.logChunks.stderr+              done = if current.phase == Sealed then Just (RunSealed current) else Nothing+          pure (WatchSnapshot next (changed <> stdout <> stderr) done)+  where+    readChunks :: Text -> Int -> Int -> (LazyByteString.ByteString -> WatchEvent) -> Text -> IO [WatchEvent]+    readChunks stream first end wrap prefix = forM [first .. end - 1] \index -> do+      let object = ObjectName (prefix <> "log/" <> stream <> "." <> showText index)+      stored <- store.getObject ref.controlBucket object+      case stored of+        Nothing -> ioError (userError ("missing cell log chunk " <> show object))+        Just (bytes, _) -> pure (wrap bytes)++watchCellRun :: ObjectStore -> CellRef -> RunId -> (WatchEvent -> IO ()) -> IO WatchTerminal+watchCellRun store ref identifier emit = loop newWatchCursor+  where+    loop cursor = do+      snapshot <- pollCellRun store ref identifier cursor+      mapM_ emit snapshot.events+      case snapshot.terminal of+        Just terminal -> pure terminal+        Nothing -> threadDelay 2000000 >> loop snapshot.cursor++readDocument :: (FromJSON document) => ObjectStore -> CellRef -> ObjectName -> IO (Maybe document)+readDocument store ref name = do+  stored <- store.getObject ref.controlBucket name+  case stored of+    Nothing -> pure Nothing+    Just (bytes, _) -> case eitherDecode bytes of+      Left failure -> ioError (userError ("invalid cell document at " <> show name <> ": " <> failure))+      Right value -> pure (Just value)++showText :: Int -> Text+showText = Text.pack . show
+ src/Kenshou/Remote/Cell/WorkJson.hs view
@@ -0,0 +1,108 @@+module Kenshou.Remote.Cell.WorkJson (decodeWorkPlan, renderPreparedWork) where++import Data.Aeson (FromJSON, Value (..))+import Data.Aeson qualified as Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time.Clock.POSIX (posixSecondsToUTCTime)+import Kenshou.Core.Id (RunId, mkSeed)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Prepare (PreparedRun (..))++data SourceRun = SourceRun+  { ordinal :: !Int,+    runId :: !RunId,+    estimateMinutes :: !Int,+    trial :: !(Maybe TrialInfo),+    spec :: !RunSpec,+    json :: !Value+  }++-- The public plan format is write-only for its provenance members. The typed+-- projection carries only fields used by cell preparation; work rendering+-- copies the original document and replaces selected specs.+decodeWorkPlan :: Value -> Either Text RunPlan+decodeWorkPlan work = do+  (_, entries, planId) <- parseWork work+  selector <- parseSelector "**"+  seed <- mkSeed 1+  let placeholder = Reason (Change (ComponentRef (ComponentId "cell-work") Nothing) Named "internal preparation placeholder") [] selector 0+      planned = [PlannedRun entry.ordinal entry.runId entry.estimateMinutes (placeholder :| []) entry.trial entry.spec | entry <- entries]+      context = PlanContext Nothing "" "" "" (PlanInputs Null) [] []+  pure (RunPlan planId (posixSecondsToUTCTime 0) context (defaultPlanPolicy seed) planned [] (sum (fmap (.estimateMinutes) planned)))++renderPreparedWork :: Value -> [PreparedRun] -> Either Text Value+renderPreparedWork work selected = do+  (fields, entries, _) <- parseWork work+  if unique (fmap (.runId) selected) && unique (fmap (.ordinal) selected)+    then pure ()+    else Left "prepared work contains duplicate run IDs or ordinals"+  let originals = Map.fromList [(entry.runId, entry) | entry <- entries]+  rendered <- traverse (renderEntry originals) selected+  let rows = fmap fst rendered+      minutes = sum (fmap snd rendered)+  pure (Object (KeyMap.insert "runs" (Aeson.toJSON rows) (KeyMap.insert "estimateMinutes" (Aeson.toJSON minutes) fields)))++parseWork :: Value -> Either Text (Aeson.Object, [SourceRun], RunId)+parseWork work = do+  fields <- case work of Object members -> Right members; _ -> Left "run plan is not a JSON object"+  case KeyMap.lookup "schema" fields of+    Just (String "kenshou.run-plan/v1") -> pure ()+    _ -> Left "run plan has an unsupported schema"+  planId <- required "planId" fields+  raw <- case KeyMap.lookup "runs" fields of+    Just (Array rows) -> Right (toList rows)+    _ -> Left "run plan has no runs array"+  entries <- traverse parseEntry raw+  if unique (fmap (.runId) entries) && unique (fmap (.ordinal) entries)+    then Right (fields, entries, planId)+    else Left "run plan contains duplicate run IDs or ordinals"++parseEntry :: Value -> Either Text SourceRun+parseEntry value = case value of+  Object fields -> do+    ordinal <- required "ordinal" fields+    runId <- required "runId" fields+    estimate <- required "estimateMinutes" fields+    spec <- required "spec" fields+    trial <- case KeyMap.lookup "trial" fields of+      Nothing -> Right Nothing+      Just Null -> Right Nothing+      Just (Object members) -> Just <$> (TrialInfo <$> required "group" members <*> required "arm" members <*> required "index" members <*> required "of" members)+      _ -> Left "planned run trial is not an object"+    if ordinal < 0 || estimate <= 0 || maybe False (/= runId) spec.runId+      then Left "planned run has an invalid ordinal, estimate or spec run ID"+      else Right (SourceRun ordinal runId estimate trial spec value)+  _ -> Left "planned run is not an object"++renderEntry :: Map.Map RunId SourceRun -> PreparedRun -> Either Text (Value, Int)+renderEntry originals prepared = case Map.lookup prepared.runId originals of+  Just source+    | source.ordinal /= prepared.ordinal -> Left "prepared run ordinal differs from the source plan"+    | source.spec.scenario /= prepared.spec.scenario -> Left "prepared run scenario differs from the source plan"+    | maybe False (/= prepared.runId) prepared.spec.runId -> Left "prepared spec run ID differs from the source plan"+    | otherwise -> case source.json of+        Object original -> Right (Object (KeyMap.insert "spec" (Aeson.toJSON prepared.spec) original), source.estimateMinutes)+        _ -> Left "prepared run is not an object"+  _ -> Left "prepared run is absent from the source plan"++required :: (FromJSON document) => Text -> Aeson.Object -> Either Text document+required name fields = case KeyMap.lookup (Key.fromText name) fields of+  Nothing -> Left ("run plan is missing " <> name)+  Just value -> case Aeson.fromJSON value of+    Aeson.Error failure -> Left (name <> ": " <> Text.pack failure)+    Aeson.Success result -> Right result++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Pair.hs view
@@ -0,0 +1,93 @@+module Kenshou.Remote.Pair+  ( PairRequest (..),+    PairState (..),+    SliceResult (..),+    pairPlan,+    judgePairs,+  )+where++import Data.List (nub, nubBy)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Word (Word64)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), mkSeed, parseRunId, renderRunId, renderScenarioId)+import Kenshou.Core.RunSpec (ComparisonMembership (..), RunSpec (..))+import Kenshou.Measure.Compare.Ordering (Arm (..), PairedOrdering, TrialSlot (..), pairedSchedule)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))++data PairRequest = PairRequest+  { candidate :: !Text,+    baseline :: !Text,+    pairs :: !Int,+    ordering :: !PairedOrdering,+    seed :: !Word64,+    maxReplacements :: !Int+  }+  deriving stock (Eq, Show)++data PairState = PairValid | PairInvalid !Text+  deriving stock (Eq, Show)++data SliceResult = SliceResult+  { group :: !Text,+    pairIndex :: !Int,+    arm :: !Arm,+    passed :: !Bool+  }+  deriving stock (Eq, Show)++-- A planned benchmark may already have planner-generated trial repeats. The+-- cell schedule replaces those repeats with adjacent, same-seed pairs.+pairPlan :: PairRequest -> RunPlan -> Either Text RunPlan+pairPlan request source = do+  if request.pairs < 3 then Left "paired comparisons need at least three pairs" else pure ()+  if request.maxReplacements < 0 then Left "max replacements cannot be negative" else pure ()+  if Text.null request.baseline || Text.null request.candidate || request.baseline == request.candidate+    then Left "baseline and candidate payload labels must be distinct"+    else pure ()+  if null source.runs then Left "paired comparison plan has no runs" else pure ()+  if any ((/= Benchmark) . (.kind) . (.scenario) . (.spec)) source.runs+    then Left "paired comparisons require benchmark runs only"+    else pure ()+  let configurations = nubBy sameConfiguration source.runs+      slots = pairedSchedule request.ordering (request.pairs + request.maxReplacements) request.seed+      selected = [(configurationIndex, entry, slot) | (configurationIndex, entry) <- zip [0 :: Int ..] configurations, slot <- slots]+  planned <- traverse makeRun (zip [1 :: Int ..] selected)+  pure source {runs = planned, estimateMinutes = sum (fmap (.estimateMinutes) planned)}+  where+    sameConfiguration left right =+      let first = left.spec+          second = right.spec+       in first.scenario == second.scenario+            && first.knobs == second.knobs+            && first.dimensions == second.dimensions+            && first.phases == second.phases+            && first.environment == second.environment++    makeRun (ordinal, (configurationIndex, original, slot)) = do+      identifier <- derivedRunId original.runId configurationIndex slot.position+      pairSeed <- mkSeed (slot.pairSeed `mod` 9007199254740992)+      let groupName = renderScenarioId original.spec.scenario <> "/config-" <> Text.pack (show configurationIndex)+          armName = case slot.arm of Baseline -> request.baseline; Candidate -> request.candidate+          trial = TrialInfo groupName armName slot.pairIndex (request.pairs + request.maxReplacements)+          spec = original.spec {runId = Just identifier, seed = Just pairSeed, comparison = Just (ComparisonMembership groupName armName slot.pairIndex slot.position)}+      pure original {ordinal, runId = identifier, trial = Just trial, spec}++derivedRunId :: RunId -> Int -> Int -> Either Text RunId+derivedRunId original configurationIndex position =+  parseRunId (Text.take 18 (renderRunId original) <> "-a" <> Text.take 3 digest <> "-" <> Text.take 12 (Text.drop 3 digest))+  where+    digest = Text.drop 7 (sha256Hex (TextEncoding.encodeUtf8 (renderRunId original <> ":" <> Text.pack (show configurationIndex) <> ":" <> Text.pack (show position))))++judgePairs :: [SliceResult] -> [((Text, Int), PairState)]+judgePairs results = fmap judge keys+  where+    keys = nub [(result.group, result.pairIndex) | result <- results]+    judge key =+      let members = [result | result <- results, (result.group, result.pairIndex) == key]+          baseline = filter ((== Baseline) . (.arm)) members+          candidate = filter ((== Candidate) . (.arm)) members+       in (key, if length baseline == 1 && length candidate == 1 && all (.passed) members then PairValid else PairInvalid "one or both cell trials failed or are missing")
+ src/Kenshou/Remote/Payload.hs view
@@ -0,0 +1,163 @@+module Kenshou.Remote.Payload+  ( Bundle (..),+    CellPayload (..),+    Harness (..),+    CohortCheck (..),+    PayloadDescriptor (..),+  )+where++import Control.Monad (unless)+import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:), (.=))+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Cohort (CohortIdentity (..), CohortName (..))++data Bundle = Bundle+  { uri :: !Text,+    sha256 :: !Text,+    bytes :: !Int64+  }+  deriving stock (Eq, Show)++data CellPayload = CellPayload+  { bundle :: !Bundle,+    storePath :: !Text,+    narHash :: !Text,+    closurePaths :: ![Text],+    system :: !Text,+    command :: ![Text]+  }+  deriving stock (Eq, Show)++data Harness = Harness+  { revision :: !Text,+    dirty :: !Bool+  }+  deriving stock (Eq, Show)++data CohortCheck = CohortCheck+  { packagesChecked :: !Int+  }+  deriving stock (Eq, Show)++data PayloadDescriptor = PayloadDescriptor+  { cohort :: !Text,+    variant :: !Text,+    flakeAttr :: !Text,+    harness :: !Harness,+    cohortIdentity :: !CohortIdentity,+    cohortCheck :: !CohortCheck,+    cell :: !CellPayload,+    createdAt :: !UTCTime+  }+  deriving stock (Eq, Show)++instance ToJSON Bundle where+  toJSON bundle = object ["uri" .= bundle.uri, "sha256" .= bundle.sha256, "bytes" .= bundle.bytes]++instance FromJSON Bundle where+  parseJSON = withObject "cell payload bundle" \value -> do+    bundle <- Bundle <$> value .: "uri" <*> value .: "sha256" <*> value .: "bytes"+    unless (bundle.bytes > 0 && validSha256 bundle.sha256 && Text.isSuffixOf ("/payloads/sha256/" <> bundle.sha256 <> ".nar.zst") bundle.uri && "gs://" `Text.isPrefixOf` bundle.uri) (fail "invalid content-addressed cell bundle")+    pure bundle++instance ToJSON CellPayload where+  toJSON payload =+    object+      [ "schema" .= ("cell.payload/v1" :: Text),+        "kind" .= ("nix-nar-bundle" :: Text),+        "bundle" .= payload.bundle,+        "storePath" .= payload.storePath,+        "narHash" .= payload.narHash,+        "closurePaths" .= payload.closurePaths,+        "system" .= payload.system,+        "command" .= payload.command+      ]++instance FromJSON CellPayload where+  parseJSON = withObject "cell payload" \value -> do+    schema <- value .: "schema"+    kind <- value .: "kind"+    unless (schema == ("cell.payload/v1" :: Text) && kind == ("nix-nar-bundle" :: Text)) (fail "unsupported cell payload")+    payload <- CellPayload <$> value .: "bundle" <*> value .: "storePath" <*> value .: "narHash" <*> value .: "closurePaths" <*> value .: "system" <*> value .: "command"+    unless (payload.system == "x86_64-linux" && payload.storePath `elem` payload.closurePaths && not (null payload.closurePaths) && unique payload.closurePaths && validStorePath payload.storePath && all validStorePath payload.closurePaths && not (Text.null payload.narHash) && validCommand payload.command) (fail "invalid cell payload")+    pure payload++instance ToJSON Harness where+  toJSON harness = object ["revision" .= harness.revision, "dirty" .= harness.dirty]++instance FromJSON Harness where+  parseJSON = withObject "payload harness" \value -> do+    harness <- Harness <$> value .: "revision" <*> value .: "dirty"+    unless (Text.length harness.revision == 40 && Text.all isHex harness.revision) (fail "invalid harness revision")+    pure harness++instance ToJSON CohortCheck where+  toJSON check = object ["status" .= ("consistent" :: Text), "packagesChecked" .= check.packagesChecked]++instance FromJSON CohortCheck where+  parseJSON = withObject "payload cohort check" \value -> do+    status <- value .: "status"+    unless (status == ("consistent" :: Text)) (fail "payload cohort check is not consistent")+    count <- value .: "packagesChecked"+    unless (count >= (0 :: Int)) (fail "negative checked package count")+    pure (CohortCheck count)++instance ToJSON PayloadDescriptor where+  toJSON descriptor =+    object+      [ "schema" .= ("kenshou.payload/v1" :: Text),+        "cohort" .= descriptor.cohort,+        "variant" .= descriptor.variant,+        "flakeAttr" .= descriptor.flakeAttr,+        "harness" .= descriptor.harness,+        "cohortIdentity" .= descriptor.cohortIdentity,+        "cohortCheck" .= descriptor.cohortCheck,+        "cell" .= descriptor.cell,+        "createdAt" .= descriptor.createdAt+      ]++instance FromJSON PayloadDescriptor where+  parseJSON = withObject "Kenshou payload" \value -> do+    schema <- value .: "schema"+    unless (schema == ("kenshou.payload/v1" :: Text)) (fail "unsupported Kenshou payload schema")+    descriptor <- PayloadDescriptor <$> value .: "cohort" <*> value .: "variant" <*> value .: "flakeAttr" <*> value .: "harness" <*> value .: "cohortIdentity" <*> value .: "cohortCheck" <*> value .: "cell" <*> value .: "createdAt"+    let expectedAttr =+          "packages.x86_64-linux.kenshou-" <> descriptor.cohort <> case descriptor.variant of+            "default" -> ""+            "info-table" -> "-info-table"+            "profiled" -> "-profiled"+            _ -> "-invalid"+    unless (descriptor.variant `elem` ["default", "info-table", "profiled"] && descriptor.cohort == unCohortName descriptor.cohortIdentity.identityCohort && descriptor.flakeAttr == expectedAttr && descriptor.cohortIdentity.identityResolver == Just "nix" && descriptor.cell.command == ["bin/kenshou", "cell", "exec"]) (fail "inconsistent Kenshou payload identity")+    pure descriptor++validSha256 :: Text -> Bool+validSha256 digest = Text.length digest == 64 && Text.all isHex digest++isHex :: Char -> Bool+isHex character = character `elem` ['0' .. '9'] || character `elem` ['a' .. 'f']++validStorePath :: Text -> Bool+validStorePath path = case Text.stripPrefix "/nix/store/" path of+  Just suffix ->+    let (digest, name) = Text.breakOn "-" suffix+     in Text.length digest == 32 && Text.all validNixBase32 digest && not (Text.null (Text.drop 1 name)) && not (Text.any (== '/') name)+  Nothing -> False+  where+    validNixBase32 character = character `elem` ("0123456789abcdfghijklmnpqrsvwxyz" :: String)++validCommand :: [Text] -> Bool+validCommand [] = False+validCommand (first : rest) =+  not (Text.null first)+    && not (Text.isPrefixOf "/" first)+    && not (Text.any (== '\0') first)+    && all (/= "..") (Text.splitOn "/" first)+    && all (not . Text.null) rest++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Payload/Nix.hs view
@@ -0,0 +1,172 @@+module Kenshou.Remote.Payload.Nix+  ( NixError (..),+    NixTools (..),+    ClosureInfo (..),+    BundleInfo (..),+    nixBuild,+    nixEvalJson,+    closureInfo,+    parseClosureInfo,+    exportBundle,+    exportBundleWith,+  )+where++import Control.Exception (IOException, displayException, try)+import Data.Aeson (FromJSON, Value (..), eitherDecodeStrict')+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Int (Int64)+import Data.List (sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Kenshou.Remote.Payload.Publish (digestFile)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.Exit (ExitCode (..))+import System.FilePath (takeDirectory)+import System.IO (hClose, openBinaryTempFile)+import System.Process (StdStream (..), createProcess, proc, readProcessWithExitCode, std_in, std_out, terminateProcess, waitForProcess)++data NixError+  = NixCommandFailed !FilePath ![String] !ExitCode !Text+  | NixCommandIoError !Text+  | NixInvalidOutput !Text+  deriving stock (Eq, Show)++data NixTools = NixTools+  { nixExecutable :: !FilePath,+    nixStoreExecutable :: !FilePath,+    zstdExecutable :: !FilePath+  }+  deriving stock (Eq, Show)++defaultTools :: NixTools+defaultTools = NixTools "nix" "nix-store" "zstd"++data ClosureInfo = ClosureInfo+  { storePath :: !FilePath,+    narHash :: !Text,+    closurePaths :: ![FilePath]+  }+  deriving stock (Eq, Show)++data BundleInfo = BundleInfo+  { file :: !FilePath,+    sha256 :: !Text,+    bytes :: !Int64,+    closure :: !ClosureInfo+  }+  deriving stock (Eq, Show)++nixBuild :: String -> IO (Either NixError FilePath)+nixBuild reference = do+  result <- runCommand defaultTools.nixExecutable ["build", "--no-link", "--print-out-paths", reference]+  pure do+    output <- result+    case lines (Text.unpack output) of+      [path] | "/nix/store/" `Text.isPrefixOf` Text.pack path -> Right path+      _ -> Left (NixInvalidOutput "nix build did not return exactly one store path")++nixEvalJson :: (FromJSON value) => String -> IO (Either NixError value)+nixEvalJson reference = do+  result <- runCommand defaultTools.nixExecutable ["eval", "--json", reference]+  pure do+    output <- result+    firstDecode (eitherDecodeStrict' (TextEncoding.encodeUtf8 output))++closureInfo :: FilePath -> IO (Either NixError ClosureInfo)+closureInfo = closureInfoWith defaultTools++closureInfoWith :: NixTools -> FilePath -> IO (Either NixError ClosureInfo)+closureInfoWith tools path = do+  result <- runCommand tools.nixExecutable ["path-info", "--json", "--recursive", path]+  pure do+    output <- result+    value <- firstDecode (eitherDecodeStrict' (TextEncoding.encodeUtf8 output))+    parseClosureInfo path value++parseClosureInfo :: FilePath -> Value -> Either NixError ClosureInfo+parseClosureInfo path (Object paths) = do+  root <- maybe (Left (NixInvalidOutput "closure metadata omits the requested store path")) Right (KeyMap.lookup (Key.fromText (Text.pack path)) paths)+  hash <- case root of+    Object fields -> case KeyMap.lookup "narHash" fields of+      Just (String value) | "sha256-" `Text.isPrefixOf` value -> Right value+      _ -> Left (NixInvalidOutput "closure metadata omits the root NAR hash")+    _ -> Left (NixInvalidOutput "closure metadata has a malformed root entry")+  let names = sort (map (Text.unpack . Key.toText . fst) (KeyMap.toList paths))+  if null names || any (not . Text.isPrefixOf "/nix/store/" . Text.pack) names+    then Left (NixInvalidOutput "closure metadata contains an invalid store path")+    else Right (ClosureInfo path hash names)+parseClosureInfo _ _ = Left (NixInvalidOutput "closure metadata must be a JSON object")++exportBundle :: FilePath -> FilePath -> IO (Either NixError BundleInfo)+exportBundle = exportBundleWith defaultTools++exportBundleWith :: NixTools -> FilePath -> FilePath -> IO (Either NixError BundleInfo)+exportBundleWith tools path destination = do+  closureResult <- closureInfoWith tools path+  case closureResult of+    Left problem -> pure (Left problem)+    Right closure -> do+      createDirectoryIfMissing True (takeDirectory destination)+      (temporary, handle) <- openBinaryTempFile (takeDirectory destination) ".kenshou-payload-"+      hClose handle+      exportResult <- try (exportToFile tools closure.closurePaths temporary) :: IO (Either IOException (Either NixError ()))+      case exportResult of+        Left problem -> do+          removeIfPresent temporary+          pure (Left (NixCommandIoError (Text.pack (displayException problem))))+        Right (Left problem) -> do+          removeIfPresent temporary+          pure (Left problem)+        Right (Right ()) -> do+          finished <- try @IOException do+            (sha256, size) <- digestFile temporary+            renameFile temporary destination+            pure (BundleInfo destination sha256 size closure)+          case finished of+            Left problem -> do+              removeIfPresent temporary+              pure (Left (NixCommandIoError (Text.pack (displayException problem))))+            Right bundle -> pure (Right bundle)++exportToFile :: NixTools -> [FilePath] -> FilePath -> IO (Either NixError ())+exportToFile tools paths destination = do+  let exportArgs = "--export" : paths+      compressArgs = ["-19", "-T1", "-q", "-f", "-o", destination]+  (_, exportOutput, _, exportProcess) <- createProcess (proc tools.nixStoreExecutable exportArgs) {std_out = CreatePipe}+  case exportOutput of+    Nothing -> pure (Left (NixInvalidOutput "nix-store export did not create a pipe"))+    Just pipe -> do+      compression <- try @IOException (createProcess (proc tools.zstdExecutable compressArgs) {std_in = UseHandle pipe})+      case compression of+        Left problem -> do+          hClose pipe+          terminateProcess exportProcess+          _ <- waitForProcess exportProcess+          pure (Left (NixCommandIoError (Text.pack (displayException problem))))+        Right (_, _, _, compressProcess) -> do+          hClose pipe+          compressExit <- waitForProcess compressProcess+          exportExit <- waitForProcess exportProcess+          pure case (exportExit, compressExit) of+            (ExitSuccess, ExitSuccess) -> Right ()+            (_, failure) | failure /= ExitSuccess -> Left (NixCommandFailed tools.zstdExecutable compressArgs failure "bundle compression failed")+            (failure, _) -> Left (NixCommandFailed tools.nixStoreExecutable exportArgs failure "closure export failed")++runCommand :: FilePath -> [String] -> IO (Either NixError Text)+runCommand executable arguments = do+  result <- try (readProcessWithExitCode executable arguments "") :: IO (Either IOException (ExitCode, String, String))+  pure case result of+    Left problem -> Left (NixCommandIoError (Text.pack (displayException problem)))+    Right (ExitSuccess, output, _) -> Right (Text.pack output)+    Right (failure, _, errors) -> Left (NixCommandFailed executable arguments failure (Text.pack errors))++firstDecode :: Either String value -> Either NixError value+firstDecode = either (Left . NixInvalidOutput . Text.pack) Right++removeIfPresent :: FilePath -> IO ()+removeIfPresent path = do+  exists <- doesFileExist path+  if exists then removeFile path else pure ()
+ src/Kenshou/Remote/Payload/Publish.hs view
@@ -0,0 +1,67 @@+module Kenshou.Remote.Payload.Publish+  ( BundlePublishError (..),+    digestFile,+    publishBundle,+  )+where++import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (eitherDecode, encode)+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import System.IO (IOMode (..), withBinaryFile)++data BundlePublishError+  = InvalidPayloadDescriptor !Text+  | BundleOutsideControlBucket !Text+  | BundleBytesMismatch !Int64 !Int64+  | BundleDigestMismatch !Text !Text+  | ExistingBundleSizeMismatch !Int64 !Int64+  deriving stock (Eq, Show)++-- The caller supplies a finished export file and descriptor. The object name is+-- derived from the checked digest, so a retry can only target the same key.+publishBundle :: ObjectStore -> Bucket -> PayloadDescriptor -> FilePath -> IO (Either BundlePublishError PayloadDescriptor)+publishBundle store bucket descriptor source = case eitherDecode (encode descriptor) :: Either String PayloadDescriptor of+  Left failure -> pure (Left (InvalidPayloadDescriptor (Text.pack failure)))+  Right _ -> do+    let bundle = descriptor.cell.bundle+        expectedUri = "gs://" <> bucket.unBucket <> "/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"+        object = ObjectName ("payloads/sha256/" <> bundle.sha256 <> ".nar.zst")+    if bundle.uri /= expectedUri+      then pure (Left (BundleOutsideControlBucket bundle.uri))+      else do+        (actualDigest, actualBytes) <- digestFile source+        if actualBytes /= bundle.bytes+          then pure (Left (BundleBytesMismatch bundle.bytes actualBytes))+          else+            if actualDigest /= bundle.sha256+              then pure (Left (BundleDigestMismatch bundle.sha256 actualDigest))+              else do+                uploaded <- store.putFile bucket object "application/zstd" DoesNotExist source+                case uploaded of+                  Written meta -> pure (checkSize bundle.bytes meta.size)+                  PreconditionFailed -> do+                    existing <- store.statObject bucket object+                    pure case existing of+                      Just meta -> checkSize bundle.bytes meta.size+                      Nothing -> Left (ExistingBundleSizeMismatch bundle.bytes (-1))+  where+    checkSize expected actual+      | actual == expected = Right descriptor+      | otherwise = Left (ExistingBundleSizeMismatch expected actual)++digestFile :: FilePath -> IO (Text, Int64)+digestFile path = withBinaryFile path ReadMode (go SHA256.init 0)+  where+    go context size handle = do+      chunk <- ByteString.hGetSome handle (1024 * 1024)+      if ByteString.null chunk+        then pure (TextEncoding.decodeUtf8 (Base16.encode (SHA256.finalize context)), size)+        else go (SHA256.update context chunk) (size + fromIntegral (ByteString.length chunk)) handle
+ src/Kenshou/Remote/Payload/Publisher.hs view
@@ -0,0 +1,250 @@+module Kenshou.Remote.Payload.Publisher+  ( PublishOptions (..),+    PreparedPublishOptions (..),+    PublishError (..),+    PublisherDeps (..),+    publishPayload,+    publishPayloadWith,+    publishPreparedPayload,+    publishPreparedPayloadWith,+    validateCohortIdentity,+  )+where++import Control.Exception (IOException, displayException, finally, try)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (eitherDecodeStrict', encode)+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Time (getCurrentTime)+import Kenshou.Core.Cohort (CohortDescriptor (..), CohortIdentity (..), CohortName (..), ComponentSpec (..), checkCohort)+import Kenshou.Perf.Request (immutableRevision)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), CohortCheck (..), Harness (..), PayloadDescriptor (..))+import Kenshou.Remote.Payload.Nix (BundleInfo (..), ClosureInfo (..), NixError (..), exportBundle, nixBuild, nixEvalJson)+import Kenshou.Remote.Payload.Publish (BundlePublishError, publishBundle)+import Kenshou.Remote.Store (Bucket (..), ObjectStore)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.Exit (ExitCode (..))+import System.FilePath (takeDirectory, (</>))+import System.IO (hClose, openBinaryTempFile)+import System.Process (readProcessWithExitCode)++data PublishOptions = PublishOptions+  { root :: !FilePath,+    cohort :: !Text,+    variant :: !Text,+    bucket :: !Bucket,+    out :: !FilePath,+    allowDirty :: !Bool+  }+  deriving stock (Eq, Show)++data PreparedPublishOptions = PreparedPublishOptions+  { harnessRoot :: !FilePath,+    harnessCommit :: !Text,+    generatedFlake :: !FilePath,+    generatedDescriptor :: !FilePath,+    selectedCohort :: !Text,+    selectedVariant :: !Text,+    controlBucket :: !Bucket,+    output :: !FilePath+  }+  deriving stock (Eq, Show)++data PublishError+  = PublishInvalidSelection !Text+  | PublishDirtyWorktree+  | PublishDescriptorError !Text+  | PublishCohortMismatch !Text+  | PublishNixError !NixError+  | PublishBundleError !BundlePublishError+  | PublishIoError !Text+  deriving stock (Eq, Show)++data PublisherDeps = PublisherDeps+  { readHarness :: FilePath -> IO (Either PublishError Harness),+    evalIdentity :: String -> IO (Either NixError CohortIdentity),+    build :: String -> IO (Either NixError FilePath),+    export :: FilePath -> FilePath -> IO (Either NixError BundleInfo)+  }++defaultDeps :: PublisherDeps+defaultDeps = PublisherDeps gitHarness nixEvalJson nixBuild exportBundle++publishPayload :: ObjectStore -> PublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPayload = publishPayloadWith defaultDeps++publishPayloadWith :: PublisherDeps -> ObjectStore -> PublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPayloadWith deps store options = case flakeAttribute options.cohort options.variant of+  Left problem -> pure (Left (PublishInvalidSelection problem))+  Right attribute ->+    publishAt+      deps+      store+      options+      attribute+      (options.root <> "#" <> Text.unpack attribute)+      (options.root </> "cohort" </> Text.unpack options.cohort <> ".json")++publishPreparedPayload :: ObjectStore -> PreparedPublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPreparedPayload = publishPreparedPayloadWith defaultDeps++publishPreparedPayloadWith :: PublisherDeps -> ObjectStore -> PreparedPublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPreparedPayloadWith deps store prepared+  | prepared.selectedVariant `notElem` ["default", "info-table", "profiled"] = pure (Left (PublishInvalidSelection "invalid prepared variant"))+  | not ("perf-" `Text.isPrefixOf` prepared.selectedCohort && Text.all (\c -> c `elem` ['a' .. 'z'] || c `elem` ['0' .. '9'] || c == '-') prepared.selectedCohort) = pure (Left (PublishInvalidSelection "prepared cohort must use its generated perf name"))+  | not (immutableRevision prepared.harnessCommit) = pure (Left (PublishInvalidSelection "prepared harness commit must be immutable"))+  | otherwise = do+      harness <- deps.readHarness prepared.harnessRoot+      case harness of+        Left problem -> pure (Left problem)+        Right actual+          | actual.dirty -> pure (Left PublishDirtyWorktree)+          | actual.revision /= prepared.harnessCommit -> pure (Left (PublishDescriptorError "prepared harness commit differs from the clean build root"))+          | otherwise ->+              publishAt+                deps+                store+                (PublishOptions prepared.harnessRoot prepared.selectedCohort prepared.selectedVariant prepared.controlBucket prepared.output False)+                (preparedAttribute prepared)+                (prepared.generatedFlake <> "#" <> Text.unpack (preparedAttribute prepared))+                prepared.generatedDescriptor++preparedAttribute :: PreparedPublishOptions -> Text+preparedAttribute prepared =+  "packages.x86_64-linux.kenshou-" <> prepared.selectedCohort <> case prepared.selectedVariant of+    "default" -> ""+    other -> "-" <> other++publishAt :: PublisherDeps -> ObjectStore -> PublishOptions -> Text -> String -> FilePath -> IO (Either PublishError PayloadDescriptor)+publishAt deps store options attribute flake descriptorPath = do+  harnessResult <- deps.readHarness options.root+  case harnessResult of+    Left problem -> pure (Left problem)+    Right harness+      | harness.dirty && not options.allowDirty -> pure (Left PublishDirtyWorktree)+      | otherwise -> do+          descriptorResult <- readDescriptor descriptorPath+          case descriptorResult of+            Left problem -> pure (Left problem)+            Right (descriptor, _) | descriptor.descriptorName /= CohortName options.cohort -> pure (Left (PublishDescriptorError "selected descriptor names another cohort"))+            Right (descriptor, descriptorSha) -> do+              identityResult <- deps.evalIdentity (flake <> ".cohortIdentity")+              case identityResult of+                Left problem -> pure (Left (PublishNixError problem))+                Right identity -> case validateCohortIdentity descriptor descriptorSha identity of+                  Left problem -> pure (Left (PublishCohortMismatch problem))+                  Right checked -> do+                    built <- deps.build flake+                    case built of+                      Left problem -> pure (Left (PublishNixError problem))+                      Right storePath -> withBundleFile options.out \bundlePath -> do+                        exported <- deps.export storePath bundlePath+                        case exported of+                          Left problem -> pure (Left (PublishNixError problem))+                          Right bundle | bundle.closure.storePath /= storePath -> pure (Left (PublishNixError (NixInvalidOutput "exported closure root differs from the built store path")))+                          Right bundle -> do+                            now <- getCurrentTime+                            let digest = bundle.sha256+                                object = "payloads/sha256/" <> digest <> ".nar.zst"+                                payload =+                                  PayloadDescriptor+                                    options.cohort+                                    options.variant+                                    attribute+                                    harness+                                    identity+                                    (CohortCheck checked)+                                    ( CellPayload+                                        (Bundle ("gs://" <> options.bucket.unBucket <> "/" <> object) digest bundle.bytes)+                                        (Text.pack storePath)+                                        bundle.closure.narHash+                                        (map Text.pack bundle.closure.closurePaths)+                                        "x86_64-linux"+                                        ["bin/kenshou", "cell", "exec"]+                                    )+                                    now+                            uploaded <- publishBundle store options.bucket payload bundle.file+                            case uploaded of+                              Left problem -> pure (Left (PublishBundleError problem))+                              Right published -> do+                                written <- writeDescriptor options.out published+                                pure (published <$ written)++validateCohortIdentity :: CohortDescriptor -> Text -> CohortIdentity -> Either Text Int+validateCohortIdentity descriptor descriptorSha identity+  | identity.identityDescriptorSha256 /= descriptorSha = Left "Nix identity descriptor digest differs from the selected descriptor"+  | identity.identityResolver /= Just "nix" = Left "payload identity must use the Nix resolver"+  | identity.identityOs /= "linux" || identity.identityArch /= "x86_64" = Left "payload identity must target x86_64-linux"+  | identity.identityIndexState /= Just descriptor.descriptorIndexState = Left "Nix identity index state differs from the selected descriptor"+  | not (null mismatches) = Left (Text.pack (show mismatches))+  | otherwise = Right (sum (map (length . (.componentPackages)) descriptor.descriptorComponents))+  where+    mismatches = checkCohort descriptor identity++flakeAttribute :: Text -> Text -> Either Text Text+flakeAttribute cohort variant+  | cohort `notElem` ["released", "head"] = Left "--cohort must be released or head"+  | variant `notElem` ["default", "info-table", "profiled"] = Left "--variant must be default, info-table or profiled"+  | otherwise = Right ("packages.x86_64-linux.kenshou-" <> cohort <> if variant == "default" then "" else "-" <> variant)++readDescriptor :: FilePath -> IO (Either PublishError (CohortDescriptor, Text))+readDescriptor path = do+  contents <- try @IOException (ByteString.readFile path)+  pure case contents of+    Left problem -> Left (PublishIoError (Text.pack (displayException problem)))+    Right bytes -> case eitherDecodeStrict' bytes of+      Left problem -> Left (PublishDescriptorError (Text.pack problem))+      Right descriptor -> Right (descriptor, TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash bytes)))++gitHarness :: FilePath -> IO (Either PublishError Harness)+gitHarness path = do+  revision <- git ["-C", path, "rev-parse", "HEAD"]+  status <- git ["-C", path, "status", "--porcelain"]+  pure do+    full <- Text.strip <$> revision+    changed <- status+    if Text.length full /= 40 || not (Text.all isHex full)+      then Left (PublishDescriptorError "git did not return a full commit revision")+      else Right (Harness full (not (Text.null changed)))+  where+    isHex character = character `elem` ['0' .. '9'] || character `elem` ['a' .. 'f']++git :: [String] -> IO (Either PublishError Text)+git arguments = do+  result <- try @IOException (readProcessWithExitCode "git" arguments "")+  pure case result of+    Left problem -> Left (PublishIoError (Text.pack (displayException problem)))+    Right (ExitSuccess, output, _) -> Right (Text.pack output)+    Right (_, _, errors) -> Left (PublishIoError (Text.pack errors))++withBundleFile :: FilePath -> (FilePath -> IO result) -> IO result+withBundleFile destination operation = do+  let directory = takeDirectory destination+  createDirectoryIfMissing True directory+  (temporary, handle) <- openBinaryTempFile directory ".kenshou-export-"+  hClose handle+  operation temporary `finally` removeIfPresent temporary++writeDescriptor :: FilePath -> PayloadDescriptor -> IO (Either PublishError ())+writeDescriptor destination descriptor = do+  let directory = takeDirectory destination+  createDirectoryIfMissing True directory+  (temporary, handle) <- openBinaryTempFile directory ".kenshou-descriptor-"+  result <- try @IOException do+    LazyByteString.hPut handle (encode descriptor) `finally` hClose handle+    renameFile temporary destination+  case result of+    Left problem -> do+      removeIfPresent temporary+      pure (Left (PublishIoError (Text.pack (displayException problem))))+    Right () -> pure (Right ())++removeIfPresent :: FilePath -> IO ()+removeIfPresent path = do+  exists <- doesFileExist path+  if exists then removeFile path else pure ()
+ src/Kenshou/Remote/Perf/Attempt.hs view
@@ -0,0 +1,283 @@+module Kenshou.Remote.Perf.Attempt+  ( TrialExecutor,+    executeAttempt,+    verifyAttempt,+    verifyAttemptInputs,+  )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseEither)+import Data.List (find, nub, sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (renderRunId, renderScenarioId)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.RunSpec (ComparisonMembership (..), RunSpec (..))+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility (VaryingAxis)+import Kenshou.Measure.Compare.Policy (Policy)+import Kenshou.Measure.Perf+import Kenshou.Perf.Plan (PerfProfile (..), PlannedProfile (..))+import Kenshou.Perf.Result+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Remote.Perf.Journal (LeaseIdentity)+import Kenshou.Remote.Perf.Runner (ComparisonHooks (..))+import System.Directory (canonicalizePath, createDirectoryIfMissing, doesFileExist, renameFile)+import System.FilePath (isAbsolute, makeRelative, splitDirectories, (</>))++-- The adapter persists cell work before submission. Calling it again for the+-- same entry reconciles the existing low-level journal under the SAME lease.+-- It returns a verified nested run directory, retaining the outer cell tree.+type TrialExecutor = PlannedRun -> IO (Either Text FilePath)++data Observation = Observation PlannedRun (Maybe FilePath) [AdmissionProblem]++executeAttempt :: LeaseIdentity -> PlannedProfile -> RunPlan -> Policy -> NonEmpty VaryingAxis -> FilePath -> ComparisonHooks -> TrialExecutor -> IO ConfigurationResult+executeAttempt lease entry schedule policy axes directory callbacks execute = do+  createDirectoryIfMissing True directory+  let inputs = attemptInputs lease entry schedule+  immutable (directory </> "attempt.json") inputs+  validateSchedule entry schedule+  paths <- loadPaths directory+  validatePaths directory schedule paths+  failures <- loadFailures directory schedule paths+  atomicPaths directory paths+  collect paths failures [] [0 .. entry.profile.minimumPairs - 1] >>= continue entry.profile.minimumPairs+  where+    collect paths failures observations [] = pure (paths, failures, observations)+    collect paths failures observations (index : rest) = do+      let runs = filter (maybe False ((== index) . (.index)) . (.trial)) schedule.runs+      (changed, rejected, appended) <- runEntries paths failures runs+      collect changed rejected (observations <> appended) rest+    runEntries paths failures [] = pure (paths, failures, [])+    runEntries paths failures (run : rest) = do+      let identifier = renderRunId run.runId+      located <- case (Map.lookup identifier paths, Map.lookup identifier failures) of+        (Just relative, _) -> pure (Right (directory </> relative))+        (_, Just reason) -> pure (Left reason)+        _ -> execute run+      (changed, rejected, observation) <- case located of+        Left reason -> do+          let rejected = Map.insert identifier reason failures+          atomicJSON (directory </> "trial-failures.json") rejected+          pure (paths, rejected, Observation run Nothing [InvalidPerformanceEvidence reason])+        Right path -> do+          relative <- relativePath directory path+          let changed = Map.insert (renderRunId run.runId) relative paths+          atomicPaths directory changed+          observation <- admitTrial entry.profile run path+          mapM_ callbacks.recordRuntimeFailure (runtimeProblems observation)+          pure (changed, failures, observation)+      (final, finalFailures, appended) <- runEntries changed rejected rest+      pure (final, finalFailures, observation : appended)+    continue next (paths, failures, observations)+      | length (validPairs observations) >= entry.profile.minimumPairs = finish directory entry policy axes observations+      | next >= entry.profile.minimumPairs + entry.profile.replacementPairs = finish directory entry policy axes observations+      | otherwise = do+          callbacks.reserveReplacement next+          updated <- collect paths failures observations [next]+          continue (next + 1) updated++-- Replay uses only retained paths and rechecks every trial, including failures.+-- It never submits work or overwrites the original comparison bytes.+verifyAttempt :: PlannedProfile -> RunPlan -> Policy -> NonEmpty VaryingAxis -> FilePath -> ConfigurationResult -> IO Bool+verifyAttempt entry schedule policy axes directory retained = do+  validateSchedule entry schedule+  paths <- loadPaths directory+  validatePaths directory schedule paths+  failures <- loadFailures directory schedule paths+  observations <-+    traverse+      ( \run -> case Map.lookup (renderRunId run.runId) paths of+          Nothing -> pure (Observation run Nothing [InvalidPerformanceEvidence (Map.findWithDefault "trial has no retained result" (renderRunId run.runId) failures)])+          Just relative -> admitTrial entry.profile run (directory </> relative)+      )+      [run | run <- schedule.runs, Map.member (renderRunId run.runId) paths || Map.member (renderRunId run.runId) failures]+  recomputed <- finish directory entry policy axes observations+  pure (same recomputed retained)+  where+    same left right =+      left.profile == right.profile+        && left.workloadSha256 == right.workloadSha256+        && left.performanceVerdict == right.performanceVerdict+        && left.infrastructureState == right.infrastructureState+        && all (`elem` right.runtimeFailures) left.runtimeFailures+        && left.validPairs == right.validPairs+        && left.attempts == right.attempts+        && case (left.comparison, right.comparison) of+          (Just a, Just b) -> semanticComparison a == semanticComparison b+          (Nothing, Nothing) -> True+          _ -> False++verifyAttemptInputs :: LeaseIdentity -> PlannedProfile -> RunPlan -> FilePath -> IO ()+verifyAttemptInputs lease entry schedule directory = do+  validateSchedule entry schedule+  header <- eitherDecodeFileStrict' (directory </> "attempt.json") >>= either (ioError . userError) pure+  unless (header == attemptInputs lease entry schedule) (ioError (userError "retained attempt header differs from its journaled lease or pinned workload"))+  paths <- loadPaths directory+  validatePaths directory schedule paths+  _ <- loadFailures directory schedule paths+  pure ()++attemptInputs :: LeaseIdentity -> PlannedProfile -> RunPlan -> Value+attemptInputs lease entry schedule = object ["schema" .= ("kenshou.perf-attempt/v1" :: Text), "lease" .= lease, "workloadSha256" .= entry.workloadSha256, "plan" .= schedule]++admitTrial :: PerfProfile -> PlannedRun -> FilePath -> IO Observation+admitTrial profile run directory = do+  admitted <- verifyPerformanceRun profile directory+  result <- eitherDecodeFileStrict' (directory </> "run-result.json")+  spec <- eitherDecodeFileStrict' (directory </> "run-spec.json") :: IO (Either String RunSpec)+  let binding = do+        document <- either (Left . Text.pack) Right result+        effective <- either (Left . Text.pack) Right spec+        either (Left . Text.pack) Right (parseEither (withObject "runtime result" (.: "runId")) document) >>= \identifier ->+          unless (identifier == run.runId) (Left "retained result has a different run ID")+        unless+          ( effective.runId == Just run.runId+              && effective.scenario == run.spec.scenario+              && effective.seed == run.spec.seed+              && effective.scenarioRevision == Just profile.scenarioRevision+              && effective.phases == Just zeroPhases+              && effective.timeoutSeconds == Just profile.timeoutSeconds+              && knobs (toJSON effective) == knobs (toJSON run.spec)+              && Map.fromList effective.dimensions == Map.fromList run.spec.dimensions+          )+          (Left "retained runtime inputs differ from the pinned trial")+      problems = either id (const []) admitted <> either (\reason -> [InvalidPerformanceEvidence reason]) (const []) binding+  pure (Observation run (Just directory) problems)+  where+    knobs (Object fields) = KeyMap.lookup "knobs" fields+    knobs _ = Nothing++runtimeProblems :: Observation -> [Text]+runtimeProblems (Observation _ _ problems) = [reason | RuntimeWorkFailure reason <- problems]++validPairs :: [Observation] -> [Int]+validPairs observations = filter valid (sort (nub [trial.index | Observation run _ _ <- observations, Just trial <- [run.trial]]))+  where+    valid index = case [observation | observation@(Observation run _ _) <- observations, maybe False ((== index) . (.index)) run.trial] of+      [Observation a (Just _) ap, Observation b (Just _) bp] ->+        fmap (.arm) a.trial /= fmap (.arm) b.trial && all runtimeOnly (ap <> bp)+      _ -> False+    runtimeOnly (RuntimeWorkFailure _) = True+    runtimeOnly _ = False++finish :: FilePath -> PlannedProfile -> Policy -> NonEmpty VaryingAxis -> [Observation] -> IO ConfigurationResult+finish root entry policy axes observations = do+  let chosen = take entry.profile.minimumPairs (validPairs observations)+      path index arm = case find (\(Observation run _ _) -> maybe False (\trial -> trial.index == index && trial.arm == arm) run.trial) observations of+        Just (Observation _ (Just directory) _) -> directory+        _ -> error "valid pair lacks a retained arm"+      failures = nub (concatMap runtimeProblems observations)+      attempts = [object ["runId" .= run.runId, "path" .= fmap (makeRelative root) directory, "problems" .= map (Text.replace (Text.pack root) "." . Text.pack . show) problems] | Observation run directory problems <- observations]+      make verdict infrastructure comparison = ConfigurationResult entry.profile.name entry.workloadSha256 verdict infrastructure failures (length chosen) entry.profile.minimumPairs attempts comparison+  if length chosen < entry.profile.minimumPairs+    then pure (make "inconclusive" "failed" Nothing)+    else do+      compared <- compareRuns policy axes (map (`path` "baseline") chosen) (map (`path` "candidate") chosen)+      case compared of+        Left (CompareError _) -> pure (make "inconclusive" "failed" Nothing)+        Right comparison ->+          pure+            ( make+                (case comparison.verdict of VerdictPass -> "pass"; VerdictRegression -> "regression"; _ -> "inconclusive")+                (if comparison.verdict == VerdictInfrastructureFailure || length chosen < entry.profile.minimumPairs then "failed" else "healthy")+                (Just (toJSON comparison))+            )++validateSchedule :: PlannedProfile -> RunPlan -> IO ()+validateSchedule entry schedule = do+  let runs = schedule.runs+      identifiers = map (.runId) runs+      expected = entry.profile.minimumPairs + entry.profile.replacementPairs+      trials = [(trial.index, trial.arm) | run <- runs, Just trial <- [run.trial]]+      ordering = [(index, arm) | index <- [0 .. expected - 1], arm <- if even index then ["baseline", "candidate"] else ["candidate", "baseline"]]+      seeds = [nub [run.spec.seed | run <- runs, maybe False ((== index) . (.index)) run.trial] | index <- [0 .. expected - 1]]+  unless+    ( length runs == 2 * expected+        && nub identifiers == identifiers+        && trials == ordering+        && map (.ordinal) runs == [1 .. 2 * expected]+        && all ((== 1) . length) seeds+        && all (\run -> run.spec.runId == Just run.runId && maybe False ((== expected) . (.of_)) run.trial) runs+        && all memberMatches runs+    )+    (ioError (userError "attempt must contain the full fixed paired schedule"))+  unless (all matches runs) (ioError (userError "attempt workload differs from its pinned profile"))+  where+    memberMatches run = case (run.trial, run.spec.comparison) of+      (Just trial, Just membership) -> membership.group == trial.group && membership.arm == trial.arm && membership.trial == trial.index && membership.position == run.ordinal - 1+      _ -> False+    matches run =+      renderScenarioId run.spec.scenario == entry.profile.scenario+        && run.spec.scenarioRevision == Just entry.profile.scenarioRevision+        && run.spec.seed /= Nothing+        && run.spec.phases == Just zeroPhases+        && run.spec.timeoutSeconds == Just entry.profile.timeoutSeconds+        && Map.fromList run.spec.dimensions == entry.profile.dimensions+        && case toJSON run.spec of+          Object fields -> KeyMap.lookup "knobs" fields == Just (toJSON entry.profile.knobs)+          _ -> False++validatePaths :: FilePath -> RunPlan -> Map Text FilePath -> IO ()+validatePaths directory schedule paths = do+  unless+    (all (`elem` map (renderRunId . (.runId)) schedule.runs) (Map.keys paths))+    (ioError (userError "retained trial path names an unplanned run"))+  mapM_+    ( \relative -> do+        actual <- relativePath directory (directory </> relative)+        unless (actual == relative) (ioError (userError "retained trial path contains a directory alias"))+    )+    (Map.elems paths)++loadPaths :: FilePath -> IO (Map Text FilePath)+loadPaths directory = do+  let path = directory </> "trial-paths.json"+  exists <- doesFileExist path+  paths <- if exists then eitherDecodeFileStrict' path >>= either (ioError . userError) pure else pure Map.empty+  unless (all safe (Map.elems paths)) (ioError (userError "unsafe retained trial path"))+  pure paths+  where+    safe path = not (null path) && not (isAbsolute path) && all (`notElem` ["..", ".", ""]) (splitDirectories path)++loadFailures :: FilePath -> RunPlan -> Map Text FilePath -> IO (Map Text Text)+loadFailures directory schedule paths = do+  let path = directory </> "trial-failures.json"+  exists <- doesFileExist path+  failures <- if exists then eitherDecodeFileStrict' path >>= either (ioError . userError) pure else pure Map.empty+  unless (all (`elem` map (renderRunId . (.runId)) schedule.runs) (Map.keys failures) && all (`Map.notMember` paths) (Map.keys failures) && all (not . Text.null) (Map.elems failures)) (ioError (userError "retained trial failure has an unknown, conflicting or empty identity"))+  pure failures++relativePath :: FilePath -> FilePath -> IO FilePath+relativePath root path = do+  absoluteRoot <- canonicalizePath root+  absolutePath <- canonicalizePath path+  let relative = makeRelative absoluteRoot absolutePath+  unless (relative /= absolutePath && ".." `notElem` splitDirectories relative) (ioError (userError "trial evidence escaped its attempt directory"))+  pure relative++atomicPaths :: FilePath -> Map Text FilePath -> IO ()+atomicPaths directory paths = do+  atomicJSON (directory </> "trial-paths.json") paths++atomicJSON :: (ToJSON document) => FilePath -> document -> IO ()+atomicJSON destination document = do+  let temporary = destination <> ".next"+  encodeFile temporary document+  renameFile temporary destination++immutable :: FilePath -> Value -> IO ()+immutable path document = do+  exists <- doesFileExist path+  if exists+    then do+      retained <- eitherDecodeFileStrict' path >>= either (ioError . userError) pure+      unless (retained == document) (ioError (userError "attempt lease, workload or schedule changed"))+    else atomicJSON path document
+ src/Kenshou/Remote/Perf/Cell.hs view
@@ -0,0 +1,167 @@+module Kenshou.Remote.Perf.Cell+  ( CellAdapter (..),+    executeCellTrial,+    verifyCellTrial,+    retainedRuntimeProblems,+  )+where++import Control.Concurrent (threadDelay)+import Control.Exception (IOException, catch, displayException)+import Control.Monad (unless)+import Data.Aeson (Value, eitherDecodeFileStrict', encode)+import Data.ByteString.Lazy qualified as Bytes+import Data.Map.Strict (Map)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (Registry)+import Kenshou.Core.Id (renderRunId)+import Kenshou.Core.RunSpec (EnvironmentSpec (..), RunSpec (..))+import Kenshou.Measure.Perf (AdmissionProblem (..), runtimeResultProblems, verifyPerformanceRun)+import Kenshou.Perf.Plan (PerfProfile)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Remote.Cell.Docs (CellBuckets (..), CellDescriptor (..), CellImages (..), CellNodes (..), CellOutcome (..), Requirements (..), ResetBlock (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..), EvidenceLink (..), RunLink (..), deriveCellRunIndex)+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseSnapshot, withHeartbeat)+import Kenshou.Remote.Cell.Prepare (cellMachineProfile)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities, RoutingRule)+import Kenshou.Remote.Cell.Session.Build+import Kenshou.Remote.Cell.Session.Journal+import Kenshou.Remote.Cell.Session.Resume+import Kenshou.Remote.Cell.Session.Runner+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.Watch (WatchEvent)+import Kenshou.Remote.Payload (PayloadDescriptor)+import Kenshou.Remote.Perf.Conditions (validateCellConditions, validateCellObservations)+import Kenshou.Remote.Store (Bucket (..), ObjectStore)+import System.Directory (createDirectoryIfMissing, doesFileExist)+import System.FilePath (takeDirectory, (</>))++data CellAdapter = CellAdapter+  { registry :: Registry,+    store :: ObjectStore,+    ref :: CellRef,+    descriptor :: CellDescriptor,+    healthPolicy :: Value,+    capabilities :: Maybe CellCapabilities,+    rules :: [RoutingRule],+    payloads :: Map Text PayloadDescriptor,+    options :: BuildOptions,+    storeLabel :: Text,+    emit :: WatchEvent -> IO ()+  }++-- Each trial has a separate low-level cell journal. Its work digest is checked+-- against the immutable paired schedule before any new or resumed submission.+executeCellTrial :: CellAdapter -> LeaseHandle -> RunPlan -> FilePath -> PerfProfile -> (Text -> IO ()) -> PlannedRun -> IO (Either Text FilePath)+executeCellTrial adapter handle schedule attempt profile observeRuntime run = do+  lease <- leaseSnapshot handle+  let directory = attempt </> "trials" </> Text.unpack (renderRunId run.runId)+      journalPath = directory </> "session.json"+      bytes = encode schedule {runs = [run], estimateMinutes = run.estimateMinutes}+      expected = (workObjectFor "application/json" bytes).sha256+      results = Bucket adapter.descriptor.buckets.results+  exists <- doesFileExist journalPath+  result <- withHeartbeat adapter.store adapter.ref handle \_ ->+    if exists+      then do+        saved <- readSessionJournal journalPath+        case saved of+          Left problem -> pure (Left problem)+          Right journal+            | journal.leaseId /= lease.leaseId || journal.planSha256 /= expected || journal.payloads /= adapter.payloads ->+                pure (Left "retained cell journal differs from the pinned trial, payloads or lease")+          Right _ -> continue results journalPath+      else do+        built <- buildSession adapter.registry adapter.descriptor adapter.capabilities adapter.rules adapter.payloads adapter.options adapter.storeLabel lease.leaseId bytes+        case built of+          Left problem -> pure (Left problem)+          Right session -> do+            createDirectoryIfMissing True directory+            Bytes.writeFile (directory </> "plan.json") bytes+            mapM_+              ( \(relative, content) -> do+                  createDirectoryIfMissing True (takeDirectory (directory </> relative))+                  Bytes.writeFile (directory </> relative) content+              )+              session.workFiles+            either (Left . Text.pack . show) Right <$> runPlannedSlices adapter.store adapter.ref results handle journalPath session.journal adapter.emit+  case result of+    Left problem -> pure (Left problem)+    Right journal -> do+      problems <- retainedRuntimeProblems profile run directory journal+      mapM_ observeRuntime problems+      verifyCellTrial adapter.healthPolicy adapter.descriptor run directory journal+  where+    continue results journalPath = do+      resumed <- resumeHeldSession adapter.store adapter.ref results handle journalPath adapter.emit+      case resumed of+        Left (ContinuationError (UnresolvedSlice _ SliceSubmitted)) -> threadDelay 2000000 >> continue results journalPath+        Left failure -> pure (Left (Text.pack (show failure)))+        Right journal -> pure (Right journal)++-- A bad outer health gate cannot erase a sealed nested runtime assertion or+-- failed/incomplete steady work. Reconstruct these before rejecting the pair.+retainedRuntimeProblems :: PerfProfile -> PlannedRun -> FilePath -> SessionJournal -> IO [Text]+retainedRuntimeProblems profile run directory journal = case journal.slices of+  [slice] | slice.state `elem` [SliceFetched, SliceVerified] -> do+    let tree = takeDirectory directory </> Text.unpack (renderRunId slice.cellRun) </> "tree"+    verified <- deriveCellRunIndex (Bucket journal.resultsBucket) Nothing tree+    case verified of+      Right index | index.cellRun == slice.cellRun && index.leaseId == journal.leaseId && index.cell == journal.cell && slice.manifestSha256 == Just index.cellManifest.sha256 && slice.cellOutcome == Just index.cellOutcome && any (\link -> link.runId == run.runId && link.path == "output/" <> renderRunId run.runId) index.runs -> do+        let nested = tree </> "output" </> Text.unpack (renderRunId run.runId)+        result <- eitherDecodeFileStrict' (nested </> "run-result.json")+        measured <- verifyPerformanceRun profile nested+        pure ([reason | RuntimeWorkFailure reason <- either (const []) runtimeResultProblems result <> either id (const []) measured])+      _ -> pure []+  _ -> pure []++-- Recheck the outer cell seal and health even when a low-level journal already+-- says verified. An exit-one runtime result is still returned for independent+-- runtime classification; failed cell health cannot become a valid trial.+verifyCellTrial :: Value -> CellDescriptor -> PlannedRun -> FilePath -> SessionJournal -> IO (Either Text FilePath)+verifyCellTrial healthPolicy descriptor run directory journal =+  catch (verifyTrial healthPolicy descriptor run directory journal) (\(problem :: IOException) -> pure (Left (Text.pack (displayException problem))))++verifyTrial :: Value -> CellDescriptor -> PlannedRun -> FilePath -> SessionJournal -> IO (Either Text FilePath)+verifyTrial healthPolicy descriptor run directory journal = case journal.slices of+  [slice] | slice.state == SliceVerified && slice.runIds == [run.runId] -> do+    let tree = takeDirectory directory </> Text.unpack (renderRunId slice.cellRun) </> "tree"+    verified <- deriveCellRunIndex (Bucket journal.resultsBucket) Nothing tree+    let evidencePresent = either (const False) (\index -> all (`elem` map (.kind) index.evidence) ["cell-health", "cell-reset-evidence"]) verified+    conditions <-+      if evidencePresent+        then do+          health <- eitherDecodeFileStrict' (tree </> "cell/health.json")+          reset <- eitherDecodeFileStrict' (tree </> "cell/reset-evidence.json")+          interval <- eitherDecodeFileStrict' (tree </> "cell/execution-window.json")+          let roles = [("driver", (case descriptor.instances.drivers of driver : _ -> driver; [] -> ""), descriptor.images.driver), ("postgres", descriptor.instances.postgres, descriptor.images.postgres), ("monitoring", descriptor.instances.monitoring, descriptor.images.monitoring)] <> case (slice.submission.reset.broker, descriptor.instances.broker) of (Just _, Just broker) -> [("broker", broker, descriptor.images.broker)]; _ -> []+          raw <- traverse (\(role, machine, image) -> do value <- eitherDecodeFileStrict' (tree </> "cell/observations" </> Text.unpack role <> ".json"); pure (role, machine, image, value)) roles+          let machines =+                descriptor.instances.drivers+                  <> [descriptor.instances.postgres, descriptor.instances.monitoring]+                  <> case (slice.submission.reset.broker, descriptor.instances.broker) of (Just _, Just broker) -> [broker]; _ -> []+          pure do+            healthDocument <- either (Left . Text.pack) Right health+            resetDocument <- either (Left . Text.pack) Right reset+            validateCellConditions slice.cellRun descriptor.name machines healthDocument resetDocument+            executionWindow <- either (Left . Text.pack) Right interval+            windows <- traverse (\(role, machine, image, value) -> (role,machine,image,) <$> either (Left . Text.pack) Right value) raw+            validateCellObservations slice.cellRun slice.submission.requires.minAgentVersion healthPolicy healthDocument executionWindow windows+        else pure (Left "outer cell is missing sealed health or reset evidence")+    effective <- eitherDecodeFileStrict' (tree </> "output" </> Text.unpack (renderRunId run.runId) </> "run-spec.json") :: IO (Either String RunSpec)+    pure do+      index <- either (Left . Text.pack . show) Right verified+      conditions+      spec <- either (Left . Text.pack) Right effective+      unless (spec.environment.machineProfile == Just (cellMachineProfile descriptor)) (Left "effective machine profile differs from the frozen cell descriptor")+      unless+        ( index.cellOutcome == Completed+            && index.leaseId == journal.leaseId+            && index.cell == journal.cell+            && index.cellRun == slice.cellRun+            && slice.manifestSha256 == Just index.cellManifest.sha256+        )+        (Left "outer cell identity, seal or health differs from its journal")+      pure (tree </> "output" </> Text.unpack (renderRunId run.runId))+  _ -> pure (Left "cell trial is incomplete, rejected or has unexpected runs")
+ src/Kenshou/Remote/Perf/Cleanup.hs view
@@ -0,0 +1,52 @@+module Kenshou.Remote.Perf.Cleanup (ownedWorkSettled, waitForOwnedWork) where++import Control.Concurrent (threadDelay)+import Control.Monad (unless)+import Data.Aeson (eitherDecode)+import Kenshou.Core.Id (renderRunId)+import Kenshou.Remote.Cell.Docs (CellPhase (Sealed), CellStatus (..), Rejected (..), Submission (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..), SliceState (..))+import Kenshou.Remote.Perf.Journal (LeaseIdentity (..))+import Kenshou.Remote.Store (ObjectName (..), ObjectStore (..))+import Kenshou.Remote.Store qualified as Store++-- Caller bounds this wait with the original cleanup deadline. Do not clear a+-- lease while an ambiguous or active submitted unit still belongs to it.+waitForOwnedWork :: ObjectStore -> CellRef -> LeaseIdentity -> [SessionJournal] -> IO ()+waitForOwnedWork store ref identity journals = do+  settled <- ownedWorkSettled store ref identity journals+  unless settled (threadDelay 1000000 >> waitForOwnedWork store ref identity journals)++ownedWorkSettled :: ObjectStore -> CellRef -> LeaseIdentity -> [SessionJournal] -> IO Bool+ownedWorkSettled store ref identity journals = do+  unless (ref.cellName == identity.cell) (ioError (userError "owned cleanup reference names another cell"))+  unless (all ((== refBucket) . (.controlBucket)) owned) (ioError (userError "owned cleanup journal names another control bucket"))+  and <$> traverse inspect slices+  where+    Store.Bucket refBucket = ref.controlBucket+    owned = [journal | journal <- journals, journal.cell == identity.cell, journal.leaseId == identity.leaseId]+    slices = concatMap (.slices) owned+    inspect slice = do+      let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/"+          get name = store.getObject ref.controlBucket (ObjectName (prefix <> name))+      marker <- get "submission.json"+      case marker of+        Nothing -> pure (slice.state == SlicePlanned)+        Just (bytes, _) -> do+          submission <- either (ioError . userError) pure (eitherDecode bytes)+          unless ((submission :: Submission).leaseId == identity.leaseId && submission == slice.submission) (ioError (userError "owned cleanup submission differs from its retained journal"))+          rejected <- get "rejected.json"+          case rejected of+            Just (body, _) -> do+              record <- either (ioError . userError) pure (eitherDecode body)+              unless ((record :: Rejected).runId == slice.cellRun) (ioError (userError "owned cleanup rejection names another run"))+              pure True+            Nothing -> do+              status <- get "status.json"+              case status of+                Nothing -> pure False+                Just (body, _) -> do+                  record <- either (ioError . userError) pure (eitherDecode body)+                  unless ((record :: CellStatus).runId == slice.cellRun) (ioError (userError "owned cleanup status names another run"))+                  pure (record.phase == Sealed && record.manifestSha256 /= Nothing && record.outcome /= Nothing)
+ src/Kenshou/Remote/Perf/Conditions.hs view
@@ -0,0 +1,180 @@+module Kenshou.Remote.Perf.Conditions (validateCellConditions, validateCellObservations, validateHealthPolicy) where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser, parseEither)+import Data.List (nub, sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, diffUTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Perf.Request (strictObject)++data Sample = Sample {at :: UTCTime, total :: Integer, idle :: Integer, steal :: Integer, disk :: Double}++-- Canonical algorithm: mori://shinzui/load-testing-infra, project-relative+-- nixos/pkgs/cell-agent/src/src/health.rs and runner.rs (artifact URIs pending).+-- The immutable execution interval supplies the actual payload start; nested+-- runtime timestamps and mutable status timestamps cannot substitute for it.+validateCellObservations :: RunId -> Text -> Value -> Value -> Value -> [(Text, Text, Text, Value)] -> Either Text ()+validateCellObservations runId agentVersion policy health interval windows = either (Left . Text.pack) Right do+  let roles = [role | (role, _, _, _) <- windows]+      machines = [machine | (_, machine, _, _) <- windows]+  unless (sort roles `elem` [sort ["driver", "postgres", "monitoring"], sort ["driver", "postgres", "monitoring", "broker"]] && nub machines == machines) (Left "raw observation role coverage is missing or duplicated")+  parseEither replay interval+  where+    replay = withObject "execution window" \v -> do+      strictObject ["schema", "runId", "resetStarted", "runStarted", "runFinished"] v+      schema <- v .: "schema"+      identifier <- v .: "runId"+      resetStarted <- v .: "resetStarted"+      started <- v .: "runStarted"+      finished <- v .: "runFinished"+      unless (schema == ("cell.execution-window/v1" :: Text) && identifier == runId && resetStarted <= addUTCTime (-10) started && started < finished) (fail "invalid sealed execution interval")+      (healthStart, healthEnd, gates) <- withObject "health report" (\h -> (,,) <$> h .: "windowStart" <*> h .: "windowEnd" <*> h .: "gates") health+      unless (healthStart == resetStarted && healthEnd == finished) (fail "health and execution intervals differ")+      limits <- healthLimits policy+      mapM_ (window resetStarted started finished limits gates) windows+    window resetStarted started finished (busyLimit, stealLimit, startLimit, runLimit) gates (role, machine, image, document) =+      withObject+        "raw machine observations"+        ( \v -> do+            strictObject ["schema", "machine", "image", "agentVersion", "systemClosure", "from", "to", "samples", "maintenanceEvents", "samplingErrors"] v+            schema <- v .: "schema"+            actualMachine <- v .: "machine"+            actualImage <- v .: "image"+            actualVersion <- v .: "agentVersion"+            closure <- v .: "systemClosure"+            from <- v .: "from"+            to <- v .: "to"+            maintenance <- v .: "maintenanceEvents" :: Parser [Value]+            errors <- v .: "samplingErrors" :: Parser [Value]+            unless (schema == ("cell.observations/v1" :: Text) && actualMachine == machine && actualImage == image && actualVersion == agentVersion && "/nix/store/" `Text.isPrefixOf` closure && Text.length closure > 11 && abs (diffUTCTime from resetStarted) <= 0.001 && abs (diffUTCTime to finished) <= 0.001 && null maintenance && null errors) (fail "raw machine identity, window, maintenance or sampling errors invalidate the trial")+            samples <- v .: "samples" >>= traverse sample+            unless (not (null samples) && all (\s -> s.at <= to) samples && and [a.at < b.at && diffUTCTime b.at a.at <= 2 && a.total < b.total && a.idle <= b.idle && a.steal <= b.steal | (a, b) <- zip samples (drop 1 samples)]) (fail "raw samples are missing, unordered, interrupted or have invalid counters")+            let settle = filter (\s -> s.at >= addUTCTime (-10) started && s.at <= started) samples+                running = filter (\s -> s.at >= started && s.at <= finished) samples+            (firstSettle, lastSettle) <- endpoints settle+            (firstRun, lastRun) <- endpoints running+            unless (diffUTCTime lastSettle.at firstSettle.at >= 8 && diffUTCTime firstRun.at started <= 2 && diffUTCTime finished lastRun.at <= 2) (fail "raw settle or run sample coverage is incomplete")+            busy <- fraction True firstSettle lastSettle+            steals <- traverse (\(prior, latest) -> fraction False (rolling prior latest) latest) [(take i running, latest) | (i, latest) <- zip [1 ..] (drop 1 running)]+            let settleBusy = if role `elem` ["monitoring", "broker"] then 0 else busy+                runSteal = if role == "monitoring" then 0 else maximum (0 : steals)+                startFree = firstRun.disk+                runFree = minimum (map (.disk) running)+            unless (settleBusy <= busyLimit && runSteal <= stealLimit && startFree >= startLimit && runFree >= runLimit) (fail "raw observations violate the frozen health policy")+            checkGate gates machine "background-load" (object ["settleCpuPercent" .= settleBusy, "runStealCpuPercent" .= runSteal]) (object ["settleCpuMaxPercent" .= busyLimit, "runStealCpuMaxPercent" .= stealLimit]) finished+            checkGate gates machine "storage-pressure" (object ["startFreePercent" .= startFree, "runFreePercent" .= runFree, "outputExceeded" .= False]) (object ["startFreeMinPercent" .= startLimit, "runFreeMinPercent" .= runLimit, "outputExceeded" .= False]) finished+        )+        document+    sample = withObject "health sample" \v -> do+      strictObject ["at", "cpuTotalTicks", "cpuIdleTicks", "cpuStealTicks", "diskFreePercent"] v+      value <- Sample <$> v .: "at" <*> v .: "cpuTotalTicks" <*> v .: "cpuIdleTicks" <*> v .: "cpuStealTicks" <*> v .: "diskFreePercent"+      unless (value.total >= 0 && value.idle >= 0 && value.steal >= 0 && value.idle + value.steal <= value.total && validPercent value.disk) (fail "invalid health counters or disk percentage")+      pure value+    endpoints [] = fail "missing health samples"+    endpoints (first : rest) = pure (first, foldl (\_ next -> next) first rest)+    rolling prior latest = case reverse (filter (\s -> diffUTCTime latest.at s.at >= 30) prior) of+      first : _ -> first+      [] -> case prior of first : _ -> first; [] -> latest+    fraction countIdle first latest = do+      let total = latest.total - first.total+          counter = if countIdle then latest.idle - first.idle else latest.steal - first.steal+      unless (total > 0 && counter >= 0 && counter <= total) (fail "CPU counters did not advance consistently")+      pure (100 * fromIntegral (if countIdle then total - counter else counter) / fromIntegral total :: Double)+    checkGate :: [Value] -> Text -> Text -> Value -> Value -> UTCTime -> Parser ()+    checkGate gates machine name observed threshold finished = do+      matching <- filter (\(m, n, _, _, _) -> m == machine && n == name) <$> traverse (withObject "gate" \v -> (,,,,) <$> v .: "machine" <*> v .: "name" <*> v .: "observed" <*> v .: "threshold" <*> v .: "at") gates+      case matching of+        [(_, _, actual, actualThreshold, at)] | near actual observed && actualThreshold == threshold && at == finished -> pure ()+        _ -> fail "reported health gate differs from raw observations or frozen policy"+    near (Number a) (Number b) = abs (realToFrac a - realToFrac b :: Double) <= 0.000001+    near (Object a) (Object b) = a == b || (length a == length b && and [maybe False (near value) (KeyMap.lookup key b) | (key, value) <- KeyMap.toList a])+    near a b = a == b++validateHealthPolicy :: Value -> Either Text ()+validateHealthPolicy = either (Left . Text.pack) (const (Right ())) . parseEither healthLimits++healthLimits :: Value -> Parser (Double, Double, Double, Double)+healthLimits = withObject "cell policy" \v -> do+  schema <- v .: "schema"+  unless (schema == ("cell.policy/v1" :: Text)) (fail "unsupported cell health policy")+  limits <- v .: "health" >>= withObject "health limits" (\h -> (,,,) <$> h .: "settleCpuMaxPercent" <*> h .: "runStealCpuMaxPercent" <*> h .: "startFreeDiskMinPercent" <*> h .: "runFreeDiskMinPercent")+  let (a, b, c, d) = limits+  unless (all validPercent [a, b, c, d]) (fail "invalid frozen health thresholds")+  pure limits++validPercent :: Double -> Bool+validPercent value = not (isNaN value || isInfinite value) && value >= 0 && value <= 100++-- Canonical owner: mori://shinzui/load-testing-infra, artifact URI pending;+-- project-relative nixos/pkgs/cell-agent/src/src/health.rs. Check the reported+-- observations against their thresholds, rather than trusting passed alone.+validateCellConditions :: RunId -> Text -> [Text] -> Value -> Value -> Either Text ()+validateCellConditions runId cell machines health reset = either (Left . Text.pack) Right do+  parseEither healthDocument health+  parseEither resetDocument reset+  where+    names = ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"]+    healthDocument = withObject "cell health" \v -> do+      schema <- v .: "schema"+      identifier <- v .: "runId"+      passed <- v .: "passed"+      start <- v .: "windowStart" :: Parser UTCTime+      end <- v .: "windowEnd" :: Parser UTCTime+      unless (schema == ("cell.health/v1" :: Text) && identifier == runId && passed && start <= end && not (null machines)) (fail "cell health identity, window or outcome is invalid")+      gates <- v .: "gates" >>= traverse (gate start end)+      unless (sort gates == sort [(machine, name) | machine <- machines, name <- names]) (fail "cell health gate coverage is missing, duplicated or unexpected")+    gate start end = withObject "cell health gate" \v -> do+      machine <- v .: "machine"+      name <- v .: "name"+      status <- v .: "status"+      at <- v .: "at" :: Parser UTCTime+      observed <- v .: "observed"+      threshold <- v .: "threshold"+      unless (status == ("ok" :: Text) && at >= start && at <= end) (fail "cell health gate tripped or lies outside its window")+      valid <- case name :: Text of+        "host-maintenance" -> pure (observed == String "NONE" && threshold == String "NONE")+        "incomplete-reset" -> pure (observed == Bool True && threshold == Bool True)+        "version-mismatch" -> pure (observed == Bool True && threshold == Bool True)+        "background-load" -> do+          busy <- percent "settleCpuPercent" observed+          steal <- percent "runStealCpuPercent" observed+          busyLimit <- percent "settleCpuMaxPercent" threshold+          stealLimit <- percent "runStealCpuMaxPercent" threshold+          pure (busy <= busyLimit && steal <= stealLimit)+        "storage-pressure" -> do+          initial <- percent "startFreePercent" observed+          final <- percent "runFreePercent" observed+          initialLimit <- percent "startFreeMinPercent" threshold+          finalLimit <- percent "runFreeMinPercent" threshold+          exceeded <- withObject "disk observation" (.: "outputExceeded") observed+          expectedExceeded <- withObject "disk threshold" (.: "outputExceeded") threshold+          pure (initial >= initialLimit && final >= finalLimit && not exceeded && not expectedExceeded)+        _ -> fail "unknown cell health gate"+      unless valid (fail "cell health observation violates its threshold")+      pure (machine, name)+    percent key = withObject "health percentage" \v -> do+      value <- v .: key :: Parser Double+      unless (not (isNaN value || isInfinite value) && value >= 0 && value <= 100) (fail "invalid cell health percentage")+      pure value+    resetDocument = withObject "cell reset" \v -> do+      schema <- v .: "schema"+      identifier <- v .: "runId"+      owner <- v .: "cell"+      cache <- v .: "cachePolicy"+      verified <- v .: "verified"+      start <- v .: "startedAt" :: Parser UTCTime+      end <- v .: "finishedAt" :: Parser UTCTime+      steps <- v .: "steps" :: Parser [Value]+      unless (schema == ("cell.reset-evidence/v1" :: Text) && identifier == runId && owner == cell && cache == ("cold" :: Text) && verified && start <= end && not (null steps)) (fail "cell reset identity, cache policy, window or verification is invalid")+      mapM_+        ( withObject "reset step" \step -> do+            status <- step .: "status"+            from <- step .: "startedAt" :: Parser UTCTime+            to <- step .: "finishedAt" :: Parser UTCTime+            unless (status == ("ok" :: Text) && from >= start && to <= end && from <= to) (fail "cell reset step failed or lies outside its window")+        )+        steps
+ src/Kenshou/Remote/Perf/Journal.hs view
@@ -0,0 +1,287 @@+module Kenshou.Remote.Perf.Journal+  ( PerfSession (..),+    SessionEvent (..),+    SessionAction (..),+    LeaseIdentity (..),+    JournalState (..),+    initialSession,+    journalState,+    appendEvent,+    readPerfSession,+    writePerfSession,+    validateSessionInputs,+  )+where++import Control.Exception (bracket)+import Control.Monad (foldM, unless)+import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString.Lazy qualified as ByteString+import Data.List (find, nub)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, diffUTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Perf.Contracts (PerfContracts)+import Kenshou.Perf.Plan+import Kenshou.Perf.Prepared qualified as Prepared+import Kenshou.Perf.Request (PerfRequest (..), strictObject)+import Kenshou.Perf.Result (ConfigurationResult (..))+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.FilePath (takeDirectory)+import System.IO (hClose, openBinaryTempFile)++data LeaseIdentity = LeaseIdentity {cell :: Text, leaseId :: RunId}+  deriving stock (Eq, Show)++data SessionAction+  = AdmissionReserved Int+  | LeaseClaimProposed Text LeaseIdentity+  | LeaseClaimWithdrawn+  | LeaseClaimReleased+  | LeaseAcquired LeaseIdentity+  | LeaseLost Text+  | LeaseReleased+  | ComparisonStarted Text Int+  | ReplacementReserved Text+  | ReplacementScheduled Text Int Int+  | RuntimeBlocked Text Text+  | ComparisonCompleted ConfigurationResult+  deriving stock (Eq, Show)++data SessionEvent = SessionEvent {at :: UTCTime, action :: SessionAction}+  deriving stock (Eq, Show)++-- Immutable header plus an append-only transition history. State is derived+-- from that history, so resume cannot silently reset allowances or a deadline.+data PerfSession = PerfSession+  { sessionId :: RunId,+    preparedSha256 :: Text,+    plan :: PerfPlan,+    evidenceBaseUri :: Text,+    createdAt :: UTCTime,+    events :: [SessionEvent],+    executionInputs :: Maybe Value+  }+  deriving stock (Eq, Show)++data JournalState = JournalState+  { admissionSpentSeconds :: Int,+    executionDeadline :: Maybe UTCTime,+    cleanupDeadline :: Maybe UTCTime,+    lease :: Maybe LeaseIdentity,+    pendingClaim :: Maybe (Text, LeaseIdentity, UTCTime),+    activeComparison :: Maybe (Text, Int),+    attempts :: Map Text Int,+    replacements :: Map Text Int,+    replacementIdentities :: [(Text, Int, Int)],+    completed :: Map Text ConfigurationResult,+    runtimeBlockers :: Map Text [Text],+    lastAt :: UTCTime+  }+  deriving stock (Eq, Show)++initialSession :: RunId -> Text -> PerfPlan -> Text -> UTCTime -> Either Text PerfSession+initialSession identifier digest plan uri now = do+  let (bucket, path) = Text.breakOn "/" (Text.drop 5 uri)+  unless (validDigest digest && "gs://" `Text.isPrefixOf` uri && not (Text.null bucket) && Text.length path > 1 && not (Text.any (`elem` ['\n', '\r', ' ', '?', '#']) uri)) (Left "invalid prepared digest or durable evidence destination")+  case fromJSON (toJSON plan) :: Result PerfPlan of+    Error problem -> Left (Text.pack problem)+    Success _ -> pure ()+  pure (PerfSession identifier digest plan uri now [] Nothing)++validateSessionInputs :: PerfContracts -> Prepared.PerfPrepared -> PerfSession -> Either Text ()+validateSessionInputs contracts prepared session = do+  Prepared.validatePrepared contracts prepared+  unless+    (session.preparedSha256 == Prepared.preparedDigest prepared && toJSON session.plan == toJSON prepared.plan)+    (Left "session inputs differ from the verified immutable preparation")+  _ <- journalState session+  pure ()++journalState :: PerfSession -> Either Text JournalState+journalState session = foldM (advance session) initial session.events+  where+    initial = JournalState 0 Nothing Nothing Nothing Nothing Nothing Map.empty Map.empty [] Map.empty Map.empty session.createdAt++appendEvent :: SessionEvent -> PerfSession -> Either Text PerfSession+appendEvent event session = do+  before <- journalState session+  _ <- advance session before event+  pure session {events = session.events <> [event]}++advance :: PerfSession -> JournalState -> SessionEvent -> Either Text JournalState+advance session state event = do+  unless (event.at >= state.lastAt) (Left "performance journal clock moved backwards")+  changed <- case event.action of+    AdmissionReserved seconds -> do+      unless (state.executionDeadline == Nothing && state.lease == Nothing && seconds > 0 && state.admissionSpentSeconds + seconds <= session.plan.request.waitSeconds) (Left "initial admission allowance exhausted or already measured")+      pure state {admissionSpentSeconds = state.admissionSpentSeconds + seconds}+    LeaseClaimProposed name identity -> do+      _ <- selected name+      measurementTime+      unless (state.lease == Nothing && state.pendingClaim == Nothing && identity.cell `elem` session.plan.request.cells && Map.notMember name state.completed) (Left "unexpected or overlapping proposed cell claim")+      pure state {pendingClaim = Just (name, identity, event.at)}+    LeaseClaimWithdrawn -> do+      unless (state.lease == Nothing && state.pendingClaim /= Nothing) (Left "no proposed cell claim to withdraw")+      pure state {pendingClaim = Nothing}+    LeaseClaimReleased -> case state.pendingClaim of+      Just (_, _, proposedAt) | state.lease == Nothing -> do+        let deadline = maybe (addUTCTime (fromIntegral (session.plan.request.budgetMinutes * 60)) proposedAt) id state.executionDeadline+        pure state {pendingClaim = Nothing, executionDeadline = Just deadline, cleanupDeadline = Just (addUTCTime 60 deadline)}+      _ -> Left "no proposed owned cell claim to release"+    LeaseAcquired identity -> do+      unless (state.lease == Nothing && identity.cell `elem` session.plan.request.cells) (Left "unexpected or overlapping cell lease")+      measurementTime+      proposedAt <- case state.pendingClaim of+        Just (_, proposed, at) | proposed == identity -> Right at+        Just _ -> Left "confirmed lease differs from its proposed identity"+        Nothing -> Right event.at -- retained pre-claim-journal sessions+      let deadline = maybe (addUTCTime (fromIntegral (session.plan.request.budgetMinutes * 60)) proposedAt) id state.executionDeadline+      unless (event.at <= deadline) (Left "confirmation exceeds the original proposed-claim deadline")+      pure state {lease = Just identity, pendingClaim = Nothing, executionDeadline = Just deadline, cleanupDeadline = Just (addUTCTime 60 deadline)}+    LeaseLost reason -> do+      unless (state.lease /= Nothing && not (Text.null reason)) (Left "cannot lose an absent lease without a diagnosis")+      -- Retained low-level journals belong to the abandoned attempt. The next+      -- comparison starts a full schedule and cannot pool those old pairs.+      pure state {lease = Nothing, activeComparison = Nothing}+    LeaseReleased -> do+      unless (state.lease /= Nothing && state.activeComparison == Nothing) (Left "cannot release a lease with unresolved active comparison")+      pure state {lease = Nothing}+    ComparisonStarted name index -> do+      entry <- selected name+      measurementTime+      let count = Map.findWithDefault 0 name state.attempts+          remaining = maybe 0 (floor . (`diffUTCTime` event.at)) state.executionDeadline :: Integer+          fixedCost = 2 * fromIntegral entry.profile.minimumPairs * fromIntegral (entry.profile.trialSeconds + entry.profile.resetSeconds + entry.profile.sealSeconds)+      unless (state.lease /= Nothing && state.activeComparison == Nothing && Map.notMember name state.completed && index == count && count <= entry.profile.replacementPairs && fixedCost <= remaining) (Left "comparison has no lease, is already complete, or exceeds persistent attempt/deadline budget")+      pure state {activeComparison = Just (name, index), attempts = Map.insert name (count + 1) state.attempts}+    ReplacementReserved name -> do+      entry <- selected name+      measurementTime+      let spent = Map.findWithDefault 0 name state.replacements+      unless (maybe False ((== name) . fst) state.activeComparison && spent < entry.profile.replacementPairs) (Left "infrastructure replacement allowance exhausted or wrong active configuration")+      pure state {replacements = Map.insert name (spent + 1) state.replacements}+    ReplacementScheduled name attempt pair -> do+      entry <- selected name+      measurementTime+      let spent = Map.findWithDefault 0 name state.replacements+      unless (state.activeComparison == Just (name, attempt) && pair >= entry.profile.minimumPairs && pair < entry.profile.minimumPairs + entry.profile.replacementPairs && spent < entry.profile.replacementPairs && (name, attempt, pair) `notElem` state.replacementIdentities) (Left "replacement identity is invalid, duplicated or exceeds its persistent allowance")+      pure state {replacements = Map.insert name (spent + 1) state.replacements, replacementIdentities = state.replacementIdentities <> [(name, attempt, pair)]}+    RuntimeBlocked name reason -> do+      _ <- selected name+      -- Recovery may discover a sealed observation after the client lost its+      -- lease or finished a comparison. It still belongs to the started profile.+      unless (Map.findWithDefault 0 name state.attempts > 0 && not (Text.null reason)) (Left "runtime blocker must name a previously started configuration and diagnosis")+      let blockers = nub (Map.findWithDefault [] name state.runtimeBlockers <> [reason])+      pure state {runtimeBlockers = Map.insert name blockers state.runtimeBlockers, completed = Map.adjust (\result -> result {runtimeFailures = nub (result.runtimeFailures <> [reason])}) name state.completed}+    ComparisonCompleted result -> do+      entry <- selected result.profile+      unless (maybe False ((== result.profile) . fst) state.activeComparison && result.workloadSha256 == entry.workloadSha256 && result.requiredPairs == entry.profile.minimumPairs && Map.notMember result.profile state.completed) (Left "completion differs from active pinned configuration")+      -- Sealing/retrieval may finish in cleanup grace; result assembly still+      -- reports deadline exhaustion and cannot admit more measurements.+      unless (maybe False (event.at <=) state.cleanupDeadline) (Left "comparison completion exceeded cleanup grace")+      case fromJSON (toJSON result) :: Result ConfigurationResult of+        Error problem -> Left (Text.pack problem)+        Success _ -> pure ()+      let retained = result {runtimeFailures = nub (Map.findWithDefault [] result.profile state.runtimeBlockers <> result.runtimeFailures)}+      pure state {activeComparison = Nothing, completed = Map.insert result.profile retained state.completed, runtimeBlockers = Map.insert result.profile retained.runtimeFailures state.runtimeBlockers}+  pure changed {lastAt = event.at}+  where+    selected name = maybe (Left "journal names an unselected configuration") Right (find ((== name) . (.name) . (.profile)) session.plan.selected)+    measurementTime = unless (maybe True (event.at <) state.executionDeadline) (Left "original performance deadline exhausted")++validDigest :: Text -> Bool+validDigest digest = Text.length digest == 71 && "sha256:" `Text.isPrefixOf` digest && Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) (Text.drop 7 digest)++readPerfSession :: FilePath -> IO (Either Text PerfSession)+readPerfSession path = either (Left . Text.pack) Right <$> eitherDecodeFileStrict' path++-- The caller holds the session's exclusive orchestration lock and writes this+-- transition before its corresponding side effect. Atomic replacement alone+-- does not serialize two clients.+writePerfSession :: FilePath -> PerfSession -> IO ()+writePerfSession path session = do+  either (ioError . userError . Text.unpack) (const (pure ())) (journalState session)+  present <- doesFileExist path+  if present+    then do+      previous <- readPerfSession path >>= either (ioError . userError . Text.unpack) pure+      unless (toJSON previous {events = []} == toJSON session {events = []}) (ioError (userError "immutable performance session header differs"))+      unless (toJSON previous.events == toJSON (take (length previous.events) session.events)) (ioError (userError "performance journal history is not an append-only continuation"))+    else pure ()+  let directory = takeDirectory path+  createDirectoryIfMissing True directory+  bracket (openBinaryTempFile directory ".kenshou-perf-session-") cleanup \(temporary, handle) -> do+    hClose handle+    ByteString.writeFile temporary (encode session)+    renameFile temporary path+  where+    cleanup (temporary, _) = do+      exists <- doesFileExist temporary+      if exists then removeFile temporary else pure ()++instance ToJSON LeaseIdentity where+  toJSON identity = object ["cell" .= identity.cell, "leaseId" .= identity.leaseId]++instance FromJSON LeaseIdentity where+  parseJSON = withObject "PerformanceLease" \v -> do+    strictObject ["cell", "leaseId"] v+    LeaseIdentity <$> v .: "cell" <*> v .: "leaseId"++instance ToJSON SessionAction where+  toJSON = \case+    AdmissionReserved seconds -> object ["kind" .= ("admission-reserved" :: Text), "seconds" .= seconds]+    LeaseClaimProposed name lease -> object ["kind" .= ("lease-claim-proposed" :: Text), "profile" .= name, "lease" .= lease]+    LeaseClaimWithdrawn -> object ["kind" .= ("lease-claim-withdrawn" :: Text)]+    LeaseClaimReleased -> object ["kind" .= ("lease-claim-released" :: Text)]+    LeaseAcquired lease -> object ["kind" .= ("lease-acquired" :: Text), "lease" .= lease]+    LeaseLost reason -> object ["kind" .= ("lease-lost" :: Text), "reason" .= reason]+    LeaseReleased -> object ["kind" .= ("lease-released" :: Text)]+    ComparisonStarted name index -> object ["kind" .= ("comparison-started" :: Text), "profile" .= name, "index" .= index]+    ReplacementReserved name -> object ["kind" .= ("replacement-reserved" :: Text), "profile" .= name]+    ReplacementScheduled name attempt pair -> object ["kind" .= ("replacement-scheduled" :: Text), "profile" .= name, "attempt" .= attempt, "pair" .= pair]+    RuntimeBlocked name reason -> object ["kind" .= ("runtime-blocked" :: Text), "profile" .= name, "reason" .= reason]+    ComparisonCompleted result -> object ["kind" .= ("comparison-completed" :: Text), "result" .= result]++instance FromJSON SessionAction where+  parseJSON = withObject "PerformanceAction" \v -> do+    kind <- v .: "kind" :: Parser Text+    case kind of+      "admission-reserved" -> strictObject ["kind", "seconds"] v >> AdmissionReserved <$> v .: "seconds"+      "lease-claim-proposed" -> strictObject ["kind", "profile", "lease"] v >> LeaseClaimProposed <$> v .: "profile" <*> v .: "lease"+      "lease-claim-withdrawn" -> strictObject ["kind"] v >> pure LeaseClaimWithdrawn+      "lease-claim-released" -> strictObject ["kind"] v >> pure LeaseClaimReleased+      "lease-acquired" -> strictObject ["kind", "lease"] v >> LeaseAcquired <$> v .: "lease"+      "lease-lost" -> strictObject ["kind", "reason"] v >> LeaseLost <$> v .: "reason"+      "lease-released" -> strictObject ["kind"] v >> pure LeaseReleased+      "comparison-started" -> strictObject ["kind", "profile", "index"] v >> ComparisonStarted <$> v .: "profile" <*> v .: "index"+      "replacement-reserved" -> strictObject ["kind", "profile"] v >> ReplacementReserved <$> v .: "profile"+      "replacement-scheduled" -> strictObject ["kind", "profile", "attempt", "pair"] v >> ReplacementScheduled <$> v .: "profile" <*> v .: "attempt" <*> v .: "pair"+      "runtime-blocked" -> strictObject ["kind", "profile", "reason"] v >> RuntimeBlocked <$> v .: "profile" <*> v .: "reason"+      "comparison-completed" -> strictObject ["kind", "result"] v >> ComparisonCompleted <$> v .: "result"+      _ -> fail "unknown performance journal action"++instance ToJSON SessionEvent where+  toJSON event = object ["at" .= event.at, "action" .= event.action]++instance FromJSON SessionEvent where+  parseJSON = withObject "PerformanceEvent" \v -> do+    strictObject ["at", "action"] v+    SessionEvent <$> v .: "at" <*> v .: "action"++instance ToJSON PerfSession where+  toJSON session = object ["schema" .= ("kenshou.perf-session/v1" :: Text), "sessionId" .= session.sessionId, "preparedSha256" .= session.preparedSha256, "plan" .= session.plan, "evidenceBaseUri" .= session.evidenceBaseUri, "createdAt" .= session.createdAt, "events" .= session.events, "executionInputs" .= session.executionInputs]++instance FromJSON PerfSession where+  parseJSON = withObject "PerfSession" \v -> do+    strictObject ["schema", "sessionId", "preparedSha256", "plan", "evidenceBaseUri", "createdAt", "events", "executionInputs"] v+    schema <- v .: "schema"+    unless (schema == ("kenshou.perf-session/v1" :: Text)) (fail "unsupported performance session schema")+    session <- PerfSession <$> v .: "sessionId" <*> v .: "preparedSha256" <*> v .: "plan" <*> v .: "evidenceBaseUri" <*> v .: "createdAt" <*> v .: "events" <*> v .:? "executionInputs"+    _ <- either (fail . Text.unpack) pure (initialSession session.sessionId session.preparedSha256 session.plan session.evidenceBaseUri session.createdAt)+    _ <- either (fail . Text.unpack) pure (journalState session)+    pure session
+ src/Kenshou/Remote/Perf/Lock.hs view
@@ -0,0 +1,33 @@+module Kenshou.Remote.Perf.Lock (withPerformanceLock) where++import Control.Concurrent.MVar+import Control.Exception (bracket, bracket_)+import Data.Map.Strict qualified as Map+import System.Directory (canonicalizePath)+import System.FilePath ((</>))+import System.IO (SeekMode (AbsoluteSeek))+import System.IO.Unsafe (unsafePerformIO)+import System.Posix.IO (LockRequest (WriteLock), OpenFileFlags (creat), OpenMode (ReadWrite), closeFd, defaultFileFlags, openFd, setLock)++-- POSIX record locks serialize processes, while the MVar closes their same-+-- process loophole. Neither lock leaves stale ownership after client death.+-- Canonical dependency owner: mori://haskell/unix (artifact URI pending),+-- project-relative System/Posix/IO/Common.hsc.+withPerformanceLock :: FilePath -> IO a -> IO a+withPerformanceLock directory action = do+  canonical <- canonicalizePath directory+  lock <- modifyMVar locks \known -> case Map.lookup canonical known of+    Just existing -> pure (known, existing)+    Nothing -> do+      created <- newMVar ()+      pure (Map.insert canonical created known, created)+  bracket_ (claim lock) (putMVar lock ()) $+    bracket (openFd (canonical </> ".client.lock") ReadWrite defaultFileFlags {creat = Just 0o600}) closeFd \fd -> do+      setLock fd (WriteLock, AbsoluteSeek, 0, 0)+      action+  where+    claim lock = tryTakeMVar lock >>= maybe (ioError (userError "performance session is already controlled by another client")) pure++{-# NOINLINE locks #-}+locks :: MVar (Map.Map FilePath (MVar ()))+locks = unsafePerformIO (newMVar Map.empty)
+ src/Kenshou/Remote/Perf/Publication.hs view
@@ -0,0 +1,95 @@+module Kenshou.Remote.Perf.Publication (publishEvidence, verifyPublication, retrieveEvidence) where++import Control.Exception (IOException, bracket, displayException, try)+import Control.Monad (unless)+import Data.Aeson (eitherDecodeFileStrict')+import Data.List (nub)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..))+import Kenshou.Perf.Artifacts (safePath, verifyPerfManifest)+import Kenshou.Remote.Payload.Publish (digestFile)+import Kenshou.Remote.Store+import System.Directory (createDirectoryIfMissing, getTemporaryDirectory, removeFile)+import System.FilePath (takeDirectory, (</>))+import System.IO (hClose, openBinaryTempFile)++-- Manifest is published last. Every object is create-only, and even existing+-- objects are retrieved and hashed. Successful upload metadata is insufficient.+publishEvidence :: ObjectStore -> Bucket -> Text -> FilePath -> IO (Either Text Text)+publishEvidence store bucket prefix directory = protected do+  validatePrefix prefix+  checked <- verifyPerfManifest directory >>= require+  checked `seq` pure ()+  files <- publicationFiles directory+  mapM_ publish files+  require =<< verifyPublication store bucket prefix directory+  pure ("gs://" <> bucket.unBucket <> "/" <> prefix <> "/manifest.json")+  where+    publish (relative, _, _, media) = do+      result <- store.putFile bucket (object relative) media DoesNotExist (directory </> relative)+      case result of+        Written _ -> pure ()+        PreconditionFailed -> pure ()+    object relative = ObjectName (prefix <> "/" <> Text.pack relative)++verifyPublication :: ObjectStore -> Bucket -> Text -> FilePath -> IO (Either Text ())+verifyPublication store bucket prefix directory = protected do+  validatePrefix prefix+  verifyPerfManifest directory >>= require+  files <- publicationFiles directory+  temporaryDirectory <- getTemporaryDirectory+  bracket (openBinaryTempFile temporaryDirectory "kenshou-perf-retrieval-") (removeFile . fst) \(temporary, handle) -> do+    hClose handle+    mapM_ (check temporary) files+  where+    check temporary (relative, expected, bytes, _) = do+      present <- store.downloadTo bucket (ObjectName (prefix <> "/" <> Text.pack relative)) temporary+      unless (present /= Nothing) (ioError (userError ("published artifact is missing: " <> relative)))+      (actual, actualBytes) <- digestFile temporary+      unless (actual == expected && toInteger actualBytes == bytes) (ioError (userError ("published artifact differs: " <> relative)))++-- Retrieve into a fresh caller-owned directory. Validate the complete inventory+-- before allowing any remote name to become a local output path.+retrieveEvidence :: ObjectStore -> Bucket -> Text -> Text -> FilePath -> IO (Either Text ())+retrieveEvidence store bucket prefix expectedManifest directory = protected do+  validatePrefix prefix+  createDirectoryIfMissing True directory+  fetch "manifest.json"+  (digest, _) <- digestFile (directory </> "manifest.json")+  unless ("sha256:" <> digest == expectedManifest) (ioError (userError "retrieved performance manifest differs from its receipt"))+  manifest <- eitherDecodeFileStrict' (directory </> "manifest.json") >>= either (ioError . userError) pure+  let paths = map (.path) (manifest :: Manifest).files+  unless (all safePath paths && "manifest.json" `notElem` paths && nub paths == paths && all ((>= 0) . (.bytes)) manifest.files) (ioError (userError "unsafe calibration report inventory"))+  mapM_ fetch paths+  verifyPerfManifest directory >>= require+  where+    fetch relative = do+      let path = directory </> relative+      createDirectoryIfMissing True (takeDirectory path)+      present <- store.downloadTo bucket (ObjectName (prefix <> "/" <> Text.pack relative)) path+      unless (present /= Nothing) (ioError (userError ("missing calibration report artifact: " <> relative)))++publicationFiles :: FilePath -> IO [(FilePath, Text, Integer, Text)]+publicationFiles directory = do+  manifest <- eitherDecodeFileStrict' (directory </> "manifest.json") >>= either (ioError . userError) pure+  (digest, bytes) <- digestFile (directory </> "manifest.json")+  pure+    ( [(file.path, Text.drop 7 file.sha256, file.bytes, file.mediaType) | file <- (manifest :: Manifest).files]+        <> [("manifest.json", digest, toInteger bytes, "application/json")]+    )++validatePrefix :: Text -> IO ()+validatePrefix prefix =+  unless+    ( not (Text.null prefix)+        && all (\part -> not (Text.null part) && part `notElem` [".", ".."]) (Text.splitOn "/" prefix)+        && not (Text.any (`elem` ['\\', '\n', '\r', '?', '#']) prefix)+    )+    (ioError (userError "unsafe performance publication prefix"))++require :: Either Text a -> IO a+require = either (ioError . userError . Text.unpack) pure++protected :: IO a -> IO (Either Text a)+protected action = either (Left . Text.pack . displayException) Right <$> try @IOException action
+ src/Kenshou/Remote/Perf/Runner.hs view
@@ -0,0 +1,226 @@+module Kenshou.Remote.Perf.Runner+  ( RunnerHooks (..),+    ComparisonHooks (..),+    Admission (..),+    AdmissionHooks (..),+    runPerformance,+  )+where++import Control.Exception (IOException, mask_, onException, try)+import Control.Monad (unless, when)+import Data.Aeson (toJSON)+import Data.IORef+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, diffUTCTime)+import Kenshou.Perf.Contracts (PerfContracts)+import Kenshou.Perf.Plan (PerfPlan (..), PerfProfile (..), PlannedProfile (..))+import Kenshou.Perf.Prepared qualified as Prepared+import Kenshou.Perf.Request (PerfRequest (..))+import Kenshou.Perf.Result+import Kenshou.Remote.Perf.Journal+import System.Timeout (timeout)++data Admission = Admitted LeaseIdentity | Occupied | AdmissionFailed Text+  deriving stock (Eq, Show)++-- Replacement and runtime observations are persisted before another trial can+-- be dispatched. The cell adapter also persists its low-level work/submission+-- journal before effects and reconciles that journal when resuming an attempt.+data ComparisonHooks = ComparisonHooks+  { reserveReplacement :: Int -> IO (),+    recordRuntimeFailure :: Text -> IO ()+  }++-- Persist the proposed ID before CAS; withdraw only on a definite non-claim.+-- An unresolved proposal is reconciled read-only on resume, never replaced.+data AdmissionHooks = AdmissionHooks {proposeClaim :: LeaseIdentity -> IO (), withdrawClaim :: IO ()}++data RunnerHooks = RunnerHooks+  { -- | The suite's performance contracts, against which the preparation is+    -- re-validated before any work resumes.+    contracts :: PerfContracts,+    clock :: IO UTCTime,+    admit :: PlannedProfile -> Int -> Maybe (Text, LeaseIdentity, UTCTime) -> AdmissionHooks -> IO Admission,+    ownsLease :: LeaseIdentity -> IO Bool,+    canRunConfiguration :: LeaseIdentity -> PlannedProfile -> IO Bool,+    compareConfiguration :: LeaseIdentity -> PlannedProfile -> Int -> Bool -> ComparisonHooks -> IO ConfigurationResult,+    verifyCompleted :: ConfigurationResult -> IO Bool,+    cancelOwnedWork :: LeaseIdentity -> IO (),+    releaseOwnedLease :: LeaseIdentity -> IO Bool,+    recoverRuntimeFailures :: LeaseIdentity -> IO [(Text, Text)],+    reportMetadata :: PerfSession -> PerfResult -> IO PerfResult,+    publishAndVerify :: PerfSession -> PerfResult -> IO Bool+  }++-- Caller holds the session's exclusive orchestration lock. All inputs have+-- already passed manifest and payload retrieval verification. Admission hooks+-- include their bounded waiting; measurement callbacks must never pool pairs+-- across different lease identities or extend the supplied session deadline.+runPerformance :: RunnerHooks -> Prepared.PerfPrepared -> FilePath -> IO PerfResult+runPerformance hooks prepared path = do+  loaded <- readPerfSession path >>= require+  require (validateSessionInputs hooks.contracts prepared loaded)+  retained <- require (journalState loaded)+  mapM_ verify (Map.elems retained.completed)+  current <- newIORef loaded+  let checkpoint action = mask_ do+        now <- hooks.clock+        before <- readIORef current+        changed <- require (appendEvent (SessionEvent now action) before)+        writePerfSession path changed+        writeIORef current changed+      state = readIORef current >>= require . journalState+      cleanup = do+        saved <- state+        let proposed = case saved.pendingClaim of Just (_, identity, _) -> Just identity; Nothing -> Nothing+        case (case saved.lease of Just identity -> Just identity; Nothing -> proposed) of+          Nothing -> pure ()+          Just identity -> do+            now <- hooks.clock+            let deadline = case saved.pendingClaim of Just (_, _, at) -> Just (addUTCTime (fromIntegral (prepared.plan.request.budgetMinutes * 60 + 60)) at); Nothing -> saved.cleanupDeadline+                micros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` now)) deadline+            released <- timeout micros (hooks.cancelOwnedWork identity >> hooks.releaseOwnedLease identity >>= \released -> when released (recover identity) >> pure released)+            case saved.lease of+              Just _ -> checkpoint (LeaseLost "client interrupted; owned cleanup requested")+              Nothing -> when (released == Just True) (checkpoint LeaseClaimReleased)+      recover identity = do+        faults <- hooks.recoverRuntimeFailures identity+        mapM_ (\(profile, reason) -> do observed <- state; unless (reason `elem` Map.findWithDefault [] profile observed.runtimeBlockers) (checkpoint (RuntimeBlocked profile reason))) faults+      bounded :: String -> IO value -> IO value+      bounded label action = do+        observed <- state+        started <- hooks.clock+        let micros = maybe 10000000 (min 10000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` started)) observed.executionDeadline+        timeout micros action >>= maybe (ioError (userError (label <> " exceeded its bounded transport allowance"))) pure+      execute = do+        saved <- state+        now <- hooks.clock+        session <- readIORef current+        case saved.lease of+          Just identity -> do+            owned <- bounded "lease ownership check" (hooks.ownsLease identity)+            if not owned+              then do+                released <- bounded "lost lease reconciliation" (hooks.releaseOwnedLease identity)+                unless released (ioError (userError "lost lease still has unsettled owned work"))+                bounded "sealed lost-lease observations" (recover identity)+                checkpoint (LeaseLost "lease ownership lost; owned work reconciled")+                execute+              else case saved.activeComparison of+                Just (name, index) -> do+                  entry <- require (selected session name)+                  runOne identity entry index True+                Nothing -> case filter (\entry -> Map.notMember entry.profile.name saved.completed) session.plan.selected of+                  [] -> release identity+                  entry : _ -> do+                    let index = Map.findWithDefault 0 entry.profile.name saved.attempts+                    compatible <- bounded "configuration compatibility check" (hooks.canRunConfiguration identity entry)+                    if not compatible+                      then release identity >> execute+                      else case appendEvent (SessionEvent now (ComparisonStarted entry.profile.name index)) session of+                        Left _ -> release identity+                        Right _ -> checkpoint (ComparisonStarted entry.profile.name index) >> runOne identity entry index False+          Nothing -> case filter (\entry -> Map.notMember entry.profile.name saved.completed) session.plan.selected of+            [] -> pure ()+            entry : _ -> do+              let admissionLeft = session.plan.request.waitSeconds - saved.admissionSpentSeconds+                  measured = saved.executionDeadline /= Nothing+                  enough = maybe True (\deadline -> cost entry <= floor (diffUTCTime deadline now)) saved.executionDeadline+                  initialAdmissionAt = case [event.at | event <- session.events, AdmissionReserved _ <- [event.action]] of first : _ -> first; [] -> now+                  wallLeft = max 0 (session.plan.request.waitSeconds - ceiling (diffUTCTime now initialAdmissionAt))+                  seconds = if measured then maybe 1 (min 10 . max 0 . floor . (`diffUTCTime` now)) saved.executionDeadline else case saved.pendingClaim of Just _ -> 10; Nothing -> min 10 (min admissionLeft wallLeft)+              if not enough || seconds <= 0+                then case saved.pendingClaim of Nothing -> pure (); Just _ -> ioError (userError "unresolved cell claim exceeds the original session budget")+                else do+                  unless (measured || saved.pendingClaim /= Nothing) (checkpoint (AdmissionReserved seconds))+                  let callbacks = AdmissionHooks (checkpoint . LeaseClaimProposed entry.profile.name) (checkpoint LeaseClaimWithdrawn)+                  granted <- timeout (seconds * 1000000) (hooks.admit entry seconds saved.pendingClaim callbacks)+                  receivedAt <- hooks.clock+                  observed <- state+                  admission <- case granted of+                    Nothing | observed.pendingClaim == Nothing -> pure Occupied+                    Nothing -> ioError (userError "cell admission transport exceeded its reserved allowance; proposed claim retained")+                    Just response -> do+                      when (receivedAt > addUTCTime (fromIntegral seconds) now) (ioError (userError "cell admission returned after its reserved allowance"))+                      pure response+                  case admission of+                    Admitted identity -> checkpoint (LeaseAcquired identity) >> execute+                    Occupied -> do+                      unless (observed.pendingClaim == Nothing) (ioError (userError "occupied admission left an unresolved claim"))+                      if measured then pure () else execute+                    AdmissionFailed problem -> ioError (userError (Text.unpack problem))+      runOne identity entry index resuming = do+        saved <- state+        now <- hooks.clock+        let micros = maybe 0 (max 0 . floor . (* 1000000) . (`diffUTCTime` now)) saved.executionDeadline+            reserve pair = do+              session <- readIORef current+              let action = ReplacementScheduled entry.profile.name index pair+              unless (action `elem` map (.action) session.events) (checkpoint action)+            blocked reason = do+              observed <- state+              unless (reason `elem` Map.findWithDefault [] entry.profile.name observed.runtimeBlockers) (checkpoint (RuntimeBlocked entry.profile.name reason))+            callbacks = ComparisonHooks reserve blocked+        result <- timeout micros (hooks.compareConfiguration identity entry index resuming callbacks)+        case result of+          Nothing -> do+            cleanupAt <- hooks.clock+            let cleanupMicros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` cleanupAt)) saved.cleanupDeadline+            _ <- timeout cleanupMicros (hooks.cancelOwnedWork identity)+            checkpoint (ComparisonCompleted (ConfigurationResult entry.profile.name entry.workloadSha256 "inconclusive" "pending" [] 0 entry.profile.minimumPairs [] Nothing))+            release identity+          Just comparison -> checkpoint (ComparisonCompleted comparison) >> execute+      release identity = do+        saved <- state+        now <- hooks.clock+        let micros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` now)) saved.cleanupDeadline+        released <- timeout micros (hooks.releaseOwnedLease identity >>= \released -> when released (recover identity) >> pure released)+        if released == Just True then checkpoint LeaseReleased else ioError (userError "owned lease release failed or exceeded cleanup grace")+  execution <- try @IOException (execute `onException` cleanup)+  session <- readIORef current+  saved <- require (journalState session)+  let unfinished entry =+        ConfigurationResult+          entry.profile.name+          entry.workloadSha256+          "not-run"+          (either (const "failed") (const "pending") execution)+          (Map.findWithDefault [] entry.profile.name saved.runtimeBlockers)+          0+          entry.profile.minimumPairs+          (map toJSON session.events)+          Nothing+      completed = [Map.findWithDefault (unfinished entry) entry.profile.name saved.completed | entry <- session.plan.selected]+  preliminary <- require (aggregateResult session.plan session.preparedSha256 (either (const False) (const True) execution) completed)+  publicationStarted <- hooks.clock+  let publicationMicros =+        maybe+          (session.plan.request.budgetMinutes * 60000000)+          (min (session.plan.request.budgetMinutes * 60000000) . max 0 . floor . (* 1000000) . (`diffUTCTime` publicationStarted))+          saved.executionDeadline+  publication <-+    try @IOException+      ( timeout publicationMicros do+          reported <- hooks.reportMetadata session preliminary+          require (validateResult reported)+          unless (reported {stageSeconds = Map.empty, artifactReferences = []} == preliminary) (ioError (userError "report metadata changed a performance decision"))+          published <- hooks.publishAndVerify session reported+          pure (reported, published)+      )+  let (reported, published) = either (const (preliminary, False)) (maybe (preliminary, False) id) publication+  verifiedAt <- hooks.clock+  let withinBudget = maybe True (verifiedAt <=) saved.executionDeadline+  final <- require (aggregateResult session.plan session.preparedSha256 (published && withinBudget && either (const False) (const True) execution) completed)+  pure final {stageSeconds = reported.stageSeconds, artifactReferences = reported.artifactReferences}+  where+    verify result = hooks.verifyCompleted result >>= \valid -> unless valid (ioError (userError "retained completed configuration failed evidence verification"))+    cost entry = 2 * entry.profile.minimumPairs * (entry.profile.trialSeconds + entry.profile.resetSeconds + entry.profile.sealSeconds)+    selected session name = case filter ((== name) . (.name) . (.profile)) session.plan.selected of+      [entry] -> Right entry+      _ -> Left "active comparison has no pinned profile"++require :: Either Text a -> IO a+require = either (ioError . userError . Text.unpack) pure
+ src/Kenshou/Remote/Selftest.hs view
@@ -0,0 +1,136 @@+module Kenshou.Remote.Selftest (bundle) where++import Control.Exception (SomeAsyncException, SomeException, bracket, fromException, throwIO, try)+import Data.Aeson (object, (.=))+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Context (RunContext (..), SummarySection (..), putSummary, requirePostgres)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), noDimensions, postgresDimensions)+import Kenshou.Core.Env (PostgresRequirement (..), noEnvironment)+import Kenshou.Core.Env qualified as Env+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot (..), PostgresEnv (..), PostgresMode (..))+import Kenshou.Core.Id (parseScenarioId, selftestLayer)+import Kenshou.Core.Knob (Allowed (..), KnobName, KnobSpec (..), KnobType (..), KnobValue (..), knobText, mkKnobName)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.RunSpec (EnvironmentSpec (..), SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Scenario (..), ScenarioReport, Tier (..), failedWith, passed)+import Kenshou.Core.Selftest.Schema (selftestSchema)+import Network.HTTP.Client (Request (..), RequestBody (..), httpLbs, newManager, parseRequest, responseStatus, responseTimeoutMicro)+import Network.HTTP.Client.TLS (tlsManagerSettings)+import Network.HTTP.Types.Status (statusCode)+import Network.Socket (SocketType (Stream), addrAddress, addrFamily, addrSocketType, close, connect, defaultHints, defaultProtocol, getAddrInfo, socket)+import System.Environment (lookupEnv)+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)+import System.Timeout (timeout)++bundle :: LayerBundle+bundle = mkBundle selftestLayer [cellEnvironment] []++cellEnvironment :: Scenario+cellEnvironment =+  Scenario+    { id = either (error . show) id (parseScenarioId "selftest/remote/correctness/cell-environment"),+      revision = 2,+      summary = "Checks the environment supplied to a local or cell run.",+      tier = TierSmoke,+      placement = PlaceEither,+      knobs =+        [ KnobSpec expectPlacement "Expected execution placement" KnobText (VText "any") (OneOf (VText "any" :| [VText "local", VText "cell"])) [],+          KnobSpec otlpEndpoint "Optional OTLP HTTP endpoint to check" KnobText (VText "") AnyValue [],+          KnobSpec tcpEndpoint "Optional host:port to check for TCP reachability" KnobText (VText "") AnyValue []+        ],+      dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) noDimensions,+      phases = zeroPhases,+      requires = noEnvironment {Env.postgres = Just (PostgresRequirement [selftestSchema] [] False)},+      knownDefect = Nothing,+      run = runCellEnvironment+    }++runCellEnvironment :: RunContext -> IO ScenarioReport+runCellEnvironment context = do+  cellId <- lookupEnv "CELL_RUN_ID"+  let postgres = requirePostgres context+      cell = context.environmentSpec.placement == RunOnCell+      requestedPlacement = knobText context.knobs expectPlacement+      placementOk = (requestedPlacement == "any" || requestedPlacement == if cell then "cell" else "local") && (cell == maybe False (not . null) cellId)+      modeOk = postgres.mode == if cell then PgExternal else PgEphemeral+      major = postgres.snapshot.serverVersionNum `div` 10000+      versionOk = major == case context.dimensions.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> 0+      durabilityOk = Map.lookup "fsync" postgres.snapshot.settings == Just (if context.dimensions.pgDurability == Just PgDurable then "on" else "off")+  fresh <- (== Right "0") <$> psql postgres.connectionString "SELECT count(*) FROM kenshou_selftest.roundtrip"+  otlp <- reachableOtlp (knobText context.knobs otlpEndpoint)+  tcp <- reachableTcp (knobText context.knobs tcpEndpoint)+  let checks =+        [ ("placement-as-expected", placementOk),+          ("postgres-mode-as-expected", modeOk),+          ("pg-version-honoured", versionOk),+          ("pg-durability-honoured", durabilityOk),+          ("database-is-fresh", fresh),+          ("otlp-reachable", otlp),+          ("tcp-endpoint-reachable", tcp)+        ]+      failures = [label | (label, False) <- checks]+  putSummary context Verdicts "cell-environment" (object ["checks" .= Map.fromList checks])+  pure (if null failures then passed else failedWith failures "cell environment differs from the declared run specification")++psql :: Text -> Text -> IO (Either Text Text)+psql connection query = do+  (code, output, errors) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+  pure case code of+    ExitSuccess -> Right (Text.strip (Text.pack output))+    _ -> Left (Text.strip (Text.pack errors))++reachableOtlp :: Text -> IO Bool+reachableOtlp endpoint+  | Text.null endpoint = pure True+  | otherwise = do+      result <- try @SomeException do+        request <- parseRequest (Text.unpack (Text.dropWhileEnd (== '/') endpoint <> "/v1/traces"))+        manager <- newManager tlsManagerSettings+        response <-+          httpLbs+            request+              { method = "POST",+                requestHeaders = [("Content-Type", "application/json")],+                requestBody = RequestBodyLBS "{\"resourceSpans\":[]}",+                responseTimeout = responseTimeoutMicro 2000000+              }+            manager+        pure (statusCode (responseStatus response) `elem` [200, 202, 204])+      recovered result++-- | Whether the first of a comma-separated list of host:port addresses accepts a+-- TCP connection within two seconds; an empty list is trivially reachable.+reachableTcp :: Text -> IO Bool+reachableTcp endpoints+  | Text.null endpoints = pure True+  | otherwise = do+      let firstAddress = Text.takeWhile (/= ',') endpoints+          (host, colonPort) = Text.breakOnEnd ":" firstAddress+      if Text.null host || Text.null colonPort+        then pure False+        else do+          result <- try @SomeException $ timeout 2000000 do+            addresses <- getAddrInfo (Just defaultHints {addrSocketType = Stream}) (Just (Text.unpack (Text.dropEnd 1 host))) (Just (Text.unpack colonPort))+            case addresses of+              [] -> pure False+              address : _ -> bracket (socket (addrFamily address) Stream defaultProtocol) close (\connection -> connect connection (addrAddress address) >> pure True)+          recovered (fmap (maybe False id) result)++recovered :: Either SomeException Bool -> IO Bool+recovered (Right available) = pure available+recovered (Left failure)+  | Just asynchronous <- fromException failure :: Maybe SomeAsyncException = throwIO asynchronous+  | otherwise = pure False++expectPlacement, otlpEndpoint, tcpEndpoint :: KnobName+expectPlacement = knobName "remote.expect-placement"+otlpEndpoint = knobName "remote.otlp-endpoint"+tcpEndpoint = knobName "remote.tcp-endpoint"++knobName :: Text -> KnobName+knobName = either (error . show) id . mkKnobName
+ src/Kenshou/Remote/Store.hs view
@@ -0,0 +1,43 @@+module Kenshou.Remote.Store+  ( Bucket (..),+    ObjectName (..),+    Precondition (..),+    ObjectMeta (..),+    PutOutcome (..),+    ObjectStore (..),+  )+where++import Data.ByteString.Lazy (ByteString)+import Data.Int (Int64)+import Data.Text (Text)+import Data.Time (UTCTime)++newtype Bucket = Bucket {unBucket :: Text} deriving stock (Eq, Ord, Show)++newtype ObjectName = ObjectName {unObjectName :: Text} deriving stock (Eq, Ord, Show)++data Precondition = NoPrecondition | DoesNotExist | GenerationIs !Int64+  deriving stock (Eq, Show)++data ObjectMeta = ObjectMeta+  { generation :: !Int64,+    size :: !Int64,+    updated :: !UTCTime,+    contentType :: !Text+  }+  deriving stock (Eq, Show)++data PutOutcome = Written !ObjectMeta | PreconditionFailed+  deriving stock (Eq, Show)++data ObjectStore = ObjectStore+  { getObject :: Bucket -> ObjectName -> IO (Maybe (ByteString, ObjectMeta)),+    statObject :: Bucket -> ObjectName -> IO (Maybe ObjectMeta),+    putObject :: Bucket -> ObjectName -> Text -> Precondition -> ByteString -> IO PutOutcome,+    putFile :: Bucket -> ObjectName -> Text -> Precondition -> FilePath -> IO PutOutcome,+    downloadTo :: Bucket -> ObjectName -> FilePath -> IO (Maybe ObjectMeta),+    deleteObject :: Bucket -> ObjectName -> Precondition -> IO Bool,+    listObjects :: Bucket -> Text -> IO [(ObjectName, ObjectMeta)],+    serverTime :: IO UTCTime+  }
+ src/Kenshou/Remote/Store/File.hs view
@@ -0,0 +1,187 @@+module Kenshou.Remote.Store.File (newFileStore) where++import Control.Concurrent.MVar (MVar, modifyMVar, newMVar)+import Control.Exception (bracket)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (FromJSON (..), ToJSON, eitherDecodeFileStrict', encodeFile, withObject, (.!=), (.:), (.:?))+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List (sortOn)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import GHC.Generics (Generic)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import System.Directory (copyFile, createDirectoryIfMissing, doesFileExist, getFileSize, listDirectory, renameFile)+import System.FilePath (takeDirectory, takeExtension, (</>))+import System.IO (SeekMode (AbsoluteSeek))+import System.IO.Unsafe (unsafePerformIO)+import System.Posix.IO (LockRequest (..), OpenFileFlags (creat), OpenMode (ReadWrite), closeFd, defaultFileFlags, openFd, waitToSetLock)++data StoredMeta = StoredMeta+  { name :: !Text,+    generation :: !Int64,+    size :: !Int64,+    updated :: !UTCTime,+    present :: !Bool,+    contentType :: !Text+  }+  deriving stock (Generic)+  deriving anyclass (ToJSON)++instance FromJSON StoredMeta where+  parseJSON = withObject "cell object metadata" \value ->+    StoredMeta+      <$> value .: "name"+      <*> value .: "generation"+      <*> value .: "size"+      <*> value .: "updated"+      <*> value .: "present"+      <*> (value .:? "contentType" .!= "application/octet-stream")++newFileStore :: FilePath -> IO ObjectStore+newFileStore root = do+  let localLock = processLock+  let stat bucket object = withObjectLock localLock root bucket object \bucketRoot hash -> fmap public <$> readMeta bucketRoot object hash+      get bucket object = withObjectLock localLock root bucket object \bucketRoot hash -> do+        current <- readMeta bucketRoot object hash+        case current of+          Nothing -> pure Nothing+          Just meta -> do+            bytes <- ByteString.readFile (dataPath bucketRoot hash meta.generation)+            pure (Just (LazyByteString.fromStrict bytes, public meta))+      write bucket object media pre writeData = withObjectLock localLock root bucket object \bucketRoot hash -> do+        current <- readStoredMeta bucketRoot object hash+        if not (preconditionHolds current pre)+          then pure PreconditionFailed+          else do+            now <- getCurrentTime+            let next = maybe 1 ((+ 1) . (.generation)) current+                destination = dataPath bucketRoot hash next+            createDirectoryIfMissing True (takeDirectory destination)+            bytes <- writeData destination+            let meta = StoredMeta object.unObjectName next bytes now True media+            writeMeta bucketRoot hash meta+            pure (Written (public meta))+      delete bucket object pre = withObjectLock localLock root bucket object \bucketRoot hash -> do+        current <- readStoredMeta bucketRoot object hash+        if not (preconditionHolds current pre)+          then pure False+          else case current of+            Nothing -> pure True+            Just prior -> do+              now <- getCurrentTime+              writeMeta bucketRoot hash prior {generation = prior.generation + 1, updated = now, present = False}+              pure True+      list bucket prefix = withBucketLock localLock root bucket \bucketRoot -> do+        let metaDirectory = bucketRoot </> "meta"+        createDirectoryIfMissing True metaDirectory+        files <- filter ((== ".json") . takeExtension) <$> listDirectory metaDirectory+        records <- mapM (readMetaFile . (metaDirectory </>)) files+        pure (sortOn fst [(ObjectName meta.name, public meta) | meta <- records, meta.present, prefix `Text.isPrefixOf` meta.name])+  pure+    ObjectStore+      { getObject = get,+        statObject = stat,+        putObject = \bucket object media pre bytes -> write bucket object media pre \destination -> do+          LazyByteString.writeFile destination bytes+          pure (LazyByteString.length bytes),+        putFile = \bucket object media pre source -> write bucket object media pre \destination -> do+          copyFile source destination+          fromIntegral <$> getFileSize destination,+        downloadTo = \bucket object destination -> withObjectLock localLock root bucket object \bucketRoot hash -> do+          current <- readMeta bucketRoot object hash+          case current of+            Nothing -> pure Nothing+            Just meta -> do+              createDirectoryIfMissing True (takeDirectory destination)+              copyFile (dataPath bucketRoot hash meta.generation) destination+              pure (Just (public meta)),+        deleteObject = delete,+        listObjects = list,+        serverTime = getCurrentTime+      }++-- POSIX record locks are per process, so separate handles in one process also+-- share a local mutex before taking the cross-process lock.+{-# NOINLINE processLock #-}+processLock :: MVar ()+processLock = unsafePerformIO (newMVar ())++withObjectLock :: MVar () -> FilePath -> Bucket -> ObjectName -> (FilePath -> FilePath -> IO a) -> IO a+withObjectLock localLock root bucket object action = withBucketLock localLock root bucket \bucketRoot -> do+  validateName object+  action bucketRoot (key object)++withBucketLock :: MVar () -> FilePath -> Bucket -> (FilePath -> IO a) -> IO a+withBucketLock localLock root bucket action = do+  bucketRoot <- bucketPath root bucket+  createDirectoryIfMissing True bucketRoot+  modifyMVar localLock \state -> do+    result <- bracket (openFd (bucketRoot </> ".lock") ReadWrite defaultFileFlags {creat = Just 0o600}) closeFd \fd -> do+      waitToSetLock fd (WriteLock, AbsoluteSeek, 0, 0)+      action bucketRoot+    pure (state, result)++bucketPath :: FilePath -> Bucket -> IO FilePath+bucketPath root bucket+  | not (Text.null bucket.unBucket) && Text.all valid bucket.unBucket = pure (root </> "buckets" </> Text.unpack bucket.unBucket)+  | otherwise = ioError (userError "invalid cell bucket name")+  where+    valid character = ('a' <= character && character <= 'z') || ('A' <= character && character <= 'Z') || ('0' <= character && character <= '9') || character `elem` ['-', '_']++validateName :: ObjectName -> IO ()+validateName object+  | Text.null value || Text.any (== '\\') value || any invalid (Text.splitOn "/" value) = ioError (userError "invalid cell object name")+  | otherwise = pure ()+  where+    value = object.unObjectName+    invalid part = Text.null part || part == "." || part == ".."++key :: ObjectName -> FilePath+key = Text.unpack . Text.decodeUtf8 . Base16.encode . SHA256.hash . Text.encodeUtf8 . (.unObjectName)++metaPath :: FilePath -> FilePath -> FilePath+metaPath bucketRoot hash = bucketRoot </> "meta" </> hash <> ".json"++dataPath :: FilePath -> FilePath -> Int64 -> FilePath+dataPath bucketRoot hash generation = bucketRoot </> "objects" </> hash </> show generation++readStoredMeta :: FilePath -> ObjectName -> FilePath -> IO (Maybe StoredMeta)+readStoredMeta bucketRoot object hash = do+  let path = metaPath bucketRoot hash+  exists <- doesFileExist path+  if not exists+    then pure Nothing+    else do+      meta <- readMetaFile path+      if meta.name == object.unObjectName+        then pure (Just meta)+        else ioError (userError "cell object metadata key collision")++readMeta :: FilePath -> ObjectName -> FilePath -> IO (Maybe StoredMeta)+readMeta bucketRoot object hash = do+  meta <- readStoredMeta bucketRoot object hash+  pure (meta >>= \record -> if record.present then Just record else Nothing)++readMetaFile :: FilePath -> IO StoredMeta+readMetaFile path = either (ioError . userError) pure =<< eitherDecodeFileStrict' path++writeMeta :: FilePath -> FilePath -> StoredMeta -> IO ()+writeMeta bucketRoot hash meta = do+  let destination = metaPath bucketRoot hash+      temporary = destination <> ".writing"+  createDirectoryIfMissing True (takeDirectory destination)+  encodeFile temporary meta+  renameFile temporary destination++public :: StoredMeta -> ObjectMeta+public meta = ObjectMeta meta.generation meta.size meta.updated meta.contentType++preconditionHolds :: Maybe StoredMeta -> Precondition -> Bool+preconditionHolds _ NoPrecondition = True+preconditionHolds current DoesNotExist = maybe True (not . (.present)) current+preconditionHolds current (GenerationIs expected) = maybe False (\meta -> meta.present && meta.generation == expected) current
+ src/Kenshou/Remote/Store/Gcs.hs view
@@ -0,0 +1,409 @@+module Kenshou.Remote.Store.Gcs+  ( TokenProvider (..),+    newTokenProvider,+    newTokenProviderWith,+    newGcsStore,+    newGcsStoreAt,+  )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.MVar (modifyMVar, modifyMVar_, newMVar)+import Control.Exception (bracket, try)+import Control.Monad (unless)+import Data.Aeson (FromJSON (..), decode, eitherDecode, withObject, (.:), (.:?))+import Data.ByteString qualified as ByteString+import Data.ByteString.Char8 qualified as ByteString.Char8+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Int (Int64)+import Data.Maybe (fromMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Time.Clock (addUTCTime, getCurrentTime)+import Data.Time.Format (defaultTimeLocale, parseTimeM)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Network.HTTP.Client (BodyReader, HttpException, Request (..), RequestBody (..), Response, brRead, httpLbs, newManager, parseRequest, responseBody, responseHeaders, responseStatus, responseTimeoutMicro, withResponse)+import Network.HTTP.Client.TLS (tlsManagerSettings)+import Network.HTTP.Types.Status (statusCode)+import Numeric (showHex)+import System.Directory (createDirectoryIfMissing, doesFileExist, getFileSize, removeFile, renameFile)+import System.Environment (lookupEnv)+import System.Exit (ExitCode (ExitSuccess))+import System.FilePath (takeDirectory)+import System.IO (Handle, IOMode (..), SeekMode (AbsoluteSeek), hClose, hIsClosed, hSeek, openBinaryTempFile, withBinaryFile)+import System.Process (readProcessWithExitCode)+import Text.Read (readMaybe)++data TokenProvider = TokenProvider+  { controlBucket :: !Bucket,+    accessToken :: IO Text,+    invalidateToken :: IO ()+  }++newTokenProvider :: Bucket -> IO TokenProvider+newTokenProvider bucket = do+  manager <- newManager tlsManagerSettings+  request <- parseRequest "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"+  newTokenProviderWith+    bucket+    (fmap Text.pack <$> lookupEnv "KENSHOU_GCS_TOKEN")+    ( do+        let metadataRequest = request {requestHeaders = [("Metadata-Flavor", "Google")], responseTimeout = responseTimeoutMicro 2000000}+        response <- try (httpLbs metadataRequest manager) :: IO (Either HttpException (Response LazyByteString.ByteString))+        pure case response of+          Right value | statusCode (responseStatus value) == 200 -> (\token -> token.tokenResponse) <$> (decode (responseBody value) :: Maybe MetadataToken)+          _ -> Nothing+    )+    ( do+        (exit, output, _) <- readProcessWithExitCode "gcloud" ["auth", "print-access-token"] ""+        unless (exit == ExitSuccess) (ioError (userError "gcloud access-token command failed"))+        pure (Text.strip (Text.pack output))+    )++-- | Injection seam for the three credential sources; only CLI tokens are cached.+newTokenProviderWith :: Bucket -> IO (Maybe Text) -> IO (Maybe Text) -> IO Text -> IO TokenProvider+newTokenProviderWith bucket environment metadata cli = do+  cache <- newMVar Nothing+  let choose = do+        environment >>= \case+          Just token | validToken token -> pure token+          Just _ -> ioError (userError "invalid GCS environment token")+          Nothing -> modifyMVar cache \cached -> do+            now <- getCurrentTime+            case cached of+              Just (expiry, token) | now < expiry -> pure (cached, token)+              _ ->+                metadata >>= \case+                  Just token | validToken token -> pure (Nothing, token)+                  Just _ -> ioError (userError "invalid GCS metadata token")+                  Nothing -> do+                    token <- cli+                    unless (validToken token) (ioError (userError "invalid gcloud access token"))+                    pure (Just (addUTCTime (45 * 60) now, token), token)+  pure (TokenProvider bucket choose (modifyMVar_ cache (const (pure Nothing))))++newtype MetadataToken = MetadataToken {tokenResponse :: Text}++instance FromJSON MetadataToken where+  parseJSON = withObject "GCE access token" \value -> MetadataToken <$> value .: "access_token"++data GcsObject = GcsObject+  { objectName :: !Text,+    objectMeta :: !ObjectMeta+  }++instance FromJSON GcsObject where+  parseJSON = withObject "GCS object" \value -> do+    name <- value .: "name"+    generationText <- value .: "generation"+    sizeText <- value .: "size"+    generation <- maybe (fail "invalid GCS generation") pure (readMaybe (Text.unpack generationText))+    size <- maybe (fail "invalid GCS size") pure (readMaybe (Text.unpack sizeText))+    updated <- value .: "updated"+    contentType <- fromMaybe "application/octet-stream" <$> value .:? "contentType"+    pure (GcsObject name (ObjectMeta generation size updated contentType))++data GcsPage = GcsPage+  { items :: ![GcsObject],+    nextPageToken :: !(Maybe Text)+  }++instance FromJSON GcsPage where+  parseJSON = withObject "GCS objects page" \value -> GcsPage <$> (fromMaybe [] <$> value .:? "items") <*> value .:? "nextPageToken"++newGcsStore :: TokenProvider -> IO ObjectStore+newGcsStore = newGcsStoreAt "https://storage.googleapis.com"++-- | The endpoint seam keeps the HTTP protocol testable against a local server.+newGcsStoreAt :: Text -> TokenProvider -> IO ObjectStore+newGcsStoreAt endpoint provider = do+  unless ("http://" `Text.isPrefixOf` endpoint || "https://" `Text.isPrefixOf` endpoint) (ioError (userError "invalid GCS endpoint"))+  manager <- newManager tlsManagerSettings+  let base = Text.dropWhileEnd (== '/') endpoint+      runRequest verb url body headers = do+        initial <- parseRequest (Text.unpack url)+        snd+          <$> withAuthorizedRetry+            provider+            ( \token -> do+                response <-+                  retryResponse $+                    httpLbs+                      initial+                        { method = verb,+                          requestHeaders = ("Authorization", "Bearer " <> TextEncoding.encodeUtf8 token) : headers,+                          requestBody = body+                        }+                      manager+                pure (statusCode (responseStatus response), response)+            )+      metadata bucket object = do+        url <- objectUrl base bucket object+        response <- runRequest "GET" url (RequestBodyBS ByteString.empty) []+        case statusCode (responseStatus response) of+          404 -> pure Nothing+          200 -> Just . (\item -> item.objectMeta) <$> decodeObject object (responseBody response)+          code -> badStatus "reading GCS metadata" code+      get bucket object = do+        current <- metadata bucket object+        case current of+          Nothing -> pure Nothing+          Just meta -> do+            url <- objectUrl base bucket object+            response <- runRequest "GET" (url <> "?alt=media&generation=" <> tshow meta.generation) (RequestBodyBS ByteString.empty) []+            case statusCode (responseStatus response) of+              200 -> do+                let bytes = responseBody response+                unless (LazyByteString.length bytes == meta.size) (ioError (userError "GCS media length differs from metadata"))+                pure (Just (bytes, meta))+              code -> badStatus "reading GCS media" code+      upload bucket object media pre body = do+        url <- uploadUrl base bucket object pre "media"+        response <- runRequest "POST" url body [("Content-Type", TextEncoding.encodeUtf8 media)]+        putResponse object response+      download bucket object destination = do+        current <- metadata bucket object+        case current of+          Nothing -> pure Nothing+          Just meta -> do+            url <- objectUrl base bucket object+            initial <- parseRequest (Text.unpack (url <> "?alt=media&generation=" <> tshow meta.generation))+            createDirectoryIfMissing True (takeDirectory destination)+            bracket+              (openBinaryTempFile (takeDirectory destination) ".kenshou-gcs-download")+              ( \(temporary, handle) -> do+                  closed <- hIsClosed handle+                  unless closed (hClose handle)+                  exists <- doesFileExist temporary+                  if exists then removeFile temporary else pure ()+              )+              \(temporary, handle) -> do+                (code, bytes) <-+                  withAuthorizedRetry+                    provider+                    ( \token -> retryStatus $ withResponse initial {requestHeaders = [("Authorization", "Bearer " <> TextEncoding.encodeUtf8 token)]} manager \response -> do+                        let code = statusCode (responseStatus response)+                        if code == 200+                          then (\count -> (code, count)) <$> copyBody (responseBody response) handle+                          else pure (code, 0)+                    )+                unless (code == 200) (badStatus "downloading GCS media" code)+                unless (bytes == meta.size) (ioError (userError "GCS downloaded length differs from metadata"))+                hClose handle+                renameFile temporary destination+            pure (Just meta)+      putLargeFile bucket object media pre source size = do+        url <- uploadUrl base bucket object pre "resumable"+        response <- runRequest "POST" url (RequestBodyBS ByteString.empty) [("X-Upload-Content-Type", TextEncoding.encodeUtf8 media), ("X-Upload-Content-Length", ByteString.Char8.pack (show size))]+        case statusCode (responseStatus response) of+          412 -> pure PreconditionFailed+          code | code >= 200 && code < 300 -> do+            location <- maybe (ioError (userError "GCS resumable upload omitted Location")) (pure . TextEncoding.decodeUtf8) (lookup "Location" (responseHeaders response))+            initial <- parseRequest (Text.unpack location)+            let sendAuthorized method range body =+                  snd+                    <$> withAuthorizedRetry+                      provider+                      ( \token -> do+                          chunkResponse <- httpLbs initial {method, requestHeaders = [("Content-Range", range), ("Authorization", "Bearer " <> TextEncoding.encodeUtf8 token), ("Content-Type", TextEncoding.encodeUtf8 media)], requestBody = body} manager+                          pure (statusCode (responseStatus chunkResponse), chunkResponse)+                      )+                sendChunk offset failures = do+                  let lengthBytes = min (8 * 1024 * 1024) (size - offset)+                      lastByte = offset + lengthBytes - 1+                      range = "bytes " <> tshow offset <> "-" <> tshow lastByte <> "/" <> tshow size+                  result <- sendAuthorized "PUT" (TextEncoding.encodeUtf8 range) (streamFileChunk source offset lengthBytes)+                  case statusCode (responseStatus result) of+                    200 -> putResponse object result+                    201 -> putResponse object result+                    412 -> pure PreconditionFailed+                    308 -> continueFrom offset failures result+                    transient | transient == 429 || transient >= 500 && transient <= 599 -> do+                      statusResult <- retryResponse $ sendAuthorized "PUT" ("bytes */" <> ByteString.Char8.pack (show size)) (RequestBodyBS ByteString.empty)+                      case statusCode (responseStatus statusResult) of+                        200 -> putResponse object statusResult+                        201 -> putResponse object statusResult+                        308 -> continueFrom offset failures statusResult+                        other -> badStatus "querying GCS resumable upload" other+                    other -> badStatus "sending GCS resumable chunk" other+                continueFrom offset failures result = do+                  next <- case lookup "Range" (responseHeaders result) of+                    Nothing -> pure 0+                    Just value -> maybe (ioError (userError "invalid GCS resumable Range")) pure (parseRangeNext value)+                  unless (next <= size) (ioError (userError "GCS resumable Range exceeds file size"))+                  let stalled = if next <= offset then failures + 1 else 0+                  unless (stalled < 4) (ioError (userError "GCS resumable upload made no progress"))+                  unless (next < size) (ioError (userError "GCS resumable upload lacks final metadata"))+                  sendChunk next stalled+            sendChunk 0 (0 :: Int)+          code -> badStatus "starting GCS resumable upload" code+      list bucket prefix = do+        listUrl <- bucketUrl base bucket+        let loop token accumulated = do+              let parameters = "?prefix=" <> encodeComponent prefix <> "&maxResults=1000" <> maybe "" ("&pageToken=" <>) (encodeComponent <$> token)+              response <- runRequest "GET" (listUrl <> parameters) (RequestBodyBS ByteString.empty) []+              case statusCode (responseStatus response) of+                200 -> do+                  page <- decodeJson "GCS object list" (responseBody response) :: IO GcsPage+                  let entries = [(ObjectName item.objectName, item.objectMeta) | item <- page.items]+                  maybe (pure (accumulated <> entries)) (\next -> loop (Just next) (accumulated <> entries)) page.nextPageToken+                code -> badStatus "listing GCS objects" code+        loop Nothing []+      clock = do+        url <- bucketUrl base provider.controlBucket+        response <- runRequest "GET" (url <> "?maxResults=1") (RequestBodyBS ByteString.empty) []+        case statusCode (responseStatus response) of+          200 -> do+            date <- maybe (ioError (userError "GCS response omitted Date")) pure (lookup "Date" (responseHeaders response))+            maybe (ioError (userError "invalid GCS Date header")) pure (parseTimeM True defaultTimeLocale "%a, %d %b %Y %H:%M:%S GMT" (ByteString.Char8.unpack date))+          code -> badStatus "reading GCS server time" code+  pure+    ObjectStore+      { getObject = get,+        statObject = metadata,+        putObject = \bucket object media pre bytes -> upload bucket object media pre (RequestBodyLBS bytes),+        putFile = \bucket object media pre source -> do+          size <- getFileSize source+          if size <= 8 * 1024 * 1024+            then do+              bytes <- LazyByteString.readFile source+              upload bucket object media pre (RequestBodyLBS bytes)+            else putLargeFile bucket object media pre source (fromIntegral size),+        downloadTo = download,+        deleteObject = \bucket object pre -> do+          url <- objectUrl base bucket object+          response <- runRequest "DELETE" (url <> preconditionQuery pre) (RequestBodyBS ByteString.empty) []+          case statusCode (responseStatus response) of+            204 -> pure True+            200 -> pure True+            404 -> pure False+            412 -> pure False+            code -> badStatus "deleting GCS object" code,+        listObjects = list,+        serverTime = clock+      }++putResponse :: ObjectName -> Response LazyByteString.ByteString -> IO PutOutcome+putResponse object response = case statusCode (responseStatus response) of+  412 -> pure PreconditionFailed+  code | code == 200 || code == 201 -> (\item -> Written item.objectMeta) <$> decodeObject object (responseBody response)+  code -> badStatus "uploading GCS object" code++retryResponse :: IO (Response body) -> IO (Response body)+retryResponse action = retryStatus ((\response -> (statusCode (responseStatus response), response)) <$> action) >>= pure . snd++retryStatus :: IO (Int, value) -> IO (Int, value)+retryStatus action = go (0 :: Int)+  where+    go attempts = do+      result@(code, _) <- action+      if attempts < 4 && (code == 429 || code >= 500 && code <= 599)+        then threadDelay (100000 * (2 ^ attempts)) >> go (attempts + 1)+        else pure result++-- A gcloud token can already be near expiry when first returned. Refresh once+-- on authorization failure instead of assuming a fixed lifetime in our cache.+withAuthorizedRetry :: TokenProvider -> (Text -> IO (Int, value)) -> IO (Int, value)+withAuthorizedRetry provider action = do+  token <- provider.accessToken+  unless (validToken token) (ioError (userError "invalid GCS token"))+  result@(code, _) <- action token+  if code == 401+    then do+      provider.invalidateToken+      refreshed <- provider.accessToken+      unless (validToken refreshed) (ioError (userError "invalid GCS token"))+      action refreshed+    else pure result++decodeObject :: ObjectName -> LazyByteString.ByteString -> IO GcsObject+decodeObject object body = do+  result <- decodeJson "GCS object" body+  unless (result.objectName == object.unObjectName) (ioError (userError "GCS returned a different object name"))+  pure result++decodeJson :: (FromJSON value) => Text -> LazyByteString.ByteString -> IO value+decodeJson label bytes = either (ioError . userError . ((Text.unpack label <> ": ") <>)) pure (eitherDecode bytes)++badStatus :: Text -> Int -> IO value+badStatus operation code = ioError (userError (Text.unpack operation <> ": HTTP " <> show code))++bucketUrl :: Text -> Bucket -> IO Text+bucketUrl endpoint bucket = do+  unless (not (Text.null bucket.unBucket) && Text.all validBucketCharacter bucket.unBucket) (ioError (userError "invalid GCS bucket name"))+  pure (endpoint <> "/storage/v1/b/" <> bucket.unBucket <> "/o")++objectUrl :: Text -> Bucket -> ObjectName -> IO Text+objectUrl endpoint bucket object = do+  validateName object+  (<> "/" <> encodeComponent object.unObjectName) <$> bucketUrl endpoint bucket++uploadUrl :: Text -> Bucket -> ObjectName -> Precondition -> Text -> IO Text+uploadUrl endpoint bucket object pre uploadType = do+  validateName object+  root <- bucketUrl endpoint bucket+  pure (Text.replace "/storage/v1/" "/upload/storage/v1/" root <> "?uploadType=" <> uploadType <> "&name=" <> encodeComponent object.unObjectName <> preconditionSuffix pre)++preconditionQuery :: Precondition -> Text+preconditionQuery NoPrecondition = ""+preconditionQuery pre = "?" <> Text.drop 1 (preconditionSuffix pre)++preconditionSuffix :: Precondition -> Text+preconditionSuffix NoPrecondition = ""+preconditionSuffix DoesNotExist = "&ifGenerationMatch=0"+preconditionSuffix (GenerationIs generation) = "&ifGenerationMatch=" <> tshow generation++validateName :: ObjectName -> IO ()+validateName object = unless (not (Text.null value) && all validPart (Text.splitOn "/" value) && not (Text.any (== '\\') value)) (ioError (userError "invalid GCS object name"))+  where+    value = object.unObjectName+    validPart part = not (Text.null part) && part /= "." && part /= ".."++validBucketCharacter :: Char -> Bool+validBucketCharacter character = ('a' <= character && character <= 'z') || ('A' <= character && character <= 'Z') || ('0' <= character && character <= '9') || character `elem` ['.', '_', '-']++encodeComponent :: Text -> Text+encodeComponent = Text.concatMap encodeByte . TextEncoding.decodeLatin1 . TextEncoding.encodeUtf8+  where+    encodeByte character+      | ('a' <= character && character <= 'z') || ('A' <= character && character <= 'Z') || ('0' <= character && character <= '9') || character `elem` ['-', '_', '.', '~'] = Text.singleton character+      | otherwise = let raw = showHex (fromEnum character) "" in "%" <> Text.toUpper (Text.justifyRight 2 '0' (Text.pack raw))++validToken :: Text -> Bool+validToken token = not (Text.null (Text.strip token)) && not (Text.any (`elem` ['\r', '\n']) token)++tshow :: (Show value) => value -> Text+tshow = Text.pack . show++streamFileChunk :: FilePath -> Int64 -> Int64 -> RequestBody+streamFileChunk source offset size = RequestBodyStream size \consume -> withBinaryFile source ReadMode \handle -> do+  hSeek handle AbsoluteSeek (fromIntegral offset)+  remaining <- newIORef size+  consume do+    left <- readIORef remaining+    if left == 0+      then pure ByteString.empty+      else do+        chunk <- ByteString.hGetSome handle (fromIntegral (min left (1024 * 1024)))+        whenEmpty chunk+        writeIORef remaining (left - fromIntegral (ByteString.length chunk))+        pure chunk+  where+    whenEmpty chunk = unless (not (ByteString.null chunk)) (ioError (userError "GCS upload source shrank during transfer"))++parseRangeNext :: ByteString.ByteString -> Maybe Int64+parseRangeNext value = do+  suffix <- ByteString.Char8.stripPrefix "bytes=0-" value+  lastByte <- readMaybe (ByteString.Char8.unpack suffix)+  if lastByte >= 0 then Just (lastByte + 1) else Nothing++copyBody :: BodyReader -> Handle -> IO Int64+copyBody reader handle = go 0+  where+    go total = do+      chunk <- brRead reader+      if ByteString.null chunk+        then pure total+        else ByteString.hPut handle chunk >> go (total + fromIntegral (ByteString.length chunk))
+ test/DebugSpec.hs view
@@ -0,0 +1,26 @@+module DebugSpec (spec) where++import Data.Aeson (eitherDecodeFileStrict')+import Kenshou.Remote.Cell.Debug (remoteCommand, resolveDebugRole)+import Kenshou.Remote.Cell.Docs (CellDescriptor)+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)+import Test.Hspec++spec :: Spec+spec = describe "cell debug target" do+  it "selects only instances named by the verified descriptor" do+    decoded <- eitherDecodeFileStrict' "test/golden/cell/cell.descriptor.v1.json"+    descriptor <- either (fail . ("descriptor fixture: " <>)) pure (decoded :: Either String CellDescriptor)+    resolveDebugRole descriptor "driver" `shouldBe` Right "cell-alpha-driver-0"+    resolveDebugRole descriptor "driver-0" `shouldBe` Right "cell-alpha-driver-0"+    resolveDebugRole descriptor "postgres" `shouldBe` Right "cell-alpha-postgres"+    resolveDebugRole descriptor "monitoring" `shouldBe` Right "cell-alpha-monitoring"+    resolveDebugRole descriptor "driver-1" `shouldBe` Left "cell descriptor has no driver at that index"+    resolveDebugRole descriptor "broker" `shouldBe` Left "role must be postgres, monitoring, driver, or driver-N"++  it "preserves spaces and apostrophes in remote command arguments" do+    (code, output, failure) <- readProcessWithExitCode "/bin/sh" ["-c", remoteCommand ["printf", "%s", "two words", "it's"]] ""+    code `shouldBe` ExitSuccess+    failure `shouldBe` ""+    output `shouldBe` "two wordsit's"
+ test/DocsSpec.hs view
@@ -0,0 +1,98 @@+module DocsSpec (spec) where++import Data.Aeson (ToJSON, Value (..), eitherDecode, encode, object, toJSON, (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isLeft, isRight)+import Data.Text (Text)+import Kenshou.Remote.Cell.Docs (Artifact, CellDescriptor, CellEnvironment, CellManifest (..), CellRunResult, CellStatus, Rejected, Submission)+import Test.Hspec++spec :: Spec+spec = describe "cell owner documents" do+  it "decodes status, rejection, run result and manifest fixtures without changing their JSON" do+    checkFixture "cell.status.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellStatus)+    checkFixture "cell.rejected.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String Rejected)+    checkFixture "cell.run-result.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellRunResult)+    checkFixture "cell.artifact-manifest.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellManifest)++  it "decodes the owner descriptor and environment fixtures without changing their JSON" do+    checkFixture "cell.descriptor.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellDescriptor)+    checkFixture "cell.environment.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellEnvironment)++  it "decodes the generic owner submission, including its non-Kenshou payload" do+    checkFixture "cell.submission.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String Submission)++  it "ignores future status fields while enforcing sealed terminal fields" do+    bytes <- LazyByteString.readFile "test/golden/cell/cell.status.v1.json"+    case eitherDecode bytes :: Either String Value of+      Right (Object fields) -> do+        let extended = Object (KeyMap.insert "futureField" (Bool True) fields)+        (eitherDecode (encode extended) :: Either String CellStatus) `shouldSatisfy` isRight+      _ -> expectationFailure "invalid status fixture"+    let incomplete = "{\"schema\":\"cell.status/v1\",\"runId\":\"0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9\",\"phase\":\"sealed\",\"updatedAt\":\"2026-09-25T12:00:00Z\",\"logChunks\":{\"stdout\":0,\"stderr\":0},\"outcome\":\"completed\"}"+    (eitherDecode incomplete :: Either String CellStatus) `shouldSatisfy` isLeft++  it "accepts a nonterminal status without a verdict" do+    bytes <- LazyByteString.readFile "test/golden/cell/cell.status.v1.json"+    case eitherDecode bytes :: Either String Value of+      Right (Object fields) -> do+        let running = Object (KeyMap.insert "phase" (String "running") (KeyMap.delete "manifestSha256" (KeyMap.delete "outcome" fields)))+        case eitherDecode (encode running) :: Either String CellStatus of+          Left failure -> expectationFailure failure+          Right status -> toJSON status `shouldBe` running+      _ -> expectationFailure "invalid status fixture"++  it "preserves optional broker, OTLP, clock and fault-hook environment fields" do+    bytes <- LazyByteString.readFile "test/golden/cell/cell.environment.v1.json"+    case eitherDecode bytes :: Either String Value of+      Right (Object fields) -> do+        let endpoint = object ["grpc" .= ("http://10.0.0.4:4317" :: String), "http" .= ("http://10.0.0.4:4318" :: String)]+            broker = object ["bootstrapServers" .= ("10.0.0.5:9092" :: String), "adminUrl" .= ("http://10.0.0.5:9644" :: String), "implementation" .= ("redpanda" :: String), "version" .= ("25.1.0" :: String)]+            extended = Object (KeyMap.insert "faultHook" (String "/run/cell/fault-hook") (KeyMap.insert "clock" (object ["skewBoundMicros" .= (250 :: Int)]) (KeyMap.insert "otlp" (object ["null" .= endpoint, "file" .= endpoint]) (KeyMap.insert "broker" broker fields))))+        case eitherDecode (encode extended) :: Either String CellEnvironment of+          Left failure -> expectationFailure failure+          Right environment -> toJSON environment `shouldBe` extended+      _ -> expectationFailure "invalid environment fixture"++  it "round-trips the generic resources block of a cell environment" do+    bytes <- LazyByteString.readFile "test/golden/cell/cell.environment.v1.json"+    case eitherDecode bytes :: Either String Value of+      Right (Object fields) -> do+        let extended = Object (KeyMap.insert "resources" (object ["object-store" .= object ["endpoint" .= ("http://10.0.0.6:9000" :: String)]]) fields)+        case eitherDecode (encode extended) :: Either String CellEnvironment of+          Left failure -> expectationFailure failure+          Right environment -> toJSON environment `shouldBe` extended+      _ -> expectationFailure "invalid environment fixture"++  it "keeps a submission's resource resets beside the legacy broker reset" do+    bytes <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+    case eitherDecode bytes :: Either String Value of+      Right (Object fields) | Just (Object reset) <- KeyMap.lookup "reset" fields -> do+        let resets = KeyMap.insert "resources" (object ["object-store" .= object ["empty" .= True]]) (KeyMap.insert "broker" (object ["wipe" .= True]) reset)+            extended = Object (KeyMap.insert "reset" (Object resets) fields)+        case eitherDecode (encode extended) :: Either String Submission of+          Left failure -> expectationFailure failure+          Right submission -> toJSON submission `shouldBe` extended+      _ -> expectationFailure "invalid submission fixture"++  it "rejects an artifact path that escapes the fetched tree" do+    let escaped = "{\"path\":\"../outside\",\"sha256\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"bytes\":1,\"mediaType\":\"application/json\"}"+    (eitherDecode escaped :: Either String Artifact) `shouldSatisfy` isLeft++  it "rejects ambiguous artifact paths and duplicate manifest entries" do+    bytes <- LazyByteString.readFile "test/golden/cell/cell.artifact-manifest.v1.json"+    let badPaths = ["a//b", "a/./b", "a/../b", "a\\b", "/a/b"] :: [Text]+    mapM_ (\path -> (eitherDecode (encode (object ["path" .= path, "sha256" .= ("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" :: String), "bytes" .= (1 :: Int), "mediaType" .= ("application/json" :: String)])) :: Either String Artifact) `shouldSatisfy` isLeft) badPaths+    case eitherDecode bytes :: Either String CellManifest of+      Right manifest -> do+        let duplicate = manifest {artifacts = manifest.artifacts <> take 1 manifest.artifacts}+        (eitherDecode (encode duplicate) :: Either String CellManifest) `shouldSatisfy` isLeft+      _ -> expectationFailure "invalid cell manifest fixture"++checkFixture :: (ToJSON document) => FilePath -> (LazyByteString.ByteString -> Either String document) -> IO ()+checkFixture name decoder = do+  bytes <- LazyByteString.readFile ("test/golden/cell/" <> name)+  case decoder bytes of+    Left failure -> expectationFailure failure+    Right document -> Right (toJSON document) `shouldBe` (eitherDecode bytes :: Either String Value)
+ test/ExecSpec.hs view
@@ -0,0 +1,208 @@+module ExecSpec (execSpec) where++import Control.Exception (bracket)+import Data.Aeson (Value (..), eitherDecodeFileStrict', object, (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Bundle (LayerBundle (..), Registry, lookupScenario, mkBundle, mkRegistry)+import Kenshou.Core.Bundle qualified as Bundle+import Kenshou.Core.Dimension (allTelemetryArms)+import Kenshou.Core.Env (EnvRequirements (..))+import Kenshou.Core.Id (mkSeed, newRunId, parseScenarioId, renderRunId, selftestLayer)+import Kenshou.Core.Knob (Allowed (..), KnobSpec (..), KnobType (..), KnobValue (..), RawKnob (..), mkKnobName)+import Kenshou.Core.Resource (CellBinding (..), ProvisionedResource (..), ResourceProvider (..))+import Kenshou.Core.Resource.Types (legacyKafkaName)+import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..), minimalRunSpec)+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Core.Selftest qualified as Selftest+import Kenshou.Core.Selftest.Schema (selftestMigrationSet)+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (..), RunPlan (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Docs (CellBroker (..), CellEnvironment (..), CellPostgres (..), OtlpEndpoint (..), OtlpSinks (..))+import Kenshou.Remote.Cell.Exec (cellExec, resolveOnCell, resolveWorkJson)+import Kenshou.Remote.Cell.Prepare (OtlpSink (..))+import Kenshou.Remote.Selftest qualified as RemoteSelftest+import System.Environment (lookupEnv, setEnv, unsetEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++execSpec :: Spec+execSpec = describe "cell-side run-plan resolution" do+  it "binds durable PostgreSQL to the cell connection without changing run identity" do+    (plan, environment) <- fixtures+    let durable = withDimensions [("pg.durability", "durable"), ("pg.version", "18")] plan+    case resolveOnCell registry environment NullSink durable of+      Left problem -> expectationFailure (show problem)+      Right resolved -> do+        resolved.planId `shouldBe` durable.planId+        fmap (.runId) resolved.runs `shouldBe` fmap (.runId) durable.runs+        resolveWorkJson registry environment NullSink (Aeson.toJSON durable) `shouldBe` Right (Aeson.toJSON resolved)+        case resolved.runs of+          [run] -> do+            run.spec.environment.placement `shouldBe` RunOnCell+            run.spec.environment.postgres `shouldBe` Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))+          _ -> expectationFailure "expected one resolved run"++  it "refuses a non-durable or wrong-major PostgreSQL run" do+    (plan, environment) <- fixtures+    resolveOnCell registry environment NullSink plan `shouldSatisfy` isLeft+    let pg = environment.postgres+        pg17 = CellPostgres 17 pg.host pg.port pg.database pg.user pg.connectionString+        cell17 = CellEnvironment environment.cell environment.runId environment.leaseId pg17 environment.broker environment.otlp environment.victoriaMetricsUrl environment.drivers environment.clockSkewBoundMicros environment.faultHook environment.resources+        durable = withDimensions [("pg.durability", "durable"), ("pg.version", "18")] plan+    resolveOnCell registry cell17 NullSink durable `shouldSatisfy` isLeft++  it "binds the cell broker and selected OTLP endpoint for a tracing run" do+    (plan, environment) <- fixtures+    let broker = CellBroker "10.0.0.5:9092" "http://10.0.0.5:9644" "redpanda" "v25"+        sinks = OtlpSinks (OtlpEndpoint "http://10.0.0.4:4317" "http://10.0.0.4:4318") (OtlpEndpoint "http://10.0.0.4:5317" "http://10.0.0.4:5318")+        cell = withEndpoints environment (Just broker) (Just sinks)+        tracing = withDimensions [("pg.durability", "durable"), ("pg.version", "18"), ("telemetry.tracing", "sdk-otlp")] plan+        endpoint = either (error . Text.unpack) id (mkKnobName "otel.endpoint")+    case resolveOnCell tracingRegistry cell FileSink tracing of+      Left problem -> expectationFailure (show problem)+      Right resolved -> case resolved.runs of+        [run] -> do+          Map.lookup legacyKafkaName run.spec.environment.resources `shouldBe` Just (object ["backend" .= ("external" :: Text), "brokers" .= (["10.0.0.5:9092"] :: [Text]), "lanes" .= (0 :: Int)])+          lookup endpoint run.spec.knobs `shouldBe` Just (RawText "http://10.0.0.4:5318")+        _ -> expectationFailure "expected one tracing run"+    resolveOnCell tracingRegistry environment FileSink tracing `shouldSatisfy` isLeft+    resolveOnCell tracingRegistry (withEndpoints environment (Just broker) Nothing) FileSink tracing `shouldSatisfy` isLeft++  it "fills the remote environment probe from the cell descriptor" do+    (plan, environment) <- fixtures+    let scenario = either (error . Text.unpack) id (parseScenarioId "selftest/remote/correctness/cell-environment")+        broker = CellBroker "10.0.0.5:9092" "http://10.0.0.5:9644" "redpanda" "v25"+        sinks = OtlpSinks (OtlpEndpoint "http://10.0.0.4:4317" "http://10.0.0.4:4318") (OtlpEndpoint "http://10.0.0.4:5317" "http://10.0.0.4:5318")+        cell = withEndpoints environment (Just broker) (Just sinks)+        change entry =+          let old = entry.spec+              spec = RunSpec old.runId scenario old.scenarioRevision old.knobs old.dimensions old.seed old.phases old.timeoutSeconds old.environment old.cohortExpectation old.comparison old.labels+           in PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial spec+        changed = RunPlan plan.planId plan.createdAt plan.context plan.policy (fmap change plan.runs) plan.skipped plan.estimateMinutes+        durable = withDimensions [("pg.durability", "durable"), ("pg.version", "18")] changed+        name text = either (error . Text.unpack) id (mkKnobName text)+    case resolveOnCell remoteRegistry cell FileSink durable of+      Left problem -> expectationFailure (show problem)+      Right resolved -> case resolved.runs of+        [run] -> do+          lookup (name "remote.expect-placement") run.spec.knobs `shouldBe` Nothing+          lookup (name "remote.otlp-endpoint") run.spec.knobs `shouldBe` Just (RawText "http://10.0.0.4:5318")+          lookup (name "remote.tcp-endpoint") run.spec.knobs `shouldBe` Nothing+        _ -> expectationFailure "expected one remote probe run"++  it "leaves a secondary driver idle without requiring the work or environment" $+    withSystemTempDirectory "kenshou-cell-exec" \root ->+      withEnv "CELL_DRIVER_INDEX" (Just "1") $+        withEnv "CELL_DRIVER_COUNT" (Just "2") do+          let out = root </> "output"+          cellExec registry (root </> "missing-work.json") out `shouldReturn` ExitSuccess+          idle <- eitherDecodeFileStrict' (out </> "idle-driver.json")+          idle `shouldBe` Right (object ["schema" .= ("kenshou.cell-idle-driver/v1" :: Text), "index" .= (1 :: Int), "count" .= (2 :: Int)])++  it "records a cell adapter failure when the owner environment is missing" $+    withSystemTempDirectory "kenshou-cell-exec" \root ->+      withEnv "CELL_DRIVER_INDEX" (Just "0") $+        withEnv "CELL_DRIVER_COUNT" (Just "1") $+          withEnv "CELL_ENV_FILE" Nothing do+            let out = root </> "output"+            cellExec registry (root </> "missing-work.json") out `shouldReturn` ExitFailure 4+            failure <- eitherDecodeFileStrict' (out </> "kenshou-cell" </> "adapter-error.json")+            failure `shouldBe` Right (object ["schema" .= ("kenshou.cell-adapter-error/v1" :: Text), "reason" .= ("cell-environment-missing" :: Text), "detail" .= ("user error (CELL_ENV_FILE is required by the cell payload)" :: Text)])++  it "reports missing wrapper identity before reading cell work" $+    withSystemTempDirectory "kenshou-cell-exec" \root -> do+      (_, environment) <- fixtures+      withEnv "CELL_DRIVER_INDEX" (Just "0") $+        withEnv "CELL_DRIVER_COUNT" (Just "1") $+          withEnv "CELL_ENV_FILE" (Just "test/golden/cell/cell.environment.v1.json") $+            withEnv "CELL_RUN_ID" (Just (Text.unpack (renderRunId environment.runId))) $+              withEnv "CELL_SCRATCH_DIR" (Just (root </> "scratch")) $+                withEnv "KENSHOU_OTLP_SINK" (Just "null") $+                  withEnv "KENSHOU_COHORT_IDENTITY" Nothing do+                    let out = root </> "output"+                    cellExec registry (root </> "missing-work.json") out `shouldReturn` ExitFailure 4+                    failure <- eitherDecodeFileStrict' (out </> "kenshou-cell" </> "adapter-error.json")+                    case (failure :: Either String Value) of+                      Right (Object fields) -> KeyMap.lookup "reason" fields `shouldBe` Just (String "payload-identity-missing")+                      _ -> expectationFailure (show failure)++fixtures :: IO (RunPlan, CellEnvironment)+fixtures = do+  planId <- newRunId+  runId <- newRunId+  now <- getCurrentTime+  let scenario = either (error . Text.unpack) id (parseScenarioId "selftest/kernel/correctness/postgres-roundtrip")+      selector = either (error . Text.unpack) id (parseSelector "selftest/kernel/**")+      reason = Reason (Change (ComponentRef (ComponentId "kernel") Nothing) Named "cell test") [] selector 0+      base = minimalRunSpec scenario+      spec = RunSpec (Just runId) scenario base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+      entry = PlannedRun 0 runId 1 (reason :| []) Nothing spec+      planContext = PlanContext Nothing "fixture" "released" "sha256:fixture" (PlanInputs Null) [] []+      seed = either (error . Text.unpack) id (mkSeed 7)+      plan = RunPlan planId now planContext (defaultPlanPolicy seed) [entry] [] 1+  environment <- eitherDecodeFileStrict' "test/golden/cell/cell.environment.v1.json" >>= either fail pure+  pure (plan, environment)++registry :: Registry+registry = either (error . show) id (mkRegistry [Selftest.bundle])++remoteRegistry :: Registry+remoteRegistry = either (error . show) id (mkRegistry [Selftest.bundle, RemoteSelftest.bundle])++tracingRegistry :: Registry+tracingRegistry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [enhanced] []) {migrations = [selftestMigrationSet], Bundle.resources = [brokerProvider]}])+  where+    scenarioId = either (error . Text.unpack) id (parseScenarioId "selftest/kernel/correctness/postgres-roundtrip")+    original = maybe (error "missing PostgreSQL selftest") id (lookupScenario registry scenarioId)+    endpoint = either (error . Text.unpack) id (mkKnobName "otel.endpoint")+    knob = KnobSpec endpoint "OTLP endpoint" KnobText (VText "") AnyValue []+    requirements = EnvRequirements original.requires.postgres original.requires.extraPostgres [legacyKafkaName]+    enhanced = Scenario original.id original.revision original.summary original.tier original.placement (knob : original.knobs) (allTelemetryArms original.dimensions) original.phases requirements original.knownDefect original.run++withEndpoints :: CellEnvironment -> Maybe CellBroker -> Maybe OtlpSinks -> CellEnvironment+withEndpoints environment broker otlp =+  CellEnvironment environment.cell environment.runId environment.leaseId environment.postgres broker otlp environment.victoriaMetricsUrl environment.drivers environment.clockSkewBoundMicros environment.faultHook environment.resources++withDimensions :: [(Text, Text)] -> RunPlan -> RunPlan+withDimensions dimensions plan =+  let runs = fmap update plan.runs+   in RunPlan plan.planId plan.createdAt plan.context plan.policy runs plan.skipped plan.estimateMinutes+  where+    update entry =+      let spec = entry.spec+          changed = RunSpec spec.runId spec.scenario spec.scenarioRevision spec.knobs dimensions spec.seed spec.phases spec.timeoutSeconds spec.environment spec.cohortExpectation spec.comparison spec.labels+       in PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial changed++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft _ = False++withEnv :: String -> Maybe String -> IO result -> IO result+withEnv name value operation = bracket (lookupEnv name) (restore name) (const (apply name value >> operation))+  where+    apply key Nothing = unsetEnv key+    apply key (Just setting) = setEnv key setting+    restore key prior = apply key prior++-- A provider for the resource the legacy cell broker binds; the cell supplies the+-- broker, so the local bracket only passes the value through.+brokerProvider :: ResourceProvider+brokerProvider =+  ResourceProvider+    { name = legacyKafkaName,+      summary = "Cell broker",+      normalize = Right . maybe Null id,+      withLocal = \_ value action -> action (ProvisionedResource value Null),+      cell = Just (CellBinding "broker" (Just (object ["wipe" .= True])))+    }
+ test/FetchSpec.hs view
@@ -0,0 +1,258 @@+module FetchSpec (spec, completeTreeWith, completeTreeWithNested) where++import Data.Aeson (Value, eitherDecode, encode, object, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, newRunId, parseScenarioId, renderRunId)+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..))+import Kenshou.Core.Outcome qualified as Outcome+import Kenshou.Core.RunSpec (minimalRunSpec)+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellPhase (..), CellRunResult (..), CellStatus (..), LogChunks (..), ManifestPayload (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Fetch (FetchError (..), VerifyProblem (..), effectiveOutcome, fetchCellRun, verifyCellRun, verifyCellRunWithStatus, verifyCellTree)+import Kenshou.Remote.Cell.Index (CellRunIndex (..), RunLink (..), deriveCellRunIndex, writeCellRunIndex)+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile)+import System.FilePath (takeDirectory, (</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "sealed cell tree fetch" do+  it "round-trips the derived cell-run index fixture" do+    bytes <- LazyByteString.readFile "test/golden/cell-run.json"+    case eitherDecode bytes :: Either String CellRunIndex of+      Left failure -> expectationFailure failure+      Right index -> eitherDecode (encode index) `shouldBe` (eitherDecode bytes :: Either String Value)++  it "maps cell failures onto effective nested-run outcomes" do+    effectiveOutcome Completed True Outcome.Failed `shouldBe` Outcome.Failed+    effectiveOutcome InfrastructureFailure True Outcome.Passed `shouldBe` Outcome.InfrastructureFailure+    effectiveOutcome Cancelled True Outcome.Passed `shouldBe` Outcome.Passed+    effectiveOutcome Cancelled False Outcome.Passed `shouldBe` Outcome.Errored+    effectiveOutcome TimedOut False Outcome.Inconclusive `shouldBe` Outcome.Errored++  it "requires a seal, verifies bytes, and repairs a damaged local artifact on replay" $ withSystemTempDirectory "kenshou-fetch" \root -> do+    store <- newFileStore (root </> "store")+    identifier <- newRunId+    fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Left Unsealed+    manifest <- publishFixture store identifier+    fetched <- fetchCellRun store bucket identifier (root </> "out")+    tree <- case fetched of+      Right path -> pure path+      Left failure -> expectationFailure (show failure) >> error "unreachable"+    verifyCellTree tree `shouldReturn` Right manifest+    LazyByteString.writeFile (tree </> "cell" </> "result.json") "bad"+    damaged <- verifyCellTree tree+    damaged `shouldSatisfy` hasDigestMismatch+    fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Right tree+    verifyCellTree tree `shouldReturn` Right manifest+    LazyByteString.writeFile (tree </> "unexpected.txt") "extra"+    withExtra <- verifyCellTree tree+    withExtra `shouldSatisfy` hasExtra+    removeFile (tree </> "unexpected.txt")+    removeFile (tree </> "cell" </> "result.json")+    missing <- verifyCellTree tree+    missing `shouldSatisfy` hasMissing++  it "refuses an unlisted result object and a corrupt published artifact" $ withSystemTempDirectory "kenshou-fetch" \root -> do+    store <- newFileStore (root </> "store")+    identifier <- newRunId+    _ <- publishFixture store identifier+    put store identifier "stray.txt" "extra"+    result <- fetchCellRun store bucket identifier (root </> "out")+    result `shouldSatisfy` \case+      Left (ObjectSetMismatch _ [unexpected]) -> unexpected == prefix identifier <> "stray.txt"+      _ -> False+    store.deleteObject bucket (ObjectName (prefix identifier <> "stray.txt")) NoPrecondition `shouldReturn` True+    put store identifier "cell/result.json" (LazyByteString.replicate (LazyByteString.length resultBytes) 120)+    fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Left (ObjectCorrupt "cell/result.json")+    doesFileExist (root </> "out" </> Text.unpack (renderRunId identifier) </> "tree" </> "cell" </> "result.json") `shouldReturn` False++  it "cross-checks the submission, cell result and nested Kenshou manifest" $ withSystemTempDirectory "kenshou-verify" \root -> do+    (tree, manifest) <- completeTree root False False False+    verifyCellRun tree `shouldReturn` Right manifest++  it "accepts cell support directories without nested run manifests" $ withSystemTempDirectory "kenshou-verify" \root -> do+    (tree, manifest) <- completeTree root False False False+    let supporting = [("output/kenshou-cell/context.json", "{}"), ("output/specs/0001.json", "{}")]+        expanded = manifest {artifacts = manifest.artifacts <> [artifactFor name bytes | (name, bytes) <- supporting]}+    mapM_+      ( \(name, bytes) -> do+          let destination = tree </> Text.unpack name+          createDirectoryIfMissing True (takeDirectory destination)+          LazyByteString.writeFile destination bytes+      )+      supporting+    LazyByteString.writeFile (tree </> "manifest.json") (encode expanded)+    verifyCellRun tree `shouldReturn` Right expanded++  it "rejects a payload identity mismatch after all file digests pass" $ withSystemTempDirectory "kenshou-verify" \root -> do+    (tree, _) <- completeTree root True False False+    result <- verifyCellRun tree+    result `shouldSatisfy` \case+      Left problems -> any isPayloadMismatch problems+      Right _ -> False++  it "requires a nested manifest and a matching nested run identifier" $ withSystemTempDirectory "kenshou-verify" \root -> do+    (tree, _) <- completeTree (root </> "wrong-id") False True False+    mismatch <- verifyCellRun tree+    mismatch `shouldSatisfy` \case+      Left problems -> any isRunResultMismatch problems+      Right _ -> False+    (otherTree, manifest) <- completeTree (root </> "missing") False False False+    case [artifact.path | artifact <- manifest.artifacts, "/manifest.json" `Text.isSuffixOf` artifact.path] of+      [nestedPath] -> do+        removeFile (otherTree </> Text.unpack nestedPath)+        let withoutNested = manifest {artifacts = filter ((/= nestedPath) . (.path)) manifest.artifacts}+        LazyByteString.writeFile (otherTree </> "manifest.json") (encode withoutNested)+        absent <- verifyCellRun otherTree+        absent `shouldSatisfy` \case+          Left problems -> any isNestedManifestProblem problems+          Right _ -> False+      _ -> expectationFailure "expected one nested manifest"++  it "rejects an unsafe nested path before calling the kernel verifier" $ withSystemTempDirectory "kenshou-verify" \root -> do+    (tree, _) <- completeTree root False False True+    result <- verifyCellRun tree+    result `shouldSatisfy` \case+      Left problems -> any isNestedManifestProblem problems+      Right _ -> False++  it "requires the final status to agree with the fetched seal" $ withSystemTempDirectory "kenshou-verify" \root -> do+    (tree, manifest) <- completeTree root False False False+    bytes <- LazyByteString.readFile (tree </> "manifest.json")+    let sealed = CellStatus manifest.runId Sealed (Just manifest.leaseSequence) manifest.sealedAt Nothing (LogChunks 0 0) (Just manifest.outcome) (Just (digestOf bytes)) (Just [])+    verifyCellRunWithStatus sealed tree `shouldReturn` Right manifest+    let wrong = CellStatus sealed.runId sealed.phase sealed.leaseSequence sealed.updatedAt sealed.phaseStartedAt sealed.logChunks sealed.outcome (Just (Text.replicate 64 "f")) sealed.reasons+    result <- verifyCellRunWithStatus wrong tree+    result `shouldSatisfy` \case+      Left problems -> StatusDigestMismatch `elem` problems+      Right _ -> False++  it "derives a durable index beside a verified cell tree" $ withSystemTempDirectory "kenshou-index" \root -> do+    (tree, manifest) <- completeTree root False False False+    derived <- deriveCellRunIndex (Bucket "results") Nothing tree+    case derived of+      Left failure -> expectationFailure (show failure)+      Right index -> do+        index.cellRun `shouldBe` manifest.runId+        index.resultsBaseUri `shouldBe` ("gs://results/runs/" <> renderRunId manifest.runId)+        index.dataBaseUri `shouldBe` (index.resultsBaseUri <> "/output")+        length index.runs `shouldBe` 1+        map (.recordedOutcome) index.runs `shouldBe` [Outcome.Passed]+        map (.effectiveOutcome) index.runs `shouldBe` [Outcome.Passed]+        destination <- writeCellRunIndex tree index+        destination `shouldBe` root </> "cell-run.json"+        stored <- LazyByteString.readFile destination+        eitherDecode stored `shouldBe` Right index++publishFixture :: ObjectStore -> RunId -> IO CellManifest+publishFixture store identifier = do+  lease <- newRunId+  now <- getCurrentTime+  let payload = ManifestPayload "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-fixture"+      work = workObjectFor "application/json" resultBytes+      artifact = Artifact "cell/result.json" work.sha256 work.bytes "application/json"+      manifest = CellManifest identifier "alpha" lease 1 now "0.1.0" payload Completed 3600 [artifact]+  put store identifier "cell/result.json" resultBytes+  put store identifier "manifest.json" (encode manifest)+  pure manifest++put :: ObjectStore -> RunId -> Text -> LazyByteString.ByteString -> IO ()+put store identifier suffix bytes = do+  _ <- store.putObject bucket (ObjectName (prefix identifier <> suffix)) "application/json" NoPrecondition bytes+  pure ()++prefix :: RunId -> Text+prefix identifier = "runs/" <> renderRunId identifier <> "/"++bucket :: Bucket+bucket = Bucket "results"++resultBytes :: LazyByteString.ByteString+resultBytes = "{\"schema\":\"cell.run-result/v1\"}"++completeTree :: FilePath -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)+completeTree root wrongPayload wrongRunId unsafeNested = do+  identifier <- newRunId+  lease <- newRunId+  completeTreeWith root identifier lease wrongPayload wrongRunId unsafeNested++completeTreeWith :: FilePath -> RunId -> RunId -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)+completeTreeWith root identifier lease wrongPayload wrongRunId unsafeNested = do+  nestedId <- newRunId+  completeTreeWithNested root identifier lease nestedId wrongPayload wrongRunId unsafeNested++completeTreeWithNested :: FilePath -> RunId -> RunId -> RunId -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)+completeTreeWithNested root identifier lease nestedId wrongPayload wrongRunId unsafeNested = do+  otherId <- newRunId+  now <- getCurrentTime+  scenario <- either (ioError . userError . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/always-pass")+  source <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+  fixture <- either (ioError . userError) pure (eitherDecode source :: Either String Submission)+  let workBytes = encode (object ["schema" .= ("kenshou.run-plan/v1" :: Text), "planId" .= identifier, "runs" .= [object ["ordinal" .= (1 :: Int), "runId" .= nestedId, "estimateMinutes" .= (1 :: Int), "spec" .= minimalRunSpec scenario]]])+      workInfo = workObjectFor "application/json" workBytes+      submitted = Submission identifier lease fixture.payload workInfo fixture.env fixture.reset fixture.limits fixture.requires Nothing fixture.labels+      payloadDigest = fixture.payload.bundle.sha256+      manifestDigest = if wrongPayload then Text.replicate 64 "b" else payloadDigest+      manifestPayload = ManifestPayload manifestDigest fixture.payload.storePath+      cellResult = CellRunResult identifier "alpha" lease 1 Completed (Just 0) Nothing []+      runBytes = encode (object ["schema" .= ("kenshou.run-result/v1" :: Text), "runId" .= (if wrongRunId then otherId else nestedId), "scenario" .= ("selftest/kernel/correctness/always-pass" :: Text), "outcome" .= Outcome.Passed, "fingerprint" .= object ["cell" .= object ["payload" .= object ["bundleSha256" .= payloadDigest]]]])+      nestedFile = ManifestFile (if unsafeNested then "../escape" else "run-result.json") ("sha256:" <> digestOf runBytes) (fromIntegral (LazyByteString.length runBytes)) "application/json"+      nestedManifest = Manifest nestedId now [nestedFile]+      files =+        [ ("submission/work", workBytes),+          ("submission/submission.json", encode submitted),+          ("cell/result.json", encode cellResult),+          ("output/" <> renderRunId nestedId <> "/run-result.json", runBytes),+          ("output/" <> renderRunId nestedId <> "/manifest.json", encode nestedManifest)+        ]+      artifacts = [artifactFor name bytes | (name, bytes) <- files]+      manifest = CellManifest identifier "alpha" lease 1 now "0.1.0" manifestPayload Completed 3600 artifacts+      tree = root </> "tree"+  mapM_+    ( \(name, bytes) -> do+        let destination = tree </> Text.unpack name+        createDirectoryIfMissing True (takeDirectory destination)+        LazyByteString.writeFile destination bytes+    )+    files+  LazyByteString.writeFile (tree </> "manifest.json") (encode manifest)+  pure (tree, manifest)++artifactFor :: Text -> LazyByteString.ByteString -> Artifact+artifactFor name bytes = let info = workObjectFor "application/json" bytes in Artifact name info.sha256 info.bytes "application/json"++digestOf :: LazyByteString.ByteString -> Text+digestOf bytes = (workObjectFor "application/json" bytes).sha256++isPayloadMismatch :: VerifyProblem -> Bool+isPayloadMismatch (PayloadMismatch _) = True+isPayloadMismatch _ = False++isRunResultMismatch :: VerifyProblem -> Bool+isRunResultMismatch (RunResultMismatch _) = True+isRunResultMismatch _ = False++isNestedManifestProblem :: VerifyProblem -> Bool+isNestedManifestProblem (NestedManifestProblem _) = True+isNestedManifestProblem _ = False++hasDigestMismatch :: Either (NonEmpty VerifyProblem) CellManifest -> Bool+hasDigestMismatch (Left problems) = DigestMismatch "cell/result.json" `elem` toList problems+hasDigestMismatch _ = False++hasExtra :: Either (NonEmpty VerifyProblem) CellManifest -> Bool+hasExtra (Left problems) = Extra "unexpected.txt" `elem` toList problems+hasExtra _ = False++hasMissing :: Either (NonEmpty VerifyProblem) CellManifest -> Bool+hasMissing (Left problems) = Missing "cell/result.json" `elem` toList problems+hasMissing _ = False
+ test/GcsSpec.hs view
@@ -0,0 +1,256 @@+module GcsSpec (spec) where++import Control.Concurrent (forkIO, killThread)+import Control.Exception (bracket, finally)+import Control.Monad (forever, unless, void)+import Data.ByteString qualified as ByteString+import Data.ByteString.Char8 qualified as Char8+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.List (sort)+import Data.Text qualified as Text+import Data.Time (UTCTime (..), fromGregorian, secondsToDiffTime)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.Gcs (TokenProvider (..), newGcsStoreAt, newTokenProviderWith)+import Network.Socket (Family (AF_INET), SockAddr (SockAddrInet), SocketType (Stream), accept, bind, close, defaultProtocol, getSocketName, listen, socket, socketToHandle, tupleToHostAddress)+import System.Directory (doesFileExist)+import System.FilePath ((</>))+import System.IO (BufferMode (NoBuffering), Handle, IOMode (ReadWriteMode), hClose, hSetBuffering)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++data MockResponse = MockResponse !Int ![(ByteString.ByteString, ByteString.ByteString)] !LazyByteString.ByteString++spec :: Spec+spec = describe "GCS cell object store" do+  it "prefers the explicit token, then VM metadata, and caches gcloud tokens" do+    calls <- newIORef (0 :: Int)+    let cli = modifyIORef' calls (+ 1) >> pure "cli-token"+    explicit <- newTokenProviderWith (Bucket "control") (pure (Just "environment-token")) (pure (Just "metadata-token")) cli+    explicit.accessToken `shouldReturn` "environment-token"+    machine <- newTokenProviderWith (Bucket "control") (pure Nothing) (pure (Just "metadata-token")) cli+    machine.accessToken `shouldReturn` "metadata-token"+    laptop <- newTokenProviderWith (Bucket "control") (pure Nothing) (pure Nothing) cli+    laptop.accessToken `shouldReturn` "cli-token"+    laptop.accessToken `shouldReturn` "cli-token"+    readIORef calls `shouldReturn` 1++  it "encodes the object name and create-only precondition, and maps HTTP 412" do+    seen <- newIORef []+    withMock+      ( \_ target headers -> do+          modifyIORef' seen (<> [(target, lookup "Authorization" headers)])+          pure (MockResponse 412 [] "")+      )+      \port -> do+        store <- testStore port+        store.putObject (Bucket "control") (ObjectName "cells/alpha/lease space.json") "application/json" DoesNotExist "claim" `shouldReturn` PreconditionFailed+    readIORef seen `shouldReturn` [("/upload/storage/v1/b/control/o?uploadType=media&name=cells%2Falpha%2Flease%20space.json&ifGenerationMatch=0", Just "Bearer test-token")]++  it "pins media reads and downloads to the metadata generation" $ withSystemTempDirectory "kenshou-gcs" \root -> do+    seen <- newIORef []+    let object = ObjectName "cells/alpha/work"+        destination = root </> "nested" </> "work"+        metaPath = "/storage/v1/b/control/o/cells%2Falpha%2Fwork"+    withMock+      ( \_ target _ -> do+          modifyIORef' seen (<> [target])+          pure+            if "alt=media" `ByteString.isInfixOf` target+              then MockResponse 200 [] "hello"+              else MockResponse 200 [] objectMetadata+      )+      \port -> do+        store <- testStore port+        fmap fst <$> store.getObject (Bucket "control") object `shouldReturn` Just "hello"+        result <- store.downloadTo (Bucket "control") object destination+        result `shouldSatisfy` maybe False (const True)+    LazyByteString.readFile destination `shouldReturn` "hello"+    readIORef seen `shouldReturn` [metaPath, metaPath <> "?alt=media&generation=7", metaPath, metaPath <> "?alt=media&generation=7"]++  it "does not leave a truncated download as the destination" $ withSystemTempDirectory "kenshou-gcs" \root -> do+    let destination = root </> "work"+    withMock+      ( \_ target _ ->+          pure+            if "alt=media" `ByteString.isInfixOf` target+              then MockResponse 200 [] "short"+              else MockResponse 200 [] "{\"name\":\"cells/alpha/work\",\"generation\":\"7\",\"size\":\"6\",\"updated\":\"2026-09-27T00:00:00Z\"}"+      )+      \port -> do+        store <- testStore port+        store.downloadTo (Bucket "control") (ObjectName "cells/alpha/work") destination `shouldThrow` anyIOException+    doesFileExist destination `shouldReturn` False++  it "paginates listings and reads server time from the Date header" do+    withMock+      ( \_ target _ ->+          pure+            if target == "/storage/v1/b/control/o?maxResults=1"+              then MockResponse 200 [("Date", "Sun, 27 Sep 2026 12:34:56 GMT")] "{}"+              else+                if target == "/storage/v1/b/control/o?prefix=cells%2F&maxResults=1000"+                  then MockResponse 200 [] "{\"items\":[{\"name\":\"cells/a\",\"generation\":\"1\",\"size\":\"0\",\"updated\":\"2026-09-27T00:00:00Z\"}],\"nextPageToken\":\"page 2\"}"+                  else MockResponse 200 [] "{\"items\":[{\"name\":\"cells/b\",\"generation\":\"2\",\"size\":\"0\",\"updated\":\"2026-09-27T00:00:00Z\"}]}"+      )+      \port -> do+        store <- testStore port+        objects <- store.listObjects (Bucket "control") "cells/"+        sort (map fst objects) `shouldBe` [ObjectName "cells/a", ObjectName "cells/b"]+        store.serverTime `shouldReturn` UTCTime (fromGregorian 2026 9 27) (secondsToDiffTime (12 * 3600 + 34 * 60 + 56))++  it "retries a transient metadata HTTP 503" do+    calls <- newIORef (0 :: Int)+    withMock+      ( \_ _ _ -> do+          modifyIORef' calls (+ 1)+          count <- readIORef calls+          pure if count == 1 then MockResponse 503 [] "" else MockResponse 200 [] objectMetadata+      )+      \port -> do+        store <- testStore port+        store.statObject (Bucket "control") (ObjectName "cells/alpha/work") `shouldSatisfyIO` maybe False (const True)+    readIORef calls `shouldReturn` 2++  it "refreshes a cached gcloud token after HTTP 401" do+    calls <- newIORef (0 :: Int)+    seen <- newIORef []+    let cli = do+          modifyIORef' calls (+ 1)+          count <- readIORef calls+          pure (if count == 1 then "stale-token" else "fresh-token")+    withMock+      ( \_ _ headers -> do+          let authorization = lookup "Authorization" headers+          modifyIORef' seen (<> [authorization])+          pure if authorization == Just "Bearer stale-token" then MockResponse 401 [] "" else MockResponse 200 [] objectMetadata+      )+      \port -> do+        provider <- newTokenProviderWith (Bucket "control") (pure Nothing) (pure Nothing) cli+        store <- newGcsStoreAt ("http://127.0.0.1:" <> Text.pack (show port)) provider+        store.statObject (Bucket "control") (ObjectName "cells/alpha/work") `shouldSatisfyIO` maybe False (const True)+        store.statObject (Bucket "control") (ObjectName "cells/alpha/work") `shouldSatisfyIO` maybe False (const True)+    readIORef calls `shouldReturn` 2+    readIORef seen `shouldReturn` [Just "Bearer stale-token", Just "Bearer fresh-token", Just "Bearer fresh-token"]++  it "sends a large file in 8 MiB resumable chunks" $ withSystemTempDirectory "kenshou-gcs" \root -> do+    let source = root </> "payload.nar.zst"+        size = 8 * 1024 * 1024 + 5+    LazyByteString.writeFile source (LazyByteString.replicate size 120)+    ranges <- newIORef []+    withMock+      ( \port target headers ->+          if "uploadType=resumable" `ByteString.isInfixOf` target+            then pure (MockResponse 200 [("Location", Char8.pack ("http://127.0.0.1:" <> show port <> "/upload-session"))] "")+            else do+              modifyIORef' ranges (<> [lookup "Content-Range" headers])+              pure+                if lookup "Content-Range" headers == Just "bytes 0-8388607/8388613"+                  then MockResponse 308 [("Range", "bytes=0-8388607")] ""+                  else MockResponse 201 [] "{\"name\":\"payloads/sha256/abc.nar.zst\",\"generation\":\"9\",\"size\":\"8388613\",\"updated\":\"2026-09-27T00:00:00Z\"}"+      )+      \port -> do+        store <- testStore port+        result <- store.putFile (Bucket "control") (ObjectName "payloads/sha256/abc.nar.zst") "application/zstd" DoesNotExist source+        result `shouldSatisfy` \case+          Written meta -> meta.size == size+          _ -> False+    readIORef ranges `shouldReturn` [Just "bytes 0-8388607/8388613", Just "bytes 8388608-8388612/8388613"]++  it "queries resumable progress after HTTP 503 and resends an unpersisted chunk" $ withSystemTempDirectory "kenshou-gcs" \root -> do+    let source = root </> "payload.nar.zst"+    LazyByteString.writeFile source (LazyByteString.replicate (8 * 1024 * 1024 + 1) 120)+    ranges <- newIORef []+    withMock+      ( \port target headers ->+          if "uploadType=resumable" `ByteString.isInfixOf` target+            then pure (MockResponse 200 [("Location", Char8.pack ("http://127.0.0.1:" <> show port <> "/upload-session"))] "")+            else do+              let range = lookup "Content-Range" headers+              modifyIORef' ranges (<> [range])+              history <- readIORef ranges+              pure case range of+                Just "bytes */8388609" -> MockResponse 308 [] ""+                Just "bytes 0-8388607/8388609" | length history == 1 -> MockResponse 503 [] ""+                Just "bytes 0-8388607/8388609" -> MockResponse 308 [("Range", "bytes=0-8388607")] ""+                _ -> MockResponse 201 [] "{\"name\":\"payloads/sha256/abc.nar.zst\",\"generation\":\"9\",\"size\":\"8388609\",\"updated\":\"2026-09-27T00:00:00Z\"}"+      )+      \port -> do+        store <- testStore port+        result <- store.putFile (Bucket "control") (ObjectName "payloads/sha256/abc.nar.zst") "application/zstd" DoesNotExist source+        result `shouldSatisfy` \case+          Written _ -> True+          _ -> False+    readIORef ranges `shouldReturn` [Just "bytes 0-8388607/8388609", Just "bytes */8388609", Just "bytes 0-8388607/8388609", Just "bytes 8388608-8388608/8388609"]++testStore :: Int -> IO ObjectStore+testStore port = newGcsStoreAt ("http://127.0.0.1:" <> Text.pack (show port)) (TokenProvider (Bucket "control") (pure "test-token") (pure ()))++objectMetadata :: LazyByteString.ByteString+objectMetadata = "{\"name\":\"cells/alpha/work\",\"generation\":\"7\",\"size\":\"5\",\"updated\":\"2026-09-27T00:00:00Z\",\"contentType\":\"application/octet-stream\"}"++withMock :: (Int -> ByteString.ByteString -> [(ByteString.ByteString, ByteString.ByteString)] -> IO MockResponse) -> (Int -> IO value) -> IO value+withMock respond action = bracket open close \listener -> do+  port <-+    getSocketName listener >>= \case+      SockAddrInet number _ -> pure (fromIntegral number)+      _ -> error "expected IPv4 listener"+  thread <-+    forkIO+      ( forever do+          (connection, _) <- accept listener+          void (forkIO (bracket (socketToHandle connection ReadWriteMode) hClose (serve (respond port))))+      )+  action port `finally` killThread thread+  where+    open = do+      listener <- socket AF_INET Stream defaultProtocol+      bind listener (SockAddrInet 0 (tupleToHostAddress (127, 0, 0, 1)))+      listen listener 16+      pure listener++serve :: (ByteString.ByteString -> [(ByteString.ByteString, ByteString.ByteString)] -> IO MockResponse) -> Handle -> IO ()+serve handler connection = do+  hSetBuffering connection NoBuffering+  requestLine <- stripCr <$> Char8.hGetLine connection+  requestHeaders <- readHeaders connection+  let lengthBytes = maybe 0 (read . Char8.unpack) (lookup "Content-Length" requestHeaders)+  consumeBody connection lengthBytes+  MockResponse status headers body <- handler (requestTarget requestLine) requestHeaders+  let statusText = case status of+        200 -> "OK"+        201 -> "Created"+        204 -> "No Content"+        412 -> "Precondition Failed"+        _ -> "Error"+      responseHeaders = headers <> [("Content-Length", Char8.pack (show (LazyByteString.length body))), ("Connection", "close")]+  Char8.hPutStr connection ("HTTP/1.1 " <> Char8.pack (show status) <> " " <> statusText <> "\r\n")+  mapM_ (\(name, value) -> Char8.hPutStr connection (name <> ": " <> value <> "\r\n")) responseHeaders+  Char8.hPutStr connection "\r\n"+  LazyByteString.hPut connection body++requestTarget :: ByteString.ByteString -> ByteString.ByteString+requestTarget line = case Char8.words line of+  _ : target : _ -> target+  _ -> error "invalid mock request line"++readHeaders :: Handle -> IO [(ByteString.ByteString, ByteString.ByteString)]+readHeaders connection = do+  line <- stripCr <$> Char8.hGetLine connection+  if ByteString.null line+    then pure []+    else do+      let (name, rest) = Char8.break (== ':') line+      ((name, Char8.dropWhile (== ' ') (ByteString.drop 1 rest)) :) <$> readHeaders connection++stripCr :: ByteString.ByteString -> ByteString.ByteString+stripCr = Char8.dropWhileEnd (== '\r')++consumeBody :: Handle -> Int -> IO ()+consumeBody connection remaining = unless (remaining == 0) do+  chunk <- ByteString.hGetSome connection (min remaining (64 * 1024))+  unless (not (ByteString.null chunk)) (error "mock request body ended early")+  consumeBody connection (remaining - ByteString.length chunk)++shouldSatisfyIO :: (Show value) => IO value -> (value -> Bool) -> IO ()+shouldSatisfyIO action predicate = action >>= (`shouldSatisfy` predicate)
+ test/HealthSpec.hs view
@@ -0,0 +1,50 @@+module HealthSpec (spec) where++import Control.Concurrent (threadDelay)+import Data.Aeson (Value (..), eitherDecodeFileStrict', encode)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Kenshou.Core.Id (parseRunId)+import Kenshou.Remote.Cell.Health (mapCellHealth, withCellHealthFile)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell health mapping" do+  it "maps a tripped maintenance gate into a hard run notice" do+    document <- ownerHealth+    identifier <- either (error . show) pure (parseRunId "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9")+    case mapCellHealth identifier document of+      Left problem -> expectationFailure (show problem)+      Right [Object fields] -> do+        KeyMap.lookup "schema" fields `shouldBe` Just (String "kenshou.health-notice/v1")+        KeyMap.lookup "source" fields `shouldBe` Just (String "cell-health:host-maintenance:cell-alpha-postgres")+        KeyMap.lookup "severity" fields `shouldBe` Just (String "hard")+      Right notices -> expectationFailure ("expected one notice: " <> show notices)++  it "rejects another cell run's health document" do+    document <- ownerHealth+    identifier <- either (error . show) pure (parseRunId "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011")+    mapCellHealth identifier document `shouldSatisfy` isLeft++  it "reads new complete lines while a run executes and does not duplicate notices" $+    withSystemTempDirectory "kenshou-cell-health" \root -> do+      identifier <- either (error . show) pure (parseRunId "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9")+      let source = root </> "cell-health.jsonl"+          destination = root </> "health-notices.jsonl"+      LazyByteString.writeFile destination ""+      line <- encode <$> ownerHealth+      withCellHealthFile (Just source) destination identifier do+        LazyByteString.writeFile source (line <> "\n" <> line <> "\n")+        threadDelay 1500000+      notices <- LazyByteString.readFile destination+      length (filter (not . LazyByteString.null) (LazyByteString.split 10 notices)) `shouldBe` 1++ownerHealth :: IO Value+ownerHealth = eitherDecodeFileStrict' "test/golden/cell/cell.health.v1.json" >>= either fail pure++isLeft :: Either Text [Value] -> Bool+isLeft (Left _) = True+isLeft _ = False
+ test/LeaseSpec.hs view
@@ -0,0 +1,198 @@+module LeaseSpec (spec) where++import Control.Concurrent (forkIO, threadDelay)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Monad (replicateM)+import Data.Aeson (Value, eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isRight)+import Data.IORef (atomicModifyIORef', newIORef)+import Data.Maybe (isJust)+import Data.Time (addUTCTime, getCurrentTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Remote.Cell.Control (CellSnapshot (..), readCellSnapshot)+import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), Quarantine (..), acquireLease, acquireLeaseWithId, leaseHeld, leaseSnapshot, reattachLease, releaseLease, renewLease, requestCancel, resizeLease, withHeartbeat)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell lease protocol" do+  it "reconciles a caller-retained claim without another write or reviving an expired lease" $ withSystemTempDirectory "kenshou-retained-claim" \root -> do+    store <- newFileStore root+    identifier <- newRunId+    handle <- acquireLeaseWithId store cellRef request identifier >>= expectAcquired+    original <- store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json")+    recovered <- acquireLeaseWithId store cellRef request identifier >>= expectAcquired+    (leaseSnapshot recovered >>= pure . (.leaseId)) `shouldReturn` identifier+    store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` original+    now <- store.serverTime+    let future = store {serverTime = pure (addUTCTime 1000 now)}+    expired <- acquireLeaseWithId future cellRef request identifier+    case expired of Busy lease -> lease.leaseId `shouldBe` identifier; _ -> expectationFailure "expired identity was revived"+    store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` original+    releaseLease store cellRef handle `shouldReturn` True++  it "gives exactly one contender a create-only lease" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    replies <- replicateM 12 newEmptyMVar+    mapM_ (\reply -> forkIO (acquireLease store cellRef request >>= putMVar reply)) replies+    outcomes <- mapM takeMVar replies+    length [() | Acquired _ <- outcomes] `shouldBe` 1+    length [() | Busy _ <- outcomes] `shouldBe` 11++  it "renews with the latest generation, reattaches by id, and releases" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    first <- leaseSnapshot handle+    renewLease store cellRef handle `shouldReturn` True+    renewed <- leaseSnapshot handle+    renewed.heartbeatAt `shouldSatisfy` (>= first.heartbeatAt)+    reattached <- reattachLease store cellRef first.leaseId+    isJust reattached `shouldBe` True+    releaseLease store cellRef handle `shouldReturn` True+    leaseHeld handle `shouldReturn` False+    store.statObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing++  it "retains the lease when the cell agent advances runsStarted" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    lease <- leaseSnapshot handle+    let objectName = ObjectName "cells/alpha/lease.json"+    meta <- store.statObject cellRef.controlBucket objectName >>= maybe (expectationFailure "missing lease" >> error "unreachable") pure+    updated <- store.putObject cellRef.controlBucket objectName "application/json" (GenerationIs meta.generation) (encode (lease {runsStarted = 1}))+    updated `shouldSatisfy` \case Written _ -> True; _ -> False+    renewLease store cellRef handle `shouldReturn` True+    refreshed <- leaseSnapshot handle+    refreshed.runsStarted `shouldBe` 1+    releaseLease store cellRef handle `shouldReturn` True++  it "takes over only after server time exceeds updated plus TTL and grace" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    old <- acquireLease store cellRef shortRequest >>= expectAcquired+    let later = store {serverTime = addUTCTime 100 <$> store.serverTime}+    replacement <- acquireLease later cellRef request >>= expectAcquired+    leaseHeld replacement `shouldReturn` True+    renewLease store cellRef old `shouldReturn` False+    releaseLease later cellRef replacement `shouldReturn` True++  it "fences a detached TTL extension against a stale lease generation" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    resizeLease store cellRef handle 900 `shouldReturn` True+    snapshot <- leaseSnapshot handle+    snapshot.ttlSeconds `shouldBe` 900+    stale <- reattachLease store cellRef snapshot.leaseId+    case stale of+      Nothing -> expectationFailure "extended lease disappeared"+      Just old -> do+        renewLease store cellRef handle `shouldReturn` True+        resizeLease store cellRef old 1200 `shouldReturn` False+        leaseHeld old `shouldReturn` False+    releaseLease store cellRef handle `shouldReturn` True++  it "releases a newly won lease when quarantine exists" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    now <- getCurrentTime+    let record = Quarantine "alpha" "reset-failed" Nothing now+    _ <- store.putObject cellRef.controlBucket (ObjectName "cells/alpha/quarantine.json") "application/json" DoesNotExist (encode record)+    outcome <- acquireLease store cellRef request+    case outcome of+      Quarantined observed -> observed `shouldBe` record+      _ -> expectationFailure "expected quarantined cell"+    store.statObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing++  it "keeps a heartbeat and allows release after its thread stops" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef shortRequest >>= expectAcquired+    withHeartbeat store cellRef handle \stillHeld -> do+      threadDelay 1200000+      stillHeld `shouldReturn` True+    releaseLease store cellRef handle `shouldReturn` True++  it "retries one transient heartbeat write failure while the lease remains valid" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef shortRequest >>= expectAcquired+    initial <- leaseSnapshot handle+    failNext <- newIORef True+    let flaky =+          store+            { putObject = \bucket object mediaType precondition bytes -> do+                shouldFail <-+                  if object == ObjectName "cells/alpha/lease.json"+                    then atomicModifyIORef' failNext (\failPending -> (False, failPending))+                    else pure False+                if shouldFail+                  then ioError (userError "transient heartbeat write failure")+                  else store.putObject bucket object mediaType precondition bytes+            }+    withHeartbeat flaky cellRef handle \stillHeld -> do+      threadDelay 1700000+      stillHeld `shouldReturn` True+    renewed <- leaseSnapshot handle+    renewed.heartbeatAt `shouldSatisfy` (> initial.heartbeatAt)+    releaseLease store cellRef handle `shouldReturn` True++  it "cancels only the expected lease unless force is explicit" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    current <- leaseSnapshot handle+    otherId <- newRunId+    requestCancel store cellRef Nothing False `shouldThrow` anyIOException+    requestCancel store cellRef (Just otherId) False `shouldReturn` False+    requestCancel store cellRef (Just current.leaseId) False `shouldReturn` True+    renewLease store cellRef handle `shouldReturn` False+    reattached <- reattachLease store cellRef current.leaseId+    case reattached of+      Just fresh -> releaseLease store cellRef fresh `shouldReturn` True+      Nothing -> expectationFailure "cancelled lease could not be reattached for release"+    second <- acquireLease store cellRef request >>= expectAcquired+    requestCancel store cellRef Nothing True `shouldReturn` True+    leaseHeld second `shouldReturn` True+    renewLease store cellRef second `shouldReturn` False++  it "round-trips the cell lease document" $ withSystemTempDirectory "kenshou-lease" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    lease <- leaseSnapshot handle+    eitherDecode (encode lease) `shouldBe` Right lease++  it "decodes the cell owner's lease and quarantine fixtures and preserves their JSON" do+    leaseBytes <- LazyByteString.readFile "test/golden/cell/cell.lease.v1.json"+    quarantineBytes <- LazyByteString.readFile "test/golden/cell/cell.quarantine.v1.json"+    let decodedLease = eitherDecode leaseBytes :: Either String Lease+        decodedQuarantine = eitherDecode quarantineBytes :: Either String Quarantine+    decodedLease `shouldSatisfy` isRight+    decodedQuarantine `shouldSatisfy` isRight+    fmap (eitherDecode . encode) decodedLease `shouldBe` Right (eitherDecode leaseBytes :: Either String Value)+    fmap (eitherDecode . encode) decodedQuarantine `shouldBe` Right (eitherDecode quarantineBytes :: Either String Value)++  it "reads owner control objects only for an allowed project and matching bucket" $ withSystemTempDirectory "kenshou-cell-control" \root -> do+    store <- newFileStore root+    descriptor <- LazyByteString.readFile "test/golden/cell/cell.descriptor.v1.json"+    let ref = CellRef "alpha" (Bucket "tan-nb-exp-cells-control")+    _ <- store.putObject ref.controlBucket (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor+    snapshot <- readCellSnapshot store ref ["tan-nb-exp"]+    case snapshot of+      Right observed -> do+        observed.lease `shouldBe` Nothing+        observed.quarantine `shouldBe` Nothing+      Left problem -> expectationFailure (show problem)+    readCellSnapshot store ref ["another-project"] `shouldReturn` Left "cell project is outside KENSHOU_GCP_ALLOWED_PROJECTS: tan-nb-exp"+    let wrongBucket = CellRef "alpha" (Bucket "other-control")+    _ <- store.putObject wrongBucket.controlBucket (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor+    readCellSnapshot store wrongBucket ["tan-nb-exp"] `shouldReturn` Left "cell descriptor control bucket differs from selected bucket"++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")++request :: LeaseRequest+request = LeaseRequest "tester@workstation" "verification" 120++shortRequest :: LeaseRequest+shortRequest = LeaseRequest "tester@workstation" "verification" 1++expectAcquired :: AcquireOutcome -> IO LeaseHandle+expectAcquired (Acquired handle) = pure handle+expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"
+ test/Main.hs view
@@ -0,0 +1,200 @@+module Main (main) where++import Control.Concurrent (forkIO)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Monad (replicateM)+import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy.Char8 qualified as LazyByteString+import Data.Either (isLeft)+import Data.Text qualified as Text+import Data.Time (UTCTime (..), fromGregorian, secondsToDiffTime)+import DebugSpec qualified+import DocsSpec qualified+import ExecSpec qualified+import FetchSpec qualified+import GcsSpec qualified+import HealthSpec qualified+import Kenshou.Core.Cohort (CohortIdentity (..), CohortName (..), PlanHash (..))+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), CohortCheck (..), Harness (..), PayloadDescriptor (..))+import Kenshou.Remote.Perf.Lock (withPerformanceLock)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.File (newFileStore)+import LeaseSpec qualified+import PairSpec qualified+import ParitySpec qualified+import PayloadSpec qualified+import PerfAttemptSpec qualified+import PerfCellSpec qualified+import PerfCleanupSpec qualified+import PerfConditionsSpec qualified+import PerfJournalSpec qualified+import PerfLockSpec qualified+import PerfPublicationSpec qualified+import PerfRunnerSpec qualified+import PrepareSpec qualified+import SessionSpec qualified+import SubmitSpec qualified+import System.Directory (getFileSize)+import System.Environment (getArgs, getExecutablePath)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (proc, readCreateProcessWithExitCode)+import Test.Hspec+import WatchSpec qualified++main :: IO ()+main =+  getArgs >>= \case+    ["--perf-lock", root] -> withPerformanceLock root (putStrLn "locked")+    ["--claim", root] -> do+      store <- newFileStore root+      result <- store.putObject (Bucket "control") (ObjectName "cells/alpha/lease.json") "application/json" DoesNotExist "claim"+      putStrLn case result of+        Written _ -> "written"+        PreconditionFailed -> "precondition-failed"+    _ -> hspec tests++tests :: Spec+tests = do+  DebugSpec.spec+  DocsSpec.spec+  ExecSpec.execSpec+  FetchSpec.spec+  GcsSpec.spec+  HealthSpec.spec+  LeaseSpec.spec+  PairSpec.pairSpec+  ParitySpec.spec+  PayloadSpec.spec+  PerfJournalSpec.spec+  PerfRunnerSpec.spec+  PerfPublicationSpec.spec+  PerfConditionsSpec.spec+  PerfAttemptSpec.spec+  PerfCleanupSpec.spec+  PerfCellSpec.spec+  PerfLockSpec.spec+  PrepareSpec.spec+  SessionSpec.spec+  SubmitSpec.spec+  WatchSpec.spec+  describe "Kenshou payload descriptor" do+    it "round-trips the complete cell payload and Nix cohort identity" do+      eitherDecode (encode examplePayload) `shouldBe` Right examplePayload++    it "rejects a bundle URI that disagrees with its content digest" do+      let badBundle = examplePayload.cell.bundle {uri = "gs://control/payloads/sha256/other.nar.zst"}+          bad = examplePayload {cell = examplePayload.cell {bundle = badBundle}}+      (eitherDecode (encode bad) :: Either String PayloadDescriptor) `shouldSatisfy` isLeft++    it "requires the Kenshou entry point even though the cell protocol accepts other commands" do+      let generic = examplePayload {cell = examplePayload.cell {command = ["bin/cell-fixture-hello"]}}+      (eitherDecode (encode generic) :: Either String PayloadDescriptor) `shouldSatisfy` isLeft++  describe "file cell object store" do+    it "fences stale generations and preserves tombstones" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+      store <- newFileStore root+      let bucket = Bucket "control"+          object = ObjectName "cells/alpha/lease.json"+      first <- store.putObject bucket object "application/json" DoesNotExist "first"+      firstMeta <- expectWritten first+      firstMeta.generation `shouldBe` 1+      stale <- store.putObject bucket object "application/json" DoesNotExist "stale"+      stale `shouldBe` PreconditionFailed+      next <- store.putObject bucket object "application/json" (GenerationIs firstMeta.generation) "second"+      secondMeta <- expectWritten next+      secondMeta.generation `shouldBe` 2+      store.deleteObject bucket object (GenerationIs firstMeta.generation) `shouldReturn` False+      store.deleteObject bucket object (GenerationIs secondMeta.generation) `shouldReturn` True+      store.statObject bucket object `shouldReturn` Nothing+      afterDelete <- store.putObject bucket object "application/json" DoesNotExist "third"+      thirdMeta <- expectWritten afterDelete+      thirdMeta.generation `shouldBe` 4+      stored <- store.getObject bucket object+      fmap fst stored `shouldBe` Just "third"+      store.listObjects bucket "cells/alpha/" `shouldReturn` [(object, thirdMeta)]++    it "admits exactly one concurrent create-only claim" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+      store <- newFileStore root+      let bucket = Bucket "control"+          object = ObjectName "cells/alpha/lease.json"+      outcomes <- replicateM 20 newEmptyMVar+      mapM_ (\reply -> forkIO (store.putObject bucket object "application/json" DoesNotExist "claim" >>= putMVar reply)) outcomes+      results <- mapM takeMVar outcomes+      length [() | Written _ <- results] `shouldBe` 1+      length [() | PreconditionFailed <- results] `shouldBe` 19++    it "fences independent store handles in one process" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+      stores <- replicateM 12 (newFileStore root)+      replies <- replicateM 12 newEmptyMVar+      let bucket = Bucket "control"+          object = ObjectName "cells/alpha/lease.json"+      mapM_ (\(store, reply) -> forkIO (store.putObject bucket object "application/json" DoesNotExist "claim" >>= putMVar reply)) (zip stores replies)+      results <- mapM takeMVar replies+      length [() | Written _ <- results] `shouldBe` 1+      length [() | PreconditionFailed <- results] `shouldBe` 11++    it "fences claims made by separate processes" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+      executable <- getExecutablePath+      replies <- replicateM 4 newEmptyMVar+      mapM_ (\reply -> forkIO (readCreateProcessWithExitCode (proc executable ["--claim", root]) "" >>= putMVar reply)) replies+      results <- mapM takeMVar replies+      [exit | (exit, _, _) <- results] `shouldBe` replicate 4 ExitSuccess+      length [() | (_, output, _) <- results, output == "written\n"] `shouldBe` 1+      length [() | (_, output, _) <- results, output == "precondition-failed\n"] `shouldBe` 3++    it "streams file publication and download under one object name" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+      store <- newFileStore (root </> "store")+      let bucket = Bucket "control"+          object = ObjectName "payloads/sha256/abc.nar.zst"+          source = root </> "source"+          fetched = root </> "fetched"+          bytes = LazyByteString.replicate (2 * 1024 * 1024) 'a'+      LazyByteString.writeFile source bytes+      result <- store.putFile bucket object "application/zstd" DoesNotExist source+      meta <- expectWritten result+      meta.size `shouldBe` LazyByteString.length bytes+      store.downloadTo bucket object fetched `shouldReturn` Just meta+      getFileSize fetched `shouldReturn` fromIntegral (LazyByteString.length bytes)++    it "rejects names that could escape a bucket" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+      store <- newFileStore root+      store.putObject (Bucket "control") (ObjectName "../lease") "application/json" DoesNotExist "bad" `shouldThrow` anyIOException++expectWritten :: PutOutcome -> IO ObjectMeta+expectWritten (Written meta) = pure meta+expectWritten PreconditionFailed = expectationFailure "write was unexpectedly rejected" >> error "unreachable"++examplePayload :: PayloadDescriptor+examplePayload =+  PayloadDescriptor+    { cohort = "released",+      variant = "default",+      flakeAttr = "packages.x86_64-linux.kenshou-released",+      harness = Harness (Text.replicate 40 "a") False,+      cohortIdentity =+        CohortIdentity+          { identityCohort = CohortName "released",+            identityCompiler = "ghc-9.12.4",+            identityCabalVersion = "nix",+            identityOs = "linux",+            identityArch = "x86_64",+            identityIndexState = Nothing,+            identityPlanHash = PlanHash "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+            identityDescriptorSha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+            identityComponents = [],+            identityResolver = Just "nix"+          },+      cohortCheck = CohortCheck 0,+      cell =+        CellPayload+          { bundle = Bundle "gs://control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" 1,+            storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released",+            narHash = "sha256-example",+            closurePaths = ["/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released"],+            system = "x86_64-linux",+            command = ["bin/kenshou", "cell", "exec"]+          },+      createdAt = UTCTime (fromGregorian 2026 9 27) (secondsToDiffTime 0)+    }
+ test/PairSpec.hs view
@@ -0,0 +1,59 @@+module PairSpec (pairSpec) where++import Data.Aeson (Value, eitherDecodeFileStrict', toJSON)+import Data.List (nub)+import Data.Text qualified as Text+import Kenshou.Core.Id (parseScenarioId)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Measure.Compare.Ordering (Arm (..), PairedOrdering (..))+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Pair (PairRequest (..), PairState (..), SliceResult (..), judgePairs, pairPlan)+import Test.Hspec++pairSpec :: Spec+pairSpec = describe "cell paired schedule" do+  it "interleaves fresh same-seed benchmark arms and reserves replacement pairs" do+    source <- fixture+    let request = PairRequest "candidate" "baseline" 3 ABBA 7 1+    paired <- case pairPlan request source of+      Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"+      Right result -> pure result+    length paired.runs `shouldBe` 8+    paired.estimateMinutes `shouldBe` 8+    fmap (fmap (.arm) . (.trial)) paired.runs `shouldBe` fmap Just ["baseline", "candidate", "candidate", "baseline", "baseline", "candidate", "candidate", "baseline"]+    length (nub (fmap (.runId) paired.runs)) `shouldBe` 8+    fmap (.spec.seed) (take 2 paired.runs) `shouldSatisfy` \case [Just first, Just second] -> first == second; _ -> False+    fmap (.spec.seed) (take 2 (drop 2 paired.runs)) `shouldSatisfy` \case [Just first, Just second] -> first == second; _ -> False+    fmap (.comparison) (fmap (.spec) paired.runs) `shouldSatisfy` all (/= Nothing)+    pairPlan request source `shouldBe` Right paired+    decodeWorkPlan (toJSON paired) `shouldBe` Right paired++  it "keeps one failed arm out of a valid pair" do+    let result index arm passed = SliceResult "group" index arm passed+    judgePairs [result 0 Baseline True, result 0 Candidate True, result 1 Candidate False, result 1 Baseline True]+      `shouldBe` [(("group", 0), PairValid), (("group", 1), PairInvalid "one or both cell trials failed or are missing")]++  it "rejects a correctness plan or fewer than three pairs" do+    source <- sourceFixture+    pairPlan (PairRequest "candidate" "baseline" 3 ABBA 7 0) source `shouldBe` Left "paired comparisons require benchmark runs only"+    benchmark <- fixture+    pairPlan (PairRequest "candidate" "baseline" 2 ABBA 7 0) benchmark `shouldBe` Left "paired comparisons need at least three pairs"++fixture :: IO RunPlan+fixture = do+  source <- sourceFixture+  scenario <- case parseScenarioId "selftest/measure/benchmark/sleep-service" of+    Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"+    Right identifier -> pure identifier+  pure source {runs = [entry {spec = entry.spec {scenario = scenario}} | entry <- source.runs]}++sourceFixture :: IO RunPlan+sourceFixture = do+  decoded <- eitherDecodeFileStrict' "../kenshou-core/test/golden/run-plan.minimal.json" :: IO (Either String Value)+  document <- case decoded of+    Left problem -> expectationFailure problem >> error "unreachable"+    Right value -> pure value+  case decodeWorkPlan document of+    Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"+    Right plan -> pure plan
+ test/ParitySpec.hs view
@@ -0,0 +1,82 @@+module ParitySpec (spec) where++import Data.Aeson (Value, encode, object, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text qualified as Text+import Kenshou.Remote.Cell.Parity (ParityDifference (..), ParityOptions (..), ParityReport (..), compareForParity)+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "local and cell parity" do+  it "names placement differences while preserving scenario and verdict equality" $ withSystemTempDirectory "kenshou-parity" \root -> do+    let localDir = root </> "local"+        cellDir = root </> "cell"+    writeRun localDir (result "local" "passed" 7) (runSpec "local" 7)+    writeRun cellDir (result "cell" "passed" 7) (runSpec "cell" 7)+    report <- compareForParity (ParityOptions []) localDir cellDir+    report.unexpected `shouldBe` []+    let intentionalPaths = fmap (.path) report.intentional+    mapM_ (\path -> intentionalPaths `shouldContain` [path]) ["result.fingerprint.placement", "spec.environment.placement"]+    mapM_ (\path -> report.equal `shouldContain` [path]) ["result.outcome", "spec.seed"]+    report.localRunResultSha256 `shouldNotBe` report.cellRunResultSha256++  it "rejects changed outcomes, seeds and verdict counts" $ withSystemTempDirectory "kenshou-parity" \root -> do+    let localDir = root </> "local"+        cellDir = root </> "cell"+    writeRun localDir (result "local" "passed" 7) (runSpec "local" 7)+    writeRun cellDir (result "cell" "failed" 8) (runSpec "cell" 8)+    writeVerdict localDir 3+    writeVerdict cellDir 2+    report <- compareForParity (ParityOptions []) localDir cellDir+    let unexpectedPaths = fmap (.path) report.unexpected+    mapM_ (\path -> unexpectedPaths `shouldContain` [path]) ["result.outcome", "spec.seed", "verdicts.ledger.json.counts.checked"]++  it "allows only a named volatile verdict field" $ withSystemTempDirectory "kenshou-parity" \root -> do+    let localDir = root </> "local"+        cellDir = root </> "cell"+    writeRun localDir (result "local" "passed" 7) (runSpec "local" 7)+    writeRun cellDir (result "cell" "passed" 7) (runSpec "cell" 7)+    writeVerdict localDir 3+    writeVerdict cellDir 2+    report <- compareForParity (ParityOptions ["verdicts.ledger.json.counts.checked"]) localDir cellDir+    report.unexpected `shouldBe` []+    fmap (.path) report.intentional `shouldContain` ["verdicts.ledger.json.counts.checked"]++writeRun :: FilePath -> Value -> Value -> IO ()+writeRun directory runResult specValue = do+  createDirectoryIfMissing True directory+  LazyByteString.writeFile (directory </> "run-result.json") (encode runResult)+  LazyByteString.writeFile (directory </> "run-spec.json") (encode specValue)++writeVerdict :: FilePath -> Int -> IO ()+writeVerdict directory count = do+  let verdictDir = directory </> "verdicts"+  createDirectoryIfMissing True verdictDir+  LazyByteString.writeFile (verdictDir </> "ledger.json") (encode (object ["status" .= ("passed" :: Text.Text), "counts" .= object ["checked" .= count]]))++result :: Text.Text -> Text.Text -> Int -> Value+result placement outcome seed =+  object+    [ "schema" .= ("kenshou.run-result/v1" :: Text.Text),+      "runId" .= placement,+      "scenario" .= ("selftest/kernel/correctness/postgres-roundtrip" :: Text.Text),+      "outcome" .= outcome,+      "seed" .= seed,+      "cohort" .= object ["components" .= [object ["name" .= ("synthetic-store" :: Text.Text), "version" .= ("0.8.0.1" :: Text.Text)]], "os" .= placement],+      "fingerprint" .= object ["placement" .= placement]+    ]++runSpec :: Text.Text -> Int -> Value+runSpec placement seed =+  object+    [ "runId" .= placement,+      "scenario" .= ("selftest/kernel/correctness/postgres-roundtrip" :: Text.Text),+      "seed" .= seed,+      "knobs" .= object ["selftest.events" .= (3 :: Int)],+      "dimensions" .= object ["pg.version" .= ("18" :: Text.Text)],+      "environment" .= object ["placement" .= placement],+      "labels" .= (if placement == "cell" then object ["postgresPlacement" .= ("cell-server" :: Text.Text)] else object [])+    ]
+ test/PayloadSpec.hs view
@@ -0,0 +1,212 @@+module PayloadSpec (spec) where++import Data.Aeson (eitherDecode, eitherDecodeStrict', encodeFile, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort (CohortDescriptor (..), CohortIdentity (..), CohortName (..), ComponentSpec (..), PackagePin (..), PackageSource (..), PlanHash (..), ResolvedComponent (..), ResolvedPackage (..))+import Kenshou.Remote.Cell.Docs (WorkObject (..))+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), CohortCheck (..), Harness (..), PayloadDescriptor (..))+import Kenshou.Remote.Payload.Nix (BundleInfo (..), ClosureInfo (..), NixError (..), NixTools (..), exportBundleWith, parseClosureInfo)+import Kenshou.Remote.Payload.Publish (BundlePublishError (..), publishBundle)+import Kenshou.Remote.Payload.Publisher (PreparedPublishOptions (..), PublishError (..), PublishOptions (..), PublisherDeps (..), publishPayloadWith, publishPreparedPayloadWith)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (Permissions (..), doesFileExist, getPermissions, listDirectory, setPermissions)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "content-addressed payload publication" do+  it "checks a Nix identity before build, then publishes its exported bundle and descriptor" $ withSystemTempDirectory "kenshou-publisher" \root -> do+    identity <- releasedIdentity+    store <- newFileStore (root </> "store")+    builds <- newIORef (0 :: Int)+    let output = root </> "payload.json"+        options = PublishOptions "../fixtures/consumer-baseline" "released" "default" (Bucket "control") output False+        storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+        work = workObjectFor "application/octet-stream" bundleBytes+        exported destination = do+          LazyByteString.writeFile destination bundleBytes+          pure (Right (BundleInfo destination work.sha256 work.bytes (ClosureInfo storePath "sha256-root" [storePath])))+        deps selected =+          PublisherDeps+            (\_ -> pure (Right (Harness (Text.replicate 40 "a") False)))+            (\_ -> pure (Right selected))+            (\_ -> modifyIORef' builds (+ 1) >> pure (Right storePath))+            (\_ -> exported)+    let stale = identity {identityDescriptorSha256 = Text.replicate 64 "0"}+    rejected <- publishPayloadWith (deps stale) store options+    rejected `shouldBe` Left (PublishCohortMismatch "Nix identity descriptor digest differs from the selected descriptor")+    readIORef builds `shouldReturn` 0+    store.listObjects (Bucket "control") "payloads/sha256/" `shouldReturn` []+    accepted <- publishPayloadWith (deps identity) store options+    descriptor <- case accepted of+      Left problem -> expectationFailure (show problem) >> error "unreachable"+      Right payload -> pure payload+    descriptor.cohortCheck.packagesChecked `shouldBe` 42+    descriptor.cell.bundle.sha256 `shouldBe` work.sha256+    readIORef builds `shouldReturn` 1+    stored <- store.getObject (Bucket "control") (objectFor descriptor)+    fmap fst stored `shouldBe` Just bundleBytes+    saved <- eitherDecode <$> LazyByteString.readFile output+    saved `shouldBe` Right descriptor++  it "publishes generated performance inputs while verifying a separate clean harness" $ withSystemTempDirectory "kenshou-prepared-publisher" \root -> do+    bytes <- ByteString.readFile "../fixtures/consumer-baseline/cohort/released.json"+    original <- either (ioError . userError) pure (eitherDecodeStrict' bytes :: Either String CohortDescriptor)+    let generated = original {descriptorName = CohortName "perf-fixture-aa"}+        descriptorPath = root </> "cohort.json"+        harness = root </> "clean-harness"+        flake = "path:" <> root <> "/generated"+        commit = Text.replicate 40 "a"+        storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+        work = workObjectFor "application/octet-stream" bundleBytes+    encodeFile descriptorPath generated+    descriptorBytes <- ByteString.readFile descriptorPath+    identity <- releasedIdentity+    let resolved = identity {identityCohort = generated.descriptorName, identityDescriptorSha256 = Text.drop 7 (sha256Hex descriptorBytes)}+    store <- newFileStore (root </> "store")+    builds <- newIORef (0 :: Int)+    let options = PreparedPublishOptions harness commit flake descriptorPath "perf-fixture-aa" "default" (Bucket "control") (root </> "payload.json")+        deps dirty =+          PublisherDeps+            (\path -> (path `shouldBe` harness) >> pure (Right (Harness commit dirty)))+            (\reference -> (reference `shouldBe` flake <> "#packages.x86_64-linux.kenshou-perf-fixture-aa.cohortIdentity") >> pure (Right resolved))+            (\_ -> modifyIORef' builds (+ 1) >> pure (Right storePath))+            (\_ destination -> LazyByteString.writeFile destination bundleBytes >> pure (Right (BundleInfo destination work.sha256 work.bytes (ClosureInfo storePath "sha256-root" [storePath]))))+    publishPreparedPayloadWith (deps True) store options `shouldReturn` Left PublishDirtyWorktree+    wrong <- publishPreparedPayloadWith (deps False) store options {harnessCommit = Text.replicate 40 "b"}+    wrong `shouldSatisfy` \case Left (PublishDescriptorError _) -> True; _ -> False+    readIORef builds `shouldReturn` 0+    accepted <- publishPreparedPayloadWith (deps False) store options+    accepted `shouldSatisfy` \case Right payload -> payload.cohort == "perf-fixture-aa" && payload.harness == Harness commit False; _ -> False+    readIORef builds `shouldReturn` 1++  it "reads the root NAR hash and complete closure from Nix metadata" do+    let root = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+        dependency = "/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-dependency"+        metadata =+          object+            [ Key.fromText (Text.pack root) .= object ["narHash" .= ("sha256-root" :: Text.Text)],+              Key.fromText (Text.pack dependency) .= object ["narHash" .= ("sha256-dependency" :: Text.Text)]+            ]+    parseClosureInfo root metadata `shouldBe` Right (ClosureInfo root "sha256-root" [root, dependency])+    parseClosureInfo dependency (object [Key.fromText (Text.pack root) .= object ["narHash" .= ("sha256-root" :: Text.Text)]])+      `shouldBe` Left (NixInvalidOutput "closure metadata omits the requested store path")++  it "streams a closure export through compression and removes a failed temporary bundle" $ withSystemTempDirectory "kenshou-nix-export" \root -> do+    let storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+        destination = root </> "bundle.nar.zst"+        tools = NixTools (root </> "nix") (root </> "nix-store") (root </> "zstd")+        exported = "exported bytes"+        work = workObjectFor "application/octet-stream" exported+    writeTool tools.nixExecutable (unlines ["#!/bin/sh", "printf '%s' '{\"" <> storePath <> "\":{\"narHash\":\"sha256-root\"}}'"])+    writeTool tools.nixStoreExecutable (unlines ["#!/bin/sh", "printf 'exported bytes'"])+    writeTool tools.zstdExecutable (unlines ["#!/bin/sh", "cat > \"$6\""])+    exportBundleWith tools storePath destination+      `shouldReturn` Right (BundleInfo destination work.sha256 work.bytes (ClosureInfo storePath "sha256-root" [storePath]))+    LazyByteString.readFile destination `shouldReturn` exported+    writeTool tools.zstdExecutable (unlines ["#!/bin/sh", "exit 7"])+    let failedDestination = root </> "failed.nar.zst"+    result <- exportBundleWith tools storePath failedDestination+    result `shouldSatisfy` \case+      Left (NixCommandFailed executable _ _ _) -> executable == tools.zstdExecutable+      _ -> False+    doesFileExist failedDestination `shouldReturn` False+    entries <- listDirectory root+    filter (Text.isPrefixOf ".kenshou-payload-" . Text.pack) entries `shouldBe` []++  it "publishes a checked bundle once and accepts an identical retry" $ withSystemTempDirectory "kenshou-payload" \root -> do+    store <- newFileStore (root </> "store")+    let source = root </> "bundle.nar.zst"+    LazyByteString.writeFile source bundleBytes+    descriptor <- fixtureFor bundleBytes+    publishBundle store (Bucket "control") descriptor source `shouldReturn` Right descriptor+    publishBundle store (Bucket "control") descriptor source `shouldReturn` Right descriptor+    objects <- store.listObjects (Bucket "control") "payloads/sha256/"+    length objects `shouldBe` 1+    stored <- store.getObject (Bucket "control") (objectFor descriptor)+    fmap fst stored `shouldBe` Just bundleBytes++  it "refuses a digest mismatch before creating an object" $ withSystemTempDirectory "kenshou-payload" \root -> do+    store <- newFileStore (root </> "store")+    let source = root </> "bundle.nar.zst"+    LazyByteString.writeFile source bundleBytes+    descriptor <- fixtureFor bundleBytes+    let cell = descriptor.cell+        original = cell.bundle+        wrongDigest = Text.replicate 64 "b"+        wrong = descriptor {cell = cell {bundle = original {sha256 = wrongDigest, uri = "gs://control/payloads/sha256/" <> wrongDigest <> ".nar.zst"}}}+    result <- publishBundle store (Bucket "control") wrong source+    result `shouldBe` Left (BundleDigestMismatch wrongDigest descriptor.cell.bundle.sha256)+    store.listObjects (Bucket "control") "payloads/sha256/" `shouldReturn` []++  it "refuses an incorrect declared byte count before creating an object" $ withSystemTempDirectory "kenshou-payload" \root -> do+    store <- newFileStore (root </> "store")+    let source = root </> "bundle.nar.zst"+    LazyByteString.writeFile source bundleBytes+    descriptor <- fixtureFor bundleBytes+    let cell = descriptor.cell+        original = cell.bundle+        wrong = descriptor {cell = cell {bundle = original {bytes = original.bytes + 1}}}+    publishBundle store (Bucket "control") wrong source `shouldReturn` Left (BundleBytesMismatch (original.bytes + 1) original.bytes)+    store.listObjects (Bucket "control") "payloads/sha256/" `shouldReturn` []++  it "rejects a wrong control bucket and an existing object of another size" $ withSystemTempDirectory "kenshou-payload" \root -> do+    store <- newFileStore (root </> "store")+    let source = root </> "bundle.nar.zst"+    LazyByteString.writeFile source bundleBytes+    descriptor <- fixtureFor bundleBytes+    publishBundle store (Bucket "other") descriptor source `shouldReturn` Left (BundleOutsideControlBucket descriptor.cell.bundle.uri)+    _ <- store.putObject (Bucket "control") (objectFor descriptor) "application/zstd" DoesNotExist "x"+    publishBundle store (Bucket "control") descriptor source `shouldReturn` Left (ExistingBundleSizeMismatch descriptor.cell.bundle.bytes 1)++fixtureFor :: LazyByteString.ByteString -> IO PayloadDescriptor+fixtureFor bytes = do+  encoded <- LazyByteString.readFile "test/golden/payload.json"+  fixture <- either (ioError . userError) pure (eitherDecode encoded :: Either String PayloadDescriptor)+  let work = workObjectFor "application/octet-stream" bytes+      bundle = Bundle ("gs://control/payloads/sha256/" <> work.sha256 <> ".nar.zst") work.sha256 work.bytes+      cell = fixture.cell+  pure fixture {cell = cell {bundle}}++objectFor :: PayloadDescriptor -> ObjectName+objectFor descriptor = ObjectName ("payloads/sha256/" <> descriptor.cell.bundle.sha256 <> ".nar.zst")++bundleBytes :: LazyByteString.ByteString+bundleBytes = "bundle-bytes"++writeTool :: FilePath -> String -> IO ()+writeTool path contents = do+  writeFile path contents+  permissions <- getPermissions path+  setPermissions path permissions {executable = True}++releasedIdentity :: IO CohortIdentity+releasedIdentity = do+  contents <- ByteString.readFile "../fixtures/consumer-baseline/cohort/released.json"+  descriptor <- either (ioError . userError) pure (eitherDecodeStrict' contents :: Either String CohortDescriptor)+  let resolved component =+        ResolvedComponent+          component.componentId+          component.componentMoriUri+          [ResolvedPackage pin.pinName pin.pinVersion (FromHackage Nothing) | pin <- component.componentPackages]+  pure+    ( CohortIdentity+        (CohortName "released")+        "ghc-9.12.4"+        "nix"+        "linux"+        "x86_64"+        (Just descriptor.descriptorIndexState)+        (PlanHash ("sha256:" <> Text.replicate 64 "a"))+        "5c08f30a78a7d36ab707cde925813987356bdc46d0c3626b2882814db685cf56"+        (map resolved descriptor.descriptorComponents)+        (Just "nix")+    )
+ test/PerfAttemptSpec.hs view
@@ -0,0 +1,203 @@+module PerfAttemptSpec (spec) where++import Data.Aeson+import Data.ByteString qualified as Bytes+import Data.IORef+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text qualified as Text+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Data.Word (Word64)+import Kenshou.Core.Id (mkSeed, newRunId, renderRunId)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Measure.Clock (Origin (..))+import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..))+import Kenshou.Measure.Compare.Ordering (PairedOrdering (..))+import Kenshou.Measure.Compare.Policy qualified as Policy+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (encodeHistogram)+import Kenshou.Measure.Samples+import Kenshou.Measure.Summary (summarizeRunDir)+import Kenshou.Perf.Plan+import Kenshou.Perf.Result+import Kenshou.Perf.Result qualified as Configuration (ConfigurationResult (..))+import Kenshou.Plan.RunPlan (PlannedRun, RunPlan (..), TrialInfo (..))+import Kenshou.Plan.RunPlan qualified as Plan (PlannedRun (..))+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Pair+import Kenshou.Remote.Perf.Attempt+import Kenshou.Remote.Perf.Journal+import Kenshou.Remote.Perf.Runner+import System.Directory (createDirectoryIfMissing, renameDirectory)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance paired attempts" do+  it "reconstructs five raw pairs, preserves an exit-zero defect and replays portable paths" $ withSystemTempDirectory "perf-raw-pairs" \directory -> do+    (entry, schedule, policy) <- fixture+    identifier <- newRunId+    calls <- newIORef (0 :: Int)+    blockers <- newIORef ([] :: [Text.Text])+    let attempt = directory </> "original"+        moved = directory </> "relocated"+        callbacks = ComparisonHooks (\_ -> expectationFailure "stable five-pair control must not replace pairs") (\reason -> modifyIORef' blockers (<> [reason]))+        execute run = do+          modifyIORef' calls (+ 1)+          let path = attempt </> "retained" </> Text.unpack (renderRunId run.runId)+          writeRawRun run (run.ordinal == 2) 1000000 path+          pure (Right path)+    result <- executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) attempt callbacks execute+    result.performanceVerdict `shouldBe` "pass"+    result.infrastructureState `shouldBe` "healthy"+    result.validPairs `shouldBe` 5+    result.runtimeFailures `shouldBe` ["runtime assertions failed (including nonblocking known defects)"]+    readIORef calls `shouldReturn` 10+    readIORef blockers `shouldReturn` result.runtimeFailures+    verifyAttempt entry schedule policy (VaryControl :| []) attempt result `shouldReturn` True+    encodeFile (attempt </> "comparison.json") result.comparison+    originalBytes <- Bytes.readFile (attempt </> "comparison.json")+    renameDirectory attempt moved+    verifyAttempt entry schedule policy (VaryControl :| []) moved result `shouldReturn` True+    Bytes.readFile (moved </> "comparison.json") `shouldReturn` originalBytes+    plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+    aggregate <- either (fail . show) pure (aggregateResult (plan {selected = [entry]} :: PerfPlan) ("sha256:" <> Text.replicate 64 "a") True [result])+    aggregate.exitCode `shouldBe` 1+  it "replaces an invalid raw pair once and detects a genuine retained latency regression" $ withSystemTempDirectory "perf-raw-regression" \directory -> do+    (entry, schedule, policy) <- fixture+    identifier <- newRunId+    calls <- newIORef (0 :: Int)+    replacements <- newIORef (0 :: Int)+    let callbacks = ComparisonHooks (\_ -> modifyIORef' replacements (+ 1)) (\_ -> pure ())+        execute run = do+          modifyIORef' calls (+ 1)+          let path = directory </> "retained" </> Text.unpack (renderRunId run.runId)+              latency = if maybe False ((== "candidate") . (.arm)) run.trial then 3000000 else 1000000+          writeRawRun run False latency path+          if run.ordinal == 1 then Bytes.appendFile (path </> "samples/publish-to-finalize.raw") "invalid" else pure ()+          pure (Right path)+    result <- executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) directory callbacks execute+    result.performanceVerdict `shouldBe` "regression"+    result.infrastructureState `shouldBe` "healthy"+    result.validPairs `shouldBe` 5+    readIORef calls `shouldReturn` 12+    readIORef replacements `shouldReturn` 1+    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True+  it "bounds replacements and reuses retained invalid paths without resubmitting trials" $ withSystemTempDirectory "perf-attempt" \directory -> do+    (entry, schedule, policy) <- fixture+    identifier <- newRunId+    calls <- newIORef (0 :: Int)+    replacements <- newIORef (0 :: Int)+    let lease = LeaseIdentity "alpha" identifier+        callbacks = ComparisonHooks (\_ -> modifyIORef' replacements (+ 1)) (\_ -> pure ())+        execute run = do+          modifyIORef' calls (+ 1)+          let path = directory </> "retained" </> Text.unpack (renderRunId run.runId)+          createDirectoryIfMissing True path+          encodeFile (path </> "run-result.json") (object [])+          encodeFile (path </> "run-spec.json") (object [])+          pure (Right path)+    result <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+    result.infrastructureState `shouldBe` "failed"+    result.validPairs `shouldBe` 0+    readIORef calls `shouldReturn` 14+    readIORef replacements `shouldReturn` 2+    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True+    verifyAttempt entry schedule policy (VaryControl :| []) directory result {Configuration.attempts = []} `shouldReturn` False+    _ <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+    readIORef calls `shouldReturn` 14+    another <- newRunId+    verifyAttemptInputs lease entry schedule directory+    verifyAttemptInputs (LeaseIdentity "alpha" another) entry schedule directory `shouldThrow` anyIOException+    executeAttempt (LeaseIdentity "alpha" another) entry schedule policy (VaryControl :| []) directory callbacks execute `shouldThrow` anyIOException+  it "checkpoints terminal executor failures and replays them without new work" $ withSystemTempDirectory "perf-attempt-failures" \directory -> do+    (entry, schedule, policy) <- fixture+    identifier <- newRunId+    calls <- newIORef (0 :: Int)+    let callbacks = ComparisonHooks (\_ -> pure ()) (\_ -> pure ())+        execute _ = modifyIORef' calls (+ 1) >> pure (Left "sealed outer health failed")+        lease = LeaseIdentity "alpha" identifier+    result <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+    readIORef calls `shouldReturn` 14+    repeated <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+    repeated `shouldBe` result+    readIORef calls `shouldReturn` 14+    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True+    encodeFile (directory </> "trial-failures.json") (Map.singleton ("unplanned" :: Text.Text) ("invalid" :: Text.Text))+    verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldThrow` anyIOException+  it "rejects an incomplete schedule and a forged retained path before dispatch" $ withSystemTempDirectory "perf-attempt-rejection" \directory -> do+    (entry, schedule, policy) <- fixture+    identifier <- newRunId+    let callbacks = ComparisonHooks (\_ -> pure ()) (\_ -> pure ())+        execute _ = expectationFailure "must not dispatch an invalid attempt" >> pure (Left "invalid")+    executeAttempt (LeaseIdentity "alpha" identifier) entry schedule {runs = drop 1 schedule.runs} policy (VaryControl :| []) (directory </> "incomplete") callbacks execute `shouldThrow` anyIOException+    createDirectoryIfMissing True (directory </> "forged")+    encodeFile (directory </> "forged/trial-paths.json") (Map.singleton ("unplanned" :: Text.Text) ("../escape" :: FilePath))+    executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) (directory </> "forged") callbacks execute `shouldThrow` anyIOException++fixture :: IO (PlannedProfile, RunPlan, Policy.Policy)+fixture = do+  plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+  entry <- case (plan :: PerfPlan).selected of value : _ -> pure value; _ -> fail "empty profile fixture"+  identifier <- newRunId+  seed <- either (fail . show) pure (mkSeed 37)+  source <-+    either+      (fail . show)+      pure+      ( decodeWorkPlan+          ( object+              [ "schema" .= ("kenshou.run-plan/v1" :: Text.Text),+                "planId" .= identifier,+                "runs"+                  .= [ object+                         [ "ordinal" .= (0 :: Int),+                           "runId" .= identifier,+                           "estimateMinutes" .= (1 :: Int),+                           "spec"+                             .= object+                               [ "schema" .= ("kenshou.run-spec/v1" :: Text.Text),+                                 "runId" .= identifier,+                                 "scenario" .= entry.profile.scenario,+                                 "scenarioRevision" .= entry.profile.scenarioRevision,+                                 "knobs" .= entry.profile.knobs,+                                 "dimensions" .= entry.profile.dimensions,+                                 "seed" .= seed,+                                 "phases" .= zeroPhases,+                                 "timeoutSeconds" .= entry.profile.timeoutSeconds+                               ]+                         ]+                     ]+              ]+          )+      )+  schedule <- either (fail . show) pure (pairPlan (PairRequest "candidate" "baseline" 5 ABBA 37 2) source)+  policy <- Bytes.readFile "../policies/default.json" >>= either (fail . show) pure . Policy.decodePolicy+  pure (entry, schedule, policy)++-- Synthetic protocol evidence only; this is not workload calibration.+writeRawRun :: PlannedRun -> Bool -> Word64 -> FilePath -> IO ()+writeRawRun run knownFailure latency directory = do+  createDirectoryIfMissing True (directory </> "samples")+  createDirectoryIfMissing True (directory </> "series")+  Bytes.writeFile (directory </> "series/load.csv") "mono_ns,wall,phase\n1000,0,steady\n5000001000,0,drain\n"+  encodeFile (directory </> "run-spec.json") run.spec+  mutable <- Histogram.newHistogram Histogram.defaultHistogramConfig+  mapM_ (const (Histogram.recordValue mutable latency)) [1 :: Int .. 1000]+  histogram <- Histogram.freeze mutable+  let base = directory </> "samples/publish-to-finalize"+  Bytes.writeFile (base <> ".hist") (encodeHistogram histogram)+  Bytes.writeFile (base <> ".service.hist") (encodeHistogram histogram)+  writer <- openSampleWriter (base <> ".raw") (SampleHeader (Origin 0 0) MonotonicClock "publish-to-finalize" "synthetic")+  writeSampleBlock writer 0 [SampleRecord start start (start + latency) 0 1 | index <- [0 .. 999], let start = 1000 + index * 5000000]+  closeSampleWriter writer+  encodeFile (base <> ".meta.json") (object ["operation" .= ("publish-to-finalize" :: Text.Text), "latencyBasis" .= ("intended-start" :: Text.Text), "rawSamples" .= ("full" :: Text.Text), "phaseCounts" .= object ["steady" .= object ["successes" .= (1000 :: Int), "failures" .= (0 :: Int), "units" .= (1000 :: Int)]]])+  measurements <- summarizeRunDir directory >>= either (fail . show) pure+  let origin = UTCTime (fromGregorian 2026 10 10) (secondsToDiffTime 0)+  encodeFile+    (directory </> "run-result.json")+    (object ["runId" .= run.runId, "outcome" .= (if knownFailure then "failed" else "passed" :: Text.Text), "exitCode" .= (0 :: Int), "blocking" .= False, "compatibility" .= object ["inputs" .= object ["cohortPlanHash" .= ("synthetic-control" :: Text.Text)]], "fingerprint" .= object ["machineProfile" .= ("synthetic" :: Text.Text)], "timings" .= object ["startedAt" .= addUTCTime (fromIntegral run.ordinal) origin], "summaries" .= object ["measurements" .= object ["measurements" .= measurements]]])+  manifest <- writeManifest directory run.runId Map.empty+  encodeFile (directory </> "manifest.json") manifest
+ test/PerfCellSpec.hs view
@@ -0,0 +1,57 @@+module PerfCellSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as Lazy+import Data.List qualified as List+import Data.Map.Strict qualified as Map+import Data.Text qualified as Text+import FetchSpec (completeTreeWithNested)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Perf.Plan+import Kenshou.Plan.RunPlan qualified as Plan+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (Completed), WorkObject (..))+import Kenshou.Remote.Cell.Docs qualified as Submission (Submission (..))+import Kenshou.Remote.Cell.Session.Journal+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Perf.Cell+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance sealed runtime observations" do+  it "retains an exit-zero runtime defect even before the outer trial becomes admissible" $ withSystemTempDirectory "perf-cell-defect" \root -> do+    plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+    entry <- case (plan :: PerfPlan).selected of first : _ -> pure first; _ -> fail "empty plan fixture"+    journal <- eitherDecodeFileStrict' "test/golden/cell-session.json" >>= either fail pure+    slice <- case (journal :: SessionJournal).slices of [first] -> pure first; _ -> fail "expected one slice"+    identifier <- newRunId+    cellRun <- newRunId+    schedule <- either (fail . Text.unpack) pure (decodeWorkPlan (object ["schema" .= ("kenshou.run-plan/v1" :: Text.Text), "planId" .= identifier, "runs" .= [object ["ordinal" .= (1 :: Int), "runId" .= identifier, "estimateMinutes" .= (1 :: Int), "spec" .= object ["schema" .= ("kenshou.run-spec/v1" :: Text.Text), "runId" .= identifier, "scenario" .= entry.profile.scenario]]]]))+    run <- case schedule.runs of [first] -> pure first; _ -> fail "expected one run"+    (tree, outer) <- completeTreeWithNested (root </> "trials" </> Text.unpack (renderRunId cellRun)) cellRun journal.leaseId identifier False False False+    let nested = tree </> "output" </> Text.unpack (renderRunId identifier)+    original <- eitherDecodeFileStrict' (nested </> "run-result.json") >>= either fail pure+    case original of+      Object fields -> encodeFile (nested </> "run-result.json") (Object (KeyMap.insert "outcome" (String "failed") (KeyMap.insert "exitCode" (Number 0) (KeyMap.insert "blocking" (Bool False) fields))))+      _ -> fail "expected a result object"+    manifest <- writeManifest nested identifier Map.empty+    encodeFile (nested </> "manifest.json") manifest+    artifacts <-+      traverse+        ( \artifact -> do+            bytes <- Lazy.readFile (tree </> Text.unpack artifact.path)+            let work = workObjectFor "application/json" bytes+            pure (Artifact artifact.path work.sha256 work.bytes artifact.mediaType)+        )+        outer.artifacts+    let updated = outer {artifacts}+    encodeFile (tree </> "manifest.json") updated+    bytes <- Lazy.readFile (tree </> "manifest.json")+    let retained = journal {slices = [slice {cellRun, runIds = [identifier], submission = slice.submission {Submission.runId = cellRun}, state = SliceFetched, cellOutcome = Just Completed, manifestSha256 = Just (workObjectFor "application/json" bytes).sha256, fetchedPath = Just "tree"}]}+        trial = root </> "trials" </> Text.unpack (renderRunId identifier)+    problems <- retainedRuntimeProblems entry.profile (run :: Plan.PlannedRun) trial retained+    List.nub problems `shouldBe` ["runtime assertions failed (including nonblocking known defects)"]
+ test/PerfCleanupSpec.hs view
@@ -0,0 +1,82 @@+module PerfCleanupSpec (spec) where++import Data.Aeson (ToJSON, eitherDecode, encode)+import Data.ByteString.Lazy qualified as Lazy+import Data.Maybe (isJust, isNothing)+import Data.Text qualified as Text+import Data.Time (addUTCTime)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs+import Kenshou.Remote.Cell.Docs qualified as Submission (Submission (..))+import Kenshou.Remote.Cell.Lease qualified as Lease+import Kenshou.Remote.Cell.Session.Journal+import Kenshou.Remote.Perf.Cleanup+import Kenshou.Remote.Perf.Journal (LeaseIdentity (..))+import Kenshou.Remote.Store+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance owned cleanup" do+  it "accepts an unpublished planned slice but retains an ambiguous submitted slice" $ fixture \store ref identity journal -> do+    ownedWorkSettled store ref identity [journal] `shouldReturn` True+    let submitted = journal {slices = map (\slice -> slice {state = SliceSubmitted}) journal.slices}+    ownedWorkSettled store ref identity [submitted] `shouldReturn` False+  it "waits for a submitted run to seal with both outcome and digest" $ fixture \store ref identity journal -> do+    slice <- onlySlice journal+    let status = CellStatus slice.cellRun Running Nothing journal.createdAt Nothing (LogChunks 0 0) Nothing Nothing Nothing+    put store ref slice "submission.json" slice.submission+    put store ref slice "status.json" status+    ownedWorkSettled store ref identity [journal] `shouldReturn` False+    put store ref slice "status.json" (status {phase = Sealed})+    ownedWorkSettled store ref identity [journal] `shouldThrow` anyIOException+    put store ref slice "status.json" (status {phase = Sealed, outcome = Just Cancelled, manifestSha256 = Just (Text.replicate 64 "a")})+    ownedWorkSettled store ref identity [journal] `shouldReturn` True+  it "accepts owner rejection and rejects a marker naming another lease" $ fixture \store ref identity journal -> do+    slice <- onlySlice journal+    put store ref slice "submission.json" slice.submission+    put store ref slice "rejected.json" (Rejected slice.cellRun "cancelled lease" journal.createdAt)+    ownedWorkSettled store ref identity [journal] `shouldReturn` True+    other <- newRunId+    put store ref slice "submission.json" (slice.submission {Submission.leaseId = other})+    ownedWorkSettled store ref identity [journal] `shouldThrow` anyIOException+  it "does not inspect another owner's work and rejects a mismatched control reference" $ fixture \store ref identity journal -> do+    other <- newRunId+    ownedWorkSettled store ref (LeaseIdentity identity.cell other) [journal] `shouldReturn` True+    ownedWorkSettled store (ref {Lease.controlBucket = Bucket "wrong"}) identity [journal] `shouldThrow` anyIOException+    ownedWorkSettled store (ref {Lease.cellName = "beta"}) identity [journal] `shouldThrow` anyIOException+  it "reattaches live work but treats expired and cancelled leases as lost" $ withSystemTempDirectory "perf-live-lease" \root -> do+    store <- newFileStore root+    let ref = Lease.CellRef "alpha" (Bucket "control")+    acquired <- Lease.acquireLease store ref (Lease.LeaseRequest "test" "recovery" 120)+    handle <- case acquired of+      Lease.Acquired value -> pure value+      _ -> expectationFailure "lease was not acquired" >> error "unreachable"+    lease <- Lease.leaseSnapshot handle+    isJust <$> Lease.reattachActiveLease store ref lease.leaseId `shouldReturn` True+    let later = store {serverTime = addUTCTime 151 <$> store.serverTime}+    isNothing <$> Lease.reattachActiveLease later ref lease.leaseId `shouldReturn` True+    isJust <$> Lease.reattachLease later ref lease.leaseId `shouldReturn` True+    Lease.requestCancel store ref (Just lease.leaseId) False `shouldReturn` True+    isNothing <$> Lease.reattachActiveLease store ref lease.leaseId `shouldReturn` True+    isJust <$> Lease.reattachLease store ref lease.leaseId `shouldReturn` True++fixture :: (ObjectStore -> Lease.CellRef -> LeaseIdentity -> SessionJournal -> IO ()) -> IO ()+fixture action = withSystemTempDirectory "perf-cleanup" \root -> do+  store <- newFileStore root+  bytes <- Lazy.readFile "test/golden/cell-session.json"+  journal <- either (ioError . userError) pure (eitherDecode bytes)+  action store (Lease.CellRef journal.cell (Bucket journal.controlBucket)) (LeaseIdentity journal.cell journal.leaseId) journal++onlySlice :: SessionJournal -> IO SliceJournal+onlySlice journal = case journal.slices of+  [slice] -> pure slice+  _ -> expectationFailure "cleanup fixture must have one slice" >> error "unreachable"++put :: (ToJSON value) => ObjectStore -> Lease.CellRef -> SliceJournal -> Text.Text -> value -> IO ()+put store ref slice name value = do+  let path = ObjectName ("cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/" <> name)+  current <- store.statObject ref.controlBucket path+  _ <- store.putObject ref.controlBucket path "application/json" (maybe DoesNotExist (GenerationIs . (.generation)) current) (encode value)+  pure ()
+ test/PerfConditionsSpec.hs view
@@ -0,0 +1,137 @@+module PerfConditionsSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Either (isLeft)+import Data.Text (Text)+import Data.Time (UTCTime, addUTCTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Remote.Perf.Conditions+import Test.Hspec++spec :: Spec+spec = describe "performance cell conditions" do+  it "requires all five passing gates for each machine and a verified cold reset" do+    identifier <- newRunId+    let health = report identifier (concatMap gates machines)+        reset = resetReport identifier+    validateCellConditions identifier "alpha" machines health reset `shouldBe` Right ()+    validateCellConditions identifier "alpha" machines (report identifier []) reset `shouldSatisfy` isLeft+    validateCellConditions identifier "alpha" machines (report identifier (concatMap gates (drop 1 machines))) reset `shouldSatisfy` isLeft+    validateCellConditions identifier "alpha" machines (report identifier (concatMap gates machines <> gates "driver")) reset `shouldSatisfy` isLeft+  it "rejects a lying passed flag, tripped gate and pressure beyond the reported threshold" do+    identifier <- newRunId+    let health values = report identifier (values <> concatMap gates (drop 1 machines))+        reset = resetReport identifier+        entries = gates "driver"+        tripped = map (change "status" (String "tripped")) entries+        overloaded = case entries of+          first : load : rest -> first : change "observed" (object ["settleCpuPercent" .= (95 :: Int), "runStealCpuPercent" .= (0 :: Int)]) load : rest+          _ -> entries+    validateCellConditions identifier "alpha" machines (health tripped) reset `shouldSatisfy` isLeft+    validateCellConditions identifier "alpha" machines (health overloaded) reset `shouldSatisfy` isLeft+    validateCellConditions identifier "alpha" machines (change "passed" (Bool False) (health entries)) reset `shouldSatisfy` isLeft+  it "rejects wrong run identity, warm reset, failed reset and inverted windows" do+    identifier <- newRunId+    other <- newRunId+    let health = report identifier (concatMap gates machines)+        reset = resetReport identifier+    validateCellConditions other "alpha" machines health reset `shouldSatisfy` isLeft+    mapM_+      (\changed -> validateCellConditions identifier "alpha" machines health changed `shouldSatisfy` isLeft)+      [change "cachePolicy" (String "warm") reset, change "verified" (Bool False) reset, change "finishedAt" (String "2026-10-10T23:00:00Z") reset]+  it "reconstructs health from sealed raw samples and rejects scalar laundering" do+    identifier <- newRunId+    let health = rawReport identifier+        windows = rawWindows+        check = validateCellObservations identifier "0.1.0" policy health (executionWindow identifier)+    check windows `shouldBe` Right ()+    let rounded = case windows of (role, machine, image, value) : rest -> (role, machine, image, change "from" (toJSON (addUTCTime (-0.0005) (at 0))) (change "to" (toJSON (addUTCTime (-0.0005) (at 60))) value)) : rest; [] -> []+    -- Role HTTP requests retain millisecond bounds, while the driver has the+    -- original timestamp. Samples must still fall inside the retained window.+    check (case rounded of (role, machine, image, value) : rest -> (role, machine, image, change "samples" (toJSON (map healthSample [0 .. 59])) value) : rest; [] -> []) `shouldBe` Right ()+    check [] `shouldSatisfy` isLeft+    check (drop 1 windows) `shouldSatisfy` isLeft+    let corrupt update = case windows of (role, machine, image, value) : rest -> (role, machine, image, update value) : rest; [] -> []+    check (corrupt (change "image" (String "different-image"))) `shouldSatisfy` isLeft+    check (corrupt (change "agentVersion" (String "stale-agent"))) `shouldSatisfy` isLeft+    check (corrupt (change "maintenanceEvents" (toJSON [object ["at" .= at 20, "value" .= ("maintenance" :: Text)]]))) `shouldSatisfy` isLeft+    check (corrupt (change "samples" (toJSON (filter (\i -> i < 20 || i > 25) [0 .. 60 :: Int] >>= \i -> [healthSample i])))) `shouldSatisfy` isLeft+    check (corrupt (change "samples" (toJSON [change "cpuIdleTicks" (toJSON (900 + i * 50)) (healthSample i) | i <- [0 .. 60 :: Int]]))) `shouldSatisfy` isLeft+    let alteredGates = [change "threshold" (object ["settleCpuMaxPercent" .= (100 :: Int), "runStealCpuMaxPercent" .= (100 :: Int)]) g | g <- gates "driver"] <> concatMap gates (drop 1 machines)+    validateCellObservations identifier "0.1.0" policy (report identifier alteredGates) (executionWindow identifier) windows `shouldSatisfy` isLeft+  it "rejects missing windows, stale sampling, counter reversals and changed execution intervals" do+    identifier <- newRunId+    let check interval windows = validateCellObservations identifier "0.1.0" policy (rawReport identifier) interval windows+        corrupt samples = case rawWindows of (role, machine, image, value) : rest -> (role, machine, image, change "samples" (toJSON samples) value) : rest; [] -> []+    check (executionWindow identifier) (corrupt ([] :: [Value])) `shouldSatisfy` isLeft+    check (executionWindow identifier) (corrupt (map healthSample [1 .. 59])) `shouldBe` Right ()+    check (executionWindow identifier) (corrupt (map healthSample [5 .. 55])) `shouldSatisfy` isLeft+    check (executionWindow identifier) (corrupt (map healthSample [0 .. 30] <> map healthSample [29 .. 60])) `shouldSatisfy` isLeft+    check (change "runFinished" (toJSON (at 59)) (executionWindow identifier)) rawWindows `shouldSatisfy` isLeft++policy :: Value+policy = object ["schema" .= ("cell.policy/v1" :: Text), "health" .= object ["settleCpuMaxPercent" .= (5 :: Int), "runStealCpuMaxPercent" .= (5 :: Int), "startFreeDiskMinPercent" .= (20 :: Int), "runFreeDiskMinPercent" .= (10 :: Int)]]++at :: Int -> UTCTime+at seconds = addUTCTime (fromIntegral seconds) (read "2026-10-11 00:00:00 UTC")++executionWindow :: (ToJSON a) => a -> Value+executionWindow identifier = object ["schema" .= ("cell.execution-window/v1" :: Text), "runId" .= identifier, "resetStarted" .= at 0, "runStarted" .= at 10, "runFinished" .= at 60]++rawReport :: (ToJSON a) => a -> Value+rawReport identifier = report identifier [change "at" (toJSON (at 60)) (if machine == "monitoring" && jsonName gate == Just (String "background-load") then change "observed" (object ["settleCpuPercent" .= (0 :: Int), "runStealCpuPercent" .= (0 :: Int)]) gate else gate) | machine <- machines, gate <- gates machine]+  where+    jsonName (Object value) = KeyMap.lookup "name" value+    jsonName _ = Nothing++rawWindows :: [(Text, Text, Text, Value)]+rawWindows = [(role, role, "image-" <> role, object ["schema" .= ("cell.observations/v1" :: Text), "machine" .= role, "image" .= ("image-" <> role), "agentVersion" .= ("0.1.0" :: Text), "systemClosure" .= ("/nix/store/synthetic-system-identity" :: Text), "from" .= at 0, "to" .= at 60, "samples" .= map healthSample [0 .. 60], "maintenanceEvents" .= ([] :: [Value]), "samplingErrors" .= ([] :: [Value])]) | role <- machines]++healthSample :: Int -> Value+healthSample i = object ["at" .= at i, "cpuTotalTicks" .= (1000 + i * 100), "cpuIdleTicks" .= (990 + i * 99), "cpuStealTicks" .= (0 :: Int), "diskFreePercent" .= (if i <= 10 then 80 else 75 :: Int)]++machines :: [Text]+machines = ["driver", "postgres", "monitoring"]++report :: (ToJSON a) => a -> [Value] -> Value+report identifier entries =+  object+    [ "schema" .= ("cell.health/v1" :: Text),+      "runId" .= identifier,+      "passed" .= True,+      "windowStart" .= ("2026-10-11T00:00:00Z" :: Text),+      "windowEnd" .= ("2026-10-11T00:01:00Z" :: Text),+      "gates" .= entries+    ]++gates :: Text -> [Value]+gates machine =+  [ gate "host-maintenance" (String "NONE") (String "NONE"),+    gate "background-load" (object ["settleCpuPercent" .= (1 :: Int), "runStealCpuPercent" .= (0 :: Int)]) (object ["settleCpuMaxPercent" .= (5 :: Int), "runStealCpuMaxPercent" .= (5 :: Int)]),+    gate "storage-pressure" (object ["startFreePercent" .= (80 :: Int), "runFreePercent" .= (75 :: Int), "outputExceeded" .= False]) (object ["startFreeMinPercent" .= (20 :: Int), "runFreeMinPercent" .= (10 :: Int), "outputExceeded" .= False]),+    gate "incomplete-reset" (Bool True) (Bool True),+    gate "version-mismatch" (Bool True) (Bool True)+  ]+  where+    gate name observed threshold = object ["name" .= (name :: Text), "machine" .= machine, "status" .= ("ok" :: Text), "at" .= ("2026-10-11T00:00:30Z" :: Text), "observed" .= observed, "threshold" .= threshold]++resetReport :: (ToJSON a) => a -> Value+resetReport identifier =+  object+    [ "schema" .= ("cell.reset-evidence/v1" :: Text),+      "runId" .= identifier,+      "cell" .= ("alpha" :: Text),+      "cachePolicy" .= ("cold" :: Text),+      "verified" .= True,+      "startedAt" .= start,+      "finishedAt" .= end,+      "steps" .= [object ["name" .= ("stop-driver-processes" :: Text), "status" .= ("ok" :: Text), "startedAt" .= start, "finishedAt" .= end]]+    ]+  where+    start = "2026-10-11T00:00:00Z" :: Text+    end = "2026-10-11T00:00:05Z" :: Text++change :: Key -> Value -> Value -> Value+change key value (Object fields) = Object (KeyMap.insert key value fields)+change _ _ value = value
+ test/PerfJournalSpec.hs view
@@ -0,0 +1,153 @@+module PerfJournalSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as MapKey+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Perf.Plan+import Kenshou.Perf.Request qualified as Request+import Kenshou.Perf.Result (ConfigurationResult (..))+import Kenshou.Remote.Perf.Journal+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance session budgets" do+  it "replays the synthetic interruption fixture without extending its deadline" do+    session <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-remote/test/golden/perf-session.fixture.json" >>= either fail pure+    state <- right (journalState session)+    state.executionDeadline `shouldBe` Just (time 1810)+    state.admissionSpentSeconds `shouldBe` 30+  it "retains proposed claims and anchors confirmation or owned release to their original time" do+    session <- fixture+    profile <- firstProfile session+    identifier <- newRunId+    other <- newRunId+    let identity = LeaseIdentity "alpha" identifier+    proposed <- apply 10 (LeaseClaimProposed profile identity) session+    restored <- right (eitherDecode (encode proposed) :: Either String PerfSession)+    state <- right (journalState restored)+    state.pendingClaim `shouldBe` Just (profile, identity, time 10)+    appendEvent (SessionEvent (time 11) (LeaseClaimProposed profile identity)) restored `shouldSatisfy` isLeft+    appendEvent (SessionEvent (time 11) (LeaseAcquired (LeaseIdentity "alpha" other))) restored `shouldSatisfy` isLeft+    confirmed <- apply 20 (LeaseAcquired identity) restored >>= right . journalState+    confirmed.executionDeadline `shouldBe` Just (time 1810)+    confirmed.pendingClaim `shouldBe` Nothing+    withdrawn <- apply 21 LeaseClaimWithdrawn restored >>= right . journalState+    withdrawn.executionDeadline `shouldBe` Nothing+    released <- apply 21 LeaseClaimReleased restored+    next <- apply 30 (LeaseAcquired (LeaseIdentity "alpha" other)) released >>= right . journalState+    next.executionDeadline `shouldBe` Just (time 1810)+    appendEvent (SessionEvent (time 1811) (LeaseAcquired identity)) restored `shouldSatisfy` isLeft+  it "keeps spent admission allowance through serialization and repeated interruption" do+    session <- fixture+    waited <- apply 1 (AdmissionReserved 60) session+    restored <- right (eitherDecode (encode waited) :: Either String PerfSession)+    appendEvent (SessionEvent (time 2) (AdmissionReserved 31)) restored `shouldSatisfy` isLeft+    final <- apply 2 (AdmissionReserved 30) restored >>= right . journalState+    final.admissionSpentSeconds `shouldBe` 90+  it "retains its first deadline and refuses a fresh near-deadline comparison after lease loss" do+    session <- fixture+    firstLease <- newRunId+    nextLease <- newRunId+    acquired <- apply 10 (LeaseAcquired (LeaseIdentity "alpha" firstLease)) session+    name <- firstProfile acquired+    started <- apply 11 (ComparisonStarted name 0) acquired+    lost <- apply 1790 (LeaseLost "client lease expired") started+    restored <- right (eitherDecode (encode lost) :: Either String PerfSession)+    reacquired <- apply 1791 (LeaseAcquired (LeaseIdentity "alpha" nextLease)) restored+    state <- right (journalState reacquired)+    state.executionDeadline `shouldBe` Just (time 1810)+    state.cleanupDeadline `shouldBe` Just (time 1870)+    appendEvent (SessionEvent (time 1792) (ComparisonStarted name 1)) reacquired `shouldSatisfy` isLeft+    appendEvent (SessionEvent (time 1810) (ReplacementReserved name)) reacquired `shouldSatisfy` isLeft+  it "starts a separate attempt after loss and preserves replacement counts and runtime blockers" do+    session <- fixture+    lease <- newRunId+    acquired <- apply 1 (LeaseAcquired (LeaseIdentity "alpha" lease)) session+    name <- firstProfile acquired+    started <- apply 2 (ComparisonStarted name 0) acquired+    invalid <- apply 3 (ReplacementReserved name) started+    blocked <- apply 4 (RuntimeBlocked name "known defect reproduced with exit zero") invalid+    lost <- apply 5 (LeaseLost "reset health failure") blocked+    nextLease <- newRunId+    continued <- apply 6 (LeaseAcquired (LeaseIdentity "alpha" nextLease)) lost >>= apply 7 (ComparisonStarted name 1)+    state <- right (journalState continued)+    Map.lookup name state.attempts `shouldBe` Just 2+    Map.lookup name state.replacements `shouldBe` Just 1+    state.activeComparison `shouldBe` Just (name, 1)+    twice <- apply 8 (ReplacementReserved name) continued+    appendEvent (SessionEvent (time 9) (ReplacementReserved name)) twice `shouldSatisfy` isLeft+    entry <- selectedProfile twice+    let comparison = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= ("pass" :: Text), "pairCount" .= (5 :: Int)]+        result = ConfigurationResult name entry.workloadSha256 "pass" "healthy" [] 5 5 [] (Just comparison)+    finished <- apply 10 (ComparisonCompleted result) twice+    final <- right (journalState finished)+    fmap (.runtimeFailures) (Map.lookup name final.completed) `shouldBe` Just ["known defect reproduced with exit zero"]+    appendEvent (SessionEvent (time 11) (ComparisonStarted name 2)) finished `shouldSatisfy` isLeft+  it "rejects overlapping leases, backward clocks, unknown profiles and reset deadlines" do+    session <- fixture+    lease <- newRunId+    acquired <- apply 10 (LeaseAcquired (LeaseIdentity "alpha" lease)) session+    appendEvent (SessionEvent (time 11) (LeaseAcquired (LeaseIdentity "alpha" lease))) acquired `shouldSatisfy` isLeft+    appendEvent (SessionEvent (time 9) LeaseReleased) acquired `shouldSatisfy` isLeft+    appendEvent (SessionEvent (time 11) (ComparisonStarted "unselected" 0)) acquired `shouldSatisfy` isLeft+    appendEvent (SessionEvent (time 11) (AdmissionReserved 1)) acquired `shouldSatisfy` isLeft+  it "recovers a sealed blocker after lease loss or comparison completion without duplicating it" do+    session <- fixture+    name <- firstProfile session+    appendEvent (SessionEvent (time 1) (RuntimeBlocked name "sealed defect")) session `shouldSatisfy` isLeft+    identifier <- newRunId+    acquired <- apply 1 (LeaseAcquired (LeaseIdentity "alpha" identifier)) session+    started <- apply 2 (ComparisonStarted name 0) acquired+    lost <- apply 3 (LeaseLost "interrupted before runtime checkpoint") started+    recovered <- apply 4 (RuntimeBlocked name "sealed defect") lost+    state <- right (journalState recovered)+    Map.lookup name state.runtimeBlockers `shouldBe` Just ["sealed defect"]+    entry <- selectedProfile started+    let comparison = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= ("pass" :: Text), "pairCount" .= (5 :: Int)]+        result = ConfigurationResult name entry.workloadSha256 "pass" "healthy" [] 5 5 [] (Just comparison)+    finished <- apply 3 (ComparisonCompleted result) started+    late <- apply 4 (RuntimeBlocked name "sealed defect") finished >>= apply 5 (RuntimeBlocked name "sealed defect")+    final <- right (journalState late)+    fmap (.runtimeFailures) (Map.lookup name final.completed) `shouldBe` Just ["sealed defect"]+  it "persists the transition journal atomically and refuses changed request purpose" $ withSystemTempDirectory "perf-journal" \directory -> do+    session <- fixture+    writePerfSession (directory </> "session.json") session+    readPerfSession (directory </> "session.json") `shouldReturn` Right session+    changed <- apply 1 (AdmissionReserved 10) session+    writePerfSession (directory </> "session.json") changed+    writePerfSession (directory </> "session.json") session `shouldThrow` anyIOException+    writePerfSession (directory </> "session.json") changed {evidenceBaseUri = "gs://different-evidence/performance"} `shouldThrow` anyIOException+    let mutate (Object document) = Object (MapKey.insert "schema" (String "unknown") document)+        mutate value = value+    (eitherDecode (encode (mutate (toJSON session))) :: Either String PerfSession) `shouldSatisfy` isLeft+    let pinned = session.plan+    (eitherDecode (encode session {plan = pinned {request = pinned.request {Request.purpose = "gating"}}}) :: Either String PerfSession) `shouldSatisfy` isLeft++fixture :: IO PerfSession+fixture = do+  plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+  identifier <- newRunId+  right (initialSession identifier ("sha256:" <> "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") plan "gs://fixture-evidence/performance" (time 0))++time :: Integer -> UTCTime+time seconds = addUTCTime (fromInteger seconds) (UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0))++apply :: Integer -> SessionAction -> PerfSession -> IO PerfSession+apply seconds action = right . appendEvent (SessionEvent (time seconds) action)++selectedProfile :: PerfSession -> IO PlannedProfile+selectedProfile session = case session.plan.selected of+  entry : _ -> pure entry+  [] -> fail "fixture must select a profile"++firstProfile :: PerfSession -> IO Text+firstProfile session = (.name) . (.profile) <$> selectedProfile session++right :: (Show e) => Either e a -> IO a+right = either (fail . show) pure
+ test/PerfLockSpec.hs view
@@ -0,0 +1,31 @@+module PerfLockSpec (spec) where++import Control.Exception (IOException, try)+import Kenshou.Remote.Perf.Lock+import System.Directory (createDirectory, createDirectoryLink)+import System.Environment (getExecutablePath)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (proc, readCreateProcessWithExitCode)+import Test.Hspec++spec :: Spec+spec = describe "performance client locking" do+  it "fences another process and releases ownership after a client exception" $ withSystemTempDirectory "perf-lock" \directory -> do+    executable <- getExecutablePath+    withPerformanceLock directory do+      (code, _, _) <- readCreateProcessWithExitCode (proc executable ["--perf-lock", directory]) ""+      code `shouldBe` ExitFailure 1+    _ <- try @IOException (withPerformanceLock directory (ioError (userError "interrupted client")))+    (code, output, _) <- readCreateProcessWithExitCode (proc executable ["--perf-lock", directory]) ""+    code `shouldBe` ExitSuccess+    output `shouldBe` "locked\n"+  it "fences two clients in one process, including directory aliases" $ withSystemTempDirectory "perf-lock-alias" \root -> do+    let target = root </> "target"+        alias = root </> "alias"+    createDirectory target+    createDirectoryLink target alias+    withPerformanceLock target $+      withPerformanceLock alias (pure ()) `shouldThrow` anyIOException+    withPerformanceLock target (pure ()) `shouldReturn` ()
+ test/PerfPublicationSpec.hs view
@@ -0,0 +1,60 @@+module PerfPublicationSpec (spec) where++import Data.Aeson (encode, object, (.=))+import Data.ByteString.Lazy.Char8 qualified as Bytes+import Data.Either (isLeft)+import Data.Text (Text)+import Data.Time (getCurrentTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (newRunId)+import Kenshou.Perf.Artifacts (sealPerfDirectory)+import Kenshou.Remote.Perf.Publication+import Kenshou.Remote.Store+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (createDirectory, doesPathExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance evidence publication" do+  it "retrieves every artifact, permits an identical retry and rejects a changed existing object" $ withSystemTempDirectory "perf-publication" \root -> do+    let directory = root </> "evidence"+        bucket = Bucket "evidence"+        prefix = "performance/fixture"+    createDirectory directory+    Bytes.writeFile (directory </> "result.json") "retained result"+    sealPerfDirectory directory >>= either (fail . show) pure+    store <- newFileStore (root </> "store")+    published <- publishEvidence store bucket prefix directory+    published `shouldBe` Right "gs://evidence/performance/fixture/manifest.json"+    publishEvidence store bucket prefix directory `shouldReturn` published+    verifyPublication store bucket prefix directory `shouldReturn` Right ()+    digest <- sha256Hex . Bytes.toStrict <$> Bytes.readFile (directory </> "manifest.json")+    retrieveEvidence store bucket prefix digest (root </> "retrieved") `shouldReturn` Right ()+    Bytes.readFile (root </> "retrieved/result.json") `shouldReturn` "retained result"+    retrieveEvidence store bucket prefix "sha256:wrong" (root </> "wrong") >>= (`shouldSatisfy` isLeft)+    current <- store.statObject bucket (ObjectName (prefix <> "/result.json")) >>= maybe (fail "missing published result") pure+    _ <- store.putObject bucket (ObjectName (prefix <> "/result.json")) "application/json" (GenerationIs current.generation) "changed result"+    verifyPublication store bucket prefix directory >>= (`shouldSatisfy` isLeft)+    publishEvidence store bucket prefix directory >>= (`shouldSatisfy` isLeft)+  it "rejects local tampering and unsafe destinations before uploading any object" $ withSystemTempDirectory "perf-publication-rejection" \root -> do+    let directory = root </> "evidence"+        bucket = Bucket "evidence"+    createDirectory directory+    Bytes.writeFile (directory </> "result.json") "sealed"+    sealPerfDirectory directory >>= either (fail . show) pure+    store <- newFileStore (root </> "store")+    publishEvidence store bucket "performance/../escape" directory >>= (`shouldSatisfy` isLeft)+    store.listObjects bucket "" `shouldReturn` []+    Bytes.appendFile (directory </> "result.json") "tampered"+    publishEvidence store bucket "performance/fixture" directory >>= (`shouldSatisfy` isLeft)+    store.listObjects bucket "" `shouldReturn` []+  it "rejects a pinned manifest containing an escaping path before creating its target" $ withSystemTempDirectory "perf-retrieval-escape" \root -> do+    store <- newFileStore (root </> "store")+    identifier <- newRunId+    now <- getCurrentTime+    let manifest = encode (object ["schema" .= ("kenshou.artifact-manifest/v1" :: Text), "algorithm" .= ("sha256" :: Text), "runId" .= identifier, "createdAt" .= now, "files" .= [object ["path" .= ("../escape" :: Text), "sha256" .= ("sha256:wrong" :: Text), "bytes" .= (1 :: Int), "mediaType" .= ("text/plain" :: Text)]]])+    _ <- store.putObject (Bucket "evidence") (ObjectName "performance/fixture/manifest.json") "application/json" DoesNotExist manifest+    retrieveEvidence store (Bucket "evidence") "performance/fixture" (sha256Hex (Bytes.toStrict manifest)) (root </> "retrieved") >>= (`shouldSatisfy` isLeft)+    doesPathExist (root </> "escape") `shouldReturn` False
+ test/PerfRunnerSpec.hs view
@@ -0,0 +1,310 @@+module PerfRunnerSpec (spec) where++import Control.Concurrent (threadDelay)+import Data.Aeson+import Data.IORef+import Data.Map.Strict qualified as Map+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)+import Kenshou.Perf.Plan+import Kenshou.Perf.Prepared qualified as Prepared+import Kenshou.Perf.Result+import Kenshou.Perf.Result qualified as Result+import Kenshou.Remote.Cell.Lease qualified as Lease+import Kenshou.Remote.Perf.Journal+import Kenshou.Remote.Perf.Runner+import Kenshou.Remote.Store (Bucket (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance runner orchestration" do+  it "persists a claim before CAS and reconciles a timed-out accepted claim without redispatch" $ withSystemTempDirectory "perf-admission-ambiguous" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    store <- newFileStore (directory </> "store")+    identifier <- newRunId+    writes <- newIORef (0 :: Int)+    let identity = LeaseIdentity "alpha" identifier+        ref = Lease.CellRef "alpha" (Bucket "control")+        stalled =+          hooks+            { admit = \entry _ pending callbacks -> do+                pending `shouldBe` Nothing+                callbacks.proposeClaim identity+                state <- readPerfSession path >>= right >>= right . journalState+                state.pendingClaim `shouldBe` Just (entry.profile.name, identity, now)+                claimed <- Lease.acquireLeaseWithId store ref (Lease.LeaseRequest "fixture" "ambiguous transport control" 180) identifier+                case claimed of Lease.Acquired _ -> pure (); _ -> fail "fixture claim failed"+                modifyIORef' writes (+ 1)+                threadDelay 60000000+                pure (Admitted identity),+              releaseOwnedLease = \_ -> pure False+            }+    failed <- runPerformance stalled prepared path+    failed.exitCode `shouldBe` 4+    retained <- readPerfSession path >>= right >>= right . journalState+    retained.pendingClaim `shouldSatisfy` isJust+    readIORef calls `shouldReturn` []+    let resumed =+          hooks+            { admit = \_ _ pending _ -> do+                case pending of Just (_, proposed, _) -> proposed `shouldBe` identity; _ -> fail "lost retained proposal"+                active <- Lease.reattachActiveLease store ref identifier+                isJust active `shouldBe` True+                pure (Admitted identity),+              ownsLease = \_ -> isJust <$> Lease.reattachActiveLease store ref identifier,+              releaseOwnedLease = \_ -> Lease.reattachLease store ref identifier >>= maybe (pure False) (Lease.releaseLease store ref)+            }+    recovered <- runPerformance resumed prepared path+    recovered.exitCode `shouldBe` 0+    readIORef writes `shouldReturn` 1+    final <- readPerfSession path >>= right >>= right . journalState+    final.pendingClaim `shouldBe` Nothing+    final.executionDeadline `shouldBe` Just (addUTCTime 1800 now)+  it "pins the deadline when cleanup confirms release of an unacknowledged claim" $ withSystemTempDirectory "perf-admission-release" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    identifier <- newRunId+    clockRef <- newIORef now+    let late =+          hooks+            { clock = readIORef clockRef,+              admit = \_ _ _ callbacks -> do+                callbacks.proposeClaim (LeaseIdentity "alpha" identifier)+                writeIORef clockRef (addUTCTime 11 now)+                pure (Admitted (LeaseIdentity "alpha" identifier))+            }+    result <- runPerformance late prepared path+    result.exitCode `shouldBe` 4+    state <- readPerfSession path >>= right >>= right . journalState+    state.pendingClaim `shouldBe` Nothing+    state.executionDeadline `shouldBe` Just (addUTCTime 1800 now)+    readIORef calls `shouldReturn` []+  it "collects sealed runtime failures under a lost lease before starting a fresh attempt" $ withSystemTempDirectory "perf-lost-seal" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    first <- newRunId+    session <- readPerfSession path >>= right+    entry <- case prepared.plan.selected of value : _ -> pure value; _ -> fail "empty fixture"+    acquired <- right (appendEvent (SessionEvent now (LeaseAcquired (LeaseIdentity "alpha" first))) session)+    started <- right (appendEvent (SessionEvent now (ComparisonStarted entry.profile.name 0)) acquired)+    writePerfSession path started+    let recovered =+          hooks+            { ownsLease = \identity -> pure (identity.leaseId /= first),+              recoverRuntimeFailures = \identity -> pure [(entry.profile.name, "sealed exit-zero defect") | identity.leaseId == first]+            }+    result <- runPerformance recovered prepared path+    result.exitCode `shouldBe` 1+    result.blockingRuntimeFailures `shouldBe` ["sealed exit-zero defect"]+    readIORef calls >>= \executed -> map (\(_, attempt, _) -> attempt) executed `shouldBe` [1, 0]+  it "executes both pinned configurations and reuses completed evidence on resume" $ withSystemTempDirectory "perf-runner" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    result <- runPerformance hooks prepared path+    result.exitCode `shouldBe` 0+    length result.configurations `shouldBe` 2+    result.gatingEligible `shouldBe` False+    resumed <- runPerformance hooks prepared path+    resumed `shouldBe` result+    readIORef calls >>= \executed -> length executed `shouldBe` 2+  it "persists runtime blockers and retries a whole configuration after interruption" $ withSystemTempDirectory "perf-runner" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    let interrupted =+          hooks+            { compareConfiguration = \_ _ _ _ callbacks -> do+                callbacks.reserveReplacement 5+                callbacks.recordRuntimeFailure "known defect with exit zero"+                ioError (userError "simulated lost connection")+            }+    failed <- runPerformance interrupted prepared path+    failed.exitCode `shouldBe` 4+    restored <- readPerfSession path >>= right+    interruptedState <- right (journalState restored)+    Map.elems interruptedState.replacements `shouldBe` [1]+    resumed <- runPerformance hooks prepared path+    resumed.exitCode `shouldBe` 1+    resumed.blockingRuntimeFailures `shouldBe` ["known defect with exit zero"]+    resumedState <- readPerfSession path >>= right >>= right . journalState+    resumedState.executionDeadline `shouldBe` interruptedState.executionDeadline+    Map.elems resumedState.replacements `shouldBe` [1]+    readIORef calls >>= \executed -> map (\(_, index, _) -> index) executed `shouldBe` [1, 0]+  it "returns infrastructure failure when durable publication throws or rejects verification" $ withSystemTempDirectory "perf-runner" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    result <- runPerformance hooks {publishAndVerify = \_ _ -> ioError (userError "storage unavailable")} prepared path+    result.exitCode `shouldBe` 4+    recovered <- runPerformance hooks prepared path+    recovered.exitCode `shouldBe` 0+    invalid <- runPerformance hooks {publishAndVerify = \_ _ -> pure False} prepared path+    invalid.exitCode `shouldBe` 4+  it "rejects a report decorator that changes a performance decision" $ withSystemTempDirectory "perf-runner-report" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    published <- newIORef False+    result <- runPerformance hooks {reportMetadata = \_ report -> pure report {Result.changeCoverage = "complete"}, publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path+    result.exitCode `shouldBe` 4+    readIORef published `shouldReturn` False+  it "rejects changed immutable preparation before any admission" $ withSystemTempDirectory "perf-runner" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    runPerformance hooks prepared {Prepared.harnessRevision = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} path `shouldThrow` anyIOException+    readIORef calls `shouldReturn` []+  it "does not extend cleanup or publication time when a finished session is resumed" $ withSystemTempDirectory "perf-runner-deadline" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    original <- runPerformance hooks prepared path+    original.exitCode `shouldBe` 0+    published <- newIORef False+    result <- runPerformance hooks {clock = pure (addUTCTime 1870 now), publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path+    result.exitCode `shouldBe` 4+    readIORef published `shouldReturn` False+  it "does not publish during cleanup grace after the execution deadline" $ withSystemTempDirectory "perf-publication-grace" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    runPerformance hooks prepared path >>= \result -> result.exitCode `shouldBe` 0+    published <- newIORef False+    result <- runPerformance hooks {clock = pure (addUTCTime 1830 now), publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path+    result.exitCode `shouldBe` 4+    readIORef published `shouldReturn` False+  it "rejects publication that completes beyond the original execution deadline" $ withSystemTempDirectory "perf-publication-late" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    current <- newIORef now+    result <- runPerformance hooks {clock = readIORef current, publishAndVerify = \_ _ -> writeIORef current (addUTCTime 1801 now) >> pure True} prepared path+    result.exitCode `shouldBe` 4+    result.evidenceVerified `shouldBe` False+  it "reserves an identified replacement once when the same callback is reconciled" $ withSystemTempDirectory "perf-runner-reservation" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    let reconciled =+          hooks+            { compareConfiguration = \lease entry index resuming callbacks -> do+                callbacks.reserveReplacement entry.profile.minimumPairs+                callbacks.reserveReplacement entry.profile.minimumPairs+                hooks.compareConfiguration lease entry index resuming callbacks+            }+    result <- runPerformance reconciled prepared path+    result.exitCode `shouldBe` 0+    state <- readPerfSession path >>= right >>= right . journalState+    Map.elems state.replacements `shouldBe` [1, 1]+    length state.replacementIdentities `shouldBe` 2+  it "does not restart cancellation or release grace for an expired interrupted session" $ withSystemTempDirectory "perf-expired-cleanup" \directory -> do+    (prepared, path) <- fixture directory+    session <- readPerfSession path >>= right+    identifier <- newRunId+    acquired <- right (appendEvent (SessionEvent now (LeaseAcquired (LeaseIdentity "alpha" identifier))) session)+    active <- right (appendEvent (SessionEvent now (ComparisonStarted (firstProfile prepared) 0)) acquired)+    writePerfSession path active+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    cleanupCalls <- newIORef (0 :: Int)+    hooks <- fixtureHooks calls+    result <- runPerformance hooks {clock = pure (addUTCTime 1870 now), cancelOwnedWork = \_ -> modifyIORef' cleanupCalls (+ 1), releaseOwnedLease = \_ -> modifyIORef' cleanupCalls (+ 1) >> pure True} prepared path+    result.exitCode `shouldBe` 4+    readIORef calls `shouldReturn` []+    readIORef cleanupCalls `shouldReturn` 0+  it "releases an incompatible lease before admitting the next configuration" $ withSystemTempDirectory "perf-runner-cells" \directory -> do+    (prepared, path) <- fixture directory+    calls <- newIORef ([] :: [(Text, Int, Bool)])+    hooks <- fixtureHooks calls+    admissions <- newIORef (0 :: Int)+    releases <- newIORef (0 :: Int)+    first <- newRunId+    second <- newRunId+    let changed =+          hooks+            { admit = \_ _ _ _ -> do+                count <- atomicModifyIORef' admissions (\n -> (n + 1, n))+                pure (Admitted (LeaseIdentity "alpha" (if count == 0 then first else second))),+              canRunConfiguration = \lease entry -> pure (lease.leaseId /= first || entry.profile.name == firstProfile prepared),+              releaseOwnedLease = \_ -> modifyIORef' releases (+ 1) >> pure True+            }+    result <- runPerformance changed prepared path+    result.exitCode `shouldBe` 0+    readIORef admissions `shouldReturn` 2+    readIORef releases `shouldReturn` 2+    state <- readPerfSession path >>= right >>= right . journalState+    state.executionDeadline `shouldBe` Just (addUTCTime 1800 now)++fixture :: FilePath -> IO (Prepared.PerfPrepared, FilePath)+fixture directory = do+  prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= either fail pure+  identifier <- newRunId+  session <- right (initialSession identifier (Prepared.preparedDigest prepared) prepared.plan "gs://fixture-evidence/performance" now)+  let path = directory </> "session.json"+  writePerfSession path session+  pure (prepared, path)++fixtureHooks :: IORef [(Text, Int, Bool)] -> IO RunnerHooks+fixtureHooks calls = do+  contracts <- fixtureContracts+  lease <- newRunId+  pure+    RunnerHooks+      { contracts,+        clock = pure now,+        admit = \_ _ _ _ -> pure (Admitted (LeaseIdentity "alpha" lease)),+        ownsLease = \_ -> pure True,+        canRunConfiguration = \_ _ -> pure True,+        compareConfiguration = \_ entry index resuming _ -> do+          modifyIORef' calls (<> [(entry.profile.name, index, resuming)])+          let comparison = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= ("pass" :: Text), "pairCount" .= entry.profile.minimumPairs]+          pure (ConfigurationResult entry.profile.name entry.workloadSha256 "pass" "healthy" [] entry.profile.minimumPairs entry.profile.minimumPairs [] (Just comparison)),+        verifyCompleted = \_ -> pure True,+        cancelOwnedWork = \_ -> pure (),+        releaseOwnedLease = \_ -> pure True,+        recoverRuntimeFailures = \_ -> pure [],+        reportMetadata = \_ result -> pure result,+        publishAndVerify = \_ _ -> pure True+      }++now :: UTCTime+now = UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0)++right :: (Show e) => Either e a -> IO a+right = either (fail . show) pure++-- The captured preparation pins the first consumer's reviewed benchmark revisions;+-- its contracts are data in the consumer-baseline bridge.+fixtureContracts :: IO PerfContracts+fixtureContracts = do+  document <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/perf-contracts.json" >>= either fail pure+  let entries = [(entry.scenario, entry.revision) | entry <- (document :: ContractsDocument).reviewed]+  countKnob <- either (fail . show) pure (mkKnobName document.countKnob)+  pure noPerfContracts {reviewedPhaseContracts = entries, steadyCountKnob = Just countKnob}++data ContractsDocument = ContractsDocument {reviewed :: [ReviewedContract], countKnob :: Text}++data ReviewedContract = ReviewedContract {scenario :: Text, revision :: Int}++instance FromJSON ContractsDocument where+  parseJSON = withObject "PerfContracts" \value -> ContractsDocument <$> value .: "reviewedPhaseContracts" <*> value .: "steadyCountKnob"++instance FromJSON ReviewedContract where+  parseJSON = withObject "ReviewedPhaseContract" \value -> ReviewedContract <$> value .: "scenario" <*> value .: "revision"++-- The first profile the captured preparation selects.+firstProfile :: Prepared.PerfPrepared -> Text+firstProfile prepared = case prepared.plan.selected of+  entry : _ -> entry.profile.name+  [] -> error "the captured preparation selects no profile"
+ test/PrepareSpec.hs view
@@ -0,0 +1,441 @@+module PrepareSpec (spec) where++import Data.Aeson (Value (..), eitherDecode, eitherDecodeFileStrict', encode, toJSON)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Bundle (LayerBundle (..), lookupScenario, mkBundle, mkRegistry, registryRoutingRules)+import Kenshou.Core.Bundle qualified as Bundle+import Kenshou.Core.Dimension (PgDurability (..), PgVersion (..), noDimensions, postgresDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (Kind (..), mkSeed, newRunId, parseScenarioId, renderKind, selftestLayer)+import Kenshou.Core.Knob (RawKnob (..), mkKnobName)+import Kenshou.Core.Resource (CellBinding (..), ProvisionedResource (..), ResourceProvider (..), mkResourceName)+import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), minimalRunSpec)+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Core.Selftest qualified as Selftest+import Kenshou.Core.Selftest.Schema (selftestMigrationSet)+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (PlannedRun), RunPlan (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Docs (CachePolicy (..), CellBuckets (..), CellDescriptor (..), CellImages (..), CollectOptions (..), Limits (..), PgReset (..), ResetBlock (..), Submission (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Prepare (Granularity (..), OtlpSink (..), PrepareOptions (..), Prepared (..), PreparedRun (..), RejectReason (..), Routed (..), Slice (..), SubmissionInputs (..), prepareForCell, routePlan, slicePlan, sliceRuns, submissionFor)+import Kenshou.Remote.Cell.Prepare qualified as Prepare+import Kenshou.Remote.Cell.RouteJson (RouteDocuments (..), routeWorkJson)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities (..), RoutingRule (..), RuleCondition (..), descriptorDigest, loadCellCapabilities, matchingRules)+import Kenshou.Remote.Cell.Session.Build (BuildOptions (..), BuiltSession (..), buildSession)+import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..))+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan, renderPreparedWork)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))+import Kenshou.Remote.Store (Bucket (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell run slicing" do+  it "invalidates machine profiles when cell images change and preserves equivalent hardware profiles" do+    (_, descriptor, _) <- preparationFixture+    let original = Prepare.cellMachineProfile descriptor+    Prepare.cellMachineProfile descriptor {name = "same-shape-other-cell"} `shouldBe` original+    Prepare.cellMachineProfile descriptor {images = descriptor.images {driver = "different-driver-image"}} `shouldNotBe` original+    Prepare.cellMachineProfile descriptor {images = descriptor.images {postgres = "different-postgres-image"}} `shouldNotBe` original++  it "prepares durable PostgreSQL work and records reset settings and payload cohort" do+    (plan, descriptor, payload) <- preparationFixture+    let registry = either (error . show) id (mkRegistry [Selftest.bundle])+        payloads = Map.singleton "default" payload+        options = PrepareOptions False False [] Cold+        rejected = prepareForCell registry descriptor Nothing [] payloads options plan+        accepted = prepareForCell registry descriptor Nothing [] payloads (options {coerceDurable = True}) plan+    fmap snd rejected.rejected `shouldBe` [DurabilityNotDurable]+    case accepted.accepted of+      [run] -> do+        run.spec.environment.postgres `shouldBe` Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))+        case run.reset.postgres of+          Just reset -> do+            Map.lookup "fsync" reset.settings `shouldBe` Just "on"+            Map.lookup "synchronous_commit" reset.settings `shouldBe` Just "on"+          Nothing -> expectationFailure "expected PostgreSQL reset"+        run.spec.cohortExpectation `shouldSatisfy` (/= Nothing)+        accepted.warnings `shouldSatisfy` (not . null)+      _ -> expectationFailure "expected one prepared run"+    fmap snd (prepareForCell registry (descriptor {postgresMajor = 17}) Nothing [] payloads (options {coerceDurable = True}) plan).rejected `shouldBe` [PgVersionMismatch 18 17]+    fmap snd (prepareForCell registry descriptor Nothing [] payloads (options {coerceDurable = True, pgSettings = [("fsync", "off")]}) plan).rejected `shouldBe` [ConflictingPostgresSetting "fsync"]++  it "keeps image-owned preloaded libraries out of PostgreSQL reset settings" do+    (plan, descriptor, payload) <- preparationFixture+    let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+        scenarioId = case plan.runs of+          [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario+          _ -> error "expected one planned run"+        original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)+        requirement = maybe (error "missing PostgreSQL requirement") id original.requires.postgres+        required = PostgresRequirement requirement.schemas [("shared_preload_libraries", "'pg_stat_statements'")] requirement.needsServerControl+        withPreload = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements (Just required) original.requires.extraPostgres original.requires.resources) original.knownDefect original.run+        registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [withPreload] []) {migrations = [selftestMigrationSet]}])+        prepared = prepareForCell registry descriptor Nothing [] (Map.singleton "default" payload) (PrepareOptions True False [] Cold) plan+    case prepared.accepted of+      [run] -> case run.reset.postgres of+        Just reset -> Map.member "shared_preload_libraries" reset.settings `shouldBe` False+        Nothing -> expectationFailure "expected PostgreSQL reset"+      _ -> expectationFailure "expected one prepared run"++  it "keeps the owner PostgreSQL reset path for a worker with no database requirement" do+    (plan, descriptor, payload) <- preparationFixture+    worker <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseScenarioId "selftest/kernel/concurrency/worker-echo")+    let registry = either (error . show) id (mkRegistry [Selftest.bundle])+        payloads = Map.singleton "default" payload+        workerPlan = case plan.runs of+          [PlannedRun ordinal identifier estimate reasons trial _] ->+            let base = minimalRunSpec worker+                workerSpec = RunSpec (Just identifier) worker base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+             in plan {runs = [PlannedRun ordinal identifier estimate reasons trial workerSpec]}+          _ -> error "expected one planned run"+    case (prepareForCell registry descriptor Nothing [] payloads (PrepareOptions False False [] Warm) workerPlan).accepted of+      [preparedRun] -> do+        preparedRun.spec.environment.postgres `shouldBe` Nothing+        preparedRun.reset.postgres `shouldBe` Just (PgReset descriptor.postgresMajor [] Map.empty)+      _ -> expectationFailure "expected one worker run"++  it "rejects a resource the cell cannot bind or does not report" do+    (plan, descriptor, payload) <- preparationFixture+    now <- getCurrentTime+    let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+        scenarioId = case plan.runs of+          [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario+          _ -> error "expected one planned run"+        original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)+        objectStore = either (error . Text.unpack) id (mkResourceName "object-store")+        needsStore = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements original.requires.postgres original.requires.extraPostgres [objectStore]) original.knownDefect original.run+        provider binding = ResourceProvider objectStore "Object store" (Right . maybe Null id) (\_ value action -> action (ProvisionedResource value Null)) binding+        registryWith binding = either (error . show) id (mkRegistry [(mkBundle selftestLayer [needsStore] []) {migrations = [selftestMigrationSet], Bundle.resources = [provider binding]}])+        payloads = Map.singleton "default" payload+        options = PrepareOptions True False [] Cold+        binding = Just (CellBinding "object-store" Nothing)+        reporting = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "object-store" (Bool True))+    fmap snd (prepareForCell (registryWith Nothing) descriptor Nothing [] payloads options plan).rejected `shouldBe` [ResourceUnavailable objectStore]+    fmap snd (prepareForCell (registryWith binding) descriptor Nothing [] payloads options plan).rejected `shouldBe` [ResourceUnavailable objectStore]+    length (prepareForCell (registryWith binding) descriptor (Just reporting) [] payloads options plan).accepted `shouldBe` 1++  it "keeps server-control work off the cell server unless the driver is explicitly selected" do+    (plan, descriptor, payload) <- preparationFixture+    let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+        scenarioId = case plan.runs of+          [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario+          _ -> error "expected one planned run"+        original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)+        requirement = maybe (error "missing PostgreSQL requirement") id original.requires.postgres+        controlled = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements (Just (requirement {needsServerControl = True})) original.requires.extraPostgres original.requires.resources) original.knownDefect original.run+        registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [controlled] []) {migrations = [selftestMigrationSet]}])+        payloads = Map.singleton "default" payload+        options = PrepareOptions False False [] Cold+    fmap snd (prepareForCell registry descriptor Nothing [] payloads options plan).rejected `shouldBe` [NeedsServerControl "primary"]+    case (prepareForCell registry descriptor Nothing [] payloads (options {ephemeralOnDriver = True}) plan).accepted of+      [run] -> do+        run.spec.environment.postgres `shouldBe` Just (PostgresEphemeral [])+        run.reset.postgres `shouldBe` Just (PgReset descriptor.postgresMajor [] Map.empty)+      _ -> expectationFailure "expected one driver-local prepared run"+    let routed = routePlan registry ((descriptor, Nothing) :| []) [] payloads options plan+    fmap (.planId) routed.local `shouldBe` Just plan.planId+    Map.null routed.perCell `shouldBe` True++  it "routes PostgreSQL majors to matching cells while preserving plan order and run IDs" do+    (plan, descriptor18, payload) <- preparationFixture+    secondId <- newRunId+    let original = case plan.runs of+          [PlannedRun _ _ _ originalReasons originalTrial originalSpec] -> (originalReasons, originalTrial, originalSpec)+          _ -> error "expected one planned run"+        (reasons, trial, firstSpec) = original+        secondSpec = RunSpec (Just secondId) firstSpec.scenario firstSpec.scenarioRevision firstSpec.knobs [("pg.version", "17")] firstSpec.seed firstSpec.phases firstSpec.timeoutSeconds firstSpec.environment firstSpec.cohortExpectation firstSpec.comparison firstSpec.labels+        second = PlannedRun 1 secondId 1 reasons trial secondSpec+        twoRunPlan = RunPlan plan.planId plan.createdAt plan.context plan.policy (plan.runs <> [second]) plan.skipped 2+        descriptor17 = descriptor18 {name = "beta", postgresMajor = 17}+        baseRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+        scenario = maybe (error "missing selftest") id (lookupScenario baseRegistry firstSpec.scenario)+        bothMajors = Scenario scenario.id scenario.revision scenario.summary scenario.tier scenario.placement scenario.knobs (postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) noDimensions) scenario.phases (EnvRequirements (Just (PostgresRequirement [] [] False)) [] []) scenario.knownDefect scenario.run+        registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [bothMajors] []) {migrations = [selftestMigrationSet]}])+        routed = routePlan registry ((descriptor17, Nothing) :| [(descriptor18, Nothing)]) [] (Map.singleton "default" payload) (PrepareOptions True False [] Cold) twoRunPlan+        runIds selected = fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) selected.runs+    fmap runIds (Map.lookup "alpha" routed.perCell) `shouldBe` Just [case plan.runs of [PlannedRun _ identifier _ _ _ _] -> identifier; _ -> error "expected one planned run"]+    fmap runIds (Map.lookup "beta" routed.perCell) `shouldBe` Just [secondId]+    routed.local `shouldBe` Nothing+    routed.unroutable `shouldBe` []++  it "routes a public JSON plan while preserving its provenance and reporting refusals" do+    (plan, descriptor, payload) <- preparationFixture+    let registry = either (error . show) id (mkRegistry [Selftest.bundle])+        input = toJSON plan+        route = routeWorkJson registry ((descriptor, Nothing) :| []) [] (Just (Map.singleton "default" payload)) (PrepareOptions True False [] Cold) input+    documents <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure route+    documents.routeComplete `shouldBe` True+    documents.local `shouldBe` Nothing+    case (input, Map.lookup descriptor.name documents.perCell) of+      (Object source, Just (Object selected)) -> do+        KeyMap.lookup "planId" selected `shouldBe` KeyMap.lookup "planId" source+        KeyMap.lookup "createdAt" selected `shouldBe` KeyMap.lookup "createdAt" source+        KeyMap.lookup "policy" selected `shouldBe` KeyMap.lookup "policy" source+        KeyMap.lookup "changes" selected `shouldBe` KeyMap.lookup "changes" source+        KeyMap.lookup "runs" selected `shouldNotBe` KeyMap.lookup "runs" source+      _ -> expectationFailure "expected a routed plan object"+    case input of+      Object source -> do+        let originalRun = case KeyMap.lookup "runs" source of+              Just (Array runs) -> case toList runs of+                [run] -> run+                _ -> error "expected one run"+              _ -> error "missing runs"+            duplicate = Object (KeyMap.insert "runs" (toJSON [originalRun, originalRun]) source)+        routeWorkJson registry ((descriptor, Nothing) :| []) [] (Just (Map.singleton "default" payload)) (PrepareOptions True False [] Cold) duplicate `shouldSatisfy` isLeft+      _ -> expectationFailure "expected a plan object"+    withoutPayload <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (routeWorkJson registry ((descriptor, Nothing) :| []) [] Nothing (PrepareOptions True False [] Cold) input)+    withoutPayload.routeComplete `shouldBe` True+    case Map.lookup descriptor.name withoutPayload.perCell of+      Just (Object selected) -> case KeyMap.lookup "runs" selected of+        Just (Array rows) -> case toList rows of+          [Object row] -> case KeyMap.lookup "spec" row of+            Just (Object selectedSpec) -> KeyMap.lookup "cohortExpectation" selectedSpec `shouldBe` Just Null+            _ -> expectationFailure "expected a selected spec"+          _ -> expectationFailure "expected one selected run"+        _ -> expectationFailure "expected runs"+      _ -> expectationFailure "expected a routed plan"++  it "renders prepared slices from the original plan with reset settings intact" do+    (plan, descriptor, payload) <- preparationFixture+    let input = toJSON plan+        registry = either (error . show) id (mkRegistry [Selftest.bundle])+        options = PrepareOptions True False [] Cold+    decoded <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (decodeWorkPlan input)+    decoded.planId `shouldBe` plan.planId+    let selectedWork = prepareForCell registry descriptor Nothing [] (Map.singleton "default" payload) options decoded+    case selectedWork.accepted of+      [selected] -> do+        rendered <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (renderPreparedWork input [selected])+        case (input, rendered) of+          (Object original, Object slice) -> do+            KeyMap.lookup "policy" slice `shouldBe` KeyMap.lookup "policy" original+            KeyMap.lookup "changes" slice `shouldBe` KeyMap.lookup "changes" original+            KeyMap.lookup "runs" slice `shouldNotBe` KeyMap.lookup "runs" original+          _ -> expectationFailure "expected a rendered slice"+        renderPreparedWork input [selected {ordinal = 99}] `shouldSatisfy` isLeft+      _ -> expectationFailure "expected a prepared run"++  it "builds a durable session and refuses incompatible work before publication" do+    (plan, descriptor, payload) <- preparationFixture+    lease <- newRunId+    let registry = either (error . show) id (mkRegistry [Selftest.bundle])+        payloads = Map.singleton "default" payload+        matchingDescriptor = descriptor {buckets = descriptor.buckets {control = "control"}}+        options coerce = BuildOptions (PrepareOptions coerce False [] Cold) GranularityAuto False NullSink Nothing (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0"+        source = encode plan+    rejected <- buildSession registry matchingDescriptor Nothing [] payloads (options False) "file:fixture" lease source+    rejected `shouldSatisfy` isLeft+    built <- buildSession registry matchingDescriptor Nothing [] payloads (options True) "file:fixture" lease source+    case built of+      Left problem -> expectationFailure (Text.unpack problem)+      Right session -> do+        session.journal.leaseId `shouldBe` lease+        length session.journal.slices `shouldBe` 1+        length session.workFiles `shouldBe` 1+        case (session.journal.slices, session.workFiles) of+          ([slice], [(_, bytes)]) -> do+            slice.runIds `shouldBe` fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) plan.runs+            case eitherDecode bytes of+              Right (Object rendered) -> KeyMap.lookup "policy" rendered `shouldBe` case toJSON plan of Object original -> KeyMap.lookup "policy" original; _ -> Nothing+              _ -> expectationFailure "expected a work plan"+          _ -> expectationFailure "expected one slice and work file"++  it "uses cached denials and warns when a matching capability has not been probed" do+    (plan, descriptor, payload) <- preparationFixture+    now <- getCurrentTime+    selector <- either (error . Text.unpack) pure (parseSelector "selftest/kernel/**")+    let registry = either (error . show) id (mkRegistry [Selftest.bundle])+        rule = RoutingRule (ScenarioMatches selector) "postgres.pg_partman" "requires extension"+        options = PrepareOptions True False [] Cold+        payloads = Map.singleton "default" payload+        unprobed = prepareForCell registry descriptor Nothing [rule] payloads options plan+        deniedCache = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "postgres.pg_partman" (Bool False))+        denied = prepareForCell registry descriptor (Just deniedCache) [rule] payloads options plan+        stale = prepareForCell registry (descriptor {postgresMajor = 17}) (Just deniedCache) [rule] payloads options plan+    length unprobed.accepted `shouldBe` 1+    unprobed.warnings `shouldSatisfy` any (Text.isInfixOf "kenshou cell probe")+    fmap snd denied.rejected `shouldBe` [MissingCapability "postgres.pg_partman"]+    fmap snd stale.rejected `shouldBe` [PgVersionMismatch 18 17]++  it "reads the routing rules registered bundles contribute" do+    scenario <- either (error . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/always-pass")+    knob <- either (error . Text.unpack) pure (mkKnobName "synthetic.queue-kind")+    let rule = RoutingRule (KnobEquals "synthetic.queue-kind" "partitioned") "postgres.pg_partman" "Partitioned queues need pg_partman on the cell PostgreSQL server."+        registry = either (error . show) id (mkRegistry [Selftest.bundle, (mkBundle selftestLayer [] []) {routingRules = [rule]}])+        base = minimalRunSpec scenario+        withKnob = RunSpec base.runId base.scenario base.scenarioRevision [(knob, RawText "partitioned")] base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+    registryRoutingRules registry `shouldBe` [rule]+    fmap (.requires) (matchingRules (registryRoutingRules registry) withKnob) `shouldBe` ["postgres.pg_partman"]+    matchingRules (registryRoutingRules registry) base `shouldBe` []++  it "round-trips a probed capability cache" do+    cache <- eitherDecodeFileStrict' "test/golden/cell-capabilities.json" >>= either fail pure+    eitherDecode (encode cache) `shouldBe` Right (cache :: CellCapabilities)++  it "loads only a valid cache for the current cell descriptor" $+    withSystemTempDirectory "kenshou-capabilities" \directory -> do+      (plan, descriptor, _) <- preparationFixture+      now <- getCurrentTime+      let path = directory </> Text.unpack descriptor.name <> ".capabilities.json"+          cache = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "postgres.pg_partman" (Bool False))+      loadCellCapabilities directory descriptor `shouldReturn` Right Nothing+      LazyByteString.writeFile path (encode cache)+      loadCellCapabilities directory descriptor `shouldReturn` Right (Just cache)+      loadCellCapabilities directory (descriptor {postgresMajor = 17}) `shouldReturn` Right Nothing+      LazyByteString.writeFile path "invalid JSON"+      malformed <- loadCellCapabilities directory descriptor+      malformed `shouldSatisfy` isLeft++  it "shares adjacent correctness resets and isolates benchmark and soak runs" do+    first <- prepared 0 Correctness "default" cold+    second <- prepared 1 Concurrency "default" cold+    benchmark <- prepared 2 Benchmark "default" cold+    third <- prepared 3 Correctness "default" cold+    soak <- prepared 4 Soak "default" cold+    fourth <- prepared 5 Correctness "default" cold+    slices <- expectSlices (sliceRuns GranularityAuto [first, second, benchmark, third, soak, fourth])+    fmap (NonEmpty.length . (.entries)) slices `shouldBe` [2, 1, 1, 1, 1]+    fmap (.index) slices `shouldBe` [0, 1, 2, 3, 4]+    fmap (.wallClockSeconds) slices `shouldBe` [320, 310, 310, 910, 310]++  it "leaves bounded time for full-soak drain and diagnosis without extending benchmark limits" do+    soak <- prepared 0 Soak "default" cold+    benchmark <- prepared 1 Benchmark "default" cold+    slices <- expectSlices (sliceRuns GranularityRun [soak {Prepare.timeoutSeconds = 14400}, benchmark {Prepare.timeoutSeconds = 600}])+    fmap (.wallClockSeconds) slices `shouldBe` [15300, 900]++  it "budgets finalization for each soak in a grouped slice and rejects overflow" do+    first <- prepared 0 Soak "default" cold+    second <- prepared 1 Soak "default" cold+    slices <- expectSlices (sliceRuns GranularityPlan [first {Prepare.timeoutSeconds = 14400}, second {Prepare.timeoutSeconds = 14400}])+    fmap (.wallClockSeconds) slices `shouldBe` [30300]+    sliceRuns GranularityRun [first {Prepare.timeoutSeconds = maxBound}] `shouldSatisfy` isLeft++  it "starts a new automatic slice when the payload or reset changes" do+    first <- prepared 0 Correctness "default" cold+    second <- prepared 1 Correctness "candidate" cold+    third <- prepared 2 Correctness "candidate" warm+    fourth <- prepared 3 Correctness "candidate" warm+    slices <- expectSlices (sliceRuns GranularityAuto [first, second, third, fourth])+    fmap (NonEmpty.length . (.entries)) slices `shouldBe` [1, 1, 2]+    fmap (.payloadLabel) slices `shouldBe` ["default", "candidate", "candidate"]++  it "requires a homogeneous plan slice and refuses duplicate identities" do+    first <- prepared 0 Correctness "default" cold+    second <- prepared 1 Correctness "default" cold+    runSlices <- expectSlices (sliceRuns GranularityRun [first, second])+    fmap (NonEmpty.length . (.entries)) runSlices `shouldBe` [1, 1]+    planSlices <- expectSlices (sliceRuns GranularityPlan [first, second])+    fmap (NonEmpty.length . (.entries)) planSlices `shouldBe` [2]+    sliceRuns GranularityPlan [first, PreparedRun second.ordinal second.runId second.spec second.payloadLabel warm second.timeoutSeconds] `shouldBe` Left "plan granularity requires one payload and reset for every run"+    sliceRuns GranularityRun [first, first {ordinal = 1}] `shouldSatisfy` isLeft++  it "keeps the original plan metadata while replacing only selected run specs" do+    first <- prepared 0 Correctness "default" cold+    second <- prepared 1 Correctness "default" cold+    planId <- newRunId+    now <- getCurrentTime+    selector <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseSelector "selftest/**")+    seed <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (mkSeed 1)+    let reason = Reason (Change (ComponentRef (ComponentId "remote") Nothing) Everything "fixture") [] selector 0+        planned run = PlannedRun run.ordinal run.runId 1 (reason :| []) Nothing run.spec+        planContext = PlanContext Nothing "graph" "released" "plan-hash" (PlanInputs Null) [] []+        original = RunPlan planId now planContext (defaultPlanPolicy seed) [planned first, planned second] [] 2+        changed = PreparedRun first.ordinal first.runId (first.spec {RunSpec.timeoutSeconds = Just 42}) first.payloadLabel first.reset first.timeoutSeconds+    slices <- expectSlices (sliceRuns GranularityAuto [changed, second])+    case slices of+      [slice] -> do+        sliced <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (slicePlan original slice)+        sliced.planId `shouldBe` original.planId+        fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) sliced.runs `shouldBe` [first.runId, second.runId]+        fmap (\(PlannedRun _ _ _ _ _ runSpec) -> runSpec.timeoutSeconds) sliced.runs `shouldBe` [Just 42, Nothing]+        sliced.estimateMinutes `shouldBe` 2+        slicePlan (original {runs = [planned first]}) slice `shouldSatisfy` isLeft+      _ -> expectationFailure "expected one slice"++  it "builds a cell submission with payload transport, limits and session labels" do+    planned <- prepared 0 Correctness "default" cold+    slices <- expectSlices (sliceRuns GranularityRun [planned])+    slice <- case slices of+      [value] -> pure value+      _ -> expectationFailure "expected one slice" >> error "unreachable"+    bytes <- LazyByteString.readFile "test/golden/payload.json"+    payload <- either (\problem -> expectationFailure problem >> error "unreachable") pure (eitherDecode bytes :: Either String PayloadDescriptor)+    cellRun <- newRunId+    lease <- newRunId+    session <- newRunId+    plan <- newRunId+    let inputs = SubmissionInputs cellRun lease session plan FileSink (Just "-N2 -T") (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0" Nothing+        ref = CellRef "alpha" (Bucket "control")+        work = workObjectFor "application/json" "{}"+    submission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref inputs payload slice work)+    submission.runId `shouldBe` cellRun+    submission.payload `shouldBe` payload.cell+    submission.limits.wallClockSeconds `shouldBe` 310+    Map.lookup "KENSHOU_PAYLOAD_BUNDLE_SHA256" submission.env `shouldBe` Just payload.cell.bundle.sha256+    Map.lookup "KENSHOU_OTLP_SINK" submission.env `shouldBe` Just "file"+    fmap (.traces) submission.collect `shouldBe` Just True+    fmap (.profiles) submission.collect `shouldBe` Just False+    Map.lookup "GHCRTS" submission.env `shouldBe` Just "-N2 -T"+    Map.lookup "payload" submission.labels `shouldBe` Just "default"+    defaultSubmission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref (inputs {otlpSink = NullSink, rtsOptions = Nothing}) payload slice work)+    Map.lookup "KENSHOU_OTLP_SINK" defaultSubmission.env `shouldBe` Just "null"+    defaultSubmission.collect `shouldBe` Nothing+    Map.lookup "GHCRTS" defaultSubmission.env `shouldBe` Nothing+    profileSubmission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref (inputs {otlpSink = NullSink, rtsOptions = Just "-p"}) payload slice work)+    profileSubmission.collect `shouldBe` Just (CollectOptions False True)+    eitherDecode (encode profileSubmission) `shouldBe` Right profileSubmission+    submissionFor (CellRef "alpha" (Bucket "other")) inputs payload slice work `shouldBe` Left "payload bundle is outside the cell control bucket"+    submissionFor ref inputs payload slice (workObjectFor "text/plain" "{}") `shouldBe` Left "cell work must be a nonempty JSON run plan"++prepared :: Int -> Kind -> Text -> ResetBlock -> IO PreparedRun+prepared ordinal kind label reset = do+  identifier <- newRunId+  scenario <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseScenarioId ("selftest/remote/" <> renderKind kind <> "/slice"))+  pure (PreparedRun ordinal identifier (minimalRunSpec scenario) label reset 10)++expectSlices :: Either Text [Slice] -> IO [Slice]+expectSlices = either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft _ = False++cold :: ResetBlock+cold = ResetBlock Cold Nothing Nothing Map.empty++warm :: ResetBlock+warm = ResetBlock Warm Nothing Nothing Map.empty++preparationFixture :: IO (RunPlan, CellDescriptor, PayloadDescriptor)+preparationFixture = do+  planId <- newRunId+  runId <- newRunId+  now <- getCurrentTime+  selector <- either (error . Text.unpack) pure (parseSelector "selftest/kernel/**")+  scenario <- either (error . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/postgres-roundtrip")+  seed <- either (error . Text.unpack) pure (mkSeed 7)+  descriptor <- eitherDecodeFileStrict' "test/golden/cell/cell.descriptor.v1.json" >>= either fail pure+  payload <- eitherDecodeFileStrict' "test/golden/payload.json" >>= either fail pure+  let reason = Reason (Change (ComponentRef (ComponentId "kernel") Nothing) Named "cell prepare") [] selector 0+      base = minimalRunSpec scenario+      runSpec = RunSpec (Just runId) scenario base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+      run = PlannedRun 0 runId 1 (reason :| []) Nothing runSpec+      planContext = PlanContext Nothing "fixture" "released" "sha256:fixture" (PlanInputs Null) [] []+  pure (RunPlan planId now planContext (defaultPlanPolicy seed) [run] [] 1, descriptor, payload)
+ test/SessionSpec.hs view
@@ -0,0 +1,365 @@+module SessionSpec (spec) where++import Control.Concurrent (forkIO, threadDelay)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Exception (SomeException, try)+import Control.Monad (forM_)+import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isLeft)+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import FetchSpec (completeTreeWithNested)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellPhase (..), CellStatus (..), LogChunks (..), Rejected (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..))+import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), acquireLease, leaseSnapshot, releaseLease, requestCancel)+import Kenshou.Remote.Cell.Session (SessionError (..), runSubmission)+import Kenshou.Remote.Cell.Session qualified as Session+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Cell.Session.Rebind (RebindError (RemoteMarkerExists), rebindPlannedSlices)+import Kenshou.Remote.Cell.Session.Rebind qualified as Rebind+import Kenshou.Remote.Cell.Session.Resume (HeldResumeError (..), resumeHeldSession, resumeObservedSlices)+import Kenshou.Remote.Cell.Session.Runner (SessionRunError (..), runPendingSlices, runPlannedSlices)+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (createDirectoryIfMissing, doesFileExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Timeout (timeout)+import Test.Hspec++spec :: Spec+spec = describe "one leased cell submission" do+  it "round-trips a durable journal and rejects skipped transitions" do+    journal <- readSessionJournal "test/golden/cell-session.json" >>= expectRight+    now <- getCurrentTime+    identifier <- case journal.slices of+      [slice] -> pure slice.cellRun+      _ -> expectationFailure "expected one golden slice" >> error "unreachable"+    applyTransition now identifier (Session.ResultsFetched "tree") journal `shouldSatisfy` isLeft+    submitted <- expectRight (applyTransition now identifier Session.SubmissionPublished journal)+    fmap (.state) submitted.slices `shouldBe` [SliceSubmitted]+    applyTransition now identifier Session.SubmissionPublished submitted `shouldSatisfy` isLeft++  it "reconciles an exact marker after interruption before its publication checkpoint without writing another marker" $ withSystemTempDirectory "kenshou-marker-recovery" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    let path = root </> "out" </> "session.json"+        marker = ObjectName (submissionPrefix submission <> "submission.json")+    (_, _) <- journalCheckpoint path submission+    absent <- resumeObservedSlices store cellRef (Bucket "results") path (const (pure ())) >>= either (fail . show) pure+    fmap (.state) absent.slices `shouldBe` [SlicePlanned]+    store.getObject cellRef.controlBucket marker `shouldReturn` Nothing+    _ <- store.putObject cellRef.controlBucket marker "application/json" DoesNotExist (encode submission)+    retained <- store.getObject cellRef.controlBucket marker+    observed <- resumeObservedSlices store cellRef (Bucket "results") path (const (pure ())) >>= either (fail . show) pure+    fmap (.state) observed.slices `shouldBe` [SliceSubmitted]+    store.getObject cellRef.controlBucket marker `shouldReturn` retained+    releaseLease store cellRef handle `shouldReturn` True++  it "stops before publication when the lease was cancelled" $ withSystemTempDirectory "kenshou-session" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    requestCancel store cellRef (Just submission.leaseId) False `shouldReturn` True+    runSubmission store cellRef (Bucket "results") handle submission workBytes (root <> "/out") (const (pure ())) `shouldReturn` Left LeaseLost+    store.listObjects cellRef.controlBucket (submissionPrefix submission) `shouldReturn` []++  it "returns the cell's rejection reason after publishing the marker" $ withSystemTempDirectory "kenshou-session" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    let journalPath = root </> "out" </> "session.json"+    (_, checkpoint) <- journalCheckpoint journalPath submission+    _ <- forkIO do+      let marker = ObjectName (submissionPrefix submission <> "submission.json")+      let awaitMarker = do+            visible <- store.statObject cellRef.controlBucket marker+            case visible of+              Nothing -> threadDelay 10000 >> awaitMarker+              Just _ -> pure ()+      awaitMarker+      now <- getCurrentTime+      _ <- store.putObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "rejected.json")) "application/json" DoesNotExist (encode (Rejected submission.runId "unsafe-command" now))+      pure ()+    Session.runSubmissionWithTransitions store cellRef (Bucket "results") handle submission workBytes (root </> "out") (const (pure ())) checkpoint `shouldReturn` Left (SubmissionRejected "unsafe-command")+    journal <- readSessionJournal journalPath >>= expectRight+    fmap (.state) journal.slices `shouldBe` [SliceRejected]+    fmap (.rejectionReason) journal.slices `shouldBe` [Just "unsafe-command"]++  it "takes one submission through a sealed local protocol agent and writes the index" $ withSystemTempDirectory "kenshou-session" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    let journalPath = root </> "out" </> "session.json"+    (nested, checkpoint) <- journalCheckpoint journalPath submission+    workerResult <- newEmptyMVar+    _ <- forkIO do+      completed <- try (publishSealedFixture store root submission nested) :: IO (Either SomeException ())+      putMVar workerResult completed+    result <- timeout 10000000 (Session.runSubmissionWithTransitions store cellRef (Bucket "results") handle submission workBytes (root </> "out") (const (pure ())) checkpoint)+    case result of+      Nothing -> expectationFailure "session did not observe the sealed result within ten seconds"+      Just (Left failure) -> expectationFailure (show failure)+      Just (Right index) -> do+        index.cellRun `shouldBe` submission.runId+        indexed <- doesFileExist (root </> "out" </> Text.unpack (renderRunId submission.runId) </> "cell-run.json")+        indexed `shouldBe` True+        journal <- readSessionJournal journalPath >>= expectRight+        fmap (.state) journal.slices `shouldBe` [SliceVerified]+        fmap (.manifestSha256) journal.slices `shouldBe` [Just index.cellManifest.sha256]+        otherRun <- newRunId+        now <- getCurrentTime+        case journal.slices of+          [slice] -> do+            let mismatched = journal {slices = [slice {state = SliceFetched, entryExitCode = Nothing, runIds = [otherRun]}]}+            applyTransition now submission.runId (Session.ResultsVerified index) mismatched `shouldBe` Left "verified index nested runs differ from planned slice"+            let preEntryFailure = index {cellOutcome = InfrastructureFailure, runs = []}+                fetchedFailure = journal {slices = [slice {state = SliceFetched, entryExitCode = Nothing, cellOutcome = Just InfrastructureFailure}]}+            case applyTransition now submission.runId (Session.ResultsVerified preEntryFailure) fetchedFailure of+              Right completed -> fmap (.state) completed.slices `shouldBe` [SliceVerified]+              Left problem -> expectationFailure (Text.unpack problem)+          _ -> expectationFailure "expected one journal slice"+        worker <- takeMVar workerResult+        case worker of+          Left failure -> expectationFailure (show failure)+          Right () -> pure ()++  it "runs two prepared slices under one lease and checkpoints both results" $ withSystemTempDirectory "kenshou-session" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    first <- fixtureFor handle+    second <- fixtureFor handle+    firstNested <- newRunId+    secondNested <- newRunId+    sessionId <- newRunId+    now <- getCurrentTime+    let sessionDir = root </> "out" </> Text.unpack (renderRunId sessionId)+        journalPath = sessionDir </> "session.json"+        firstWork = "slice-0/work.json"+        secondWork = "slice-1/work.json"+        makeSlice index submission nested workPath = SliceJournal index submission.runId [index] [nested] submission.reset submission workPath SlicePlanned Nothing Nothing Nothing Nothing Nothing+        initial = SessionJournal sessionId "alpha" "file" Nothing "control" "results" first.leaseId Held Map.empty (Text.replicate 64 "a") [] [makeSlice 0 first firstNested firstWork, makeSlice 1 second secondNested secondWork] now now+    createDirectoryIfMissing True (sessionDir </> "slice-0")+    createDirectoryIfMissing True (sessionDir </> "slice-1")+    LazyByteString.writeFile (sessionDir </> firstWork) workBytes+    LazyByteString.writeFile (sessionDir </> secondWork) "wrong"+    runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())) `shouldReturn` Left (InvalidSession "prepared work digest or size differs: slice-1/work.json")+    doesFileExist journalPath `shouldReturn` False+    LazyByteString.writeFile (sessionDir </> secondWork) workBytes+    workerResult <- newEmptyMVar+    _ <- forkIO do+      completed <- try (forM_ [(first, firstNested), (second, secondNested)] \(submission, nested) -> publishSealedFixture store root submission nested) :: IO (Either SomeException ())+      putMVar workerResult completed+    result <- timeout 20000000 (runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())))+    case result of+      Nothing -> expectationFailure "two-slice session did not finish within twenty seconds"+      Just (Left failure) -> expectationFailure (show failure)+      Just (Right journal) -> do+        fmap (.state) journal.slices `shouldBe` [SliceVerified, SliceVerified]+        persisted <- readSessionJournal journalPath >>= expectRight+        persisted `shouldBe` journal+        forM_ [first, second] \submission -> do+          indexed <- doesFileExist (root </> "out" </> Text.unpack (renderRunId submission.runId) </> "cell-run.json")+          indexed `shouldBe` True+        worker <- takeMVar workerResult+        case worker of+          Left failure -> expectationFailure (show failure)+          Right () -> pure ()+        runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())) `shouldReturn` Left (SessionFileExists journalPath)+        case journal.slices of+          [firstSlice, secondSlice] -> do+            let interrupted =+                  journal+                    { slices =+                        [ firstSlice {state = SliceSealed, entryExitCode = Nothing, fetchedPath = Nothing},+                          secondSlice {state = SliceSubmitted, cellOutcome = Nothing, entryExitCode = Nothing, manifestSha256 = Nothing, fetchedPath = Nothing}+                        ]+                    }+            writeSessionJournal journalPath interrupted+            runPendingSlices store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Left (UnresolvedSlice firstSlice.cellRun SliceSealed)+            resumed <- resumeObservedSlices store cellRef (Bucket "results") journalPath (const (pure ()))+            recovered <- case resumed of+              Left failure -> expectationFailure (show failure) >> error "unreachable"+              Right value -> pure value+            fmap (.state) recovered.slices `shouldBe` [SliceVerified, SliceVerified]+            readSessionJournal journalPath `shouldReturn` Right recovered+            resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Right recovered+            priorLeaseId <- newRunId+            let completedPriorLease = (recovered :: SessionJournal) {leaseId = priorLeaseId}+            writeSessionJournal journalPath completedPriorLease+            resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Right completedPriorLease+            readSessionJournal journalPath `shouldReturn` Right completedPriorLease+            writeSessionJournal journalPath recovered+            case recovered.slices of+              [firstRecovered, secondRecovered] -> do+                writeSessionJournal journalPath (recovered {slices = [firstRecovered {state = SliceFetched, entryExitCode = Nothing}, secondRecovered]})+                fetchedAgain <- resumeObservedSlices store cellRef (Bucket "results") journalPath (const (pure ()))+                case fetchedAgain of+                  Left failure -> expectationFailure (show failure)+                  Right value -> fmap (.state) value.slices `shouldBe` [SliceVerified, SliceVerified]+                let oldMarker = secondRecovered {state = SlicePlanned, cellOutcome = Nothing, entryExitCode = Nothing, manifestSha256 = Nothing, fetchedPath = Nothing}+                writeSessionJournal journalPath (recovered {slices = [firstRecovered, oldMarker]})+                rebindPlannedSlices store cellRef handle journalPath `shouldReturn` Left (RemoteMarkerExists secondRecovered.cellRun)+                observedMarker <- resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ()))+                case observedMarker of+                  Left failure -> expectationFailure (show failure)+                  Right collected -> fmap (.state) collected.slices `shouldBe` [SliceVerified, SliceVerified]+              _ -> expectationFailure "expected two recovered slices"+          _ -> expectationFailure "expected two journal slices"+        fresh <- fixtureFor handle+        freshNested <- newRunId+        freshSessionId <- newRunId+        at <- getCurrentTime+        let freshDir = root </> "out" </> Text.unpack (renderRunId freshSessionId)+            freshPath = freshDir </> "session.json"+            freshSlice = SliceJournal 0 fresh.runId [0] [freshNested] fresh.reset fresh "slice-0/work.json" SlicePlanned Nothing Nothing Nothing Nothing Nothing+            freshJournal = SessionJournal freshSessionId "alpha" "file" Nothing "control" "results" fresh.leaseId Held Map.empty (Text.replicate 64 "a") [] [freshSlice] at at+        writeSessionJournal freshPath freshJournal+        rebound <- rebindPlannedSlices store cellRef handle freshPath+        case rebound of+          Left failure -> expectationFailure (show failure)+          Right updated -> do+            updated.leaseId `shouldBe` fresh.leaseId+            fmap (.state) updated.slices `shouldBe` [SlicePlanned]+            fmap (.cellRun) updated.slices `shouldNotBe` [fresh.runId]+            readSessionJournal freshPath `shouldReturn` Right updated+            case updated.slices of+              [pendingSlice] -> do+                createDirectoryIfMissing True (freshDir </> "slice-0")+                LazyByteString.writeFile (freshDir </> pendingSlice.workPath) workBytes+                pendingWorker <- newEmptyMVar+                _ <- forkIO do+                  completed <- try (publishSealedFixture store root pendingSlice.submission freshNested) :: IO (Either SomeException ())+                  putMVar pendingWorker completed+                continued <- timeout 10000000 (runPendingSlices store cellRef (Bucket "results") handle freshPath (const (pure ())))+                case continued of+                  Nothing -> expectationFailure "rebound slice did not finish within ten seconds"+                  Just (Left failure) -> expectationFailure (show failure)+                  Just (Right finalJournal) -> do+                    fmap (.state) finalJournal.slices `shouldBe` [SliceVerified]+                    readSessionJournal freshPath `shouldReturn` Right finalJournal+                pendingOutcome <- takeMVar pendingWorker+                case pendingOutcome of+                  Left failure -> expectationFailure (show failure)+                  Right () -> pure ()+              _ -> expectationFailure "expected one rebound slice"++  it "retries an unaccepted submitted slice only after a fresh lease fences it" $ withSystemTempDirectory "kenshou-session" \root -> do+    store <- newFileStore root+    old <- acquireLease store cellRef request >>= expectAcquired+    submission <- fixtureFor old+    nested <- newRunId+    sessionId <- newRunId+    now <- getCurrentTime+    let sessionDir = root </> "out" </> Text.unpack (renderRunId sessionId)+        journalPath = sessionDir </> "session.json"+        slice = SliceJournal 0 submission.runId [0] [nested] submission.reset submission "slice-0/work.json" SliceSubmitted Nothing Nothing Nothing Nothing Nothing+        journal = SessionJournal sessionId "alpha" "file" Nothing "control" "results" submission.leaseId Held Map.empty (Text.replicate 64 "a") [] [slice] now now+        marker = ObjectName (submissionPrefix submission <> "submission.json")+        status = ObjectName (submissionPrefix submission <> "status.json")+        result = ObjectName ("runs/" <> renderRunId submission.runId <> "/cell/result.json")+    createDirectoryIfMissing True (sessionDir </> "slice-0")+    LazyByteString.writeFile (sessionDir </> slice.workPath) workBytes+    writeSessionJournal journalPath journal+    _ <- store.putObject cellRef.controlBucket marker "application/json" DoesNotExist (encode submission)+    rebindPlannedSlices store cellRef old journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)+    releaseLease store cellRef old `shouldReturn` True+    fresh <- acquireLease store cellRef request >>= expectAcquired+    _ <- store.putObject cellRef.controlBucket status "application/json" DoesNotExist "{}"+    rebindPlannedSlices store cellRef fresh journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)+    store.deleteObject cellRef.controlBucket status NoPrecondition `shouldReturn` True+    _ <- store.putObject (Bucket "results") result "application/json" DoesNotExist "{}"+    rebindPlannedSlices store cellRef fresh journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)+    store.deleteObject (Bucket "results") result NoPrecondition `shouldReturn` True+    rebound <- rebindPlannedSlices store cellRef fresh journalPath+    updated <- case rebound of+      Left failure -> expectationFailure (show failure) >> error "unreachable"+      Right value -> pure value+    fmap (.state) updated.slices `shouldBe` [SlicePlanned]+    updated.leaseId `shouldNotBe` submission.leaseId+    case updated.slices of+      [reboundSlice] -> do+        reboundSlice.cellRun `shouldNotBe` submission.runId+        reboundSlice.submission.leaseId `shouldBe` updated.leaseId+        store.statObject cellRef.controlBucket marker >>= (`shouldSatisfy` (maybe False (const True)))+      _ -> expectationFailure "expected one rebound slice"++publishSealedFixture :: ObjectStore -> FilePath -> Submission -> RunId -> IO ()+publishSealedFixture store root submission nested = do+  let marker = ObjectName (submissionPrefix submission <> "submission.json")+      results = Bucket "results"+      resultPrefix = "runs/" <> renderRunId submission.runId <> "/"+      awaitMarker = do+        visible <- store.statObject cellRef.controlBucket marker+        case visible of+          Nothing -> threadDelay 10000 >> awaitMarker+          Just _ -> pure ()+  awaitMarker+  (sourceTree, manifest) <- completeTreeWithNested (root </> "fixture" </> Text.unpack (renderRunId submission.runId)) submission.runId submission.leaseId nested False False False+  forM_ manifest.artifacts \artifact -> do+    bytes <- LazyByteString.readFile (sourceTree </> Text.unpack artifact.path)+    _ <- store.putObject results (ObjectName (resultPrefix <> artifact.path)) artifact.mediaType DoesNotExist bytes+    pure ()+  bytes <- LazyByteString.readFile (sourceTree </> "manifest.json")+  _ <- store.putObject results (ObjectName (resultPrefix <> "manifest.json")) "application/json" DoesNotExist bytes+  now <- getCurrentTime+  let digest = (workObjectFor "application/json" bytes).sha256+      status = CellStatus submission.runId Sealed (Just manifest.leaseSequence) now Nothing (LogChunks 0 0) (Just Completed) (Just digest) (Just [])+  _ <- store.putObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "status.json")) "application/json" DoesNotExist (encode status)+  pure ()++fixtureFor :: LeaseHandle -> IO Submission+fixtureFor handle = do+  bytes <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+  fixture <- either (ioError . userError) pure (eitherDecode bytes :: Either String Submission)+  identifier <- newRunId+  lease <- leaseSnapshot handle+  let bundle = fixture.payload.bundle+      payload = fixture.payload {bundle = bundle {uri = "gs://control/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"}}+  pure (Submission identifier lease.leaseId payload (workObjectFor "application/json" workBytes) fixture.env fixture.reset fixture.limits fixture.requires Nothing fixture.labels)++submissionPrefix :: Submission -> Text+submissionPrefix submission = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/"++workBytes :: LazyByteString.ByteString+workBytes = "{}"++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")++request :: LeaseRequest+request = LeaseRequest "tester@workstation" "verification" 120++expectAcquired :: AcquireOutcome -> IO LeaseHandle+expectAcquired (Acquired handle) = pure handle+expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"++expectRight :: Either Text value -> IO value+expectRight = either (\failure -> expectationFailure (Text.unpack failure) >> error "unreachable") pure++journalCheckpoint :: FilePath -> Submission -> IO (RunId, Session.SessionTransition -> IO ())+journalCheckpoint path submission = do+  now <- getCurrentTime+  identifier <- newRunId+  nested <- newRunId+  let slice = SliceJournal 0 submission.runId [0] [nested] submission.reset submission "slice-0/work.json" SlicePlanned Nothing Nothing Nothing Nothing Nothing+      initial = SessionJournal identifier "alpha" "file" Nothing "control" "results" submission.leaseId Held Map.empty (Text.replicate 64 "a") [] [slice] now now+  writeSessionJournal path initial+  journalRef <- newIORef initial+  pure+    ( nested,+      \transition -> do+        current <- readIORef journalRef+        at <- getCurrentTime+        changed <- either (ioError . userError . Text.unpack) pure (applyTransition at submission.runId transition current)+        writeSessionJournal path changed+        writeIORef journalRef changed+    )
+ test/SubmitSpec.hs view
@@ -0,0 +1,117 @@+module SubmitSpec (spec) where++import Data.Aeson (eitherDecode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.Text (Text)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), acquireLease, leaseSnapshot, requestCancel)+import Kenshou.Remote.Cell.Submit (PublishOutcome (..), publishSubmission, workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell submission publication" do+  it "publishes work before the create-only marker without changing the lease sequence" $ withSystemTempDirectory "kenshou-submit" \root -> do+    base <- newFileStore root+    handle <- acquireLease base cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    writes <- newIORef []+    let store =+          base+            { putObject = \bucket name media pre bytes -> do+                modifyIORef' writes (<> [(name, pre)])+                base.putObject bucket name media pre bytes+            }+        prefix = submissionPrefix submission+        workName = ObjectName (prefix <> "work")+        markerName = ObjectName (prefix <> "submission.json")+    publishSubmission store cellRef handle submission workBytes `shouldReturn` Submitted+    readIORef writes `shouldReturn` [(workName, DoesNotExist), (markerName, DoesNotExist)]+    fmap fst <$> base.getObject cellRef.controlBucket workName `shouldReturn` Just workBytes+    marker <- base.getObject cellRef.controlBucket markerName+    fmap (eitherDecode . fst) marker `shouldBe` Just (Right submission)+    lease <- leaseSnapshot handle+    lease.runsStarted `shouldBe` 0+    publishSubmission store cellRef handle submission workBytes `shouldReturn` WorkAlreadyExists+    readIORef writes `shouldReturn` [(workName, DoesNotExist), (markerName, DoesNotExist), (workName, DoesNotExist)]++  it "refuses incorrect work metadata before writing either object" $ withSystemTempDirectory "kenshou-submit" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    let badWork = WorkObject "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" submission.work.bytes submission.work.mediaType+        bad = submission {work = badWork}+    publishSubmission store cellRef handle bad workBytes `shouldThrow` anyIOException+    store.listObjects cellRef.controlBucket (submissionPrefix submission) `shouldReturn` []++  it "refuses a payload bundle from another control bucket" $ withSystemTempDirectory "kenshou-submit" \root -> do+    store <- newFileStore root+    handle <- acquireLease store cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    let bundle = submission.payload.bundle+        wrongBucket = submission {payload = submission.payload {bundle = bundle {uri = "gs://elsewhere/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"}}}+    publishSubmission store cellRef handle wrongBucket workBytes `shouldThrow` anyIOException+    store.listObjects cellRef.controlBucket (submissionPrefix submission) `shouldReturn` []++  it "leaves an orphan work object if the lease is cancelled during upload" $ withSystemTempDirectory "kenshou-submit" \root -> do+    base <- newFileStore root+    handle <- acquireLease base cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    let store =+          base+            { putObject = \bucket name media pre bytes -> do+                result <- base.putObject bucket name media pre bytes+                if name == ObjectName (submissionPrefix submission <> "work")+                  then do+                    requestCancel base cellRef (Just submission.leaseId) False `shouldReturn` True+                    pure result+                  else pure result+            }+    publishSubmission store cellRef handle submission workBytes `shouldReturn` LostLease+    work <- base.statObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "work"))+    work `shouldSatisfy` (/= Nothing)+    base.statObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "submission.json")) `shouldReturn` Nothing++  it "reports a marker collision without overwriting it" $ withSystemTempDirectory "kenshou-submit" \root -> do+    base <- newFileStore root+    handle <- acquireLease base cellRef request >>= expectAcquired+    submission <- fixtureFor handle+    let markerName = ObjectName (submissionPrefix submission <> "submission.json")+        store =+          base+            { putObject = \bucket name media pre bytes ->+                if name == markerName then pure PreconditionFailed else base.putObject bucket name media pre bytes+            }+    publishSubmission store cellRef handle submission workBytes `shouldReturn` SubmissionAlreadyExists+    base.statObject cellRef.controlBucket markerName `shouldReturn` Nothing++fixtureFor :: LeaseHandle -> IO Submission+fixtureFor handle = do+  bytes <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+  fixture <- either (ioError . userError) pure (eitherDecode bytes)+  identifier <- newRunId+  lease <- leaseSnapshot handle+  let bundle = fixture.payload.bundle+      payload = fixture.payload {bundle = bundle {uri = "gs://control/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"}}+  pure fixture {runId = identifier, leaseId = lease.leaseId, payload = payload, work = workObjectFor "application/json" workBytes}++submissionPrefix :: Submission -> Text+submissionPrefix submission = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/"++workBytes :: LazyByteString.ByteString+workBytes = "{\"schema\":\"kenshou.run-plan/v1\",\"runs\":[]}"++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")++request :: LeaseRequest+request = LeaseRequest "tester@workstation" "verification" 120++expectAcquired :: AcquireOutcome -> IO LeaseHandle+expectAcquired (Acquired handle) = pure handle+expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"
+ test/WatchSpec.hs view
@@ -0,0 +1,83 @@+module WatchSpec (spec) where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (CellPhase (..), CellStatus (..), LogChunks (..), Rejected (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Watch (WatchEvent (..), WatchSnapshot (..), WatchTerminal (..), newWatchCursor, pollCellRun)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell status and log watcher" do+  it "waits for a status and then emits each phase and chunk exactly once" $ withSystemTempDirectory "kenshou-watch" \root -> do+    store <- newFileStore root+    identifier <- newRunId+    pollCellRun store cellRef identifier newWatchCursor `shouldReturn` WatchSnapshot newWatchCursor [] Nothing+    fixture <- statusFixture identifier+    let running = fixture {phase = Running, logChunks = LogChunks 1 1, outcome = Nothing, manifestSha256 = Nothing}+    put store identifier "log/stdout.0" "first out\n"+    put store identifier "log/stderr.0" "first err\n"+    put store identifier "status.json" (encode running)+    first <- pollCellRun store cellRef identifier newWatchCursor+    first.events `shouldBe` [PhaseChanged Running, StdoutChunk "first out\n", StderrChunk "first err\n"]+    first.terminal `shouldBe` Nothing+    unchanged <- pollCellRun store cellRef identifier first.cursor+    unchanged.events `shouldBe` []+    put store identifier "log/stdout.1" "second out\n"+    put store identifier "status.json" (encode fixture {logChunks = LogChunks 2 1})+    sealed <- pollCellRun store cellRef identifier unchanged.cursor+    sealed.events `shouldBe` [PhaseChanged Sealed, StdoutChunk "second out\n"]+    sealed.terminal `shouldBe` Just (RunSealed fixture {logChunks = LogChunks 2 1})++  it "gives rejection priority over status and checks the run identity" $ withSystemTempDirectory "kenshou-watch" \root -> do+    store <- newFileStore root+    identifier <- newRunId+    fixture <- statusFixture identifier+    rejected <- rejectedFixture identifier+    put store identifier "status.json" (encode fixture)+    put store identifier "rejected.json" (encode rejected)+    result <- pollCellRun store cellRef identifier newWatchCursor+    result.events `shouldBe` []+    result.terminal `shouldBe` Just (RunRejected rejected)+    other <- newRunId+    wrong <- rejectedFixture other+    put store identifier "rejected.json" (encode wrong)+    pollCellRun store cellRef identifier newWatchCursor `shouldThrow` anyIOException++  it "refuses missing chunks and a backwards chunk count" $ withSystemTempDirectory "kenshou-watch" \root -> do+    store <- newFileStore root+    identifier <- newRunId+    fixture <- statusFixture identifier+    let running = fixture {phase = Running, logChunks = LogChunks 1 0, outcome = Nothing, manifestSha256 = Nothing}+    put store identifier "status.json" (encode running)+    pollCellRun store cellRef identifier newWatchCursor `shouldThrow` anyIOException+    put store identifier "log/stdout.0" "first out\n"+    first <- pollCellRun store cellRef identifier newWatchCursor+    put store identifier "status.json" (encode running {logChunks = LogChunks 0 0})+    pollCellRun store cellRef identifier first.cursor `shouldThrow` anyIOException++statusFixture :: RunId -> IO CellStatus+statusFixture identifier = do+  bytes <- LazyByteString.readFile "test/golden/cell/cell.status.v1.json"+  fixture <- either (ioError . userError) pure (eitherDecode bytes :: Either String CellStatus)+  pure (CellStatus identifier fixture.phase fixture.leaseSequence fixture.updatedAt fixture.phaseStartedAt fixture.logChunks fixture.outcome fixture.manifestSha256 fixture.reasons)++rejectedFixture :: RunId -> IO Rejected+rejectedFixture identifier = do+  bytes <- LazyByteString.readFile "test/golden/cell/cell.rejected.v1.json"+  fixture <- either (ioError . userError) pure (eitherDecode bytes :: Either String Rejected)+  pure (Rejected identifier fixture.reason fixture.at)++put :: ObjectStore -> RunId -> Text -> LazyByteString.ByteString -> IO ()+put store identifier suffix bytes = do+  let name = ObjectName ("cells/alpha/submissions/" <> renderRunId identifier <> "/" <> suffix)+  _ <- store.putObject cellRef.controlBucket name "application/json" NoPrecondition bytes+  pure ()++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")
+ test/golden/cell-capabilities.json view
@@ -0,0 +1,13 @@+{+  "schema": "kenshou.cell-capabilities/v1",+  "cell": "alpha",+  "descriptorSha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+  "probedBy": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "probedAt": "2026-09-25T12:00:00Z",+  "capabilities": {+    "postgres.major": "18",+    "postgres.pg_partman": false,+    "broker": "redpanda v25.2.4",+    "otlp.null": true+  }+}
+ test/golden/cell-route.json view
@@ -0,0 +1,8 @@+{+  "schema": "kenshou.cell-route/v1",+  "planId": "0199a3f2-7c1e-7b8a-9d41-3e5f6a7b8c9d",+  "perCell": {"alpha": ["0199a3f2-7c20-7f00-8a11-0c0d0e0f1011"]},+  "localRunIds": [],+  "unroutable": [],+  "warnings": []+}
+ test/golden/cell-run.json view
@@ -0,0 +1,20 @@+{+  "schema": "kenshou.cell-run/v1",+  "cellRun": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "cell": "alpha",+  "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+  "leaseSequence": 2,+  "resultsBaseUri": "gs://tan-nb-exp-cells-results/runs/0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "dataBaseUri": "gs://tan-nb-exp-cells-results/runs/0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9/output",+  "cellManifest": {"path": "manifest.json", "sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "bytes": 5210},+  "cellOutcome": "completed",+  "reasons": [],+  "entryExitCode": 0,+  "evidence": [+    {"kind": "cell-fingerprint", "path": "cell/fingerprint.json", "sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "bytes": 3904, "mediaType": "application/json"}+  ],+  "runs": [+    {"runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011", "path": "output/0199a3f2-7c20-7f00-8a11-0c0d0e0f1011", "scenario": "selftest/kernel/correctness/postgres-roundtrip", "manifestSha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", "recordedOutcome": "passed", "effectiveOutcome": "passed", "overrideReason": null}+  ],+  "verifiedAt": "2026-09-27T12:00:00Z"+}
+ test/golden/cell-session.json view
@@ -0,0 +1,118 @@+{+  "schema": "kenshou.cell-session/v1",+  "sessionId": "0199a3c3-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "cell": "alpha",+  "store": "gs://",+  "project": "tan-nb-exp",+  "controlBucket": "tan-nb-exp-cells-control",+  "resultsBucket": "tan-nb-exp-cells-results",+  "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+  "leaseMode": "held",+  "payloads": {},+  "planSha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+  "rejectedRuns": [],+  "slices": [+    {+      "index": 0,+      "cellRun": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+      "ordinals": [+        0+      ],+      "runIds": [+        "0199a3c4-7b1e-7c44-9d0a-3f5e2a61b7c9"+      ],+      "reset": {+        "postgres": {+          "major": 18,+          "databases": [+            {+              "name": "benchmark",+              "owner": "benchmark",+              "template": "template0"+            }+          ],+          "settings": {+            "checkpoint_timeout": "30min",+            "max_wal_size": "16GB"+          }+        },+        "cachePolicy": "cold",+        "broker": {+          "wipe": true+        }+      },+      "submission": {+        "schema": "cell.submission/v1",+        "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+        "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+        "payload": {+          "schema": "cell.payload/v1",+          "kind": "nix-nar-bundle",+          "bundle": {+            "uri": "gs://tan-nb-exp-cells-control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst",+            "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+            "bytes": 48211934+          },+          "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello",+          "narHash": "sha256-example",+          "closurePaths": [+            "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"+          ],+          "system": "x86_64-linux",+          "command": [+            "bin/cell-fixture-hello"+          ]+        },+        "work": {+          "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+          "bytes": 212,+          "mediaType": "application/json"+        },+        "env": {+          "EXAMPLE": "value"+        },+        "reset": {+          "postgres": {+            "major": 18,+            "databases": [+              {+                "name": "benchmark",+                "owner": "benchmark",+                "template": "template0"+              }+            ],+            "settings": {+              "checkpoint_timeout": "30min",+              "max_wal_size": "16GB"+            }+          },+          "cachePolicy": "cold",+          "broker": {+            "wipe": true+          }+        },+        "limits": {+          "wallClockSeconds": 3600,+          "memoryMaxBytes": 25769803776,+          "outputMaxBytes": 21474836480+        },+        "requires": {+          "protocol": "cell.protocol/v1",+          "minAgentVersion": "0.1.0"+        },+        "labels": {+          "arm": "candidate"+        }+      },+      "workPath": "slice-0/work.json",+      "state": "planned",+      "cellOutcome": null,+      "entryExitCode": null,+      "manifestSha256": null,+      "rejectionReason": null,+      "fetchedPath": null+    }+  ],+  "createdAt": "2026-09-27T12:00:00Z",+  "updatedAt": "2026-09-27T12:00:00Z"+}
+ test/golden/cell/SOURCE view
@@ -0,0 +1,19 @@+mori://shinzui/load-testing-infra+Commit: ef1596775d4df5a31b8de3a4b833adeeaf0f8021+Project-relative source directories (artifact-level URIs pending):+  schemas/cell/ (all cell.*.schema.json files)+  schemas/cell/examples/ (all cell.*.json files)++Execution-window schema and example extension:+  Commit: 915fb2f9ee8ca8eccafecd8341887638ac8b4218+  Project-relative paths (artifact-level URIs pending):+    schemas/cell/cell.execution-window.v1.schema.json+    schemas/cell/examples/cell.execution-window.v1.json++Kenshou extension (not in the upstream commit above):+  cell.environment.v1.schema.json and cell.submission.v1.schema.json carry an optional+  `resources` object (top level of the environment; inside `reset` of the submission),+  added by docs/plans/3-replace-kafka-specific-environment-fields-with-suite-provisioned-named-resources.md.+  It is pending adoption in mori://shinzui/load-testing-infra. Until then the client never+  sends `reset.resources` for the resource the legacy broker fields describe (see+  Kenshou.Remote.Cell.Legacy), and the example documents are unchanged.
+ test/golden/cell/cell.artifact-manifest.v1.json view
@@ -0,0 +1,21 @@+{+  "schema": "cell.artifact-manifest/v1",+  "layout": 1,+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "cell": "alpha",+  "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+  "leaseSequence": 2,+  "sealedAt": "2026-09-25T12:21:07Z",+  "agentVersion": "0.1.0",+  "payload": {+    "kind": "nix-nar-bundle",+    "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+    "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"+  },+  "outcome": "completed",+  "retention": {"policy": "bucket-retention", "retentionSeconds": 34560000},+  "artifacts": [+    {"path": "cell/result.json", "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "bytes": 412, "mediaType": "application/json"},+    {"path": "output/result.json", "sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "bytes": 96, "mediaType": "application/json"}+  ]+}
+ test/golden/cell/cell.artifact-manifest.v1.schema.json view
@@ -0,0 +1,53 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.artifact-manifest.v1.schema.json",+  "title": "Sealed verification cell result manifest",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "layout", "runId", "cell", "leaseId", "leaseSequence", "sealedAt", "agentVersion", "payload", "outcome", "retention", "artifacts"],+  "properties": {+    "schema": {"const": "cell.artifact-manifest/v1"},+    "layout": {"const": 1},+    "runId": {"$ref": "#/$defs/identifier"},+    "cell": {"$ref": "#/$defs/identifier"},+    "leaseId": {"$ref": "#/$defs/identifier"},+    "leaseSequence": {"type": "integer", "minimum": 0},+    "sealedAt": {"type": "string", "format": "date-time"},+    "agentVersion": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"},+    "payload": {+      "type": "object", "additionalProperties": false,+      "required": ["kind", "sha256", "storePath"],+      "properties": {+        "kind": {"const": "nix-nar-bundle"},+        "sha256": {"$ref": "#/$defs/sha256"},+        "storePath": {"type": "string", "pattern": "^/nix/store/[a-z0-9]{32}-[^/]+$"}+      }+    },+    "outcome": {"enum": ["completed", "infrastructure-failure", "cancelled", "timed-out"]},+    "retention": {+      "type": "object", "additionalProperties": false,+      "required": ["policy", "retentionSeconds"],+      "properties": {+        "policy": {"const": "bucket-retention"},+        "retentionSeconds": {"type": "integer", "minimum": 0}+      }+    },+    "artifacts": {+      "type": "array", "minItems": 1,+      "items": {+        "type": "object", "additionalProperties": false,+        "required": ["path", "sha256", "bytes", "mediaType"],+        "properties": {+          "path": {"type": "string", "pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[^\\u0000]+$"},+          "sha256": {"$ref": "#/$defs/sha256"},+          "bytes": {"type": "integer", "minimum": 0},+          "mediaType": {"type": "string", "minLength": 1}+        }+      }+    }+  },+  "$defs": {+    "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}+  }+}
+ test/golden/cell/cell.descriptor.v1.json view
@@ -0,0 +1,31 @@+{+  "schema": "cell.descriptor/v1",+  "name": "alpha",+  "project": "tan-nb-exp",+  "zone": "us-west1-a",+  "postgresMajor": 18,+  "instances": {+    "postgres": "cell-alpha-postgres",+    "drivers": ["cell-alpha-driver-0"],+    "monitoring": "cell-alpha-monitoring"+  },+  "addresses": {+    "postgres": "10.0.0.2",+    "drivers": ["10.0.0.3"],+    "monitoring": "10.0.0.4"+  },+  "shape": {+    "postgres": {"machineType": "n2-standard-8", "diskType": "pd-ssd", "diskSizeGb": 200},+    "driver": {"machineType": "n2-standard-8", "count": 1, "bootDiskSizeGb": 100},+    "monitoring": {"machineType": "n2-standard-4"},+    "scheduling": {"onHostMaintenance": "MIGRATE", "compactPlacement": false}+  },+  "images": {+    "driver": "projects/tan-nb-exp/global/images/example-cell-driver",+    "postgres": "projects/tan-nb-exp/global/images/example-cell-postgres18",+    "monitoring": "projects/tan-nb-exp/global/images/example-cell-monitoring",+    "broker": ""+  },+  "buckets": {"control": "tan-nb-exp-cells-control", "results": "tan-nb-exp-cells-results"},+  "agentProtocol": "cell.protocol/v1"+}
+ test/golden/cell/cell.descriptor.v1.schema.json view
@@ -0,0 +1,72 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.descriptor.v1.schema.json",+  "title": "Verification cell descriptor",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "name", "project", "zone", "postgresMajor", "instances", "addresses", "shape", "images", "buckets", "agentProtocol"],+  "properties": {+    "schema": {"const": "cell.descriptor/v1"},+    "name": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,40}$"},+    "project": {"type": "string", "minLength": 1},+    "zone": {"type": "string", "minLength": 1},+    "postgresMajor": {"enum": [17, 18]},+    "instances": {+      "type": "object", "additionalProperties": false,+      "required": ["postgres", "drivers", "monitoring"],+      "properties": {+        "postgres": {"type": "string", "minLength": 1},+        "drivers": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},+        "monitoring": {"type": "string", "minLength": 1}+      }+    },+    "addresses": {+      "type": "object", "additionalProperties": false,+      "required": ["postgres", "drivers", "monitoring"],+      "properties": {+        "postgres": {"type": "string", "format": "ipv4"},+        "drivers": {"type": "array", "minItems": 1, "items": {"type": "string", "format": "ipv4"}},+        "monitoring": {"type": "string", "format": "ipv4"}+      }+    },+    "shape": {+      "type": "object", "additionalProperties": false,+      "required": ["postgres", "driver", "monitoring", "scheduling"],+      "properties": {+        "postgres": {"type": "object", "required": ["machineType", "diskType", "diskSizeGb"], "properties": {+          "machineType": {"type": "string"}, "diskType": {"type": "string"},+          "diskSizeGb": {"type": "integer", "minimum": 1}, "provisionedIops": {"type": ["integer", "null"]}+        }},+        "driver": {"type": "object", "required": ["machineType", "count", "bootDiskSizeGb"], "properties": {+          "machineType": {"type": "string"}, "count": {"type": "integer", "minimum": 1},+          "bootDiskSizeGb": {"type": "integer", "minimum": 1}+        }},+        "monitoring": {"type": "object", "required": ["machineType"], "properties": {"machineType": {"type": "string"}}},+        "scheduling": {"type": "object", "required": ["onHostMaintenance", "compactPlacement"], "properties": {+          "onHostMaintenance": {"enum": ["MIGRATE", "TERMINATE"]},+          "minCpuPlatform": {"type": ["string", "null"]},+          "compactPlacement": {"type": "boolean"}+        }}+      }+    },+    "images": {+      "type": "object", "additionalProperties": false,+      "required": ["driver", "postgres", "monitoring", "broker"],+      "properties": {+        "driver": {"type": "string", "minLength": 1},+        "postgres": {"type": "string", "minLength": 1},+        "monitoring": {"type": "string", "minLength": 1},+        "broker": {"type": "string"}+      }+    },+    "buckets": {+      "type": "object", "additionalProperties": false,+      "required": ["control", "results"],+      "properties": {+        "control": {"type": "string", "minLength": 1},+        "results": {"type": "string", "minLength": 1}+      }+    },+    "agentProtocol": {"const": "cell.protocol/v1"}+  }+}
+ test/golden/cell/cell.environment.v1.json view
@@ -0,0 +1,18 @@+{+  "schema": "cell.environment/v1",+  "cell": "alpha",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+  "postgres": {+    "major": 18,+    "host": "10.0.0.2",+    "port": 5432,+    "database": "benchmark",+    "user": "benchmark",+    "connectionString": "host=10.0.0.2 port=5432 dbname=benchmark user=benchmark"+  },+  "broker": null,+  "otlp": null,+  "metrics": {"victoriaMetricsUrl": "http://10.0.0.4:8428"},+  "drivers": [{"index": 0, "host": "10.0.0.3"}]+}
+ test/golden/cell/cell.environment.v1.schema.json view
@@ -0,0 +1,76 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.environment.v1.schema.json",+  "title": "Environment made available to a verification cell payload",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "cell", "runId", "leaseId", "postgres", "broker", "otlp", "metrics", "drivers"],+  "properties": {+    "schema": {"const": "cell.environment/v1"},+    "cell": {"$ref": "#/$defs/identifier"},+    "runId": {"$ref": "#/$defs/identifier"},+    "leaseId": {"$ref": "#/$defs/identifier"},+    "postgres": {+      "type": "object", "additionalProperties": false,+      "required": ["major", "host", "port", "database", "user", "connectionString"],+      "properties": {+        "major": {"enum": [17, 18]},+        "host": {"type": "string", "format": "ipv4"},+        "port": {"type": "integer", "minimum": 1, "maximum": 65535},+        "database": {"type": "string", "minLength": 1},+        "user": {"type": "string", "minLength": 1},+        "connectionString": {"type": "string", "minLength": 1}+      }+    },+    "broker": {+      "oneOf": [+        {"type": "null"},+        {"type": "object", "additionalProperties": false, "required": ["bootstrapServers", "adminUrl", "implementation", "version"], "properties": {+          "bootstrapServers": {"type": "string", "minLength": 1},+          "adminUrl": {"type": "string", "format": "uri"},+          "implementation": {"type": "string", "minLength": 1},+          "version": {"type": "string", "minLength": 1}+        }}+      ]+    },+    "otlp": {+      "oneOf": [+        {"type": "null"},+        {"type": "object", "additionalProperties": false, "required": ["null", "file"], "properties": {+          "null": {"$ref": "#/$defs/otlpEndpoint"},+          "file": {"$ref": "#/$defs/otlpEndpoint"}+        }}+      ]+    },+    "metrics": {+      "type": "object", "additionalProperties": false,+      "required": ["victoriaMetricsUrl"],+      "properties": {"victoriaMetricsUrl": {"type": "string", "format": "uri"}}+    },+    "drivers": {+      "type": "array", "minItems": 1,+      "items": {"type": "object", "additionalProperties": false, "required": ["index", "host"], "properties": {+        "index": {"type": "integer", "minimum": 0},+        "host": {"type": "string", "format": "ipv4"}+      }}+    },+    "clock": {+      "type": "object", "additionalProperties": false,+      "required": ["skewBoundMicros"],+      "properties": {"skewBoundMicros": {"type": "integer", "minimum": 0}}+    },+    "faultHook": {"type": "string", "minLength": 1},+    "resources": {"type": "object", "description": "kenshou extension pending upstream adoption: bound values of named resources, keyed by resource name"}+  },+  "$defs": {+    "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "otlpEndpoint": {+      "type": "object", "additionalProperties": false,+      "required": ["grpc", "http"],+      "properties": {+        "grpc": {"type": "string", "format": "uri"},+        "http": {"type": "string", "format": "uri"}+      }+    }+  }+}
+ test/golden/cell/cell.execution-window.v1.json view
@@ -0,0 +1,7 @@+{+  "schema": "cell.execution-window/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "resetStarted": "2026-09-25T12:00:00Z",+  "runStarted": "2026-09-25T12:00:10Z",+  "runFinished": "2026-09-25T12:01:00Z"+}
+ test/golden/cell/cell.execution-window.v1.schema.json view
@@ -0,0 +1,33 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "type": "object",+  "additionalProperties": false,+  "required": [+    "schema",+    "runId",+    "resetStarted",+    "runStarted",+    "runFinished"+  ],+  "properties": {+    "schema": {+      "const": "cell.execution-window/v1"+    },+    "runId": {+      "type": "string",+      "format": "uuid"+    },+    "resetStarted": {+      "type": "string",+      "format": "date-time"+    },+    "runStarted": {+      "type": "string",+      "format": "date-time"+    },+    "runFinished": {+      "type": "string",+      "format": "date-time"+    }+  }+}
+ test/golden/cell/cell.health.v1.json view
@@ -0,0 +1,18 @@+{+  "schema": "cell.health/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "windowStart": "2026-09-25T12:00:00Z",+  "windowEnd": "2026-09-25T12:01:00Z",+  "passed": false,+  "gates": [+    {+      "name": "host-maintenance", "machine": "cell-alpha-postgres", "status": "tripped",+      "observed": "MIGRATE_ON_HOST_MAINTENANCE", "threshold": "NONE",+      "at": "2026-09-25T12:00:30Z", "detail": "maintenance event during run"+    },+    {+      "name": "background-load", "machine": "cell-alpha-driver-0", "status": "ok",+      "observed": 1.2, "threshold": 5, "at": "2026-09-25T12:00:10Z"+    }+  ]+}
+ test/golden/cell/cell.health.v1.schema.json view
@@ -0,0 +1,36 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.health.v1.schema.json",+  "title": "Verification cell health gate evidence",+  "type": "object", "additionalProperties": false,+  "required": ["schema", "runId", "windowStart", "windowEnd", "passed", "gates"],+  "properties": {+    "schema": {"const": "cell.health/v1"},+    "runId": {"$ref": "#/$defs/identifier"},+    "windowStart": {"type": "string", "format": "date-time"},+    "windowEnd": {"type": "string", "format": "date-time"},+    "passed": {"type": "boolean"},+    "gates": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/gate"}}+  },+  "allOf": [{+    "if": {"properties": {"passed": {"const": true}}, "required": ["passed"]},+    "then": {"properties": {"gates": {"items": {"properties": {"status": {"const": "ok"}}}}}},+    "else": {"properties": {"gates": {"contains": {"properties": {"status": {"const": "tripped"}}, "required": ["status"]}}}}+  }],+  "$defs": {+    "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "gate": {+      "type": "object", "additionalProperties": false,+      "required": ["name", "machine", "status", "observed", "threshold", "at"],+      "properties": {+        "name": {"enum": ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"]},+        "machine": {"$ref": "#/$defs/identifier"},+        "status": {"enum": ["ok", "tripped"]},+        "observed": {"type": ["number", "string", "boolean", "object"]},+        "threshold": {"type": ["number", "string", "boolean", "object", "null"]},+        "at": {"type": "string", "format": "date-time"},+        "detail": {"type": "string"}+      }+    }+  }+}
+ test/golden/cell/cell.lease.v1.json view
@@ -0,0 +1,12 @@+{+  "schema": "cell.lease/v1",+  "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+  "cell": "alpha",+  "owner": "nadeem@workstation",+  "purpose": "fixture-abba",+  "ttlSeconds": 120,+  "acquiredAt": "2026-09-25T12:00:00Z",+  "heartbeatAt": "2026-09-25T12:00:40Z",+  "cancelRequested": false,+  "runsStarted": 2+}
+ test/golden/cell/cell.lease.v1.schema.json view
@@ -0,0 +1,20 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.lease.v1.schema.json",+  "title": "Exclusive verification cell lease",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "leaseId", "cell", "owner", "purpose", "ttlSeconds", "acquiredAt", "heartbeatAt", "cancelRequested", "runsStarted"],+  "properties": {+    "schema": {"const": "cell.lease/v1"},+    "leaseId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "cell": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "owner": {"type": "string", "minLength": 1},+    "purpose": {"type": "string", "minLength": 1},+    "ttlSeconds": {"type": "integer", "minimum": 1},+    "acquiredAt": {"type": "string", "format": "date-time"},+    "heartbeatAt": {"type": "string", "format": "date-time"},+    "cancelRequested": {"type": "boolean"},+    "runsStarted": {"type": "integer", "minimum": 0}+  }+}
+ test/golden/cell/cell.payload.v1.json view
@@ -0,0 +1,14 @@+{+  "schema": "cell.payload/v1",+  "kind": "nix-nar-bundle",+  "bundle": {+    "uri": "gs://tan-nb-exp-cells-control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst",+    "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+    "bytes": 48211934+  },+  "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello",+  "narHash": "sha256-example",+  "closurePaths": ["/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"],+  "system": "x86_64-linux",+  "command": ["bin/cell-fixture-hello"]+}
+ test/golden/cell/cell.payload.v1.schema.json view
@@ -0,0 +1,34 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.payload.v1.schema.json",+  "title": "Content-addressed verification cell payload",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "kind", "bundle", "storePath", "narHash", "closurePaths", "system", "command"],+  "properties": {+    "schema": {"const": "cell.payload/v1"},+    "kind": {"const": "nix-nar-bundle"},+    "bundle": {+      "type": "object",+      "additionalProperties": false,+      "required": ["uri", "sha256", "bytes"],+      "properties": {+        "uri": {"type": "string", "pattern": "^gs://[^/]+/payloads/sha256/[a-f0-9]{64}\\.nar\\.zst$"},+        "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},+        "bytes": {"type": "integer", "minimum": 1}+      }+    },+    "storePath": {"$ref": "#/$defs/storePath"},+    "narHash": {"type": "string", "minLength": 1},+    "closurePaths": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/storePath"}},+    "system": {"const": "x86_64-linux"},+    "command": {+      "type": "array", "minItems": 1,+      "items": {"type": "string", "minLength": 1},+      "prefixItems": [{"type": "string", "pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[^\\u0000]+$"}]+    }+  },+  "$defs": {+    "storePath": {"type": "string", "pattern": "^/nix/store/[a-z0-9]{32}-[^/]+$"}+  }+}
+ test/golden/cell/cell.policy.v1.json view
@@ -0,0 +1,12 @@+{+  "schema": "cell.policy/v1",+  "idleStopMinutes": 30,+  "keepWarmUntil": null,+  "allowFaultInjection": false,+  "health": {+    "settleCpuMaxPercent": 5,+    "runStealCpuMaxPercent": 2,+    "startFreeDiskMinPercent": 20,+    "runFreeDiskMinPercent": 5+  }+}
+ test/golden/cell/cell.policy.v1.schema.json view
@@ -0,0 +1,24 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.policy.v1.schema.json",+  "title": "Verification cell idle and fault policy",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "idleStopMinutes", "keepWarmUntil", "allowFaultInjection", "health"],+  "properties": {+    "schema": {"const": "cell.policy/v1"},+    "idleStopMinutes": {"type": "integer", "minimum": 1, "maximum": 1440},+    "keepWarmUntil": {"type": ["string", "null"], "format": "date-time"},+    "allowFaultInjection": {"type": "boolean"},+    "health": {+      "type": "object", "additionalProperties": false,+      "required": ["settleCpuMaxPercent", "runStealCpuMaxPercent", "startFreeDiskMinPercent", "runFreeDiskMinPercent"],+      "properties": {+        "settleCpuMaxPercent": {"type": "number", "minimum": 0, "maximum": 100},+        "runStealCpuMaxPercent": {"type": "number", "minimum": 0, "maximum": 100},+        "startFreeDiskMinPercent": {"type": "number", "minimum": 0, "maximum": 100},+        "runFreeDiskMinPercent": {"type": "number", "minimum": 0, "maximum": 100}+      }+    }+  }+}
+ test/golden/cell/cell.quarantine.v1.json view
@@ -0,0 +1,7 @@+{+  "schema": "cell.quarantine/v1",+  "cell": "alpha",+  "reason": "reset-failed",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "at": "2026-09-25T12:05:00Z"+}
+ test/golden/cell/cell.quarantine.v1.schema.json view
@@ -0,0 +1,15 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.quarantine.v1.schema.json",+  "title": "Verification cell quarantine record",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "cell", "reason", "runId", "at"],+  "properties": {+    "schema": {"const": "cell.quarantine/v1"},+    "cell": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "reason": {"type": "string", "minLength": 1},+    "runId": {"type": ["string", "null"], "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "at": {"type": "string", "format": "date-time"}+  }+}
+ test/golden/cell/cell.recovery.v1.json view
@@ -0,0 +1,6 @@+{+  "schema": "cell.recovery/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "reason": "agent-restarted",+  "at": "2026-09-25T17:30:00Z"+}
+ test/golden/cell/cell.recovery.v1.schema.json view
@@ -0,0 +1,14 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.recovery.v1.schema.json",+  "title": "Verification cell interrupted-run recovery",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "runId", "reason", "at"],+  "properties": {+    "schema": {"const": "cell.recovery/v1"},+    "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "reason": {"const": "agent-restarted"},+    "at": {"type": "string", "format": "date-time"}+  }+}
+ test/golden/cell/cell.rejected.v1.json view
@@ -0,0 +1,6 @@+{+  "schema": "cell.rejected/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "reason": "unsafe-command",+  "at": "2026-09-25T12:21:07Z"+}
+ test/golden/cell/cell.rejected.v1.schema.json view
@@ -0,0 +1,14 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.rejected.v1.schema.json",+  "title": "Rejected verification cell submission",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "runId", "reason", "at"],+  "properties": {+    "schema": {"const": "cell.rejected/v1"},+    "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "reason": {"type": "string", "minLength": 1},+    "at": {"type": "string", "format": "date-time"}+  }+}
+ test/golden/cell/cell.reset-evidence.v1.json view
@@ -0,0 +1,20 @@+{+  "schema": "cell.reset-evidence/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "cell": "alpha",+  "cachePolicy": "cold",+  "startedAt": "2026-09-25T12:00:00Z",+  "finishedAt": "2026-09-25T12:00:05Z",+  "verified": true,+  "steps": [+    {"name": "stop-driver-processes", "startedAt": "2026-09-25T12:00:00Z", "finishedAt": "2026-09-25T12:00:01Z", "status": "ok"},+    {"name": "reset-postgres", "startedAt": "2026-09-25T12:00:01Z", "finishedAt": "2026-09-25T12:00:05Z", "status": "ok"}+  ],+  "postgres": {+    "version": "PostgreSQL 18.1",+    "databases": ["benchmark", "postgres"],+    "settings": [{"name": "checkpoint_timeout", "setting": "1800", "source": "configuration file"}],+    "walLsn": "0/1734400",+    "checkpoint": {"numRequested": 1, "numDone": 1}+  }+}
+ test/golden/cell/cell.reset-evidence.v1.schema.json view
@@ -0,0 +1,47 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.reset-evidence.v1.schema.json",+  "title": "Verification cell reset evidence",+  "type": "object", "additionalProperties": false,+  "required": ["schema", "runId", "cell", "cachePolicy", "startedAt", "finishedAt", "verified", "steps", "postgres"],+  "properties": {+    "schema": {"const": "cell.reset-evidence/v1"},+    "runId": {"$ref": "#/$defs/identifier"},+    "cell": {"$ref": "#/$defs/identifier"},+    "cachePolicy": {"enum": ["warm", "cold"]},+    "startedAt": {"type": "string", "format": "date-time"},+    "finishedAt": {"type": "string", "format": "date-time"},+    "verified": {"type": "boolean"},+    "steps": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/step"}},+    "postgres": {"type": ["object", "null"], "additionalProperties": false,+      "required": ["version", "databases", "settings", "walLsn", "checkpoint"],+      "properties": {+        "version": {"type": "string", "minLength": 1},+        "databases": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/identifier"}},+        "settings": {"type": "array", "items": {"type": "object", "additionalProperties": false,+          "required": ["name", "setting", "source"],+          "properties": {"name": {"type": "string"}, "setting": {"type": "string"}, "source": {"type": "string"}}}},+        "walLsn": {"type": "string"},+        "checkpoint": {"type": "object"}+      }+    }+  },+  "allOf": [{+    "if": {"properties": {"verified": {"const": true}}, "required": ["verified"]},+    "then": {"properties": {"steps": {"items": {"properties": {"status": {"const": "ok"}}}}}},+    "else": {"properties": {"steps": {"contains": {"properties": {"status": {"const": "failed"}}, "required": ["status"]}}}}+  }],+  "$defs": {+    "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "step": {"type": "object", "additionalProperties": false,+      "required": ["name", "startedAt", "finishedAt", "status"],+      "properties": {+        "name": {"type": "string", "minLength": 1},+        "startedAt": {"type": "string", "format": "date-time"},+        "finishedAt": {"type": "string", "format": "date-time"},+        "status": {"enum": ["ok", "failed"]},+        "detail": {"type": "string"}+      }+    }+  }+}
+ test/golden/cell/cell.reset-request.v1.json view
@@ -0,0 +1,6 @@+{+  "schema": "cell.reset-request/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "leaseId": "lease-001",+  "reset": {"cachePolicy": "warm"}+}
+ test/golden/cell/cell.reset-request.v1.schema.json view
@@ -0,0 +1,44 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.reset-request.v1.schema.json",+  "title": "Lease-authorized role reset request",+  "type": "object", "additionalProperties": false,+  "required": ["schema", "runId", "leaseId", "reset"],+  "properties": {+    "schema": {"const": "cell.reset-request/v1"},+    "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "leaseId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "reset": {+      "type": "object", "additionalProperties": false,+      "required": ["cachePolicy"],+      "properties": {+        "cachePolicy": {"enum": ["cold", "warm"]},+        "postgres": {+          "type": "object", "additionalProperties": false,+          "required": ["major", "databases", "settings"],+          "properties": {+            "major": {"enum": [17, 18]},+            "databases": {+              "type": "array", "uniqueItems": true,+              "items": {+                "type": "object", "additionalProperties": false,+                "required": ["name", "owner", "template"],+                "properties": {+                  "name": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+                  "owner": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+                  "template": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"}+                }+              }+            },+            "settings": {"type": "object", "additionalProperties": {"type": "string"}, "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_.-]*$"}}+          }+        },+        "broker": {+          "type": "object", "additionalProperties": false,+          "required": ["wipe"],+          "properties": {"wipe": {"type": "boolean"}}+        }+      }+    }+  }+}
+ test/golden/cell/cell.run-result.v1.json view
@@ -0,0 +1,11 @@+{+  "schema": "cell.run-result/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "cell": "alpha",+  "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+  "leaseSequence": 2,+  "outcome": "completed",+  "entryExitCode": 0,+  "entrySignal": null,+  "reasons": []+}
+ test/golden/cell/cell.run-result.v1.schema.json view
@@ -0,0 +1,22 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.run-result.v1.schema.json",+  "title": "Verification cell execution result",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "runId", "cell", "leaseId", "leaseSequence", "outcome", "entryExitCode", "entrySignal", "reasons"],+  "properties": {+    "schema": {"const": "cell.run-result/v1"},+    "runId": {"$ref": "#/$defs/identifier"},+    "cell": {"$ref": "#/$defs/identifier"},+    "leaseId": {"$ref": "#/$defs/identifier"},+    "leaseSequence": {"type": "integer", "minimum": 0},+    "outcome": {"enum": ["completed", "infrastructure-failure", "cancelled", "timed-out"]},+    "entryExitCode": {"type": ["integer", "null"]},+    "entrySignal": {"type": ["integer", "null"]},+    "reasons": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}+  },+  "$defs": {+    "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"}+  }+}
+ test/golden/cell/cell.status.v1.json view
@@ -0,0 +1,11 @@+{+  "schema": "cell.status/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "phase": "sealed",+  "updatedAt": "2026-09-25T12:00:00Z",+  "phaseStartedAt": "2026-09-25T11:59:59Z",+  "logChunks": {"stdout": 3, "stderr": 1},+  "outcome": "completed",+  "manifestSha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",+  "reasons": []+}
+ test/golden/cell/cell.status.v1.schema.json view
@@ -0,0 +1,34 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.status.v1.schema.json",+  "title": "Verification cell submission status",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "runId", "phase", "updatedAt", "logChunks"],+  "properties": {+    "schema": {"const": "cell.status/v1"},+    "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "phase": {"enum": ["accepted", "resetting", "fetching", "running", "publishing", "sealed"]},+    "leaseSequence": {"type": "integer", "minimum": 0},+    "updatedAt": {"type": "string", "format": "date-time"},+    "phaseStartedAt": {"type": "string", "format": "date-time"},+    "logChunks": {+      "type": "object", "additionalProperties": false,+      "required": ["stdout", "stderr"],+      "properties": {+        "stdout": {"type": "integer", "minimum": 0},+        "stderr": {"type": "integer", "minimum": 0}+      }+    },+    "outcome": {"enum": ["completed", "infrastructure-failure", "cancelled", "timed-out"]},+    "manifestSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},+    "reasons": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}+  },+  "allOf": [+    {+      "if": {"properties": {"phase": {"const": "sealed"}}, "required": ["phase"]},+      "then": {"required": ["outcome", "manifestSha256"]},+      "else": {"not": {"anyOf": [{"required": ["outcome"]}, {"required": ["manifestSha256"]}]}}+    }+  ]+}
+ test/golden/cell/cell.submission.v1.json view
@@ -0,0 +1,41 @@+{+  "schema": "cell.submission/v1",+  "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+  "payload": {+    "schema": "cell.payload/v1",+    "kind": "nix-nar-bundle",+    "bundle": {+      "uri": "gs://tan-nb-exp-cells-control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst",+      "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+      "bytes": 48211934+    },+    "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello",+    "narHash": "sha256-example",+    "closurePaths": ["/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"],+    "system": "x86_64-linux",+    "command": ["bin/cell-fixture-hello"]+  },+  "work": {+    "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+    "bytes": 212,+    "mediaType": "application/json"+  },+  "env": {"EXAMPLE": "value"},+  "reset": {+    "postgres": {+      "major": 18,+      "databases": [{"name": "benchmark", "owner": "benchmark", "template": "template0"}],+      "settings": {"checkpoint_timeout": "30min", "max_wal_size": "16GB"}+    },+    "cachePolicy": "cold",+    "broker": {"wipe": true}+  },+  "limits": {+    "wallClockSeconds": 3600,+    "memoryMaxBytes": 25769803776,+    "outputMaxBytes": 21474836480+  },+  "requires": {"protocol": "cell.protocol/v1", "minAgentVersion": "0.1.0"},+  "labels": {"arm": "candidate"}+}
+ test/golden/cell/cell.submission.v1.schema.json view
@@ -0,0 +1,86 @@+{+  "$schema": "https://json-schema.org/draft/2020-12/schema",+  "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.submission.v1.schema.json",+  "title": "Verification cell run submission",+  "type": "object",+  "additionalProperties": false,+  "required": ["schema", "runId", "leaseId", "payload", "work", "env", "reset", "limits", "requires", "labels"],+  "properties": {+    "schema": {"const": "cell.submission/v1"},+    "runId": {"$ref": "#/$defs/identifier"},+    "leaseId": {"$ref": "#/$defs/identifier"},+    "payload": {"$ref": "cell.payload.v1.schema.json"},+    "work": {+      "type": "object", "additionalProperties": false,+      "required": ["sha256", "bytes", "mediaType"],+      "properties": {+        "sha256": {"$ref": "#/$defs/sha256"},+        "bytes": {"type": "integer", "minimum": 0},+        "mediaType": {"type": "string", "minLength": 1}+      }+    },+    "env": {"$ref": "#/$defs/stringMap"},+    "reset": {+      "type": "object", "additionalProperties": false,+      "required": ["cachePolicy"],+      "properties": {+        "cachePolicy": {"enum": ["cold", "warm"]},+        "resources": {"type": "object", "description": "kenshou extension pending upstream adoption: per-resource reset requests, keyed by resource name"},+        "postgres": {+          "type": "object", "additionalProperties": false,+          "required": ["major", "databases", "settings"],+          "properties": {+            "major": {"enum": [17, 18]},+            "databases": {+              "type": "array", "uniqueItems": true,+              "items": {+                "type": "object", "additionalProperties": false,+                "required": ["name", "owner", "template"],+                "properties": {+                  "name": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+                  "owner": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+                  "template": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"}+                }+              }+            },+            "settings": {"$ref": "#/$defs/stringMap"}+          }+        },+        "broker": {+          "type": "object", "additionalProperties": false,+          "required": ["wipe"],+          "properties": {"wipe": {"type": "boolean"}}+        }+      }+    },+    "limits": {+      "type": "object", "additionalProperties": false,+      "required": ["wallClockSeconds", "memoryMaxBytes", "outputMaxBytes"],+      "properties": {+        "wallClockSeconds": {"type": "integer", "minimum": 1},+        "memoryMaxBytes": {"type": "integer", "minimum": 1},+        "outputMaxBytes": {"type": "integer", "minimum": 1}+      }+    },+    "requires": {+      "type": "object", "additionalProperties": false,+      "required": ["protocol", "minAgentVersion"],+      "properties": {+        "protocol": {"const": "cell.protocol/v1"},+        "minAgentVersion": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"},+        "capabilities": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}+      }+    },+    "collect": {+      "type": "object", "additionalProperties": false,+      "required": ["traces"],+      "properties": {"traces": {"type": "boolean"}, "profiles": {"type": "boolean"}}+    },+    "labels": {"$ref": "#/$defs/stringMap"}+  },+  "$defs": {+    "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+    "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},+    "stringMap": {"type": "object", "additionalProperties": {"type": "string"}, "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_.-]*$"}}+  }+}
+ test/golden/parity-report.json view
@@ -0,0 +1,10 @@+{+  "schema": "kenshou.parity-report/v1",+  "localRunResultSha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000",+  "cellRunResultSha256": "sha256:1111111111111111111111111111111111111111111111111111111111111111",+  "equal": ["result.outcome", "spec.seed"],+  "intentional": [+    {"path": "result.fingerprint.placement", "local": "local", "cell": "cell"}+  ],+  "unexpected": []+}
+ test/golden/payload-identity.json view
@@ -0,0 +1,12 @@+{+  "schema": "kenshou.payload-identity/v1",+  "cohort": "released",+  "variant": "default",+  "system": "x86_64-linux",+  "compiler": "ghc-9.12.4",+  "nixpkgsRevision": "d5dfd8e6716dde34398bc14bc87c10dece9c8c68",+  "haskellNixRevision": "7b696dc80f8aaccaf1783fda0ab6a7f978a67134",+  "packages": [+    { "name": "kenshou-cli", "version": "0.1.0.0" }+  ]+}
+ test/golden/payload.json view
@@ -0,0 +1,79 @@+{+  "cell": {+    "bundle": {+      "bytes": 1,+      "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+      "uri": "gs://control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst"+    },+    "closurePaths": [+      "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released"+    ],+    "command": [+      "bin/kenshou",+      "cell",+      "exec"+    ],+    "kind": "nix-nar-bundle",+    "narHash": "sha256-example",+    "schema": "cell.payload/v1",+    "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released",+    "system": "x86_64-linux"+  },+  "cohort": "released",+  "cohortCheck": {+    "packagesChecked": 2,+    "status": "consistent"+  },+  "cohortIdentity": {+    "arch": "x86_64",+    "cabalVersion": "nix",+    "cohort": "released",+    "compiler": "ghc-9.12.4",+    "components": [+      {+        "id": "foo",+        "moriUri": "mori://example/foo",+        "packages": [+          {+            "name": "foo",+            "source": {+              "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+              "type": "hackage"+            },+            "version": "1.0.0"+          }+        ]+      },+      {+        "id": "bar",+        "moriUri": "mori://example/bar",+        "packages": [+          {+            "name": "bar",+            "source": {+              "location": "https://example.com/bar.git",+              "rev": "0123456789abcdef0123456789abcdef01234567",+              "subdir": "bar",+              "type": "git"+            },+            "version": "2.0.0"+          }+        ]+      }+    ],+    "descriptorSha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",+    "indexState": "2026-09-20T13:44:47Z",+    "os": "linux",+    "planHash": "sha256:857e6ff76844ccdf1903dc62d6518fe59b64e34d4bc626873efe2534934f1cc7",+    "resolver": "nix",+    "schema": "kenshou.cohort-identity/v1"+  },+  "createdAt": "2026-09-27T00:00:00Z",+  "flakeAttr": "packages.x86_64-linux.kenshou-released",+  "harness": {+    "dirty": false,+    "revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+  },+  "schema": "kenshou.payload/v1",+  "variant": "default"+}