kenshou-remote (empty) → 0.1.0.0
raw patch · 105 files changed
+12010/−0 lines, 105 filesdep +aesondep +basedep +base16-bytestring
Dependencies added: aeson, base, base16-bytestring, bytestring, containers, cryptohash-sha256, directory, filepath, hspec, http-client, http-client-tls, http-types, kenshou-core, kenshou-measure, kenshou-remote, network, process, temporary, text, time, unix
Files
- CHANGELOG.md +12/−0
- LICENSE +28/−0
- kenshou-remote.cabal +155/−0
- src/Kenshou/Remote/Cell/Control.hs +55/−0
- src/Kenshou/Remote/Cell/Debug.hs +102/−0
- src/Kenshou/Remote/Cell/Docs.hs +725/−0
- src/Kenshou/Remote/Cell/Exec.hs +351/−0
- src/Kenshou/Remote/Cell/Fetch.hs +387/−0
- src/Kenshou/Remote/Cell/Health.hs +103/−0
- src/Kenshou/Remote/Cell/Index.hs +195/−0
- src/Kenshou/Remote/Cell/Lease.hs +332/−0
- src/Kenshou/Remote/Cell/Legacy.hs +41/−0
- src/Kenshou/Remote/Cell/Parity.hs +189/−0
- src/Kenshou/Remote/Cell/Prepare.hs +376/−0
- src/Kenshou/Remote/Cell/RouteJson.hs +123/−0
- src/Kenshou/Remote/Cell/RouteRules.hs +101/−0
- src/Kenshou/Remote/Cell/Session.hs +75/−0
- src/Kenshou/Remote/Cell/Session/Build.hs +118/−0
- src/Kenshou/Remote/Cell/Session/Journal.hs +232/−0
- src/Kenshou/Remote/Cell/Session/Rebind.hs +86/−0
- src/Kenshou/Remote/Cell/Session/Resume.hs +128/−0
- src/Kenshou/Remote/Cell/Session/Runner.hs +120/−0
- src/Kenshou/Remote/Cell/Submit.hs +96/−0
- src/Kenshou/Remote/Cell/Watch.hs +99/−0
- src/Kenshou/Remote/Cell/WorkJson.hs +108/−0
- src/Kenshou/Remote/Pair.hs +93/−0
- src/Kenshou/Remote/Payload.hs +163/−0
- src/Kenshou/Remote/Payload/Nix.hs +172/−0
- src/Kenshou/Remote/Payload/Publish.hs +67/−0
- src/Kenshou/Remote/Payload/Publisher.hs +250/−0
- src/Kenshou/Remote/Perf/Attempt.hs +283/−0
- src/Kenshou/Remote/Perf/Cell.hs +167/−0
- src/Kenshou/Remote/Perf/Cleanup.hs +52/−0
- src/Kenshou/Remote/Perf/Conditions.hs +180/−0
- src/Kenshou/Remote/Perf/Journal.hs +287/−0
- src/Kenshou/Remote/Perf/Lock.hs +33/−0
- src/Kenshou/Remote/Perf/Publication.hs +95/−0
- src/Kenshou/Remote/Perf/Runner.hs +226/−0
- src/Kenshou/Remote/Selftest.hs +136/−0
- src/Kenshou/Remote/Store.hs +43/−0
- src/Kenshou/Remote/Store/File.hs +187/−0
- src/Kenshou/Remote/Store/Gcs.hs +409/−0
- test/DebugSpec.hs +26/−0
- test/DocsSpec.hs +98/−0
- test/ExecSpec.hs +208/−0
- test/FetchSpec.hs +258/−0
- test/GcsSpec.hs +256/−0
- test/HealthSpec.hs +50/−0
- test/LeaseSpec.hs +198/−0
- test/Main.hs +200/−0
- test/PairSpec.hs +59/−0
- test/ParitySpec.hs +82/−0
- test/PayloadSpec.hs +212/−0
- test/PerfAttemptSpec.hs +203/−0
- test/PerfCellSpec.hs +57/−0
- test/PerfCleanupSpec.hs +82/−0
- test/PerfConditionsSpec.hs +137/−0
- test/PerfJournalSpec.hs +153/−0
- test/PerfLockSpec.hs +31/−0
- test/PerfPublicationSpec.hs +60/−0
- test/PerfRunnerSpec.hs +310/−0
- test/PrepareSpec.hs +441/−0
- test/SessionSpec.hs +365/−0
- test/SubmitSpec.hs +117/−0
- test/WatchSpec.hs +83/−0
- test/golden/cell-capabilities.json +13/−0
- test/golden/cell-route.json +8/−0
- test/golden/cell-run.json +20/−0
- test/golden/cell-session.json +118/−0
- test/golden/cell/SOURCE +19/−0
- test/golden/cell/cell.artifact-manifest.v1.json +21/−0
- test/golden/cell/cell.artifact-manifest.v1.schema.json +53/−0
- test/golden/cell/cell.descriptor.v1.json +31/−0
- test/golden/cell/cell.descriptor.v1.schema.json +72/−0
- test/golden/cell/cell.environment.v1.json +18/−0
- test/golden/cell/cell.environment.v1.schema.json +76/−0
- test/golden/cell/cell.execution-window.v1.json +7/−0
- test/golden/cell/cell.execution-window.v1.schema.json +33/−0
- test/golden/cell/cell.health.v1.json +18/−0
- test/golden/cell/cell.health.v1.schema.json +36/−0
- test/golden/cell/cell.lease.v1.json +12/−0
- test/golden/cell/cell.lease.v1.schema.json +20/−0
- test/golden/cell/cell.payload.v1.json +14/−0
- test/golden/cell/cell.payload.v1.schema.json +34/−0
- test/golden/cell/cell.policy.v1.json +12/−0
- test/golden/cell/cell.policy.v1.schema.json +24/−0
- test/golden/cell/cell.quarantine.v1.json +7/−0
- test/golden/cell/cell.quarantine.v1.schema.json +15/−0
- test/golden/cell/cell.recovery.v1.json +6/−0
- test/golden/cell/cell.recovery.v1.schema.json +14/−0
- test/golden/cell/cell.rejected.v1.json +6/−0
- test/golden/cell/cell.rejected.v1.schema.json +14/−0
- test/golden/cell/cell.reset-evidence.v1.json +20/−0
- test/golden/cell/cell.reset-evidence.v1.schema.json +47/−0
- test/golden/cell/cell.reset-request.v1.json +6/−0
- test/golden/cell/cell.reset-request.v1.schema.json +44/−0
- test/golden/cell/cell.run-result.v1.json +11/−0
- test/golden/cell/cell.run-result.v1.schema.json +22/−0
- test/golden/cell/cell.status.v1.json +11/−0
- test/golden/cell/cell.status.v1.schema.json +34/−0
- test/golden/cell/cell.submission.v1.json +41/−0
- test/golden/cell/cell.submission.v1.schema.json +86/−0
- test/golden/parity-report.json +10/−0
- test/golden/payload-identity.json +12/−0
- test/golden/payload.json +79/−0
+ CHANGELOG.md view
@@ -0,0 +1,12 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of the leased verification cell client, payload publishing, and file and GCS object stores.+- Cells bind named resources through a `resources` block; `Kenshou.Remote.Cell.Legacy` adapts cells that still describe a broker.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+ may be used to endorse or promote products derived from this software without+ specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ kenshou-remote.cabal view
@@ -0,0 +1,155 @@+cabal-version: 3.0+name: kenshou-remote+version: 0.1.0.0+synopsis: Leased verification cell client and storage protocol+description: Leases verification cells, publishes immutable run payloads, submits and recovers cell runs, and stores evidence objects in files or Google Cloud Storage.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+ test/golden/*.json+ test/golden/cell/*.json+ test/golden/cell/SOURCE++common warnings+ ghc-options:+ -Wall+ -Wcompat+ -Widentities+ -Wincomplete-record-updates+ -Wincomplete-uni-patterns+ -Wpartial-fields+ -Wredundant-constraints++ default-language: GHC2024+ default-extensions:+ BlockArguments+ DeriveAnyClass+ DeriveGeneric+ DuplicateRecordFields+ ImportQualifiedPost+ LambdaCase+ NoFieldSelectors+ OverloadedRecordDot+ OverloadedStrings++library+ import: warnings+ hs-source-dirs: src+ exposed-modules:+ Kenshou.Remote.Cell.Control+ Kenshou.Remote.Cell.Debug+ Kenshou.Remote.Cell.Docs+ Kenshou.Remote.Cell.Exec+ Kenshou.Remote.Cell.Fetch+ Kenshou.Remote.Cell.Health+ Kenshou.Remote.Cell.Index+ Kenshou.Remote.Cell.Lease+ Kenshou.Remote.Cell.Legacy+ Kenshou.Remote.Cell.Parity+ Kenshou.Remote.Cell.Prepare+ Kenshou.Remote.Cell.RouteJson+ Kenshou.Remote.Cell.RouteRules+ Kenshou.Remote.Cell.Session+ Kenshou.Remote.Cell.Session.Build+ Kenshou.Remote.Cell.Session.Journal+ Kenshou.Remote.Cell.Session.Rebind+ Kenshou.Remote.Cell.Session.Resume+ Kenshou.Remote.Cell.Session.Runner+ Kenshou.Remote.Cell.Submit+ Kenshou.Remote.Cell.Watch+ Kenshou.Remote.Cell.WorkJson+ Kenshou.Remote.Pair+ Kenshou.Remote.Payload+ Kenshou.Remote.Payload.Nix+ Kenshou.Remote.Payload.Publish+ Kenshou.Remote.Payload.Publisher+ Kenshou.Remote.Perf.Attempt+ Kenshou.Remote.Perf.Cell+ Kenshou.Remote.Perf.Cleanup+ Kenshou.Remote.Perf.Conditions+ Kenshou.Remote.Perf.Journal+ Kenshou.Remote.Perf.Lock+ Kenshou.Remote.Perf.Publication+ Kenshou.Remote.Perf.Runner+ Kenshou.Remote.Selftest+ Kenshou.Remote.Store+ Kenshou.Remote.Store.File+ Kenshou.Remote.Store.Gcs++ build-depends:+ aeson >=2.2 && <2.3,+ base >=4.21 && <4.22,+ base16-bytestring >=1.0 && <1.1,+ bytestring >=0.12 && <0.13,+ containers >=0.7 && <0.8,+ cryptohash-sha256 >=0.11 && <0.12,+ directory >=1.3 && <1.4,+ filepath >=1.5 && <1.6,+ http-client >=0.7 && <0.8,+ http-client-tls >=0.4 && <0.5,+ http-types >=0.12 && <0.13,+ kenshou-core ^>=0.1.0.0,+ kenshou-measure ^>=0.1.0.0,+ network >=3.2 && <3.3,+ process >=1.6 && <1.7,+ text >=2.1 && <2.2,+ time >=1.14 && <1.15,+ unix >=2.8 && <2.9,++test-suite kenshou-remote-test+ import: warnings+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Main.hs+ other-modules:+ DebugSpec+ DocsSpec+ ExecSpec+ FetchSpec+ GcsSpec+ HealthSpec+ LeaseSpec+ PairSpec+ ParitySpec+ PayloadSpec+ PerfAttemptSpec+ PerfCellSpec+ PerfCleanupSpec+ PerfConditionsSpec+ PerfJournalSpec+ PerfLockSpec+ PerfPublicationSpec+ PerfRunnerSpec+ PrepareSpec+ SessionSpec+ SubmitSpec+ WatchSpec++ build-depends:+ aeson,+ base >=4.21 && <4.22,+ bytestring,+ containers >=0.7 && <0.8,+ directory >=1.3 && <1.4,+ filepath,+ hspec >=2.11 && <2.12,+ kenshou-core ^>=0.1.0.0,+ kenshou-measure ^>=0.1.0.0,+ kenshou-remote,+ network >=3.2 && <3.3,+ process >=1.6 && <1.7,+ temporary,+ text,+ time >=1.14 && <1.15,++source-repository head+ type: git+ location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Remote/Cell/Control.hs view
@@ -0,0 +1,55 @@+module Kenshou.Remote.Cell.Control+ ( CellSnapshot (..),+ readCellSnapshot,+ )+where++import Data.Aeson (FromJSON, eitherDecode)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Remote.Cell.Docs (CellBuckets (..), CellDescriptor (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), Quarantine (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..))++data CellSnapshot = CellSnapshot+ { descriptor :: !CellDescriptor,+ lease :: !(Maybe Lease),+ quarantine :: !(Maybe Quarantine)+ }+ deriving stock (Eq, Show)++-- Read the same control objects as the cell owner. A descriptor is mandatory+-- before any client action so a bucket or project mismatch cannot be ignored.+readCellSnapshot :: ObjectStore -> CellRef -> [Text] -> IO (Either Text CellSnapshot)+readCellSnapshot store ref allowedProjects = do+ descriptor <- readDocument store ref "descriptor.json"+ case descriptor of+ Left problem -> pure (Left problem)+ Right Nothing -> pure (Left "cell descriptor is missing")+ Right (Just current)+ | current.name /= ref.cellName -> pure (Left "cell descriptor names another cell")+ | current.buckets.control /= ref.controlBucket.unBucket -> pure (Left "cell descriptor control bucket differs from selected bucket")+ | current.project `notElem` allowedProjects -> pure (Left ("cell project is outside KENSHOU_GCP_ALLOWED_PROJECTS: " <> current.project))+ | otherwise -> do+ lease <- readDocument store ref "lease.json"+ quarantine <- readDocument store ref "quarantine.json"+ pure do+ observedLease <- lease+ observedQuarantine <- quarantine+ case observedLease of+ Just record | record.cell /= ref.cellName -> Left "cell lease names another cell"+ _ -> pure ()+ case observedQuarantine of+ Just record | record.cell /= ref.cellName -> Left "cell quarantine names another cell"+ _ -> pure ()+ pure (CellSnapshot current observedLease observedQuarantine)++readDocument :: (FromJSON document) => ObjectStore -> CellRef -> Text -> IO (Either Text (Maybe document))+readDocument store ref name = do+ let object = ObjectName ("cells/" <> ref.cellName <> "/" <> name)+ stored <- store.getObject ref.controlBucket object+ pure case stored of+ Nothing -> Right Nothing+ Just (bytes, _) -> case eitherDecode bytes of+ Left failure -> Left ("invalid " <> name <> ": " <> Text.pack failure)+ Right document -> Right (Just document)
+ src/Kenshou/Remote/Cell/Debug.hs view
@@ -0,0 +1,102 @@+module Kenshou.Remote.Cell.Debug+ ( DebugCommand (..),+ resolveDebugRole,+ remoteCommand,+ runCellDebug,+ )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Remote.Cell.Docs (CellDescriptor (..), CellNodes (..))+import System.Directory (doesFileExist)+import System.Environment (getEnvironment, lookupEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO (hPutStrLn, stderr)+import System.Process (CreateProcess (..), StdStream (..), createProcess, proc, readProcessWithExitCode, waitForProcess)+import Text.Read (readMaybe)++data DebugCommand+ = DebugSsh !Text ![String]+ | DebugJournal+ | DebugTunnel !Text !Int !Int+ deriving stock (Eq, Show)++resolveDebugRole :: CellDescriptor -> Text -> Either Text Text+resolveDebugRole descriptor role = case role of+ "postgres" -> Right descriptor.instances.postgres+ "monitoring" -> Right descriptor.instances.monitoring+ "driver" -> driver 0+ _ -> case Text.stripPrefix "driver-" role >>= readMaybe . Text.unpack of+ Just index -> driver index+ Nothing -> Left "role must be postgres, monitoring, driver, or driver-N"+ where+ driver index = case drop index descriptor.instances.drivers of+ name : _ | index >= 0 -> Right name+ _ -> Left "cell descriptor has no driver at that index"++remoteCommand :: [String] -> String+remoteCommand = unwords . map quote+ where+ quote value = "'" <> concatMap escape value <> "'"+ escape '\'' = "'\\''"+ escape character = [character]++runCellDebug :: CellDescriptor -> DebugCommand -> IO (Either Text ExitCode)+runCellDebug descriptor selected = do+ scriptResult <- locateScript+ case scriptResult of+ Left problem -> pure (Left problem)+ Right script -> case arguments of+ Left problem -> pure (Left problem)+ Right args -> do+ environment <- getEnvironment+ let overrides =+ [ ("CLOUDSDK_CORE_PROJECT", Text.unpack descriptor.project),+ ("ZONE", Text.unpack descriptor.zone)+ ]+ inherited = filter (\(key, _) -> key `notElem` map fst overrides) environment+ hPutStrLn stderr ("cell debug: " <> unwords (script : args))+ (_, _, _, handle) <-+ createProcess+ (proc script args)+ { env = Just (overrides <> inherited),+ std_in = Inherit,+ std_out = Inherit,+ std_err = Inherit+ }+ Right <$> waitForProcess handle+ where+ arguments = case selected of+ DebugSsh role command -> do+ instanceName <- resolveDebugRole descriptor role+ pure (["ssh", Text.unpack instanceName, "--"] <> [remoteCommand command | not (null command)])+ DebugJournal -> do+ instanceName <- resolveDebugRole descriptor "driver"+ pure ["ssh", Text.unpack instanceName, "--", remoteCommand ["sudo", "journalctl", "-u", "cell-agent", "-n", "100", "--no-pager"]]+ DebugTunnel role remotePort localPort -> do+ instanceName <- resolveDebugRole descriptor role+ if validPort remotePort && validPort localPort+ then Right ["ssh-tunnel", Text.unpack instanceName, show remotePort, show localPort]+ else Left "tunnel ports must be between 1 and 65535"+ validPort port = port >= 1 && port <= 65535++locateScript :: IO (Either Text FilePath)+locateScript = do+ configured <- lookupEnv "KENSHOU_LTI_DIR"+ root <- case configured of+ Just value | not (null value) -> pure (Right value)+ _ -> do+ (code, output, failure) <- readProcessWithExitCode "mori" ["path", "shinzui/load-testing-infra"] ""+ pure case code of+ ExitSuccess -> case lines output of+ [path] -> Right path+ _ -> Left "mori did not return one load-testing-infra path"+ _ -> Left ("mori could not locate load-testing-infra: " <> Text.pack failure)+ case root of+ Left problem -> pure (Left problem)+ Right directory -> do+ let script = directory </> "scripts" </> "iap-ssh.sh"+ exists <- doesFileExist script+ pure if exists then Right script else Left ("cell debug needs the owner IAP script at " <> Text.pack script)
+ src/Kenshou/Remote/Cell/Docs.hs view
@@ -0,0 +1,725 @@+module Kenshou.Remote.Cell.Docs+ ( CellPhase (..),+ CellOutcome (..),+ CellNodes (..),+ CellImages (..),+ CellBuckets (..),+ CellDescriptor (..),+ CellPostgres (..),+ CellBroker (..),+ OtlpEndpoint (..),+ OtlpSinks (..),+ CellDriver (..),+ CellEnvironment (..),+ WorkObject (..),+ PgDatabase (..),+ PgReset (..),+ BrokerReset (..),+ CachePolicy (..),+ ResetBlock (..),+ Limits (..),+ Requirements (..),+ CollectOptions (..),+ Submission (..),+ LogChunks (..),+ CellStatus (..),+ Rejected (..),+ CellRunResult (..),+ ManifestPayload (..),+ Artifact (..),+ CellManifest (..),+ )+where++import Control.Monad (unless)+import Data.Aeson (FromJSON (..), Object, ToJSON (..), Value (..), object, withObject, withText, (.!=), (.:), (.:?), (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser)+import Data.Int (Int64)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Core.Resource.Types (ResourceName)+import Kenshou.Remote.Payload (CellPayload)+import Text.Read (readMaybe)++data CellPhase = Accepted | Resetting | Fetching | Running | Publishing | Sealed+ deriving stock (Eq, Show)++data CellOutcome = Completed | InfrastructureFailure | Cancelled | TimedOut+ deriving stock (Eq, Show)++data CellNodes = CellNodes+ { postgres :: !Text,+ drivers :: ![Text],+ monitoring :: !Text,+ -- | The optional broker role, present on cells that run one.+ broker :: !(Maybe Text)+ }+ deriving stock (Eq, Show)++data CellImages = CellImages+ { driver :: !Text,+ postgres :: !Text,+ monitoring :: !Text,+ broker :: !Text+ }+ deriving stock (Eq, Show)++data CellBuckets = CellBuckets+ { control :: !Text,+ results :: !Text+ }+ deriving stock (Eq, Show)++data CellDescriptor = CellDescriptor+ { name :: !Text,+ project :: !Text,+ zone :: !Text,+ postgresMajor :: !Int,+ instances :: !CellNodes,+ addresses :: !CellNodes,+ shape :: !Value,+ images :: !CellImages,+ buckets :: !CellBuckets+ }+ deriving stock (Eq, Show)++data CellPostgres = CellPostgres+ { major :: !Int,+ host :: !Text,+ port :: !Int,+ database :: !Text,+ user :: !Text,+ connectionString :: !Text+ }+ deriving stock (Eq, Show)++data CellBroker = CellBroker+ { bootstrapServers :: !Text,+ adminUrl :: !Text,+ implementation :: !Text,+ version :: !Text+ }+ deriving stock (Eq, Show)++data OtlpEndpoint = OtlpEndpoint+ { grpc :: !Text,+ http :: !Text+ }+ deriving stock (Eq, Show)++data OtlpSinks = OtlpSinks+ { nullEndpoint :: !OtlpEndpoint,+ fileEndpoint :: !OtlpEndpoint+ }+ deriving stock (Eq, Show)++data CellDriver = CellDriver+ { index :: !Int,+ host :: !Text+ }+ deriving stock (Eq, Show)++data CellEnvironment = CellEnvironment+ { cell :: !Text,+ runId :: !RunId,+ leaseId :: !RunId,+ postgres :: !CellPostgres,+ broker :: !(Maybe CellBroker),+ otlp :: !(Maybe OtlpSinks),+ victoriaMetricsUrl :: !Text,+ drivers :: ![CellDriver],+ clockSkewBoundMicros :: !(Maybe Int),+ faultHook :: !(Maybe Text),+ -- | Bound values of named resources, keyed by resource name. A kenshou extension+ -- of @cell.environment/v1@, written only when non-empty.+ resources :: !(Map ResourceName Value)+ }+ deriving stock (Eq, Show)++data WorkObject = WorkObject+ { sha256 :: !Text,+ bytes :: !Int64,+ mediaType :: !Text+ }+ deriving stock (Eq, Show)++data PgDatabase = PgDatabase+ { name :: !Text,+ owner :: !Text,+ template :: !Text+ }+ deriving stock (Eq, Show)++data PgReset = PgReset+ { major :: !Int,+ databases :: ![PgDatabase],+ settings :: !(Map Text Text)+ }+ deriving stock (Eq, Show)++newtype BrokerReset = BrokerReset {wipe :: Bool}+ deriving stock (Eq, Show)++data CachePolicy = Cold | Warm+ deriving stock (Eq, Show)++data ResetBlock = ResetBlock+ { cachePolicy :: !CachePolicy,+ postgres :: !(Maybe PgReset),+ broker :: !(Maybe BrokerReset),+ -- | Per-resource reset requests, keyed by resource name. A kenshou extension of+ -- @cell.submission/v1@, written only when non-empty.+ resources :: !(Map ResourceName Value)+ }+ deriving stock (Eq, Show)++data Limits = Limits+ { wallClockSeconds :: !Int64,+ memoryMaxBytes :: !Int64,+ outputMaxBytes :: !Int64+ }+ deriving stock (Eq, Show)++data Requirements = Requirements+ { minAgentVersion :: !Text,+ capabilities :: !(Maybe [Text])+ }+ deriving stock (Eq, Show)++data CollectOptions = CollectOptions+ { traces :: !Bool,+ profiles :: !Bool+ }+ deriving stock (Eq, Show)++data Submission = Submission+ { runId :: !RunId,+ leaseId :: !RunId,+ payload :: !CellPayload,+ work :: !WorkObject,+ env :: !(Map Text Text),+ reset :: !ResetBlock,+ limits :: !Limits,+ requires :: !Requirements,+ collect :: !(Maybe CollectOptions),+ labels :: !(Map Text Text)+ }+ deriving stock (Eq, Show)++data LogChunks = LogChunks+ { stdout :: !Int,+ stderr :: !Int+ }+ deriving stock (Eq, Show)++data CellStatus = CellStatus+ { runId :: !RunId,+ phase :: !CellPhase,+ leaseSequence :: !(Maybe Int),+ updatedAt :: !UTCTime,+ phaseStartedAt :: !(Maybe UTCTime),+ logChunks :: !LogChunks,+ outcome :: !(Maybe CellOutcome),+ manifestSha256 :: !(Maybe Text),+ reasons :: !(Maybe [Text])+ }+ deriving stock (Eq, Show)++data Rejected = Rejected+ { runId :: !RunId,+ reason :: !Text,+ at :: !UTCTime+ }+ deriving stock (Eq, Show)++data CellRunResult = CellRunResult+ { runId :: !RunId,+ cell :: !Text,+ leaseId :: !RunId,+ leaseSequence :: !Int,+ outcome :: !CellOutcome,+ entryExitCode :: !(Maybe Int),+ entrySignal :: !(Maybe Int),+ reasons :: ![Text]+ }+ deriving stock (Eq, Show)++data ManifestPayload = ManifestPayload+ { sha256 :: !Text,+ storePath :: !Text+ }+ deriving stock (Eq, Show)++data Artifact = Artifact+ { path :: !Text,+ sha256 :: !Text,+ bytes :: !Int64,+ mediaType :: !Text+ }+ deriving stock (Eq, Show)++data CellManifest = CellManifest+ { runId :: !RunId,+ cell :: !Text,+ leaseId :: !RunId,+ leaseSequence :: !Int,+ sealedAt :: !UTCTime,+ agentVersion :: !Text,+ payload :: !ManifestPayload,+ outcome :: !CellOutcome,+ retentionSeconds :: !Int64,+ artifacts :: ![Artifact]+ }+ deriving stock (Eq, Show)++instance ToJSON CellPhase where+ toJSON = toJSON . renderPhase++instance FromJSON CellPhase where+ parseJSON = withText "cell phase" \case+ "accepted" -> pure Accepted+ "resetting" -> pure Resetting+ "fetching" -> pure Fetching+ "running" -> pure Running+ "publishing" -> pure Publishing+ "sealed" -> pure Sealed+ _ -> fail "unknown cell phase"++renderPhase :: CellPhase -> Text+renderPhase Accepted = "accepted"+renderPhase Resetting = "resetting"+renderPhase Fetching = "fetching"+renderPhase Running = "running"+renderPhase Publishing = "publishing"+renderPhase Sealed = "sealed"++instance ToJSON CellOutcome where+ toJSON = toJSON . renderOutcome++instance FromJSON CellOutcome where+ parseJSON = withText "cell outcome" \case+ "completed" -> pure Completed+ "infrastructure-failure" -> pure InfrastructureFailure+ "cancelled" -> pure Cancelled+ "timed-out" -> pure TimedOut+ _ -> fail "unknown cell outcome"++renderOutcome :: CellOutcome -> Text+renderOutcome Completed = "completed"+renderOutcome InfrastructureFailure = "infrastructure-failure"+renderOutcome Cancelled = "cancelled"+renderOutcome TimedOut = "timed-out"++instance ToJSON CellNodes where+ toJSON nodes = object (["postgres" .= nodes.postgres, "drivers" .= nodes.drivers, "monitoring" .= nodes.monitoring] <> maybe [] (\name -> ["broker" .= name]) nodes.broker)++instance FromJSON CellNodes where+ parseJSON = withObject "cell nodes" \value -> do+ nodes <- CellNodes <$> value .: "postgres" <*> value .: "drivers" <*> value .: "monitoring" <*> value .:? "broker"+ unless (not (Text.null nodes.postgres) && not (null nodes.drivers) && all (not . Text.null) nodes.drivers && not (Text.null nodes.monitoring) && maybe True (not . Text.null) nodes.broker) (fail "invalid cell nodes")+ pure nodes++instance ToJSON CellImages where+ toJSON images = object ["driver" .= images.driver, "postgres" .= images.postgres, "monitoring" .= images.monitoring, "broker" .= images.broker]++instance FromJSON CellImages where+ parseJSON = withObject "cell images" \value -> do+ images <- CellImages <$> value .: "driver" <*> value .: "postgres" <*> value .: "monitoring" <*> value .: "broker"+ unless (all (not . Text.null) [images.driver, images.postgres, images.monitoring]) (fail "invalid cell images")+ pure images++instance ToJSON CellBuckets where+ toJSON buckets = object ["control" .= buckets.control, "results" .= buckets.results]++instance FromJSON CellBuckets where+ parseJSON = withObject "cell buckets" \value -> do+ buckets <- CellBuckets <$> value .: "control" <*> value .: "results"+ unless (not (Text.null buckets.control) && not (Text.null buckets.results)) (fail "invalid cell buckets")+ pure buckets++instance ToJSON CellDescriptor where+ toJSON descriptor =+ object+ [ "schema" .= ("cell.descriptor/v1" :: Text),+ "name" .= descriptor.name,+ "project" .= descriptor.project,+ "zone" .= descriptor.zone,+ "postgresMajor" .= descriptor.postgresMajor,+ "instances" .= descriptor.instances,+ "addresses" .= descriptor.addresses,+ "shape" .= descriptor.shape,+ "images" .= descriptor.images,+ "buckets" .= descriptor.buckets,+ "agentProtocol" .= ("cell.protocol/v1" :: Text)+ ]++instance FromJSON CellDescriptor where+ parseJSON = withObject "cell descriptor" \value -> do+ expectSchema "cell.descriptor/v1" value+ protocol <- value .: "agentProtocol"+ unless (protocol == ("cell.protocol/v1" :: Text)) (fail "unsupported cell agent protocol")+ descriptor <- CellDescriptor <$> value .: "name" <*> value .: "project" <*> value .: "zone" <*> value .: "postgresMajor" <*> value .: "instances" <*> value .: "addresses" <*> value .: "shape" <*> value .: "images" <*> value .: "buckets"+ unless (validName descriptor.name && Text.length descriptor.name <= 41 && not (Text.null descriptor.project) && not (Text.null descriptor.zone) && descriptor.postgresMajor `elem` [17, 18] && all validIpv4 (descriptor.addresses.postgres : descriptor.addresses.monitoring : descriptor.addresses.drivers) && validShape descriptor.shape) (fail "invalid cell descriptor")+ pure descriptor++instance ToJSON CellPostgres where+ toJSON postgres = object ["major" .= postgres.major, "host" .= postgres.host, "port" .= postgres.port, "database" .= postgres.database, "user" .= postgres.user, "connectionString" .= postgres.connectionString]++instance FromJSON CellPostgres where+ parseJSON = withObject "cell PostgreSQL" \value -> do+ postgres <- CellPostgres <$> value .: "major" <*> value .: "host" <*> value .: "port" <*> value .: "database" <*> value .: "user" <*> value .: "connectionString"+ unless (postgres.major `elem` [17, 18] && validIpv4 postgres.host && postgres.port >= 1 && postgres.port <= 65535 && all (not . Text.null) [postgres.database, postgres.user, postgres.connectionString]) (fail "invalid cell PostgreSQL endpoint")+ pure postgres++instance ToJSON CellBroker where+ toJSON broker = object ["bootstrapServers" .= broker.bootstrapServers, "adminUrl" .= broker.adminUrl, "implementation" .= broker.implementation, "version" .= broker.version]++instance FromJSON CellBroker where+ parseJSON = withObject "cell broker" \value -> do+ broker <- CellBroker <$> value .: "bootstrapServers" <*> value .: "adminUrl" <*> value .: "implementation" <*> value .: "version"+ unless (all (not . Text.null) [broker.bootstrapServers, broker.implementation, broker.version] && validUri broker.adminUrl) (fail "invalid cell broker")+ pure broker++instance ToJSON OtlpEndpoint where+ toJSON endpoint = object ["grpc" .= endpoint.grpc, "http" .= endpoint.http]++instance FromJSON OtlpEndpoint where+ parseJSON = withObject "cell OTLP endpoint" \value -> do+ endpoint <- OtlpEndpoint <$> value .: "grpc" <*> value .: "http"+ unless (validUri endpoint.grpc && validUri endpoint.http) (fail "invalid cell OTLP endpoint")+ pure endpoint++instance ToJSON OtlpSinks where+ toJSON sinks = object ["null" .= sinks.nullEndpoint, "file" .= sinks.fileEndpoint]++instance FromJSON OtlpSinks where+ parseJSON = withObject "cell OTLP sinks" \value -> OtlpSinks <$> value .: "null" <*> value .: "file"++instance ToJSON CellDriver where+ toJSON driver = object ["index" .= driver.index, "host" .= driver.host]++instance FromJSON CellDriver where+ parseJSON = withObject "cell driver" \value -> do+ driver <- CellDriver <$> value .: "index" <*> value .: "host"+ unless (driver.index >= 0 && validIpv4 driver.host) (fail "invalid cell driver")+ pure driver++instance ToJSON CellEnvironment where+ toJSON environment =+ object $+ [ "schema" .= ("cell.environment/v1" :: Text),+ "cell" .= environment.cell,+ "runId" .= environment.runId,+ "leaseId" .= environment.leaseId,+ "postgres" .= environment.postgres,+ "broker" .= environment.broker,+ "otlp" .= environment.otlp,+ "metrics" .= object ["victoriaMetricsUrl" .= environment.victoriaMetricsUrl],+ "drivers" .= environment.drivers+ ]+ <> catMaybes+ [ ("clock" .=) . object . pure . ("skewBoundMicros" .=) <$> environment.clockSkewBoundMicros,+ ("faultHook" .=) <$> environment.faultHook+ ]+ <> ["resources" .= environment.resources | not (Map.null environment.resources)]++instance FromJSON CellEnvironment where+ parseJSON = withObject "cell environment" \value -> do+ expectSchema "cell.environment/v1" value+ metrics <- value .: "metrics"+ clock <- value .:? "clock"+ url <- metrics .: "victoriaMetricsUrl"+ skew <- traverse (.: "skewBoundMicros") clock+ environment <- CellEnvironment <$> value .: "cell" <*> value .: "runId" <*> value .: "leaseId" <*> value .: "postgres" <*> value .: "broker" <*> value .: "otlp" <*> pure url <*> value .: "drivers" <*> pure skew <*> value .:? "faultHook" <*> value .:? "resources" .!= Map.empty+ unless (validName environment.cell && validUri environment.victoriaMetricsUrl && not (null environment.drivers) && maybe True (>= 0) environment.clockSkewBoundMicros && maybe True (not . Text.null) environment.faultHook) (fail "invalid cell environment")+ pure environment++instance ToJSON WorkObject where+ toJSON work = object ["sha256" .= work.sha256, "bytes" .= work.bytes, "mediaType" .= work.mediaType]++instance FromJSON WorkObject where+ parseJSON = withObject "cell work object" \value -> do+ work <- WorkObject <$> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"+ unless (validDigest work.sha256 && work.bytes >= 0 && not (Text.null work.mediaType)) (fail "invalid cell work object")+ pure work++instance ToJSON PgDatabase where+ toJSON database = object ["name" .= database.name, "owner" .= database.owner, "template" .= database.template]++instance FromJSON PgDatabase where+ parseJSON = withObject "cell PostgreSQL database" \value -> do+ database <- PgDatabase <$> value .: "name" <*> value .: "owner" <*> value .: "template"+ unless (all validPgName [database.name, database.owner, database.template]) (fail "invalid cell PostgreSQL database")+ pure database++instance ToJSON PgReset where+ toJSON reset = object ["major" .= reset.major, "databases" .= reset.databases, "settings" .= reset.settings]++instance FromJSON PgReset where+ parseJSON = withObject "cell PostgreSQL reset" \value -> do+ reset <- PgReset <$> value .: "major" <*> value .: "databases" <*> value .: "settings"+ unless (reset.major `elem` [17, 18] && unique reset.databases && validStringMap reset.settings) (fail "invalid cell PostgreSQL reset")+ pure reset++instance ToJSON BrokerReset where+ toJSON reset = object ["wipe" .= reset.wipe]++instance FromJSON BrokerReset where+ parseJSON = withObject "cell broker reset" \value -> BrokerReset <$> value .: "wipe"++instance ToJSON CachePolicy where+ toJSON Cold = toJSON ("cold" :: Text)+ toJSON Warm = toJSON ("warm" :: Text)++instance FromJSON CachePolicy where+ parseJSON = withText "cell cache policy" \case+ "cold" -> pure Cold+ "warm" -> pure Warm+ _ -> fail "unknown cell cache policy"++instance ToJSON ResetBlock where+ toJSON reset =+ object $+ ["cachePolicy" .= reset.cachePolicy]+ <> catMaybes+ [ ("postgres" .=) <$> reset.postgres,+ ("broker" .=) <$> reset.broker+ ]+ <> ["resources" .= reset.resources | not (Map.null reset.resources)]++instance FromJSON ResetBlock where+ parseJSON = withObject "cell reset" \value -> ResetBlock <$> value .: "cachePolicy" <*> value .:? "postgres" <*> value .:? "broker" <*> value .:? "resources" .!= Map.empty++instance ToJSON Limits where+ toJSON limits = object ["wallClockSeconds" .= limits.wallClockSeconds, "memoryMaxBytes" .= limits.memoryMaxBytes, "outputMaxBytes" .= limits.outputMaxBytes]++instance FromJSON Limits where+ parseJSON = withObject "cell limits" \value -> do+ limits <- Limits <$> value .: "wallClockSeconds" <*> value .: "memoryMaxBytes" <*> value .: "outputMaxBytes"+ unless (limits.wallClockSeconds > 0 && limits.memoryMaxBytes > 0 && limits.outputMaxBytes > 0) (fail "invalid cell limits")+ pure limits++instance ToJSON Requirements where+ toJSON requirements =+ object $+ [ "protocol" .= ("cell.protocol/v1" :: Text),+ "minAgentVersion" .= requirements.minAgentVersion+ ]+ <> catMaybes [("capabilities" .=) <$> requirements.capabilities]++instance FromJSON Requirements where+ parseJSON = withObject "cell requirements" \value -> do+ protocol <- value .: "protocol"+ unless (protocol == ("cell.protocol/v1" :: Text)) (fail "unsupported cell protocol")+ requirements <- Requirements <$> value .: "minAgentVersion" <*> value .:? "capabilities"+ unless (validVersion requirements.minAgentVersion && maybe True (\items -> unique items && all (not . Text.null) items) requirements.capabilities) (fail "invalid cell requirements")+ pure requirements++instance ToJSON Submission where+ toJSON submission =+ object+ ( [ "schema" .= ("cell.submission/v1" :: Text),+ "runId" .= submission.runId,+ "leaseId" .= submission.leaseId,+ "payload" .= submission.payload,+ "work" .= submission.work,+ "env" .= submission.env,+ "reset" .= submission.reset,+ "limits" .= submission.limits,+ "requires" .= submission.requires,+ "labels" .= submission.labels+ ]+ <> maybe [] (\options -> ["collect" .= object (["traces" .= options.traces] <> ["profiles" .= True | options.profiles])]) submission.collect+ )++instance FromJSON Submission where+ parseJSON = withObject "cell submission" \value -> do+ expectSchema "cell.submission/v1" value+ submission <- Submission <$> value .: "runId" <*> value .: "leaseId" <*> value .: "payload" <*> value .: "work" <*> value .: "env" <*> value .: "reset" <*> value .: "limits" <*> value .: "requires" <*> (value .:? "collect" >>= traverse parseCollect) <*> value .: "labels"+ unless (validStringMap submission.env && validStringMap submission.labels) (fail "invalid cell submission map keys")+ pure submission+ where+ parseCollect = withObject "cell collection options" \options -> CollectOptions <$> options .: "traces" <*> (options .:? "profiles" .!= False)++instance ToJSON LogChunks where+ toJSON chunks = object ["stdout" .= chunks.stdout, "stderr" .= chunks.stderr]++instance FromJSON LogChunks where+ parseJSON = withObject "cell log chunks" \value -> do+ chunks <- LogChunks <$> value .: "stdout" <*> value .: "stderr"+ unless (chunks.stdout >= 0 && chunks.stderr >= 0) (fail "negative cell log chunk count")+ pure chunks++instance ToJSON CellStatus where+ toJSON status =+ object $+ [ "schema" .= ("cell.status/v1" :: Text),+ "runId" .= status.runId,+ "phase" .= status.phase,+ "updatedAt" .= status.updatedAt,+ "logChunks" .= status.logChunks+ ]+ <> catMaybes+ [ ("leaseSequence" .=) <$> status.leaseSequence,+ ("phaseStartedAt" .=) <$> status.phaseStartedAt,+ ("outcome" .=) <$> status.outcome,+ ("manifestSha256" .=) <$> status.manifestSha256,+ ("reasons" .=) <$> status.reasons+ ]++instance FromJSON CellStatus where+ parseJSON = withObject "cell status" \value -> do+ expectSchema "cell.status/v1" value+ status <- CellStatus <$> value .: "runId" <*> value .: "phase" <*> value .:? "leaseSequence" <*> value .: "updatedAt" <*> value .:? "phaseStartedAt" <*> value .: "logChunks" <*> value .:? "outcome" <*> value .:? "manifestSha256" <*> value .:? "reasons"+ unless (maybe True (>= 0) status.leaseSequence && maybe True validDigest status.manifestSha256 && maybe True (all (not . Text.null)) status.reasons) (fail "invalid cell status fields")+ unless (if status.phase == Sealed then status.outcome /= Nothing && status.manifestSha256 /= Nothing else status.outcome == Nothing && status.manifestSha256 == Nothing) (fail "cell status terminal fields disagree with phase")+ pure status++instance ToJSON Rejected where+ toJSON rejected = object ["schema" .= ("cell.rejected/v1" :: Text), "runId" .= rejected.runId, "reason" .= rejected.reason, "at" .= rejected.at]++instance FromJSON Rejected where+ parseJSON = withObject "cell rejection" \value -> do+ expectSchema "cell.rejected/v1" value+ rejected <- Rejected <$> value .: "runId" <*> value .: "reason" <*> value .: "at"+ unless (not (Text.null rejected.reason)) (fail "empty cell rejection reason")+ pure rejected++instance ToJSON CellRunResult where+ toJSON result =+ object+ [ "schema" .= ("cell.run-result/v1" :: Text),+ "runId" .= result.runId,+ "cell" .= result.cell,+ "leaseId" .= result.leaseId,+ "leaseSequence" .= result.leaseSequence,+ "outcome" .= result.outcome,+ "entryExitCode" .= result.entryExitCode,+ "entrySignal" .= result.entrySignal,+ "reasons" .= result.reasons+ ]++instance FromJSON CellRunResult where+ parseJSON = withObject "cell run result" \value -> do+ expectSchema "cell.run-result/v1" value+ result <- CellRunResult <$> value .: "runId" <*> value .: "cell" <*> value .: "leaseId" <*> value .: "leaseSequence" <*> value .: "outcome" <*> value .: "entryExitCode" <*> value .: "entrySignal" <*> value .: "reasons"+ unless (validName result.cell && result.leaseSequence >= 0 && all (not . Text.null) result.reasons) (fail "invalid cell run result")+ pure result++instance ToJSON ManifestPayload where+ toJSON payload = object ["kind" .= ("nix-nar-bundle" :: Text), "sha256" .= payload.sha256, "storePath" .= payload.storePath]++instance FromJSON ManifestPayload where+ parseJSON = withObject "manifest payload" \value -> do+ kind <- value .: "kind"+ unless (kind == ("nix-nar-bundle" :: Text)) (fail "unsupported manifest payload kind")+ payload <- ManifestPayload <$> value .: "sha256" <*> value .: "storePath"+ unless (validDigest payload.sha256 && validStorePath payload.storePath) (fail "invalid manifest payload")+ pure payload++instance ToJSON Artifact where+ toJSON artifact = object ["path" .= artifact.path, "sha256" .= artifact.sha256, "bytes" .= artifact.bytes, "mediaType" .= artifact.mediaType]++instance FromJSON Artifact where+ parseJSON = withObject "cell artifact" \value -> do+ artifact <- Artifact <$> value .: "path" <*> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"+ unless (validArtifactPath artifact.path && validDigest artifact.sha256 && artifact.bytes >= 0 && not (Text.null artifact.mediaType)) (fail "invalid cell artifact")+ pure artifact++instance ToJSON CellManifest where+ toJSON manifest =+ object+ [ "schema" .= ("cell.artifact-manifest/v1" :: Text),+ "layout" .= (1 :: Int),+ "runId" .= manifest.runId,+ "cell" .= manifest.cell,+ "leaseId" .= manifest.leaseId,+ "leaseSequence" .= manifest.leaseSequence,+ "sealedAt" .= manifest.sealedAt,+ "agentVersion" .= manifest.agentVersion,+ "payload" .= manifest.payload,+ "outcome" .= manifest.outcome,+ "retention" .= object ["policy" .= ("bucket-retention" :: Text), "retentionSeconds" .= manifest.retentionSeconds],+ "artifacts" .= manifest.artifacts+ ]++instance FromJSON CellManifest where+ parseJSON = withObject "cell manifest" \value -> do+ expectSchema "cell.artifact-manifest/v1" value+ layout <- value .: "layout"+ unless (layout == (1 :: Int)) (fail "unsupported cell manifest layout")+ retention <- value .: "retention"+ policy <- retention .: "policy"+ unless (policy == ("bucket-retention" :: Text)) (fail "unsupported cell retention policy")+ seconds <- retention .: "retentionSeconds"+ manifest <- CellManifest <$> value .: "runId" <*> value .: "cell" <*> value .: "leaseId" <*> value .: "leaseSequence" <*> value .: "sealedAt" <*> value .: "agentVersion" <*> value .: "payload" <*> value .: "outcome" <*> pure seconds <*> value .: "artifacts"+ unless (validName manifest.cell && manifest.leaseSequence >= 0 && manifest.retentionSeconds >= 0 && validVersion manifest.agentVersion && not (null manifest.artifacts) && all ((/= "manifest.json") . (.path)) manifest.artifacts && Set.size (Set.fromList (map (.path) manifest.artifacts)) == length manifest.artifacts) (fail "invalid cell manifest")+ pure manifest++expectSchema :: Text -> Object -> Parser ()+expectSchema expected value = do+ schema <- value .: "schema"+ unless (schema == expected) (fail "unsupported cell document schema")++validName :: Text -> Bool+validName value = Text.length value >= 3 && Text.length value <= 63 && Text.head value `elem` (['a' .. 'z'] <> ['0' .. '9']) && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'] <> "-")) value++validShape :: Value -> Bool+validShape (Object fields) = all (`KeyMap.member` fields) ["postgres", "driver", "monitoring", "scheduling"]+validShape _ = False++validIpv4 :: Text -> Bool+validIpv4 value = case traverse (readMaybe . Text.unpack) (Text.splitOn "." value) :: Maybe [Int] of+ Just [first, second, third, fourth] -> all (\piece -> piece >= 0 && piece <= 255) [first, second, third, fourth]+ _ -> False++validUri :: Text -> Bool+validUri value = case Text.breakOn ":" value of+ (scheme, rest) ->+ not (Text.null scheme)+ && Text.length rest > 1+ && Text.head scheme `elem` (['a' .. 'z'] <> ['A' .. 'Z'])+ && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['A' .. 'Z'] <> ['0' .. '9'] <> "+.-")) scheme++validDigest :: Text -> Bool+validDigest value = Text.length value == 64 && Text.all (\character -> character `elem` (['0' .. '9'] <> ['a' .. 'f'])) value++validVersion :: Text -> Bool+validVersion value = case Text.splitOn "." value of+ [major, minor, patch] -> all (\piece -> not (Text.null piece) && Text.all (\character -> character `elem` ['0' .. '9']) piece) [major, minor, patch]+ _ -> False++validStorePath :: Text -> Bool+validStorePath value = case Text.stripPrefix "/nix/store/" value of+ Just suffix -> let (digest, name) = Text.breakOn "-" suffix in Text.length digest == 32 && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'])) digest && Text.length name > 1 && not (Text.any (== '/') name)+ Nothing -> False++validArtifactPath :: Text -> Bool+validArtifactPath value =+ not (Text.null value)+ && not (Text.isPrefixOf "/" value)+ && not (Text.any (`elem` ['\\', '\0']) value)+ && all (\part -> not (Text.null part) && part /= "." && part /= "..") (Text.splitOn "/" value)++validPgName :: Text -> Bool+validPgName value = case Text.uncons value of+ Just (first, rest) -> Text.length value <= 63 && first `elem` ['a' .. 'z'] && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'] <> "_")) rest+ Nothing -> False++validStringMap :: Map Text Text -> Bool+validStringMap = all validKey . Map.keys+ where+ validKey value = case Text.uncons value of+ Just (first, rest) -> first `elem` (['A' .. 'Z'] <> ['a' .. 'z'] <> "_") && Text.all (\character -> character `elem` (['A' .. 'Z'] <> ['a' .. 'z'] <> ['0' .. '9'] <> "_.-")) rest+ Nothing -> False++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Cell/Exec.hs view
@@ -0,0 +1,351 @@+module Kenshou.Remote.Cell.Exec (resolveOnCell, resolveWorkJson, cellExec) where++import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)+import Control.Monad (when)+import Data.Aeson (Value (..), encode, object, (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (toList)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as TextIO+import Kenshou.Core.Bundle (Registry, lookupScenario)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), TracingArm (..), resolveDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), parseRunId, renderScenarioId)+import Kenshou.Core.Knob (KnobSpec (..), RawKnob (..), mkKnobName)+import Kenshou.Core.Resource.Types (legacyKafkaName, renderResourceName)+import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Scenario (..))+import Kenshou.Plan.Execute (ExecuteOptions (..), executePlan)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Plan.Summary (summaryExitCode)+import Kenshou.Remote.Cell.Docs (CellBroker (..), CellEnvironment (..), CellPostgres (..), OtlpEndpoint (..), OtlpSinks (..))+import Kenshou.Remote.Cell.Health (withCellHealthFile)+import Kenshou.Remote.Cell.Legacy (brokerAsResource)+import Kenshou.Remote.Cell.Prepare (OtlpSink (..))+import System.Directory (Permissions (..), createDirectoryIfMissing, doesFileExist, getPermissions)+import System.Environment (lookupEnv, setEnv, unsetEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO (stderr)+import System.Process (readProcessWithExitCode)++-- Bind the generic environment in a prepared work plan to endpoints available+-- on this cell. Run IDs and plan metadata are preserved; no run is silently+-- dropped here.+resolveOnCell :: Registry -> CellEnvironment -> OtlpSink -> RunPlan -> Either Text RunPlan+resolveOnCell registry environment sink plan = do+ runs <- traverse resolveRun plan.runs+ pure (RunPlan plan.planId plan.createdAt plan.context plan.policy runs plan.skipped plan.estimateMinutes)+ where+ resolveRun entry = do+ resolved <- resolveSpec registry environment sink entry.spec+ pure (PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial resolved)++-- The executor accepts a narrow run-plan JSON document rather than decoding+-- every planning provenance type. Rewrite just the spec member of each entry.+resolveWorkJson :: Registry -> CellEnvironment -> OtlpSink -> Value -> Either Text Value+resolveWorkJson registry environment sink work = fst <$> resolveWorkDocument registry environment sink work++resolveWorkDocument :: Registry -> CellEnvironment -> OtlpSink -> Value -> Either Text (Value, [RunSpec])+resolveWorkDocument registry environment sink work = case work of+ Object fields -> case (KeyMap.lookup "schema" fields, KeyMap.lookup "runs" fields) of+ (Just (String "kenshou.run-plan/v1"), Just (Array runs)) -> do+ resolved <- traverse resolveEntry runs+ pure (Object (KeyMap.insert "runs" (Array (fmap fst resolved)) fields), toList (fmap snd resolved))+ _ -> Left "cell work is not a kenshou.run-plan/v1 document"+ _ -> Left "cell work is not a JSON object"+ where+ resolveEntry (Object entry) = do+ identifier <- case KeyMap.lookup "runId" entry of+ Just value -> (decodeValue "planned run ID" value :: Either Text RunId)+ Nothing -> Left "cell work entry has no run ID"+ spec <- case KeyMap.lookup "spec" entry of+ Just value -> (decodeValue "planned run spec" value :: Either Text RunSpec)+ Nothing -> Left "cell work entry has no run spec"+ if maybe True (== identifier) spec.runId+ then pure ()+ else Left "cell work entry and spec have different run IDs"+ resolved <- resolveSpec registry environment sink spec+ pure (Object (KeyMap.insert "spec" (Aeson.toJSON resolved) entry), resolved)+ resolveEntry _ = Left "cell work run is not an object"++ decodeValue :: (Aeson.FromJSON document) => Text -> Value -> Either Text document+ decodeValue label value = case Aeson.fromJSON value of+ Aeson.Error failure -> Left (label <> ": " <> Text.pack failure)+ Aeson.Success document -> Right document++resolveSpec :: Registry -> CellEnvironment -> OtlpSink -> RunSpec -> Either Text RunSpec+resolveSpec registry environment sink spec = do+ scenario <- maybe (Left ("unknown cell scenario: " <> renderScenarioId spec.scenario)) Right (lookupScenario registry spec.scenario)+ if scenario.placement == PlaceLocal+ then Left ("scenario is local-only: " <> renderScenarioId scenario.id)+ else pure ()+ dimensions <- either (Left . Text.intercalate "; " . NonEmpty.toList) Right (resolveDimensions scenario.dimensions spec.dimensions)+ postgres <- bindPostgres scenario dimensions+ extras <- bindExtras scenario postgres+ resources <- Map.fromList <$> traverse (bindResource scenario) scenario.requires.resources+ telemetryKnobs <- bindOtlp scenario dimensions+ knobs <- bindRemoteScenario telemetryKnobs+ let prior = spec.environment+ cellEnvironment = EnvironmentSpec RunOnCell prior.machineProfile postgres extras resources prior.telemetry+ pure (RunSpec spec.runId spec.scenario spec.scenarioRevision knobs spec.dimensions spec.seed spec.phases spec.timeoutSeconds cellEnvironment spec.cohortExpectation spec.comparison spec.labels)+ where+ bindPostgres scenario dimensions = case scenario.requires.postgres of+ Nothing -> pure spec.environment.postgres+ Just requirement+ | requirement.needsServerControl ->+ if scenario.id.kind `elem` [Correctness, Concurrency]+ then case spec.environment.postgres of+ Just ephemeral@(PostgresEphemeral _) -> pure (Just ephemeral)+ _ -> Left ("scenario needs driver-local PostgreSQL server control: " <> renderScenarioId scenario.id)+ else Left ("benchmark or soak cannot control the cell PostgreSQL server: " <> renderScenarioId scenario.id)+ | otherwise -> do+ if dimensions.pgDurability /= Just PgDurable+ then Left ("cell run requires pg.durability=durable: " <> renderScenarioId scenario.id)+ else pure ()+ let requested = case dimensions.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> 0+ if requested /= environment.postgres.major+ then Left ("cell PostgreSQL major differs from pg.version for " <> renderScenarioId scenario.id)+ else pure (Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL")))++ bindExtras scenario primary =+ if case primary of Just (PostgresEphemeral _) -> True; _ -> False+ then Map.fromList <$> traverse driverExtra scenario.requires.extraPostgres+ else do+ let requirements = scenario.requires.extraPostgres+ variableNames = fmap (variableName . fst) requirements+ if any (.needsServerControl) (fmap snd requirements)+ then Left ("extra PostgreSQL server needs driver-local control: " <> renderScenarioId scenario.id)+ else pure ()+ if length variableNames /= length (unique variableNames)+ then Left "extra PostgreSQL names collide as environment variables"+ else pure (Map.fromList [(name, PostgresExternal (ConnFromEnv ("KENSHOU_CELL_PG_URL_" <> variableName name))) | (name, _) <- requirements])+ where+ driverExtra (name, _) = case Map.lookup name spec.environment.extraPostgres of+ Nothing -> pure (name, PostgresEphemeral [])+ Just local@(PostgresEphemeral _) -> pure (name, local)+ Just (PostgresExternal _) -> Left ("driver-local PostgreSQL requires every extra server to be ephemeral: " <> name)++ -- The cell environment binds a resource by name; the legacy broker fields still+ -- bind the resource they always described.+ bindResource scenario name = case (Map.lookup name environment.resources, environment.broker) of+ (Just value, _) -> pure (name, value)+ (Nothing, Just broker) | name == legacyKafkaName -> (name,) <$> brokerAsResource broker+ _ -> Left ("cell has no resource " <> renderResourceName name <> " for " <> renderScenarioId scenario.id)++ bindOtlp scenario dimensions+ | dimensions.tracing /= Just TracingSdkOtlp = pure spec.knobs+ | otherwise = do+ name <- mkKnobName "otel.endpoint"+ if name `notElem` fmap (.name) scenario.knobs+ then Left ("tracing scenario lacks otel.endpoint: " <> renderScenarioId scenario.id)+ else pure ()+ sinks <- maybe (Left "cell has no OTLP sink") Right environment.otlp+ let endpoint = case sink of NullSink -> sinks.nullEndpoint.http; FileSink -> sinks.fileEndpoint.http+ if Text.null endpoint+ then Left "cell OTLP sink has no HTTP endpoint"+ else pure ((name, RawText endpoint) : filter ((/= name) . fst) spec.knobs)++ bindRemoteScenario knobs+ | renderScenarioId spec.scenario /= "selftest/remote/correctness/cell-environment" = pure knobs+ | otherwise = do+ otlpName <- mkKnobName "remote.otlp-endpoint"+ let selectedOtlp = case environment.otlp of+ Nothing -> ""+ Just sinks -> case sink of NullSink -> sinks.nullEndpoint.http; FileSink -> sinks.fileEndpoint.http+ pure (assignWhenBlank otlpName selectedOtlp knobs)++ assignWhenBlank name value knobs+ | Text.null value || maybe False (not . blank) (lookup name knobs) = knobs+ | otherwise = assign name value knobs+ blank (RawText value) = Text.null value+ blank (RawJson (String value)) = Text.null value+ blank _ = False+ assign name value knobs = (name, RawText value) : filter ((/= name) . fst) knobs++variableName :: Text -> Text+variableName = Text.map (\character -> if character == '-' then '_' else character) . Text.toUpper++unique :: (Eq value) => [value] -> [value]+unique [] = []+unique (first : rest) = first : unique (filter (/= first) rest)++-- The generic cell agent invokes this command on the first driver with the+-- submitted work and an environment document. Every child run receives the+-- resolved plan, the cell context and the same payload identity variables.+cellExec :: Registry -> FilePath -> FilePath -> IO ExitCode+cellExec registry workFile outDir = do+ result <- try (executeOnDriver registry workFile outDir) :: IO (Either SomeException ExitCode)+ case result of+ Left failure | Just async <- (fromException failure :: Maybe SomeAsyncException) -> throwIO async+ Left failure -> adapterFailure outDir (Text.pack (displayException failure))+ Right code -> pure code++executeOnDriver :: Registry -> FilePath -> FilePath -> IO ExitCode+executeOnDriver registry workFile outDir = do+ driverIndex <- integerEnv "CELL_DRIVER_INDEX" 0+ driverCount <- integerEnv "CELL_DRIVER_COUNT" 1+ when (driverIndex >= driverCount) (ioError (userError "CELL_DRIVER_INDEX exceeds CELL_DRIVER_COUNT"))+ let cellDirectory = outDir </> "kenshou-cell"+ createDirectoryIfMissing True cellDirectory+ if driverIndex /= 0+ then do+ LazyByteString.writeFile (outDir </> "idle-driver.json") (encode (object ["schema" .= ("kenshou.cell-idle-driver/v1" :: Text), "index" .= driverIndex, "count" .= driverCount]))+ pure ExitSuccess+ else do+ environmentPath <- requiredEnv "CELL_ENV_FILE"+ environment <- Aeson.eitherDecodeFileStrict' environmentPath >>= either (ioError . userError . ("invalid CELL_ENV_FILE: " <>)) pure+ cellRun <- requiredEnv "CELL_RUN_ID" >>= either (ioError . userError . Text.unpack) pure . parseRunId . Text.pack+ when (cellRun /= environment.runId) (ioError (userError "CELL_RUN_ID differs from the cell environment"))+ scratch <- requiredEnv "CELL_SCRATCH_DIR"+ sink <-+ requiredEnv "KENSHOU_OTLP_SINK" >>= \case+ "null" -> pure NullSink+ "file" -> pure FileSink+ _ -> ioError (userError "KENSHOU_OTLP_SINK must be null or file")+ identity <- requiredEnv "KENSHOU_COHORT_IDENTITY"+ present <- doesFileExist identity+ when (not present) (ioError (userError "payload cohort identity file is missing"))+ payload <- requiredPayloadIdentity+ work <- Aeson.eitherDecodeFileStrict' workFile >>= either (ioError . userError . ("invalid cell work: " <>)) pure+ (resolved, specs) <- either (ioError . userError . Text.unpack) pure (resolveWorkDocument registry environment sink work)+ let planPath = cellDirectory </> "plan.resolved.json"+ contextPath = cellDirectory </> "context.json"+ noticePath = cellDirectory </> "health-notices.jsonl"+ cachePath = scratch </> "cache"+ tempPath = scratch </> "tmp"+ createDirectoryIfMissing True cachePath+ createDirectoryIfMissing True tempPath+ LazyByteString.writeFile planPath (encode resolved)+ LazyByteString.writeFile noticePath ""+ setEnv "KENSHOU_CELL_PG_URL" (Text.unpack environment.postgres.connectionString)+ mapM_ (setExtraPostgres environment.postgres.connectionString) specs+ setEnv "KENSHOU_CELL_FINGERPRINT" contextPath+ setEnv "KENSHOU_HEALTH_NOTICES" noticePath+ setEnv "XDG_CACHE_HOME" cachePath+ setEnv "TMPDIR" tempPath+ faultEnabled <- case environment.faultHook of+ Nothing -> unsetEnv "KENSHOU_CELL_FAULT_HOOK" >> pure False+ Just hook -> do+ available <- doesFileExist (Text.unpack hook)+ executableHook <- if available then (.executable) <$> getPermissions (Text.unpack hook) else pure False+ if executableHook then setEnv "KENSHOU_CELL_FAULT_HOOK" (Text.unpack hook) else unsetEnv "KENSHOU_CELL_FAULT_HOOK"+ pure executableHook+ pgPartman <- probePgPartman environment.postgres.connectionString+ LazyByteString.writeFile contextPath (encode (cellContext environment driverIndex driverCount sink faultEnabled pgPartman payload))+ if driverCount > 1+ then setEnv "KENSHOU_CLOCK_SKEW_BOUND_MICROS" (show (maybe 50000 id environment.clockSkewBoundMicros))+ else unsetEnv "KENSHOU_CLOCK_SKEW_BOUND_MICROS"+ healthFile <- lookupEnv "CELL_HEALTH_FILE"+ let runPlan = executePlan (ExecuteOptions planPath outDir False False Nothing Nothing)+ withHealth = withCellHealthFile healthFile noticePath cellRun runPlan+ summary <- if faultEnabled then healAll environment.faultHook >> withHealth `finally` healAll environment.faultHook else withHealth+ pure (summaryExitCode summary)++healAll :: Maybe Text -> IO ()+healAll Nothing = pure ()+healAll (Just hook) = do+ (code, _, errors) <- readProcessWithExitCode (Text.unpack hook) ["heal-all"] ""+ when (code /= ExitSuccess) (ioError (userError ("cell fault hook heal-all failed: " <> errors)))++setExtraPostgres :: Text -> RunSpec -> IO ()+setExtraPostgres connection spec = mapM_ setOne (Map.elems spec.environment.extraPostgres)+ where+ setOne (PostgresExternal (ConnFromEnv name)) | "KENSHOU_CELL_PG_URL_" `Text.isPrefixOf` name = setEnv (Text.unpack name) (Text.unpack connection)+ setOne _ = pure ()++probePgPartman :: Text -> IO (Maybe Text)+probePgPartman connection = do+ (code, output, errors) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", "SELECT COALESCE((SELECT default_version FROM pg_available_extensions WHERE name = 'pg_partman'), '')"] ""+ case code of+ ExitSuccess -> pure case Text.strip (Text.pack output) of+ "" -> Nothing+ version -> Just version+ _ -> ioError (userError ("PostgreSQL extension probe failed: " <> errors))++cellContext :: CellEnvironment -> Int -> Int -> OtlpSink -> Bool -> Maybe Text -> Map.Map Text Text -> Value+cellContext environment driverIndex driverCount sink faultEnabled pgPartman payload =+ object+ [ "schema" .= ("kenshou.cell-context/v1" :: Text),+ "cell" .= environment.cell,+ "cellRun" .= environment.runId,+ "leaseId" .= environment.leaseId,+ "driver" .= object ["index" .= driverIndex, "count" .= driverCount],+ "postgres" .= object ["major" .= environment.postgres.major, "host" .= environment.postgres.host, "port" .= environment.postgres.port, "placement" .= ("cell-server" :: Text), "extraPostgres" .= ("shared-server" :: Text), "extensions" .= object ["pg_partman" .= object ["available" .= maybe False (const True) pgPartman, "version" .= pgPartman]]],+ "broker" .= environment.broker,+ "capabilities"+ .= object+ [ "postgres.major" .= Text.pack (show environment.postgres.major),+ "postgres.pg_partman" .= maybe False (const True) pgPartman,+ "postgres.second-server" .= False,+ "postgres.control-hook" .= False,+ "broker" .= maybe "none" (\broker -> broker.implementation <> " " <> broker.version) environment.broker,+ "otlp.null" .= maybe False (not . Text.null . (.nullEndpoint.http)) environment.otlp,+ "otlp.file" .= maybe False (not . Text.null . (.fileEndpoint.http)) environment.otlp,+ "fault-hook" .= faultEnabled+ ],+ "otlpSink" .= (case sink of NullSink -> "null" :: Text; FileSink -> "file"),+ "faultHook" .= faultEnabled,+ "clock" .= object ["skewBoundMicros" .= environment.clockSkewBoundMicros],+ "payload"+ .= object+ [ "bundleSha256" .= field "KENSHOU_PAYLOAD_BUNDLE_SHA256",+ "storePath" .= field "KENSHOU_PAYLOAD_STORE_PATH",+ "narHash" .= field "KENSHOU_PAYLOAD_NAR_HASH",+ "cohort" .= field "KENSHOU_PAYLOAD_COHORT",+ "harness" .= object ["revision" .= field "KENSHOU_HARNESS_REVISION", "dirty" .= (field "KENSHOU_HARNESS_DIRTY" == "true")]+ ]+ ]+ where+ field name = Map.findWithDefault "" name payload++requiredPayloadIdentity :: IO (Map.Map Text Text)+requiredPayloadIdentity = do+ values <- Map.fromList <$> traverse load (filter (/= "KENSHOU_COHORT_IDENTITY") payloadIdentityNames)+ when (Map.lookup "KENSHOU_HARNESS_DIRTY" values `notElem` [Just "true", Just "false"]) (ioError (userError "KENSHOU_HARNESS_DIRTY must be true or false"))+ pure values+ where+ load name = (name,) . Text.pack <$> requiredEnv (Text.unpack name)++requiredEnv :: String -> IO String+requiredEnv name =+ lookupEnv name >>= \case+ Just value | not (null value) -> pure value+ _ -> ioError (userError (name <> " is required by the cell payload"))++integerEnv :: String -> Int -> IO Int+integerEnv name fallback =+ lookupEnv name >>= \case+ Nothing -> pure fallback+ Just raw -> case reads raw of+ [(value, "")] | value >= 0 -> pure value+ _ -> ioError (userError (name <> " must be a non-negative integer"))++adapterFailure :: FilePath -> Text -> IO ExitCode+adapterFailure outDir detail = do+ let directory = outDir </> "kenshou-cell"+ reason :: Text+ reason+ | "CELL_ENV_FILE" `Text.isInfixOf` detail = "cell-environment-missing"+ | any (`Text.isInfixOf` detail) payloadIdentityNames = "payload-identity-missing"+ | otherwise = "cell-exec-failed"+ createDirectoryIfMissing True directory+ LazyByteString.writeFile (directory </> "adapter-error.json") (encode (object ["schema" .= ("kenshou.cell-adapter-error/v1" :: Text), "reason" .= reason, "detail" .= detail]))+ TextIO.hPutStrLn stderr ("kenshou cell exec: " <> detail)+ pure (ExitFailure 4)++payloadIdentityNames :: [Text]+payloadIdentityNames =+ [ "KENSHOU_COHORT_IDENTITY",+ "KENSHOU_HARNESS_REVISION",+ "KENSHOU_HARNESS_DIRTY",+ "KENSHOU_PAYLOAD_BUNDLE_SHA256",+ "KENSHOU_PAYLOAD_STORE_PATH",+ "KENSHOU_PAYLOAD_NAR_HASH",+ "KENSHOU_PAYLOAD_COHORT"+ ]
+ src/Kenshou/Remote/Cell/Fetch.hs view
@@ -0,0 +1,387 @@+module Kenshou.Remote.Cell.Fetch+ ( FetchError (..),+ VerifyProblem (..),+ fetchCellRun,+ verifyCellTree,+ verifyCellRun,+ verifyCellRunWithStatus,+ effectiveOutcome,+ )+where++import Control.Exception (bracket)+import Control.Monad (forM)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (FromJSON, Value (..), eitherDecode)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List (sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, listToMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Kenshou.Core.Id (RunId, parseRunId, renderRunId)+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..), verifyManifest)+import Kenshou.Core.Outcome qualified as Outcome+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellPhase (..), CellRunResult (..), CellStatus (..), ManifestPayload (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Docs qualified as Cell+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket, ObjectMeta (..), ObjectName (..), ObjectStore (..))+import System.Directory (createDirectoryIfMissing, doesDirectoryExist, doesFileExist, listDirectory, pathIsSymbolicLink, removeFile, renameFile)+import System.FilePath (takeDirectory, (</>))+import System.IO (IOMode (ReadMode), hClose, openBinaryTempFile, withBinaryFile)++data VerifyProblem+ = InvalidManifest !Text+ | Missing !Text+ | Extra !Text+ | DigestMismatch !Text+ | SizeMismatch !Text+ | UnsafeEntry !Text+ | PayloadMismatch !Text+ | ManifestsDisagree !Text+ | NestedManifestProblem !Text+ | RunResultMismatch !Text+ | StatusDigestMismatch+ | StatusMismatch !Text+ deriving stock (Eq, Show)++data FetchError+ = Unsealed+ | ManifestProblem !Text+ | ObjectSetMismatch ![Text] ![Text] -- missing, extra+ | ObjectChanged !Text+ | ObjectCorrupt !Text+ | VerificationFailed !(NonEmpty VerifyProblem)+ deriving stock (Eq, Show)++effectiveOutcome :: Cell.CellOutcome -> Bool -> Outcome.Outcome -> Outcome.Outcome+effectiveOutcome Cell.Completed _ recorded = recorded+effectiveOutcome Cell.InfrastructureFailure _ _ = Outcome.InfrastructureFailure+effectiveOutcome Cell.Cancelled complete recorded = if complete then recorded else Outcome.Errored+effectiveOutcome Cell.TimedOut complete recorded = if complete then recorded else Outcome.Errored++fetchCellRun :: ObjectStore -> Bucket -> RunId -> FilePath -> IO (Either FetchError FilePath)+fetchCellRun store bucket identifier outDir = do+ let prefix = "runs/" <> renderRunId identifier <> "/"+ manifestObject = ObjectName (prefix <> "manifest.json")+ tree = outDir </> Text.unpack (renderRunId identifier) </> "tree"+ sealed <- store.getObject bucket manifestObject+ case sealed of+ Nothing -> pure (Left Unsealed)+ Just (manifestBytes, manifestMeta) -> case eitherDecode manifestBytes :: Either String CellManifest of+ Left failure -> pure (Left (ManifestProblem (Text.pack failure)))+ Right manifest+ | manifest.runId /= identifier -> pure (Left (ManifestProblem "manifest run ID differs from result prefix"))+ | otherwise -> do+ listed <- store.listObjects bucket prefix+ let actual = Set.fromList [object.unObjectName | (object, _) <- listed]+ expected = Set.fromList ((prefix <> "manifest.json") : [prefix <> artifact.path | artifact <- manifest.artifacts])+ absent = Set.toList (expected Set.\\ actual)+ unexpected = Set.toList (actual Set.\\ expected)+ if not (null absent && null unexpected)+ then pure (Left (ObjectSetMismatch absent unexpected))+ else case lookup manifestObject listed of+ Nothing -> pure (Left Unsealed)+ Just listedManifest | listedManifest.generation /= manifestMeta.generation -> pure (Left (ObjectChanged "manifest.json"))+ Just _ -> do+ createDirectoryIfMissing True tree+ rootSymbolic <- pathIsSymbolicLink tree+ if rootSymbolic+ then pure (Left (VerificationFailed (UnsafeEntry "tree" :| [])))+ else do+ (_, unsafe) <- listTree tree+ case unsafe of+ first : rest -> pure (Left (VerificationFailed (UnsafeEntry first :| map UnsafeEntry rest)))+ [] -> do+ writeVerified tree "manifest.json" manifestBytes+ fetched <- traverse (fetchArtifact store bucket prefix tree listed) manifest.artifacts+ case firstFailure fetched of+ Just failure -> pure (Left failure)+ Nothing -> do+ verified <- verifyCellTree tree+ pure case verified of+ Left problems -> Left (VerificationFailed problems)+ Right _ -> Right tree++fetchArtifact :: ObjectStore -> Bucket -> Text -> FilePath -> [(ObjectName, ObjectMeta)] -> Artifact -> IO (Maybe FetchError)+fetchArtifact store bucket prefix tree listed artifact = do+ let name = ObjectName (prefix <> artifact.path)+ target = tree </> Text.unpack artifact.path+ case lookup name listed of+ Nothing -> pure (Just (ObjectChanged artifact.path))+ Just meta+ | meta.size /= artifact.bytes -> pure (Just (ObjectCorrupt artifact.path))+ | otherwise -> do+ existing <- doesFileExist target+ valid <- if existing then matches target artifact else pure False+ if valid+ then pure Nothing+ else withTemporary target \temporary -> do+ fetched <- store.downloadTo bucket name temporary+ case fetched of+ Nothing -> pure (Just (ObjectChanged artifact.path))+ Just got+ | got.generation /= meta.generation -> pure (Just (ObjectChanged artifact.path))+ | otherwise -> do+ sound <- matches temporary artifact+ if sound+ then renameFile temporary target >> pure Nothing+ else pure (Just (ObjectCorrupt artifact.path))++verifyCellTree :: FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)+verifyCellTree tree = do+ let manifestPath = tree </> "manifest.json"+ rootExists <- doesDirectoryExist tree+ if not rootExists+ then pure (Left (Missing "manifest.json" :| []))+ else do+ rootSymbolic <- pathIsSymbolicLink tree+ if rootSymbolic+ then pure (Left (UnsafeEntry "tree" :| []))+ else do+ exists <- doesFileExist manifestPath+ if not exists+ then pure (Left (Missing "manifest.json" :| []))+ else do+ symbolic <- pathIsSymbolicLink manifestPath+ if symbolic+ then pure (Left (UnsafeEntry "manifest.json" :| []))+ else do+ bytes <- LazyByteString.readFile manifestPath+ case eitherDecode bytes :: Either String CellManifest of+ Left failure -> pure (Left (InvalidManifest (Text.pack failure) :| []))+ Right manifest -> do+ (files, unsafe) <- listTree tree+ let actual = Set.fromList files+ expected = Set.fromList ("manifest.json" : map (.path) manifest.artifacts)+ missing = map Missing (Set.toList (expected Set.\\ actual))+ extra = map Extra (Set.toList (actual Set.\\ expected))+ damaged <- fmap concat $ forM manifest.artifacts \artifact -> do+ if Set.member artifact.path actual+ then do+ (digest, size) <- digestFile (tree </> Text.unpack artifact.path)+ pure ([DigestMismatch artifact.path | digest /= artifact.sha256] <> [SizeMismatch artifact.path | size /= artifact.bytes])+ else pure []+ let problems = map UnsafeEntry unsafe <> missing <> extra <> damaged+ pure case problems of+ [] -> Right manifest+ first : rest -> Left (first :| rest)++-- This is the evidence gate after the cell-level size and digest verification.+-- Validate nested paths before calling the kernel verifier, whose standalone+-- entry point assumes its manifest came from a trusted local run directory.+verifyCellRun :: FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)+verifyCellRun tree = do+ verified <- verifyCellTree tree+ case verified of+ Left problems -> pure (Left problems)+ Right manifest -> do+ let byPath = Map.fromList [(artifact.path, artifact) | artifact <- manifest.artifacts]+ submission <- readRequired tree byPath "submission/submission.json" :: IO (Either VerifyProblem Submission)+ result <- readRequired tree byPath "cell/result.json" :: IO (Either VerifyProblem CellRunResult)+ case (submission, result) of+ (Right submitted, Right cellResult) -> do+ let identityProblems =+ [PayloadMismatch "submission run, lease or payload differs from cell manifest" | submitted.runId /= manifest.runId || submitted.leaseId /= manifest.leaseId || submitted.payload.bundle.sha256 /= manifest.payload.sha256 || submitted.payload.storePath /= manifest.payload.storePath]+ <> [RunResultMismatch "cell result differs from cell manifest" | cellResult.runId /= manifest.runId || cellResult.cell /= manifest.cell || cellResult.leaseId /= manifest.leaseId || cellResult.leaseSequence /= manifest.leaseSequence || cellResult.outcome /= manifest.outcome]+ workProblems <- checkWork tree byPath submitted+ planned <- readPlannedRuns tree+ let nested = nestedManifestPaths manifest.artifacts+ nestedIds = Set.fromList (map fst nested)+ (planProblems, expectedIds) = case planned of+ Left problem -> ([problem], Set.empty)+ Right identifiers -> ([], identifiers)+ absentNested =+ [ NestedManifestProblem ("output/" <> run <> "/manifest.json: missing")+ | manifest.outcome == Cell.Completed,+ run <- Set.toList (expectedIds Set.\\ nestedIds)+ ]+ unexpectedNested =+ [NestedManifestProblem ("output/" <> run <> "/manifest.json: run was not planned") | run <- Set.toList (nestedIds Set.\\ expectedIds)]+ nestedProblems <- concat <$> traverse (checkNested tree byPath manifest.payload.sha256) nested+ let problems = identityProblems <> workProblems <> planProblems <> absentNested <> unexpectedNested <> nestedProblems+ pure case problems of+ [] -> Right manifest+ first : rest -> Left (first :| rest)+ _ -> pure (Left (firstProblems submission result))++verifyCellRunWithStatus :: CellStatus -> FilePath -> IO (Either (NonEmpty VerifyProblem) CellManifest)+verifyCellRunWithStatus status tree = do+ verified <- verifyCellRun tree+ case verified of+ Left problems -> pure (Left problems)+ Right manifest -> do+ (digest, _) <- digestFile (tree </> "manifest.json")+ let problems =+ [StatusMismatch "status is not sealed or names another cell run" | status.phase /= Sealed || status.runId /= manifest.runId]+ <> [StatusMismatch "status sequence or outcome differs from cell manifest" | status.leaseSequence /= Just manifest.leaseSequence || status.outcome /= Just manifest.outcome]+ <> [StatusDigestMismatch | status.manifestSha256 /= Just digest]+ pure case problems of+ [] -> Right manifest+ first : rest -> Left (first :| rest)++readRequired :: (FromJSON document) => FilePath -> Map.Map Text Artifact -> Text -> IO (Either VerifyProblem document)+readRequired tree byPath relative = case Map.lookup relative byPath of+ Nothing -> pure (Left (Missing relative))+ Just _ -> do+ bytes <- LazyByteString.readFile (tree </> Text.unpack relative)+ pure case eitherDecode bytes of+ Left failure -> Left (InvalidManifest (relative <> ": " <> Text.pack failure))+ Right document -> Right document++firstProblems :: Either VerifyProblem left -> Either VerifyProblem right -> NonEmpty VerifyProblem+firstProblems left right = case catMaybes [either Just (const Nothing) left, either Just (const Nothing) right] of+ first : rest -> first :| rest+ [] -> InvalidManifest "missing required cell document" :| []++checkWork :: FilePath -> Map.Map Text Artifact -> Submission -> IO [VerifyProblem]+checkWork tree byPath submitted = case Map.lookup "submission/work" byPath of+ Nothing -> pure [Missing "submission/work"]+ Just artifact -> do+ (digest, size) <- digestFile (tree </> "submission" </> "work")+ pure [PayloadMismatch "submission work differs from its declared digest or size" | digest /= submitted.work.sha256 || size /= submitted.work.bytes || artifact.sha256 /= submitted.work.sha256 || artifact.bytes /= submitted.work.bytes]++readPlannedRuns :: FilePath -> IO (Either VerifyProblem (Set.Set Text))+readPlannedRuns tree = do+ bytes <- LazyByteString.readFile (tree </> "submission" </> "work")+ pure case eitherDecode bytes :: Either String Value of+ Left failure -> Left (InvalidManifest ("submission/work: " <> Text.pack failure))+ Right value -> case decodeWorkPlan value of+ Left failure -> Left (InvalidManifest ("submission/work: " <> failure))+ Right plan -> Right (Set.fromList (map (renderRunId . (.runId)) plan.runs))++nestedManifestPaths :: [Artifact] -> [(Text, Text)]+nestedManifestPaths artifacts =+ [(run, artifact.path) | artifact <- artifacts, ["output", run, "manifest.json"] <- [Text.splitOn "/" artifact.path], Right _ <- [parseRunId run]]++checkNested :: FilePath -> Map.Map Text Artifact -> Text -> (Text, Text) -> IO [VerifyProblem]+checkNested tree byPath bundleDigest (run, path) = do+ decoded <- readRequired tree byPath path :: IO (Either VerifyProblem Manifest)+ case decoded of+ Left problem -> pure [problem]+ Right nested+ | renderRunId nested.runId /= run -> pure [NestedManifestProblem (path <> ": run ID differs from directory")]+ | otherwise -> do+ let relativeFiles = map (Text.pack . (.path)) nested.files+ if any (not . safeRelative) relativeFiles || length relativeFiles /= Set.size (Set.fromList relativeFiles)+ then pure [NestedManifestProblem (path <> ": unsafe or duplicate nested path")]+ else do+ let prefix = "output/" <> run <> "/"+ expected = sort (path : map (prefix <>) relativeFiles)+ actual = sort [name | name <- Map.keys byPath, prefix `Text.isPrefixOf` name]+ setProblem = [ManifestsDisagree (path <> ": nested file set differs from cell manifest") | expected /= actual]+ metadataProblems =+ [ ManifestsDisagree (full <> ": digest or size differs between manifests")+ | file <- nested.files,+ let full = prefix <> Text.pack file.path,+ Just artifact <- [Map.lookup full byPath],+ Just artifact.sha256 /= Text.stripPrefix "sha256:" file.sha256 || fromIntegral artifact.bytes /= file.bytes+ ]+ checked <- verifyManifest (tree </> "output" </> Text.unpack run)+ let kernelProblems = case checked of+ Left problems -> [NestedManifestProblem (path <> ": " <> Text.pack (show problems))]+ Right () -> []+ fingerprintProblems <- checkFingerprint tree prefix run bundleDigest+ pure (setProblem <> metadataProblems <> kernelProblems <> fingerprintProblems)++checkFingerprint :: FilePath -> Text -> Text -> Text -> IO [VerifyProblem]+checkFingerprint tree prefix run expected = do+ let relative = prefix <> "run-result.json"+ present <- doesFileExist (tree </> Text.unpack relative)+ if not present+ then pure [RunResultMismatch (relative <> ": missing")]+ else do+ bytes <- LazyByteString.readFile (tree </> Text.unpack relative)+ pure case eitherDecode bytes :: Either String Value of+ Left failure -> [RunResultMismatch (relative <> ": " <> Text.pack failure)]+ Right value ->+ [RunResultMismatch (relative <> ": run ID differs from directory") | resultRunId value /= Just run]+ <> [RunResultMismatch (relative <> ": payload fingerprint differs from submission") | fingerprintDigest value /= Just expected]++resultRunId :: Value -> Maybe Text+resultRunId (Object result) = do+ String identifier <- KeyMap.lookup "runId" result+ pure identifier+resultRunId _ = Nothing++fingerprintDigest :: Value -> Maybe Text+fingerprintDigest (Object result) = do+ Object fingerprint <- KeyMap.lookup "fingerprint" result+ Object cell <- KeyMap.lookup "cell" fingerprint+ Object payload <- KeyMap.lookup "payload" cell+ String digest <- KeyMap.lookup "bundleSha256" payload+ pure digest+fingerprintDigest _ = Nothing++safeRelative :: Text -> Bool+safeRelative value =+ not (Text.null value)+ && not (Text.isPrefixOf "/" value)+ && not (Text.any (`elem` ['\\', '\0']) value)+ && all (\part -> not (Text.null part) && part /= "." && part /= "..") (Text.splitOn "/" value)++listTree :: FilePath -> IO ([Text], [Text])+listTree root = go ""+ where+ go relative = do+ let directory = if Text.null relative then root else root </> Text.unpack relative+ names <- listDirectory directory+ parts <- forM names \name -> do+ let path = if Text.null relative then Text.pack name else relative <> "/" <> Text.pack name+ full = root </> Text.unpack path+ symbolic <- pathIsSymbolicLink full+ if symbolic+ then pure ([], [path])+ else do+ directoryEntry <- doesDirectoryExist full+ if directoryEntry+ then go path+ else do+ regular <- doesFileExist full+ pure (if regular then [path] else [], if regular then [] else [path])+ pure (concatMap fst parts, concatMap snd parts)++matches :: FilePath -> Artifact -> IO Bool+matches path artifact = do+ (digest, size) <- digestFile path+ pure (digest == artifact.sha256 && size == artifact.bytes)++digestFile :: FilePath -> IO (Text, Int64)+digestFile path = withBinaryFile path ReadMode (go SHA256.init 0)+ where+ go context size handle = do+ chunk <- ByteString.hGetSome handle 65536+ if ByteString.null chunk+ then pure (TextEncoding.decodeUtf8 (Base16.encode (SHA256.finalize context)), size)+ else do+ let nextContext = SHA256.update context chunk+ nextSize = size + fromIntegral (ByteString.length chunk)+ nextContext `seq` nextSize `seq` go nextContext nextSize handle++writeVerified :: FilePath -> FilePath -> LazyByteString.ByteString -> IO ()+writeVerified root relative bytes = do+ let target = root </> relative+ withTemporary target \temporary -> LazyByteString.writeFile temporary bytes >> renameFile temporary target++withTemporary :: FilePath -> (FilePath -> IO value) -> IO value+withTemporary target action = do+ createDirectoryIfMissing True (takeDirectory target)+ bracket (openBinaryTempFile (takeDirectory target) ".kenshou-fetch-") cleanup \(temporary, handle) -> do+ hClose handle+ action temporary+ where+ cleanup (temporary, _) = do+ exists <- doesFileExist temporary+ if exists then removeFile temporary else pure ()++firstFailure :: [Maybe failure] -> Maybe failure+firstFailure = listToMaybe . catMaybes
+ src/Kenshou/Remote/Cell/Health.hs view
@@ -0,0 +1,103 @@+module Kenshou.Remote.Cell.Health (mapCellHealth, withCellHealthFile) where++import Control.Concurrent (forkIO, killThread, threadDelay)+import Control.Exception (IOException, catch, finally, mask_)+import Data.Aeson (Value, eitherDecodeStrict', encode, object, withObject, (.:), (.:?), (.=))+import Data.Aeson.Types (Parser, parseEither)+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as TextIO+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId)+import System.Directory (doesFileExist)+import System.IO (stderr)++-- The cell owner emits a health document for each observation window. Keep+-- only tripped gates, because an ok gate is not a run health notice.+mapCellHealth :: RunId -> Value -> Either Text [Value]+mapCellHealth expected document = either (Left . Text.pack) Right (parseEither parseDocument document)+ where+ parseDocument = withObject "cell health" \fields -> do+ schema <- fields .: "schema"+ if schema == ("cell.health/v1" :: Text) then pure () else fail "unsupported cell health schema"+ runId <- fields .: "runId"+ if runId == expected then pure () else fail "cell health run ID differs from the active cell run"+ gates <- fields .: "gates" :: Parser [Value]+ concat <$> traverse parseGate gates++ parseGate = withObject "cell health gate" \gate -> do+ name <- gate .: "name" :: Parser Text+ machine <- gate .: "machine" :: Parser Text+ status <- gate .: "status" :: Parser Text+ at <- gate .: "at" :: Parser UTCTime+ detail <- gate .:? "detail" :: Parser (Maybe Text)+ if name `elem` ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"] && not (Text.null machine)+ then pure ()+ else fail "unknown cell health gate or empty machine"+ case status of+ "ok" -> pure []+ "tripped" ->+ pure+ [ object+ [ "schema" .= ("kenshou.health-notice/v1" :: Text),+ "source" .= ("cell-health:" <> name <> ":" <> machine),+ "severity" .= (if name == "background-load" then "soft" else "hard" :: Text),+ "at" .= at,+ "detail" .= maybe ("cell health gate tripped: " <> name) (\message -> if Text.null message then "cell health gate tripped: " <> name else message) detail+ ]+ ]+ _ -> fail "unknown cell health gate status"++-- Re-read the rolling file because its producer may replace it atomically or+-- append JSON lines. Deduplicate notices across repeated health snapshots.+withCellHealthFile :: Maybe FilePath -> FilePath -> RunId -> IO result -> IO result+withCellHealthFile Nothing _ _ action = action+withCellHealthFile (Just source) destination runId action = do+ initial <- safePoll Set.empty+ latest <- newIORef initial+ thread <- forkIO (loop latest initial)+ action `finally` do+ killThread thread+ readIORef latest >>= safePoll >>= writeIORef latest+ where+ loop latest seen = do+ threadDelay 1000000+ next <- mask_ do+ updated <- safePoll seen+ writeIORef latest updated+ pure updated+ loop latest next++ safePoll seen =+ poll seen `catch` \(failure :: IOException) -> do+ TextIO.hPutStrLn stderr ("kenshou cell health: " <> Text.pack (show failure))+ pure seen++ poll seen = do+ exists <- doesFileExist source+ if not exists+ then pure seen+ else do+ bytes <- ByteString.readFile source+ foldLines seen (completeLines bytes)++ foldLines seen [] = pure seen+ foldLines seen (line : rest) = do+ case eitherDecodeStrict' line >>= firstText . mapCellHealth runId of+ Left problem -> TextIO.hPutStrLn stderr ("kenshou cell health: " <> Text.pack problem) >> foldLines seen rest+ Right notices -> do+ let new = filter (\notice -> encode notice `Set.notMember` seen) notices+ mapM_ (LazyByteString.appendFile destination . (<> "\n") . encode) new+ foldLines (foldr (Set.insert . encode) seen new) rest++ firstText = either (Left . Text.unpack) Right++completeLines :: ByteString.ByteString -> [ByteString.ByteString]+completeLines bytes | ByteString.null bytes = []+completeLines bytes = case ByteString.split 10 bytes of+ [] -> []+ parts -> filter (not . ByteString.null) (if ByteString.last bytes == 10 then parts else init parts)
+ src/Kenshou/Remote/Cell/Index.hs view
@@ -0,0 +1,195 @@+module Kenshou.Remote.Cell.Index+ ( CellManifestLink (..),+ EvidenceLink (..),+ RunLink (..),+ CellRunIndex (..),+ deriveCellRunIndex,+ writeCellRunIndex,+ )+where++import Control.Exception (bracket)+import Control.Monad (forM)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (FromJSON (..), ToJSON (..), Value, eitherDecode, encode, object, withObject, (.:), (.:?), (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Time (UTCTime, getCurrentTime)+import Kenshou.Core.Id (RunId, parseRunId, renderRunId)+import Kenshou.Core.Outcome (Outcome)+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellRunResult (..), CellStatus)+import Kenshou.Remote.Cell.Fetch (VerifyProblem (..), effectiveOutcome, verifyCellRun, verifyCellRunWithStatus)+import Kenshou.Remote.Store (Bucket (..))+import System.Directory (doesFileExist, removeFile, renameFile)+import System.FilePath (takeDirectory, (</>))+import System.IO (hClose, openBinaryTempFile)++data CellManifestLink = CellManifestLink+ { path :: !Text,+ sha256 :: !Text,+ bytes :: !Int64+ }+ deriving stock (Eq, Show)++data EvidenceLink = EvidenceLink+ { kind :: !Text,+ path :: !Text,+ sha256 :: !Text,+ bytes :: !Int64,+ mediaType :: !Text+ }+ deriving stock (Eq, Show)++data RunLink = RunLink+ { runId :: !RunId,+ path :: !Text,+ scenario :: !Text,+ manifestSha256 :: !Text,+ recordedOutcome :: !Outcome,+ effectiveOutcome :: !Outcome,+ overrideReason :: !(Maybe Text)+ }+ deriving stock (Eq, Show)++data CellRunIndex = CellRunIndex+ { cellRun :: !RunId,+ cell :: !Text,+ leaseId :: !RunId,+ leaseSequence :: !Int,+ resultsBaseUri :: !Text,+ dataBaseUri :: !Text,+ cellManifest :: !CellManifestLink,+ cellOutcome :: !CellOutcome,+ reasons :: ![Text],+ entryExitCode :: !(Maybe Int),+ evidence :: ![EvidenceLink],+ runs :: ![RunLink],+ verifiedAt :: !UTCTime+ }+ deriving stock (Eq, Show)++instance ToJSON CellManifestLink where+ toJSON link = object ["path" .= link.path, "sha256" .= link.sha256, "bytes" .= link.bytes]++instance FromJSON CellManifestLink where+ parseJSON = withObject "cell manifest link" \value -> CellManifestLink <$> value .: "path" <*> value .: "sha256" <*> value .: "bytes"++instance ToJSON EvidenceLink where+ toJSON link = object ["kind" .= link.kind, "path" .= link.path, "sha256" .= link.sha256, "bytes" .= link.bytes, "mediaType" .= link.mediaType]++instance FromJSON EvidenceLink where+ parseJSON = withObject "cell evidence link" \value -> EvidenceLink <$> value .: "kind" <*> value .: "path" <*> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"++instance ToJSON RunLink where+ toJSON run = object ["runId" .= run.runId, "path" .= run.path, "scenario" .= run.scenario, "manifestSha256" .= run.manifestSha256, "recordedOutcome" .= run.recordedOutcome, "effectiveOutcome" .= run.effectiveOutcome, "overrideReason" .= run.overrideReason]++instance FromJSON RunLink where+ parseJSON = withObject "cell nested run" \value -> RunLink <$> value .: "runId" <*> value .: "path" <*> value .: "scenario" <*> value .: "manifestSha256" <*> value .: "recordedOutcome" <*> value .: "effectiveOutcome" <*> value .:? "overrideReason"++instance ToJSON CellRunIndex where+ toJSON index =+ object+ [ "schema" .= ("kenshou.cell-run/v1" :: Text),+ "cellRun" .= index.cellRun,+ "cell" .= index.cell,+ "leaseId" .= index.leaseId,+ "leaseSequence" .= index.leaseSequence,+ "resultsBaseUri" .= index.resultsBaseUri,+ "dataBaseUri" .= index.dataBaseUri,+ "cellManifest" .= index.cellManifest,+ "cellOutcome" .= index.cellOutcome,+ "reasons" .= index.reasons,+ "entryExitCode" .= index.entryExitCode,+ "evidence" .= index.evidence,+ "runs" .= index.runs,+ "verifiedAt" .= index.verifiedAt+ ]++instance FromJSON CellRunIndex where+ parseJSON = withObject "cell run index" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.cell-run/v1" :: Text) then fail "unsupported cell run index" else pure ()+ CellRunIndex <$> value .: "cellRun" <*> value .: "cell" <*> value .: "leaseId" <*> value .: "leaseSequence" <*> value .: "resultsBaseUri" <*> value .: "dataBaseUri" <*> value .: "cellManifest" <*> value .: "cellOutcome" <*> value .: "reasons" <*> value .:? "entryExitCode" <*> value .: "evidence" <*> value .: "runs" <*> value .: "verifiedAt"++deriveCellRunIndex :: Bucket -> Maybe CellStatus -> FilePath -> IO (Either (NonEmpty VerifyProblem) CellRunIndex)+deriveCellRunIndex bucket status tree = do+ verified <- case status of+ Nothing -> verifyCellRun tree+ Just sealed -> verifyCellRunWithStatus sealed tree+ case verified of+ Left problems -> pure (Left problems)+ Right manifest -> do+ resultBytes <- LazyByteString.readFile (tree </> "cell" </> "result.json")+ case eitherDecode resultBytes :: Either String CellRunResult of+ Left failure -> pure (Left (InvalidManifest ("cell/result.json: " <> Text.pack failure) :| []))+ Right cellResult -> do+ manifestBytes <- LazyByteString.readFile (tree </> "manifest.json")+ let base = "gs://" <> bucket.unBucket <> "/runs/" <> renderRunId manifest.runId+ linked = CellManifestLink "manifest.json" (digest manifestBytes) (LazyByteString.length manifestBytes)+ evidence = [EvidenceLink kind artifact.path artifact.sha256 artifact.bytes artifact.mediaType | artifact <- manifest.artifacts, Just kind <- [evidenceKind artifact.path]]+ indexed <- forM (nestedRuns manifest.artifacts) \(identifier, nestedPath) -> do+ bytes <- LazyByteString.readFile (tree </> Text.unpack nestedPath </> "run-result.json")+ nestedManifest <- LazyByteString.readFile (tree </> Text.unpack nestedPath </> "manifest.json")+ pure case runFields bytes of+ Left failure -> Left (RunResultMismatch (nestedPath <> ": " <> failure))+ Right (scenario, recorded) ->+ let effective = effectiveOutcome manifest.outcome True recorded+ override = if manifest.outcome == InfrastructureFailure || effective /= recorded then Just (reasonFor cellResult.reasons) else Nothing+ in Right (RunLink identifier nestedPath scenario (digest nestedManifest) recorded effective override)+ case sequence indexed of+ Left problem -> pure (Left (problem :| []))+ Right runs -> do+ now <- getCurrentTime+ pure (Right (CellRunIndex manifest.runId manifest.cell manifest.leaseId manifest.leaseSequence base (base <> "/output") linked manifest.outcome cellResult.reasons cellResult.entryExitCode evidence runs now))++writeCellRunIndex :: FilePath -> CellRunIndex -> IO FilePath+writeCellRunIndex tree index = do+ let destination = takeDirectory tree </> "cell-run.json"+ bracket (openBinaryTempFile (takeDirectory tree) ".kenshou-cell-run-") cleanup \(temporary, handle) -> do+ hClose handle+ LazyByteString.writeFile temporary (encode index)+ renameFile temporary destination+ pure destination+ where+ cleanup (temporary, _) = do+ exists <- doesFileExist temporary+ if exists then removeFile temporary else pure ()++nestedRuns :: [Artifact] -> [(RunId, Text)]+nestedRuns artifacts =+ [ (identifier, "output/" <> renderRunId identifier)+ | artifact <- artifacts,+ ["output", rendered, "manifest.json"] <- [Text.splitOn "/" artifact.path],+ Right identifier <- [parseRunId rendered]+ ]++runFields :: LazyByteString.ByteString -> Either Text (Text, Outcome)+runFields bytes = case eitherDecode bytes :: Either String Value of+ Left failure -> Left (Text.pack failure)+ Right (Aeson.Object fields) -> case (KeyMap.lookup "scenario" fields, KeyMap.lookup "outcome" fields) of+ (Just (Aeson.String scenario), Just verdict) -> case Aeson.fromJSON verdict of+ Aeson.Success outcome -> Right (scenario, outcome)+ Aeson.Error failure -> Left (Text.pack failure)+ _ -> Left "scenario or outcome is missing"+ Right _ -> Left "run result is not an object"++evidenceKind :: Text -> Maybe Text+evidenceKind "cell/fingerprint.json" = Just "cell-fingerprint"+evidenceKind "cell/health.json" = Just "cell-health"+evidenceKind "cell/reset-evidence.json" = Just "cell-reset-evidence"+evidenceKind "metrics/export.jsonl.zst" = Just "cell-metrics-export"+evidenceKind _ = Nothing++reasonFor :: [Text] -> Text+reasonFor [] = "cell-infrastructure-failure"+reasonFor reasons = Text.intercalate "; " reasons++digest :: LazyByteString.ByteString -> Text+digest = TextEncoding.decodeUtf8 . Base16.encode . SHA256.hashlazy
+ src/Kenshou/Remote/Cell/Lease.hs view
@@ -0,0 +1,332 @@+module Kenshou.Remote.Cell.Lease+ ( CellRef (..),+ LeaseId,+ Lease (..),+ Quarantine (..),+ LeaseRequest (..),+ LeaseHandle,+ AcquireOutcome (..),+ acquireLease,+ acquireLeaseWithId,+ renewLease,+ resizeLease,+ releaseLease,+ reattachLease,+ reattachActiveLease,+ requestCancel,+ leaseSnapshot,+ leaseHeld,+ withHeartbeat,+ validCellName,+ )+where++import Control.Concurrent (forkIO, killThread, threadDelay)+import Control.Concurrent.MVar (MVar, modifyMVar, modifyMVar_, newMVar, readMVar)+import Control.Exception (SomeAsyncException, SomeException, bracket, displayException, fromException, throwIO, try)+import Control.Monad (unless)+import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecode, encode, object, withObject, (.:), (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, getCurrentTime)+import Kenshou.Core.Id (RunId, newRunId)+import Kenshou.Remote.Store (Bucket, ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import System.IO (hPutStrLn, stderr)++data CellRef = CellRef+ { cellName :: !Text,+ controlBucket :: !Bucket+ }+ deriving stock (Eq, Show)++type LeaseId = RunId++data Lease = Lease+ { leaseId :: !LeaseId,+ cell :: !Text,+ owner :: !Text,+ purpose :: !Text,+ ttlSeconds :: !Int,+ acquiredAt :: !UTCTime,+ heartbeatAt :: !UTCTime,+ cancelRequested :: !Bool,+ runsStarted :: !Int+ }+ deriving stock (Eq, Show)++data Quarantine = Quarantine+ { cell :: !Text,+ reason :: !Text,+ runId :: !(Maybe Text),+ at :: !UTCTime+ }+ deriving stock (Eq, Show)++data LeaseRequest = LeaseRequest+ { owner :: !Text,+ purpose :: !Text,+ ttlSeconds :: !Int+ }+ deriving stock (Eq, Show)++data LeaseHandle = LeaseHandle+ { state :: !(MVar (Lease, Int64, Bool))+ }++data AcquireOutcome = Acquired !LeaseHandle | Busy !Lease | Quarantined !Quarantine++instance ToJSON Lease where+ toJSON lease =+ object+ [ "schema" .= ("cell.lease/v1" :: Text),+ "leaseId" .= lease.leaseId,+ "cell" .= lease.cell,+ "owner" .= lease.owner,+ "purpose" .= lease.purpose,+ "ttlSeconds" .= lease.ttlSeconds,+ "acquiredAt" .= lease.acquiredAt,+ "heartbeatAt" .= lease.heartbeatAt,+ "cancelRequested" .= lease.cancelRequested,+ "runsStarted" .= lease.runsStarted+ ]++instance FromJSON Lease where+ parseJSON = withObject "cell lease" \value -> do+ schema <- value .: "schema"+ unless (schema == ("cell.lease/v1" :: Text)) (fail "unsupported cell lease schema")+ lease <- Lease <$> value .: "leaseId" <*> value .: "cell" <*> value .: "owner" <*> value .: "purpose" <*> value .: "ttlSeconds" <*> value .: "acquiredAt" <*> value .: "heartbeatAt" <*> value .: "cancelRequested" <*> value .: "runsStarted"+ unless (validCellName lease.cell && not (Text.null lease.owner) && not (Text.null lease.purpose) && lease.ttlSeconds > 0 && lease.runsStarted >= 0) (fail "invalid cell lease")+ pure lease++instance ToJSON Quarantine where+ toJSON quarantine =+ object+ [ "schema" .= ("cell.quarantine/v1" :: Text),+ "cell" .= quarantine.cell,+ "reason" .= quarantine.reason,+ "runId" .= quarantine.runId,+ "at" .= quarantine.at+ ]++instance FromJSON Quarantine where+ parseJSON = withObject "cell quarantine" \value -> do+ schema <- value .: "schema"+ unless (schema == ("cell.quarantine/v1" :: Text)) (fail "unsupported cell quarantine schema")+ quarantine <- Quarantine <$> value .: "cell" <*> value .: "reason" <*> value .: "runId" <*> value .: "at"+ unless (validCellName quarantine.cell && not (Text.null quarantine.reason)) (fail "invalid cell quarantine")+ pure quarantine++acquireLease :: ObjectStore -> CellRef -> LeaseRequest -> IO AcquireOutcome+acquireLease store ref request = newRunId >>= acquireLeaseWithId store ref request++-- The caller may durably retain this identity before issuing a remote claim.+-- Reconciliation never renews or revives an expired/cancelled claim.+acquireLeaseWithId :: ObjectStore -> CellRef -> LeaseRequest -> LeaseId -> IO AcquireOutcome+acquireLeaseWithId store ref request proposedId = do+ unless (validCellName ref.cellName && not (Text.null request.owner) && not (Text.null request.purpose) && request.ttlSeconds > 0) (ioError (userError "invalid lease request"))+ current <- readCurrent store ref+ now <- store.serverTime+ case current of+ Just (lease, meta)+ | lease.leaseId == proposedId ->+ if not lease.cancelRequested && addUTCTime (fromIntegral lease.ttlSeconds + 30) meta.updated >= now+ then accept lease meta+ else pure (Busy lease)+ _ -> acquire proposedId (0 :: Int)+ where+ acquire identifier attempts = do+ unless (attempts < maxCasAttempts) (ioError (userError "lease claim contention exceeded retry limit"))+ now <- store.serverTime+ let proposed = Lease identifier ref.cellName request.owner request.purpose request.ttlSeconds now now False 0+ result <- store.putObject ref.controlBucket (leaseName ref) "application/json" DoesNotExist (encode proposed)+ case result of+ Written meta -> accept proposed meta+ PreconditionFailed -> do+ observed <- readCurrent store ref+ case observed of+ Nothing -> acquire identifier (attempts + 1)+ Just (current, meta) -> do+ clock <- store.serverTime+ if addUTCTime (fromIntegral current.ttlSeconds + 30) meta.updated < clock+ then do+ replaced <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs meta.generation) (encode proposed)+ case replaced of+ Written next -> accept proposed next+ PreconditionFailed -> acquire identifier (attempts + 1)+ else pure (Busy current)+ accept lease meta = do+ quarantine <- readQuarantine store ref+ case quarantine of+ Nothing -> Acquired <$> newHandle lease meta.generation+ Just record -> do+ released <- store.deleteObject ref.controlBucket (leaseName ref) (GenerationIs meta.generation)+ unless released (ioError (userError "quarantined cell lease could not be released"))+ pure (Quarantined record)++renewLease :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool+renewLease store ref handle = modifyMVar handle.state \(lease, generation, held) ->+ if not held || lease.cancelRequested+ then pure ((lease, generation, False), False)+ else renewCurrent lease generation (0 :: Int)+ where+ renewCurrent lease generation attempts = do+ unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))+ now <- store.serverTime+ let renewed = lease {heartbeatAt = now}+ result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs generation) (encode renewed)+ case result of+ Written meta -> pure ((renewed, meta.generation, True), True)+ PreconditionFailed -> do+ current <- readCurrent store ref+ case current of+ Just (observed, meta)+ | attempts + 1 < maxCasAttempts,+ observed.leaseId == lease.leaseId,+ not observed.cancelRequested,+ addUTCTime (fromIntegral observed.ttlSeconds + 30) meta.updated >= now ->+ renewCurrent observed meta.generation (attempts + 1)+ _ -> pure ((lease, generation, False), False)++-- A detached submission needs a lease that outlives its wall-clock budget.+-- Fence the TTL change before publishing its marker.+resizeLease :: ObjectStore -> CellRef -> LeaseHandle -> Int -> IO Bool+resizeLease store ref handle ttl = do+ unless (ttl > 0) (ioError (userError "lease TTL must be positive"))+ modifyMVar handle.state \(lease, generation, held) ->+ if not held || lease.cancelRequested+ then pure ((lease, generation, False), False)+ else do+ unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))+ now <- store.serverTime+ let resized = lease {ttlSeconds = ttl, heartbeatAt = now}+ result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs generation) (encode resized)+ case result of+ Written meta -> pure ((resized, meta.generation, True), True)+ PreconditionFailed -> pure ((lease, generation, False), False)++releaseLease :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool+releaseLease store ref handle = modifyMVar handle.state \(lease, generation, held) ->+ if not held+ then pure ((lease, generation, False), False)+ else do+ unless (lease.cell == ref.cellName) (ioError (userError "lease handle names another cell"))+ released <- store.deleteObject ref.controlBucket (leaseName ref) (GenerationIs generation)+ pure ((lease, generation, False), released)++reattachLease :: ObjectStore -> CellRef -> LeaseId -> IO (Maybe LeaseHandle)+reattachLease store ref identifier = do+ current <- readCurrent store ref+ case current of+ Just (lease, meta) | lease.leaseId == identifier -> Just <$> newHandle lease meta.generation+ _ -> pure Nothing++-- Performance recovery must treat an expired or cancelled lease as lost, even+-- when its old object still exists. The unfenced reattachment above remains+-- available to cancellation/release code that needs its original generation.+reattachActiveLease :: ObjectStore -> CellRef -> LeaseId -> IO (Maybe LeaseHandle)+reattachActiveLease store ref identifier = do+ current <- readCurrent store ref+ now <- store.serverTime+ case current of+ Just (lease, meta)+ | lease.leaseId == identifier,+ not lease.cancelRequested,+ addUTCTime (fromIntegral lease.ttlSeconds + 30) meta.updated >= now ->+ Just <$> newHandle lease meta.generation+ _ -> pure Nothing++requestCancel :: ObjectStore -> CellRef -> Maybe LeaseId -> Bool -> IO Bool+requestCancel store ref expected force = do+ unless (force || isJust expected) (ioError (userError "cancelling another owner's lease requires force"))+ attempt (0 :: Int)+ where+ attempt count = do+ unless (count < maxCasAttempts) (ioError (userError "lease cancellation contention exceeded retry limit"))+ current <- readCurrent store ref+ case current of+ Nothing -> pure False+ Just (lease, meta)+ | not force && Just lease.leaseId /= expected -> pure False+ | lease.cancelRequested -> pure True+ | otherwise -> do+ result <- store.putObject ref.controlBucket (leaseName ref) "application/json" (GenerationIs meta.generation) (encode (lease {cancelRequested = True}))+ case result of+ Written _ -> pure True+ PreconditionFailed -> attempt (count + 1)++leaseHeld :: LeaseHandle -> IO Bool+leaseHeld handle = (\(_, _, held) -> held) <$> readMVar handle.state++leaseSnapshot :: LeaseHandle -> IO Lease+leaseSnapshot handle = (\(lease, _, _) -> lease) <$> readMVar handle.state++withHeartbeat :: ObjectStore -> CellRef -> LeaseHandle -> (IO Bool -> IO value) -> IO value+withHeartbeat store ref handle action = bracket start killThread (const (action (leaseHeld handle)))+ where+ start = forkIO loop+ loop = do+ (lease, _, held) <- readMVar handle.state+ if held+ then do+ threadDelay (max 100000 (lease.ttlSeconds * 1000000 `div` 3))+ renew+ else pure ()+ renew = do+ result <- try (renewLease store ref handle) :: IO (Either SomeException Bool)+ case result of+ Right True -> loop+ Right False -> markLost+ Left failure | Just async <- (fromException failure :: Maybe SomeAsyncException) -> throwIO async+ Left failure -> do+ hPutStrLn stderr ("cell lease heartbeat renewal failed; retrying before TTL: " <> displayException failure)+ current <- leaseSnapshot handle+ now <- getCurrentTime+ if now <= addUTCTime (fromIntegral current.ttlSeconds + 30) current.heartbeatAt+ then threadDelay 1000000 >> renew+ else markLost+ markLost = modifyMVar_ handle.state \(current, generation, _) -> pure (current, generation, False)++newHandle :: Lease -> Int64 -> IO LeaseHandle+newHandle lease generation = LeaseHandle <$> newMVar (lease, generation, True)++readCurrent :: ObjectStore -> CellRef -> IO (Maybe (Lease, ObjectMeta))+readCurrent store ref = do+ stored <- store.getObject ref.controlBucket (leaseName ref)+ case stored of+ Nothing -> pure Nothing+ Just (bytes, meta) -> do+ lease <- decodeDocument "cell lease" bytes+ unless (lease.cell == ref.cellName) (ioError (userError "cell lease names another cell"))+ pure (Just (lease, meta))++readQuarantine :: ObjectStore -> CellRef -> IO (Maybe Quarantine)+readQuarantine store ref = do+ stored <- store.getObject ref.controlBucket (quarantineName ref)+ case stored of+ Nothing -> pure Nothing+ Just (bytes, _) -> do+ record <- decodeDocument "cell quarantine" bytes+ unless (record.cell == ref.cellName) (ioError (userError "cell quarantine names another cell"))+ pure (Just record)++decodeDocument :: (FromJSON value) => String -> LazyByteString.ByteString -> IO value+decodeDocument label bytes = either (ioError . userError . ((label <> ": ") <>)) pure (eitherDecode bytes)++leaseName :: CellRef -> ObjectName+leaseName ref = ObjectName ("cells/" <> ref.cellName <> "/lease.json")++quarantineName :: CellRef -> ObjectName+quarantineName ref = ObjectName ("cells/" <> ref.cellName <> "/quarantine.json")++validCellName :: Text -> Bool+validCellName value =+ Text.length value >= 3+ && Text.length value <= 63+ && Text.head value `elem` (['a' .. 'z'] <> ['0' .. '9'])+ && Text.all (\character -> character `elem` (['a' .. 'z'] <> ['0' .. '9'] <> "-")) value++maxCasAttempts :: Int+maxCasAttempts = 32
+ src/Kenshou/Remote/Cell/Legacy.hs view
@@ -0,0 +1,41 @@+-- | The one place that maps the cell protocol's dedicated broker fields onto the+-- named resource that documents once carried as a top-level field. The cell agent+-- in mori://shinzui/load-testing-infra still describes a broker with+-- @CellEnvironment.broker@ and resets it with @reset.broker@; this module binds and+-- requests it through the generic resource path. Delete it when load-testing-infra+-- publishes @resources@ in the cell environment and accepts @reset.resources@ in+-- the submission.+module Kenshou.Remote.Cell.Legacy+ ( brokerAsResource,+ legacyBrokerReset,+ legacyBrokerAvailable,+ )+where++import Data.Aeson (Value, object, (.=))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Resource.Types (legacyKafkaName)+import Kenshou.Remote.Cell.Docs (BrokerReset (..), CellBroker (..), CellDescriptor (..), CellImages (..), ResetBlock (..))++-- | The resource value a cell broker binds to, exactly as the cell client has+-- always passed it to scenarios.+brokerAsResource :: CellBroker -> Either Text Value+brokerAsResource broker+ | null addresses = Left "cell broker has no bootstrap servers"+ | otherwise = Right (object ["backend" .= ("external" :: Text), "brokers" .= addresses, "lanes" .= (0 :: Int)])+ where+ addresses = filter (not . Text.null) (fmap Text.strip (Text.splitOn "," broker.bootstrapServers))++-- | Move a reset of that resource into the dedicated broker wipe. The entry leaves+-- @reset.resources@, because the cell agent decodes submissions strictly and would+-- reject the extension; the submission keeps the exact shape it always had.+legacyBrokerReset :: ResetBlock -> ResetBlock+legacyBrokerReset reset+ | Map.member legacyKafkaName reset.resources = reset {broker = Just (BrokerReset True), resources = Map.delete legacyKafkaName reset.resources}+ | otherwise = reset++-- | Whether the cell's descriptor names a broker image.+legacyBrokerAvailable :: CellDescriptor -> Bool+legacyBrokerAvailable descriptor = not (Text.null descriptor.images.broker)
+ src/Kenshou/Remote/Cell/Parity.hs view
@@ -0,0 +1,189 @@+module Kenshou.Remote.Cell.Parity+ ( ParityOptions (..),+ ParityDifference (..),+ ParityReport (..),+ compareForParity,+ )+where++import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (ToJSON (..), Value (..), eitherDecodeStrict', object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.Foldable (toList)+import Data.List (isPrefixOf, sort)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import System.Directory (doesDirectoryExist, listDirectory)+import System.FilePath (takeExtension, (</>))++newtype ParityOptions = ParityOptions {volatilePaths :: [Text]}+ deriving stock (Eq, Show)++data ParityDifference = ParityDifference+ { path :: !Text,+ localValue :: !(Maybe Value),+ cellValue :: !(Maybe Value)+ }+ deriving stock (Eq, Show)++instance ToJSON ParityDifference where+ toJSON difference =+ object+ [ "path" .= difference.path,+ "local" .= difference.localValue,+ "cell" .= difference.cellValue+ ]++data ParityReport = ParityReport+ { schema :: !Text,+ localRunResultSha256 :: !Text,+ cellRunResultSha256 :: !Text,+ equal :: ![Text],+ intentional :: ![ParityDifference],+ unexpected :: ![ParityDifference]+ }+ deriving stock (Eq, Show)++instance ToJSON ParityReport where+ toJSON report =+ object+ [ "schema" .= report.schema,+ "localRunResultSha256" .= report.localRunResultSha256,+ "cellRunResultSha256" .= report.cellRunResultSha256,+ "equal" .= report.equal,+ "intentional" .= report.intentional,+ "unexpected" .= report.unexpected+ ]++compareForParity :: ParityOptions -> FilePath -> FilePath -> IO ParityReport+compareForParity options localDir cellDir = do+ localBytes <- ByteString.readFile (localDir </> "run-result.json")+ cellBytes <- ByteString.readFile (cellDir </> "run-result.json")+ localResult <- decode "local run-result.json" localBytes+ cellResult <- decode "cell run-result.json" cellBytes+ localSpec <- readJson "local run-spec.json" (localDir </> "run-spec.json")+ cellSpec <- readJson "cell run-spec.json" (cellDir </> "run-spec.json")+ localVerdicts <- readVerdicts localDir+ cellVerdicts <- readVerdicts cellDir+ let comparisons =+ compareJson ["result"] localResult cellResult+ <> compareJson ["spec"] localSpec cellSpec+ <> compareVerdicts localVerdicts cellVerdicts+ allEqual = [renderPath path | (path, Just left, Just right) <- comparisons, left == right]+ differences = [ParityDifference (renderPath path) left right | (path, left, right) <- comparisons, left /= right]+ intentionalDiffs = filter (isIntentional options . (.path)) differences+ unexpectedDiffs = filter (not . isIntentional options . (.path)) differences+ pure+ ParityReport+ { schema = "kenshou.parity-report/v1",+ localRunResultSha256 = digest localBytes,+ cellRunResultSha256 = digest cellBytes,+ equal = allEqual,+ intentional = intentionalDiffs,+ unexpected = unexpectedDiffs+ }++decode :: Text -> ByteString.ByteString -> IO Value+decode label bytes =+ case eitherDecodeStrict' bytes of+ Left problem -> ioError (userError (Text.unpack label <> ": " <> problem))+ Right value -> pure value++readJson :: Text -> FilePath -> IO Value+readJson label path = ByteString.readFile path >>= decode label++readVerdicts :: FilePath -> IO (Map.Map Text Value)+readVerdicts runDir = do+ let verdictDir = runDir </> "verdicts"+ exists <- doesDirectoryExist verdictDir+ if not exists+ then pure Map.empty+ else do+ names <- sort . filter ((== ".json") . takeExtension) <$> listDirectory verdictDir+ Map.fromList+ <$> traverse+ ( \name -> do+ value <- readJson (Text.pack name) (verdictDir </> name)+ pure (Text.pack name, projectVerdict value)+ )+ names++projectVerdict :: Value -> Value+projectVerdict (Object value) =+ Object (KeyMap.filterWithKey (\key _ -> Key.toText key `elem` ["status", "counts"]) value)+projectVerdict value = value++compareVerdicts :: Map.Map Text Value -> Map.Map Text Value -> [([Text], Maybe Value, Maybe Value)]+compareVerdicts localVerdicts cellVerdicts =+ concatMap compareName (Set.toAscList (Map.keysSet localVerdicts `Set.union` Map.keysSet cellVerdicts))+ where+ compareName name =+ case (Map.lookup name localVerdicts, Map.lookup name cellVerdicts) of+ (Just localValue, Just cellValue) -> compareJson ["verdicts", name] localValue cellValue+ (left, right) -> [(["verdicts", name], left, right)]++compareJson :: [Text] -> Value -> Value -> [([Text], Maybe Value, Maybe Value)]+compareJson prefix localValue cellValue =+ [ (path, Map.lookup path localLeaves, Map.lookup path cellLeaves)+ | path <- Set.toAscList (Map.keysSet localLeaves `Set.union` Map.keysSet cellLeaves)+ ]+ where+ localLeaves = Map.fromList (flatten prefix localValue)+ cellLeaves = Map.fromList (flatten prefix cellValue)++flatten :: [Text] -> Value -> [([Text], Value)]+flatten prefix (Object value)+ | not (KeyMap.null value) = concatMap (\(key, child) -> flatten (prefix <> [Key.toText key]) child) (KeyMap.toList value)+flatten prefix (Array value)+ | not (null value) = concatMap (\(index, child) -> flatten (prefix <> [Text.pack (show index)]) child) (zip [0 :: Int ..] (toList value))+flatten prefix value = [(prefix, value)]++renderPath :: [Text] -> Text+renderPath = Text.intercalate "."++isIntentional :: ParityOptions -> Text -> Bool+isIntentional options rendered =+ natural || volatile+ where+ path = Text.splitOn "." rendered+ starts prefix = prefix `isPrefixOf` path+ natural =+ or+ [ starts ["result", "runId"],+ starts ["result", "timings"],+ starts ["result", "compatibility"],+ starts ["result", "invocation"],+ starts ["result", "spec", "sha256"],+ starts ["result", "spec", "path"],+ starts ["result", "cohort", "arch"],+ starts ["result", "cohort", "os"],+ starts ["result", "cohort", "resolver"],+ starts ["result", "cohort", "planHash"],+ starts ["result", "cohort", "cabalVersion"],+ starts ["result", "fingerprint", "cell"],+ starts ["result", "fingerprint", "host"],+ starts ["result", "fingerprint", "runtime"],+ starts ["result", "fingerprint", "machineProfile"],+ starts ["result", "fingerprint", "placement"],+ starts ["result", "fingerprint", "postgres"],+ starts ["result", "fingerprint", "kenshou", "executableSha256"],+ starts ["result", "fingerprint", "kenshou", "revision"],+ starts ["spec", "runId"],+ starts ["spec", "cohortExpectation"],+ starts ["spec", "environment", "placement"],+ starts ["spec", "environment", "machineProfile"],+ starts ["spec", "environment", "postgres"],+ path == ["spec", "labels"],+ starts ["spec", "labels", "postgresPlacement"]+ ]+ volatileScope = starts ["result", "summaries", "verdicts"] || starts ["verdicts"]+ volatile = volatileScope && any (\selected -> selected == rendered || (selected <> ".") `Text.isPrefixOf` rendered) options.volatilePaths++digest :: ByteString.ByteString -> Text+digest bytes = "sha256:" <> TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash bytes))
+ src/Kenshou/Remote/Cell/Prepare.hs view
@@ -0,0 +1,376 @@+module Kenshou.Remote.Cell.Prepare+ ( Granularity (..),+ RejectReason (..),+ PrepareOptions (..),+ Prepared (..),+ Routed (..),+ PreparedRun (..),+ Slice (..),+ OtlpSink (..),+ SubmissionInputs (..),+ sliceRuns,+ slicePlan,+ submissionFor,+ prepareForCell,+ routePlan,+ routePlanWithoutPayload,+ cellMachineProfile,+ )+where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe, isNothing)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (Registry, lookupResource, lookupScenario)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), resolveDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), renderRunId)+import Kenshou.Core.Resource (CellBinding (..), ResourceName, ResourceProvider (..))+import Kenshou.Core.Resource.Types (legacyKafkaName)+import Kenshou.Core.RunSpec (CohortExpectation (..), ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Scenario (..))+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Remote.Cell.Docs (BrokerReset (..), CachePolicy (..), CellDescriptor (..), CellImages (..), CollectOptions (..), Limits (..), PgReset (..), Requirements (..), ResetBlock (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Legacy (legacyBrokerAvailable, legacyBrokerReset)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities (..), RoutingRule (..), capabilityKnown, descriptorDigest, matchingRules)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))+import Kenshou.Remote.Store (Bucket (..))++cellMachineProfile :: CellDescriptor -> Text+cellMachineProfile descriptor =+ "gcp/" <> descriptor.zone <> "/shape-" <> Text.take 16 (Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor.shape)))) <> "/images-" <> Text.take 16 (Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor.images)))) <> "/pg" <> Text.pack (show descriptor.postgresMajor)++data Granularity = GranularityAuto | GranularityPlan | GranularityRun+ deriving stock (Eq, Show)++data RejectReason+ = PlacementLocalOnly+ | UnknownScenario+ | PgVersionMismatch Int Int+ | DurabilityNotDurable+ | NeedsServerControl Text+ | ResourceUnavailable ResourceName+ | PayloadLabelUnknown Text+ | ConflictingPostgresSetting Text+ | CollidingExtraPostgresName Text+ | RunIdMismatch+ | InvalidTimeout+ | MissingCapability Text+ | UnsupportedDimensions Text+ deriving stock (Eq, Show)++data PrepareOptions = PrepareOptions+ { coerceDurable :: !Bool,+ ephemeralOnDriver :: !Bool,+ pgSettings :: ![(Text, Text)],+ cachePolicy :: !CachePolicy+ }+ deriving stock (Eq, Show)++data Prepared = Prepared+ { accepted :: ![PreparedRun],+ rejected :: ![(PlannedRun, RejectReason)],+ warnings :: ![Text]+ }+ deriving stock (Eq, Show)++data Routed = Routed+ { perCell :: !(Map.Map Text RunPlan),+ local :: !(Maybe RunPlan),+ unroutable :: ![(PlannedRun, RejectReason)],+ warnings :: ![Text]+ }+ deriving stock (Eq, Show)++data PreparedRun = PreparedRun+ { ordinal :: !Int,+ runId :: !RunId,+ spec :: !RunSpec,+ payloadLabel :: !Text,+ reset :: !ResetBlock,+ timeoutSeconds :: !Int+ }+ deriving stock (Eq, Show)++data Slice = Slice+ { index :: !Int,+ payloadLabel :: !Text,+ reset :: !ResetBlock,+ entries :: !(NonEmpty PreparedRun),+ wallClockSeconds :: !Int+ }+ deriving stock (Eq, Show)++data OtlpSink = NullSink | FileSink+ deriving stock (Eq, Show)++data SubmissionInputs = SubmissionInputs+ { cellRun :: !RunId,+ leaseId :: !RunId,+ sessionId :: !RunId,+ planId :: !RunId,+ otlpSink :: !OtlpSink,+ rtsOptions :: !(Maybe Text),+ memoryMaxBytes :: !Int64,+ outputMaxBytes :: !Int64,+ minAgentVersion :: !Text,+ requiredCapabilities :: !(Maybe [Text])+ }+ deriving stock (Eq, Show)++-- Preflight a plan before it is sliced. Nothing is dropped silently: every+-- rejected entry remains paired with its original run identity.+prepareForCell :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Map.Map Text PayloadDescriptor -> PrepareOptions -> RunPlan -> Prepared+prepareForCell registry descriptor cachedCapabilities routingRules payloads options plan =+ prepareWithPayload registry descriptor cachedCapabilities routingRules (Just payloads) options plan++prepareWithPayload :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Maybe (Map.Map Text PayloadDescriptor) -> PrepareOptions -> RunPlan -> Prepared+prepareWithPayload registry descriptor cachedCapabilities routingRules payloads options plan =+ Prepared+ [prepared | Right (prepared, _) <- results]+ [(entry, reason) | (entry, Left reason) <- zip plan.runs results]+ (concat [notices | Right (_, notices) <- results])+ where+ results = fmap prepare plan.runs+ activeCapabilities = case cachedCapabilities of+ Just cache | cache.cell == descriptor.name && cache.descriptorSha256 == descriptorDigest descriptor -> Just cache+ _ -> Nothing+ profile = cellMachineProfile descriptor++ prepare entry = do+ if maybe True (== entry.runId) entry.spec.runId then pure () else Left RunIdMismatch+ scenario <- maybe (Left UnknownScenario) Right (lookupScenario registry entry.spec.scenario)+ if scenario.placement == PlaceLocal then Left PlacementLocalOnly else pure ()+ let label = maybe "default" (\trial -> if maybe False (Map.member trial.arm) payloads then trial.arm else "default") entry.trial+ payload <- traverse (\available -> maybe (Left (PayloadLabelUnknown label)) Right (Map.lookup label available)) payloads+ let requirements = scenario.requires+ controlled = case requirements.postgres of+ Just requirement | requirement.needsServerControl -> Just "primary"+ _ -> fst <$> find (\(_, requirement) -> requirement.needsServerControl) requirements.extraPostgres+ driverLocal = maybe False (const (options.ephemeralOnDriver && entry.spec.scenario.kind `elem` [Correctness, Concurrency])) controlled+ case controlled of+ Just name | not driverLocal -> Left (NeedsServerControl name)+ _ -> pure ()+ bindings <- traverse cellBinding requirements.resources+ let rules = matchingRules routingRules entry.spec+ denied = find (\rule -> maybe False (\cache -> capabilityKnown cache rule.requires == Just False) activeCapabilities) rules+ case denied of+ Just rule -> Left (MissingCapability rule.requires)+ Nothing -> pure ()+ let originalSpec = entry.spec+ dimensions =+ if options.coerceDurable && not driverLocal && controlled == Nothing && (requirements.postgres /= Nothing || not (null requirements.extraPostgres))+ then ("pg.durability", "durable") : filter ((/= "pg.durability") . fst) originalSpec.dimensions+ else originalSpec.dimensions+ resolved <- either (Left . UnsupportedDimensions . Text.intercalate "; " . NonEmpty.toList) Right (resolveDimensions scenario.dimensions dimensions)+ let needsPostgres = requirements.postgres /= Nothing || not (null requirements.extraPostgres)+ if needsPostgres && not driverLocal && resolved.pgDurability /= Just PgDurable then Left DurabilityNotDurable else pure ()+ let requestedMajor = case resolved.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> descriptor.postgresMajor+ if needsPostgres && not driverLocal && requestedMajor /= descriptor.postgresMajor+ then Left (PgVersionMismatch requestedMajor descriptor.postgresMajor)+ else pure ()+ let extraVariables = fmap (variableName . fst) requirements.extraPostgres+ case find (\name -> length (filter (== name) extraVariables) > 1) extraVariables of+ Just name -> Left (CollidingExtraPostgresName name)+ Nothing -> pure ()+ settings <- if needsPostgres && not driverLocal then combinedSettings scenario originalSpec else Right Map.empty+ let prior = originalSpec.environment+ primary =+ if requirements.postgres == Nothing+ then Nothing+ else+ if driverLocal+ then Just (case prior.postgres of Just ephemeral@(PostgresEphemeral _) -> ephemeral; _ -> PostgresEphemeral [])+ else Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))+ extras =+ if driverLocal+ then Map.fromList [(name, case Map.lookup name prior.extraPostgres of Just ephemeral@(PostgresEphemeral _) -> ephemeral; _ -> PostgresEphemeral []) | (name, _) <- requirements.extraPostgres]+ else Map.fromList [(name, PostgresExternal (ConnFromEnv ("KENSHOU_CELL_PG_URL_" <> variableName name))) | (name, _) <- requirements.extraPostgres]+ env = EnvironmentSpec RunOnCell (Just profile) primary extras prior.resources prior.telemetry+ expectation = maybe originalSpec.cohortExpectation (\selected -> Just (CohortExpectation (Just selected.cohort) selected.cohortIdentity.identityPlanHash.unPlanHash)) payload+ labels = Map.insert "postgresPlacement" (if driverLocal then "driver-ephemeral" else "cell-server") originalSpec.labels+ spec = RunSpec (Just entry.runId) originalSpec.scenario originalSpec.scenarioRevision originalSpec.knobs dimensions originalSpec.seed originalSpec.phases originalSpec.timeoutSeconds env expectation originalSpec.comparison labels+ -- The released cell agent currently requires a PostgreSQL role reset+ -- even when the scenario itself has no database requirement.+ reset = legacyBrokerReset (ResetBlock options.cachePolicy (Just (PgReset descriptor.postgresMajor [] settings)) Nothing (Map.fromList [(name, value) | (name, binding) <- bindings, Just value <- [binding.reset]]))+ timeout = fromMaybe (max 60 (entry.estimateMinutes * 60)) originalSpec.timeoutSeconds+ notices =+ ["coerced pg.durability=durable for " <> renderRunId entry.runId | dimensions /= originalSpec.dimensions]+ <> ["capability " <> rule.requires <> " is unprobed for " <> renderRunId entry.runId <> "; run kenshou cell probe" | rule <- rules, maybe True (\cache -> capabilityKnown cache rule.requires == Nothing) activeCapabilities]+ <> ["payload not selected for " <> renderRunId entry.runId <> "; cohort expectation will be set at submission" | isNothing payloads]+ if timeout <= 0 then Left InvalidTimeout else Right (PreparedRun entry.ordinal entry.runId spec label reset timeout, notices)+ -- A required resource is available on the cell when its provider can bind it+ -- there and the cell reports the binding's capability; a cell broker image+ -- still counts for the resource the legacy broker fields describe.+ cellBinding name = do+ binding <- maybe (Left (ResourceUnavailable name)) Right (lookupResource registry name >>= (.cell))+ let reported = maybe Nothing (\cache -> capabilityKnown cache binding.capability) activeCapabilities+ available = reported == Just True || (name == legacyKafkaName && reported /= Just False && legacyBrokerAvailable descriptor)+ if available then Right (name, binding) else Left (ResourceUnavailable name)++ combinedSettings scenario spec = foldl add (Right Map.empty) sources+ where+ -- The cell image owns preloaded libraries. The external PostgreSQL+ -- adapter verifies required libraries against the running server.+ required = filter ((/= "shared_preload_libraries") . fst) (maybe [] (.settings) scenario.requires.postgres <> concatMap ((.settings) . snd) scenario.requires.extraPostgres)+ primary = case spec.environment.postgres of Just (PostgresEphemeral values) -> values; _ -> []+ extras = concatMap (\value -> case value of PostgresEphemeral values -> values; _ -> []) (Map.elems spec.environment.extraPostgres)+ sources = [("fsync", "on"), ("synchronous_commit", "on"), ("full_page_writes", "on")] <> required <> primary <> extras <> options.pgSettings+ add previous (name, value) = do+ settings <- previous+ case Map.lookup name settings of+ Nothing -> Right (Map.insert name value settings)+ Just existing | existing == value -> Right settings+ _ -> Left (ConflictingPostgresSetting name)++variableName :: Text -> Text+variableName = Text.map (\character -> if character == '-' then '_' else character) . Text.toUpper++-- Prefer the caller's cell order. A rejected run is either retained for local+-- execution or named as unroutable; its original identity never disappears.+routePlan :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Map.Map Text PayloadDescriptor -> PrepareOptions -> RunPlan -> Routed+routePlan registry cells routingRules payloads options plan = routePlanWith registry cells routingRules (Just payloads) options plan++routePlanWithoutPayload :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> PrepareOptions -> RunPlan -> Routed+routePlanWithoutPayload registry cells routingRules options plan = routePlanWith registry cells routingRules Nothing options plan++routePlanWith :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Maybe (Map.Map Text PayloadDescriptor) -> PrepareOptions -> RunPlan -> Routed+routePlanWith registry cells routingRules payloads options plan =+ Routed (Map.map makePlan assigned) (if null localRuns then Nothing else Just (makePlan localRuns)) rejectedRuns notices+ where+ choices = fmap routeEntry plan.runs+ assigned = foldl addCell Map.empty [(name, selected) | Right (name, selected, _) <- choices]+ localRuns = [entry | Left (entry, reason) <- choices, localReason reason]+ rejectedRuns = [(entry, reason) | Left (entry, reason) <- choices, not (localReason reason)]+ notices = concat [messages | Right (_, _, messages) <- choices]++ routeEntry entry = choose (NonEmpty.toList cells) Nothing+ where+ choose [] firstFailure = Left (entry, fromMaybe UnknownScenario firstFailure)+ choose ((descriptor, capabilities) : rest) firstFailure =+ let one = RunPlan plan.planId plan.createdAt plan.context plan.policy [entry] [] entry.estimateMinutes+ prepared = prepareWithPayload registry descriptor capabilities routingRules payloads options one+ in case prepared.accepted of+ [accepted] -> Right (descriptor.name, preparedEntry accepted, prepared.warnings)+ _ -> case prepared.rejected of+ [(_, reason)] -> choose rest (Just (fromMaybe reason firstFailure))+ _ -> choose rest (Just (fromMaybe UnknownScenario firstFailure))+ preparedEntry accepted = PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial accepted.spec++ addCell grouped (name, entry) = Map.insertWith (flip (<>)) name [entry] grouped+ makePlan entries = RunPlan plan.planId plan.createdAt plan.context plan.policy entries plan.skipped (sum (fmap (.estimateMinutes) entries))+ localReason PlacementLocalOnly = True+ localReason (NeedsServerControl _) = True+ localReason _ = False++sliceRuns :: Granularity -> [PreparedRun] -> Either Text [Slice]+sliceRuns granularity prepared = do+ if all valid prepared && unique (fmap (.runId) prepared) && unique (fmap (.ordinal) prepared)+ then pure ()+ else Left "prepared runs have an empty payload, invalid timeout, duplicate identity or invalid ordinal"+ groups <- case granularity of+ GranularityRun -> pure (fmap (:| []) prepared)+ GranularityAuto -> pure (autoGroups prepared)+ GranularityPlan -> case NonEmpty.nonEmpty prepared of+ Nothing -> pure []+ Just group+ | all (compatible (NonEmpty.head group)) (NonEmpty.tail group) -> pure [group]+ | otherwise -> Left "plan granularity requires one payload and reset for every run"+ traverse makeSlice (zip [0 ..] groups)+ where+ valid run = run.ordinal >= 0 && run.timeoutSeconds > 0 && not (Text.null run.payloadLabel) && maybe True (== run.runId) run.spec.runId+ makeSlice (index, group) =+ let runBudget :: PreparedRun -> Integer+ runBudget run = toInteger run.timeoutSeconds + if run.spec.scenario.kind == Soak then 600 else 0+ -- Soaks drain delayed work and compute resource verdicts after load ends.+ -- Finding 49 records a full inbox run exhausting the old shared margin.+ total = 300 + sum (fmap runBudget (NonEmpty.toList group))+ in if total > toInteger (maxBound :: Int)+ then Left "cell slice wall-clock limit is too large"+ else Right (Slice index (NonEmpty.head group).payloadLabel (NonEmpty.head group).reset group (fromInteger total))++slicePlan :: RunPlan -> Slice -> Either Text RunPlan+slicePlan plan slice = do+ selected <- traverse select (NonEmpty.toList slice.entries)+ pure (RunPlan plan.planId plan.createdAt plan.context plan.policy selected plan.skipped (sum (fmap (.estimateMinutes) selected)))+ where+ select prepared = case find ((== prepared.runId) . (.runId)) plan.runs of+ Nothing -> Left "slice contains a run absent from its plan"+ Just original+ | original.ordinal /= prepared.ordinal -> Left "slice ordinal differs from its plan"+ | original.spec.scenario /= prepared.spec.scenario -> Left "slice scenario differs from its plan"+ | otherwise -> Right (PlannedRun original.ordinal original.runId original.estimateMinutes original.reasons original.trial prepared.spec)++submissionFor :: CellRef -> SubmissionInputs -> PayloadDescriptor -> Slice -> WorkObject -> Either Text Submission+submissionFor ref inputs descriptor slice work = do+ case eitherDecode (encode descriptor) :: Either String PayloadDescriptor of+ Left failure -> Left ("invalid payload descriptor: " <> Text.pack failure)+ Right _ -> pure ()+ case eitherDecode (encode work) :: Either String WorkObject of+ Left failure -> Left ("invalid work object: " <> Text.pack failure)+ Right _ -> pure ()+ if work.mediaType /= "application/json" || work.bytes <= 0+ then Left "cell work must be a nonempty JSON run plan"+ else pure ()+ let bundle = descriptor.cell.bundle+ expectedPrefix = "gs://" <> ref.controlBucket.unBucket <> "/payloads/sha256/"+ if not (expectedPrefix `Text.isPrefixOf` bundle.uri)+ then Left "payload bundle is outside the cell control bucket"+ else pure ()+ if inputs.memoryMaxBytes <= 0 || inputs.outputMaxBytes <= 0 || Text.null inputs.minAgentVersion || maybe False Text.null inputs.rtsOptions+ then Left "invalid cell submission limits or options"+ else pure ()+ let env =+ Map.fromList+ ( [ ("KENSHOU_PAYLOAD_BUNDLE_SHA256", bundle.sha256),+ ("KENSHOU_PAYLOAD_STORE_PATH", descriptor.cell.storePath),+ ("KENSHOU_PAYLOAD_NAR_HASH", descriptor.cell.narHash),+ ("KENSHOU_PAYLOAD_COHORT", descriptor.cohort),+ ("KENSHOU_OTLP_SINK", case inputs.otlpSink of NullSink -> "null"; FileSink -> "file")+ ]+ <> maybe [] (\options -> [("GHCRTS", options)]) inputs.rtsOptions+ )+ labels =+ Map.fromList+ [ ("session", renderRunId inputs.sessionId),+ ("plan", renderRunId inputs.planId),+ ("slice", Text.pack (show slice.index)),+ ("payload", slice.payloadLabel)+ ]+ profileCollection = maybe False (any profileFlag . Text.words) inputs.rtsOptions+ traceCollection = inputs.otlpSink == FileSink+ collection = if profileCollection || traceCollection then Just (CollectOptions traceCollection profileCollection) else Nothing+ submission = Submission inputs.cellRun inputs.leaseId descriptor.cell work env slice.reset (Limits (fromIntegral slice.wallClockSeconds) inputs.memoryMaxBytes inputs.outputMaxBytes) (Requirements inputs.minAgentVersion inputs.requiredCapabilities) collection labels+ case eitherDecode (encode submission) :: Either String Submission of+ Left failure -> Left ("invalid cell submission: " <> Text.pack failure)+ Right _ -> Right submission+ where+ profileFlag flag = any (`Text.isPrefixOf` flag) ["-p", "-h", "-l"]++autoGroups :: [PreparedRun] -> [NonEmpty PreparedRun]+autoGroups [] = []+autoGroups (first : rest)+ | isolated first = (first :| []) : autoGroups rest+ | otherwise =+ let (same, later) = span (\candidate -> not (isolated candidate) && compatible first candidate) rest+ in (first :| same) : autoGroups later++compatible :: PreparedRun -> PreparedRun -> Bool+compatible first second = first.payloadLabel == second.payloadLabel && first.reset == second.reset++isolated :: PreparedRun -> Bool+isolated run = run.spec.scenario.kind `elem` [Benchmark, Soak]++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Cell/RouteJson.hs view
@@ -0,0 +1,123 @@+module Kenshou.Remote.Cell.RouteJson (RouteDocuments (..), routeWorkJson) where++import Data.Aeson (FromJSON, Value (..), object, (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time.Clock.POSIX (posixSecondsToUTCTime)+import Kenshou.Core.Bundle (Registry)+import Kenshou.Core.Id (RunId, mkSeed)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Docs (CellDescriptor)+import Kenshou.Remote.Cell.Prepare (PrepareOptions, Routed (..), routePlan, routePlanWithoutPayload)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities, RoutingRule)+import Kenshou.Remote.Payload (PayloadDescriptor)++data RouteDocuments = RouteDocuments+ { perCell :: !(Map Text Value),+ local :: !(Maybe Value),+ report :: !Value,+ routeComplete :: !Bool+ }+ deriving stock (Eq, Show)++-- Route the public JSON plan without decoding and re-encoding its provenance+-- types. The typed plan below carries only the fields used by preparation; the+-- output copies each original entry and replaces its prepared spec.+routeWorkJson :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Maybe (Map Text PayloadDescriptor) -> PrepareOptions -> Value -> Either Text RouteDocuments+routeWorkJson registry cells rules payloads options work = do+ fields <- case work of Object value -> Right value; _ -> Left "run plan is not a JSON object"+ case KeyMap.lookup "schema" fields of+ Just (String "kenshou.run-plan/v1") -> pure ()+ _ -> Left "run plan has an unsupported schema"+ planId <- required "planId" fields+ rawRuns <- case KeyMap.lookup "runs" fields of+ Just (Array entries) -> Right (toList entries)+ _ -> Left "run plan has no runs array"+ entries <- traverse parseEntry rawRuns+ let identifiers = fmap (\(entry, _) -> entry.runId) entries+ ordinals = fmap (\(entry, _) -> entry.ordinal) entries+ if unique identifiers && unique ordinals then pure () else Left "run plan contains duplicate run IDs or ordinals"+ selector <- parseSelector "**"+ seed <- mkSeed 1+ let placeholder = Reason (Change (ComponentRef (ComponentId "cell-route") Nothing) Named "internal routing placeholder") [] selector 0+ planned = fmap (toPlanned placeholder . fst) entries+ context = PlanContext Nothing "" "" "" (PlanInputs Null) [] []+ typed = RunPlan planId (posixSecondsToUTCTime 0) context (defaultPlanPolicy seed) planned [] (sum (fmap (.estimateMinutes) planned))+ routed = case payloads of+ Just selected -> routePlan registry cells rules selected options typed+ Nothing -> routePlanWithoutPayload registry cells rules options typed+ originals = Map.fromList [(entry.runId, value) | (entry, value) <- entries]+ cellDocs <- traverse (renderPlan fields originals True) routed.perCell+ localDoc <- traverse (renderPlan fields originals False) routed.local+ let routeReport =+ object+ [ "schema" .= ("kenshou.cell-route/v1" :: Text),+ "planId" .= planId,+ "perCell" .= Map.map (fmap (.runId) . (.runs)) routed.perCell,+ "localRunIds" .= maybe [] (fmap (.runId) . (.runs)) routed.local,+ "unroutable" .= [object ["runId" .= entry.runId, "reason" .= Text.pack (show reason)] | (entry, reason) <- routed.unroutable],+ "warnings" .= routed.warnings+ ]+ pure (RouteDocuments cellDocs localDoc routeReport (null routed.unroutable))+ where+ required :: (FromJSON document) => Text -> Aeson.Object -> Either Text document+ required name fields = case KeyMap.lookup (Key.fromText name) fields of+ Nothing -> Left ("run plan is missing " <> name)+ Just value -> decode name value++ decode :: (FromJSON document) => Text -> Value -> Either Text document+ decode label value = case Aeson.fromJSON value of+ Aeson.Error failure -> Left (label <> ": " <> Text.pack failure)+ Aeson.Success result -> Right result++ parseEntry value = case value of+ Object fields -> do+ ordinal <- required "ordinal" fields+ runId <- required "runId" fields+ estimate <- required "estimateMinutes" fields+ spec <- required "spec" fields+ trial <- case KeyMap.lookup "trial" fields of+ Nothing -> Right Nothing+ Just Null -> Right Nothing+ Just (Object trialFields) -> Just <$> (TrialInfo <$> required "group" trialFields <*> required "arm" trialFields <*> required "index" trialFields <*> required "of" trialFields)+ _ -> Left "planned run trial is not an object"+ if ordinal < 0 || estimate <= 0 || maybe False (/= runId) spec.runId+ then Left "planned run has an invalid ordinal, estimate or spec run ID"+ else Right (RouteEntry ordinal runId estimate trial spec, value)+ _ -> Left "planned run is not an object"++ toPlanned placeholder entry = PlannedRun entry.ordinal entry.runId entry.estimateMinutes (placeholder :| []) entry.trial entry.spec++ renderPlan fields originals prepared plan = do+ rows <- traverse (renderEntry originals prepared) plan.runs+ let changed = KeyMap.insert "runs" (Aeson.toJSON rows) (KeyMap.insert "estimateMinutes" (Aeson.toJSON plan.estimateMinutes) fields)+ pure (Object changed)++ renderEntry originals prepared entry = case Map.lookup entry.runId originals of+ Just (Object original) ->+ Right (Object (if prepared then KeyMap.insert "spec" (Aeson.toJSON entry.spec) original else original))+ _ -> Left "routed run is absent from the input plan"++data RouteEntry = RouteEntry+ { ordinal :: !Int,+ runId :: !RunId,+ estimateMinutes :: !Int,+ trial :: !(Maybe TrialInfo),+ spec :: !RunSpec+ }++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Cell/RouteRules.hs view
@@ -0,0 +1,101 @@+module Kenshou.Remote.Cell.RouteRules+ ( CellCapabilities (..),+ RuleCondition (..),+ RoutingRule (..),+ defaultRoutingRules,+ matchingRules,+ capabilityKnown,+ descriptorDigest,+ loadCellCapabilities,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), eitherDecodeFileStrict', encode, object, withObject, (.:), (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Contribution (RoutingRule (..), RuleCondition (..))+import Kenshou.Core.Id (RunId)+import Kenshou.Core.Knob (RawKnob (..), mkKnobName)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Plan.Selector (Selector, matches, parseSelector)+import Kenshou.Remote.Cell.Docs (CellDescriptor (..))+import System.Directory (doesFileExist)+import System.FilePath ((</>))++data CellCapabilities = CellCapabilities+ { cell :: !Text,+ descriptorSha256 :: !Text,+ probedBy :: !RunId,+ probedAt :: !UTCTime,+ capabilities :: !(Map Text Value)+ }+ deriving stock (Eq, Show)++-- | Superseded by 'Kenshou.Core.Bundle.registryRoutingRules': suites contribute+-- their routing rules on their bundles. Kept, empty, so existing imports compile.+defaultRoutingRules :: [RoutingRule]+defaultRoutingRules = []++instance FromJSON CellCapabilities where+ parseJSON = withObject "cell capabilities" \fields -> do+ schema <- fields .: "schema"+ if schema == ("kenshou.cell-capabilities/v1" :: Text) then pure () else fail "unsupported cell capabilities schema"+ document <- CellCapabilities <$> fields .: "cell" <*> fields .: "descriptorSha256" <*> fields .: "probedBy" <*> fields .: "probedAt" <*> fields .: "capabilities"+ if Text.null document.cell || Text.length document.descriptorSha256 /= 64 || not (Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) document.descriptorSha256)+ then fail "invalid cell capabilities identity"+ else pure document++instance ToJSON CellCapabilities where+ toJSON cache =+ object+ [ "schema" .= ("kenshou.cell-capabilities/v1" :: Text),+ "cell" .= cache.cell,+ "descriptorSha256" .= cache.descriptorSha256,+ "probedBy" .= cache.probedBy,+ "probedAt" .= cache.probedAt,+ "capabilities" .= cache.capabilities+ ]++matchingRules :: [RoutingRule] -> RunSpec -> [RoutingRule]+matchingRules rules spec = filter (matchesRule spec) rules+ where+ matchesRule run rule = case rule.condition of+ ScenarioMatches selector -> matches selector run.scenario+ KnobEquals name expected -> case mkKnobName name of+ Left _ -> False+ Right key -> case lookup key run.knobs of+ Just (RawText actual) -> actual == expected+ Just (RawJson (String actual)) -> actual == expected+ _ -> False++capabilityKnown :: CellCapabilities -> Text -> Maybe Bool+capabilityKnown cache name = case Map.lookup name cache.capabilities of+ Just (Bool available) -> Just available+ _ -> Nothing++-- The cache uses the canonical codec form, so object member order in the+-- downloaded descriptor cannot invalidate an otherwise identical cell shape.+descriptorDigest :: CellDescriptor -> Text+descriptorDigest = Text.drop 7 . sha256Hex . LazyByteString.toStrict . encode++-- A cache belongs to one descriptor revision. Missing and stale files leave+-- routing in its explicit unprobed state; malformed files must be repaired.+loadCellCapabilities :: FilePath -> CellDescriptor -> IO (Either Text (Maybe CellCapabilities))+loadCellCapabilities directory descriptor = do+ let path = directory </> Text.unpack descriptor.name <> ".capabilities.json"+ present <- doesFileExist path+ if not present+ then pure (Right Nothing)+ else do+ decoded <- eitherDecodeFileStrict' path+ pure case decoded of+ Left problem -> Left ("invalid capability cache " <> Text.pack path <> ": " <> Text.pack problem)+ Right cache+ | cache.cell == descriptor.name && cache.descriptorSha256 == descriptorDigest descriptor -> Right (Just cache)+ | otherwise -> Right Nothing
+ src/Kenshou/Remote/Cell/Session.hs view
@@ -0,0 +1,75 @@+module Kenshou.Remote.Cell.Session+ ( SessionError (..),+ SessionTransition (..),+ runSubmission,+ runSubmissionWithTransitions,+ )+where++import Control.Concurrent (threadDelay)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty)+import Data.Text (Text)+import Kenshou.Remote.Cell.Docs (CellStatus, Rejected (..), Submission (..))+import Kenshou.Remote.Cell.Fetch (FetchError, VerifyProblem, fetchCellRun)+import Kenshou.Remote.Cell.Index (CellRunIndex, deriveCellRunIndex, writeCellRunIndex)+import Kenshou.Remote.Cell.Lease (CellRef, LeaseHandle, withHeartbeat)+import Kenshou.Remote.Cell.Submit (PublishOutcome (..), publishSubmission)+import Kenshou.Remote.Cell.Watch (WatchEvent, WatchSnapshot (..), WatchTerminal (..), newWatchCursor, pollCellRun)+import Kenshou.Remote.Store (Bucket, ObjectStore)++data SessionError+ = LeaseLost+ | PublicationFailed !PublishOutcome+ | SubmissionRejected !Text+ | FetchFailed !FetchError+ | VerificationFailed !(NonEmpty VerifyProblem)+ deriving stock (Eq, Show)++data SessionTransition+ = SubmissionPublished+ | SubmissionSealed !CellStatus+ | SubmissionRejectedByCell !Rejected+ | ResultsFetched !FilePath+ | ResultsVerified !CellRunIndex+ deriving stock (Eq, Show)++-- One already prepared slice under an existing held lease. Its caller owns the+-- session document and advances its state after each returned transition.+runSubmission :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> Submission -> LazyByteString.ByteString -> FilePath -> (WatchEvent -> IO ()) -> IO (Either SessionError CellRunIndex)+runSubmission store ref resultsBucket handle submission workBytes outDir emit =+ runSubmissionWithTransitions store ref resultsBucket handle submission workBytes outDir emit (const (pure ()))++runSubmissionWithTransitions :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> Submission -> LazyByteString.ByteString -> FilePath -> (WatchEvent -> IO ()) -> (SessionTransition -> IO ()) -> IO (Either SessionError CellRunIndex)+runSubmissionWithTransitions store ref resultsBucket handle submission workBytes outDir emit checkpoint =+ withHeartbeat store ref handle \stillHeld -> do+ published <- publishSubmission store ref handle submission workBytes+ case published of+ Submitted -> checkpoint SubmissionPublished >> waitForSeal stillHeld newWatchCursor+ LostLease -> pure (Left LeaseLost)+ other -> pure (Left (PublicationFailed other))+ where+ waitForSeal stillHeld cursor = do+ held <- stillHeld+ if not held+ then pure (Left LeaseLost)+ else do+ snapshot <- pollCellRun store ref submission.runId cursor+ mapM_ emit snapshot.events+ case snapshot.terminal of+ Just (RunRejected rejected) -> checkpoint (SubmissionRejectedByCell rejected) >> pure (Left (SubmissionRejected rejected.reason))+ Just (RunSealed status) -> do+ checkpoint (SubmissionSealed status)+ fetched <- fetchCellRun store resultsBucket submission.runId outDir+ case fetched of+ Left failure -> pure (Left (FetchFailed failure))+ Right tree -> do+ checkpoint (ResultsFetched tree)+ indexed <- deriveCellRunIndex resultsBucket (Just status) tree+ case indexed of+ Left problems -> pure (Left (VerificationFailed problems))+ Right index -> do+ _ <- writeCellRunIndex tree index+ checkpoint (ResultsVerified index)+ pure (Right index)+ Nothing -> threadDelay 2000000 >> waitForSeal stillHeld snapshot.cursor
+ src/Kenshou/Remote/Cell/Session/Build.hs view
@@ -0,0 +1,118 @@+module Kenshou.Remote.Cell.Session.Build+ ( BuildOptions (..),+ BuiltSession (..),+ buildSession,+ )+where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Bundle (Registry)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Remote.Cell.Docs (CellBuckets (..), CellDescriptor (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Prepare (Granularity, OtlpSink, PrepareOptions, Prepared (..), PreparedRun (..), Slice (..), SubmissionInputs (..), prepareForCell, sliceRuns, submissionFor)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities, RoutingRule)+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), RejectedRun (..), SessionJournal (..), SliceJournal (..), SliceState (..))+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan, renderPreparedWork)+import Kenshou.Remote.Payload (PayloadDescriptor)+import Kenshou.Remote.Store (Bucket (..))++data BuildOptions = BuildOptions+ { preparation :: !PrepareOptions,+ granularity :: !Granularity,+ skipIncompatible :: !Bool,+ otlpSink :: !OtlpSink,+ rtsOptions :: !(Maybe Text),+ memoryMaxBytes :: !Int64,+ outputMaxBytes :: !Int64,+ minAgentVersion :: !Text+ }+ deriving stock (Eq, Show)++data BuiltSession = BuiltSession+ { journal :: !SessionJournal,+ workFiles :: ![(FilePath, LazyByteString.ByteString)]+ }+ deriving stock (Eq, Show)++-- Plan bytes are kept for their original digest, while each work file is+-- rendered from the original JSON so that provenance survives slicing.+buildSession :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Map Text PayloadDescriptor -> BuildOptions -> Text -> RunId -> LazyByteString.ByteString -> IO (Either Text BuiltSession)+buildSession registry descriptor capabilities rules payloads options storeLabel leaseId planBytes =+ case eitherDecode planBytes of+ Left failure -> pure (Left ("run plan: " <> Text.pack failure))+ Right document -> case decodeWorkPlan document of+ Left problem -> pure (Left problem)+ Right plan -> do+ let prepared = prepareForCell registry descriptor capabilities rules payloads options.preparation plan+ rejected = [RejectedRun entry.runId (Text.pack (show reason)) | (entry, reason) <- prepared.rejected]+ if not options.skipIncompatible && not (null rejected)+ then pure (Left ("cell cannot run " <> Text.intercalate ", " [renderRunId run.runId <> ": " <> run.reason | run <- rejected]))+ else case sliceRuns options.granularity prepared.accepted of+ Left problem -> pure (Left problem)+ Right [] -> pure (Left "no compatible runs remain in the plan")+ Right slices -> case traverse (buildWork document payloads) slices of+ Left problem -> pure (Left problem)+ Right work -> do+ sessionId <- newRunId+ cellRuns <- traverse (const newRunId) slices+ now <- getCurrentTime+ let ref = CellRef descriptor.name (Bucket descriptor.buckets.control)+ planHash = (workObjectFor "application/json" planBytes).sha256+ results = Bucket descriptor.buckets.results+ mkSlice slice (payload, path, bytes) cellRun = do+ let inputs =+ SubmissionInputs+ cellRun+ leaseId+ sessionId+ plan.planId+ options.otlpSink+ options.rtsOptions+ options.memoryMaxBytes+ options.outputMaxBytes+ options.minAgentVersion+ Nothing+ submission <- submissionFor ref inputs payload slice (workObjectFor "application/json" bytes)+ pure+ ( SliceJournal slice.index cellRun (fmap (.ordinal) (NonEmpty.toList slice.entries)) (fmap (.runId) (NonEmpty.toList slice.entries)) slice.reset submission path SlicePlanned Nothing Nothing Nothing Nothing Nothing,+ (path, bytes)+ )+ pure do+ if options.memoryMaxBytes <= 0 || options.outputMaxBytes <= 0+ then Left "cell memory and output limits must be positive"+ else pure ()+ assembled <- sequence (zipWith3 mkSlice slices work cellRuns)+ let journal =+ SessionJournal+ sessionId+ descriptor.name+ storeLabel+ (Just descriptor.project)+ ref.controlBucket.unBucket+ results.unBucket+ leaseId+ Held+ payloads+ planHash+ rejected+ (fmap fst assembled)+ now+ now+ pure (BuiltSession journal (fmap snd assembled))+ where+ buildWork document available slice = do+ payload <- maybe (Left ("slice payload label is absent: " <> slice.payloadLabel)) Right (Map.lookup slice.payloadLabel available)+ rendered <- renderPreparedWork document (NonEmpty.toList slice.entries)+ let path = "slice-" <> show slice.index <> "/work.json"+ pure (payload, path, encode rendered)
+ src/Kenshou/Remote/Cell/Session/Journal.hs view
@@ -0,0 +1,232 @@+module Kenshou.Remote.Cell.Session.Journal+ ( LeaseMode (..),+ SliceState (..),+ RejectedRun (..),+ SliceJournal (..),+ SessionJournal (..),+ applyTransition,+ readSessionJournal,+ writeSessionJournal,+ )+where++import Control.Exception (bracket)+import Control.Monad (unless)+import Data.Aeson (FromJSON (..), ToJSON (..), eitherDecode, encode, object, withObject, withText, (.:), (.:?), (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (find, sort)+import Data.Map.Strict (Map)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Remote.Cell.Docs (CellOutcome (..), CellStatus (..), Rejected (..), ResetBlock, Submission (..))+import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..), RunLink (..))+import Kenshou.Remote.Cell.Session (SessionTransition (..))+import Kenshou.Remote.Payload (PayloadDescriptor)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.FilePath (isAbsolute, splitDirectories, takeDirectory)+import System.IO (hClose, openBinaryTempFile)++data LeaseMode = Held | Detached+ deriving stock (Eq, Show)++data SliceState = SlicePlanned | SliceSubmitted | SliceSealed | SliceRejected | SliceFetched | SliceVerified+ deriving stock (Eq, Show)++data RejectedRun = RejectedRun+ { runId :: !RunId,+ reason :: !Text+ }+ deriving stock (Eq, Show)++data SliceJournal = SliceJournal+ { index :: !Int,+ cellRun :: !RunId,+ ordinals :: ![Int],+ runIds :: ![RunId],+ reset :: !ResetBlock,+ submission :: !Submission,+ workPath :: !FilePath,+ state :: !SliceState,+ cellOutcome :: !(Maybe CellOutcome),+ entryExitCode :: !(Maybe Int),+ manifestSha256 :: !(Maybe Text),+ rejectionReason :: !(Maybe Text),+ fetchedPath :: !(Maybe FilePath)+ }+ deriving stock (Eq, Show)++data SessionJournal = SessionJournal+ { sessionId :: !RunId,+ cell :: !Text,+ store :: !Text,+ project :: !(Maybe Text),+ controlBucket :: !Text,+ resultsBucket :: !Text,+ leaseId :: !RunId,+ leaseMode :: !LeaseMode,+ payloads :: !(Map Text PayloadDescriptor),+ planSha256 :: !Text,+ rejectedRuns :: ![RejectedRun],+ slices :: ![SliceJournal],+ createdAt :: !UTCTime,+ updatedAt :: !UTCTime+ }+ deriving stock (Eq, Show)++instance ToJSON LeaseMode where+ toJSON Held = toJSON ("held" :: Text)+ toJSON Detached = toJSON ("detached" :: Text)++instance FromJSON LeaseMode where+ parseJSON = withText "cell lease mode" \case+ "held" -> pure Held+ "detached" -> pure Detached+ _ -> fail "unknown cell lease mode"++instance ToJSON SliceState where+ toJSON SlicePlanned = toJSON ("planned" :: Text)+ toJSON SliceSubmitted = toJSON ("submitted" :: Text)+ toJSON SliceSealed = toJSON ("sealed" :: Text)+ toJSON SliceRejected = toJSON ("rejected" :: Text)+ toJSON SliceFetched = toJSON ("fetched" :: Text)+ toJSON SliceVerified = toJSON ("verified" :: Text)++instance FromJSON SliceState where+ parseJSON = withText "cell slice state" \case+ "planned" -> pure SlicePlanned+ "submitted" -> pure SliceSubmitted+ "sealed" -> pure SliceSealed+ "rejected" -> pure SliceRejected+ "fetched" -> pure SliceFetched+ "verified" -> pure SliceVerified+ _ -> fail "unknown cell slice state"++instance ToJSON RejectedRun where+ toJSON run = object ["runId" .= run.runId, "reason" .= run.reason]++instance FromJSON RejectedRun where+ parseJSON = withObject "rejected planned run" \value -> RejectedRun <$> value .: "runId" <*> value .: "reason"++instance ToJSON SliceJournal where+ toJSON slice =+ object+ [ "index" .= slice.index,+ "cellRun" .= slice.cellRun,+ "ordinals" .= slice.ordinals,+ "runIds" .= slice.runIds,+ "reset" .= slice.reset,+ "submission" .= slice.submission,+ "workPath" .= slice.workPath,+ "state" .= slice.state,+ "cellOutcome" .= slice.cellOutcome,+ "entryExitCode" .= slice.entryExitCode,+ "manifestSha256" .= slice.manifestSha256,+ "rejectionReason" .= slice.rejectionReason,+ "fetchedPath" .= slice.fetchedPath+ ]++instance FromJSON SliceJournal where+ parseJSON = withObject "cell slice journal" \value -> do+ slice <- SliceJournal <$> value .: "index" <*> value .: "cellRun" <*> value .: "ordinals" <*> value .: "runIds" <*> value .: "reset" <*> value .: "submission" <*> value .: "workPath" <*> value .: "state" <*> value .:? "cellOutcome" <*> value .:? "entryExitCode" <*> value .:? "manifestSha256" <*> value .:? "rejectionReason" <*> value .:? "fetchedPath"+ unless (slice.index >= 0 && slice.cellRun == slice.submission.runId && slice.reset == slice.submission.reset && not (null slice.ordinals) && not (null slice.runIds) && unique slice.ordinals && unique slice.runIds && validWorkPath slice.workPath && validSliceState slice) (fail "inconsistent cell slice journal")+ pure slice++instance ToJSON SessionJournal where+ toJSON journal =+ object+ [ "schema" .= ("kenshou.cell-session/v1" :: Text),+ "sessionId" .= journal.sessionId,+ "cell" .= journal.cell,+ "store" .= journal.store,+ "project" .= journal.project,+ "controlBucket" .= journal.controlBucket,+ "resultsBucket" .= journal.resultsBucket,+ "leaseId" .= journal.leaseId,+ "leaseMode" .= journal.leaseMode,+ "payloads" .= journal.payloads,+ "planSha256" .= journal.planSha256,+ "rejectedRuns" .= journal.rejectedRuns,+ "slices" .= journal.slices,+ "createdAt" .= journal.createdAt,+ "updatedAt" .= journal.updatedAt+ ]++instance FromJSON SessionJournal where+ parseJSON = withObject "cell session journal" \value -> do+ schema <- value .: "schema"+ unless (schema == ("kenshou.cell-session/v1" :: Text)) (fail "unsupported cell session journal")+ journal <- SessionJournal <$> value .: "sessionId" <*> value .: "cell" <*> value .: "store" <*> value .:? "project" <*> value .: "controlBucket" <*> value .: "resultsBucket" <*> value .: "leaseId" <*> value .: "leaseMode" <*> value .: "payloads" <*> value .: "planSha256" <*> value .: "rejectedRuns" <*> value .: "slices" <*> value .: "createdAt" <*> value .: "updatedAt"+ unless (unique (fmap (.index) journal.slices) && unique (fmap (.cellRun) journal.slices) && validDigest journal.planSha256 && journal.createdAt <= journal.updatedAt) (fail "inconsistent cell session journal")+ pure journal++applyTransition :: UTCTime -> RunId -> SessionTransition -> SessionJournal -> Either Text SessionJournal+applyTransition now identifier transition journal = do+ slice <- maybe (Left "cell run is not in session") Right (find ((== identifier) . (.cellRun)) journal.slices)+ changed <- advance slice transition+ pure journal {slices = fmap (\current -> if current.cellRun == identifier then changed else current) journal.slices, updatedAt = now}+ where+ advance slice SubmissionPublished = require SlicePlanned slice $ slice {state = SliceSubmitted}+ advance slice (SubmissionRejectedByCell rejected)+ | rejected.runId /= identifier = Left "rejection names another cell run"+ | otherwise = require SliceSubmitted slice $ slice {state = SliceRejected, rejectionReason = Just rejected.reason}+ advance slice (SubmissionSealed status)+ | status.runId /= identifier = Left "seal names another cell run"+ | otherwise = case (status.outcome, status.manifestSha256) of+ (Just outcome, Just digest) -> require SliceSubmitted slice $ slice {state = SliceSealed, cellOutcome = Just outcome, manifestSha256 = Just digest}+ _ -> Left "sealed status is missing outcome or manifest digest"+ advance slice (ResultsFetched path)+ | null path = Left "fetched tree path is empty"+ | otherwise = require SliceSealed slice $ slice {state = SliceFetched, fetchedPath = Just path}+ advance slice (ResultsVerified index)+ | index.cellRun /= identifier || index.leaseId /= slice.submission.leaseId = Left "verified index names another run or lease"+ | Just index.cellOutcome /= slice.cellOutcome || Just index.cellManifest.sha256 /= slice.manifestSha256 = Left "verified index disagrees with seal"+ | index.cellOutcome == Completed && sort (fmap (.runId) index.runs) /= sort slice.runIds = Left "verified index nested runs differ from planned slice"+ | any (\run -> run.runId `notElem` slice.runIds) index.runs = Left "verified index nested runs differ from planned slice"+ | otherwise = require SliceFetched slice $ slice {state = SliceVerified, entryExitCode = index.entryExitCode}+ require expected slice next+ | slice.state == expected = Right next+ | otherwise = Left ("invalid cell slice transition from " <> Text.pack (show slice.state))++readSessionJournal :: FilePath -> IO (Either Text SessionJournal)+readSessionJournal path = do+ bytes <- LazyByteString.readFile path+ pure case eitherDecode bytes of+ Left failure -> Left (Text.pack failure)+ Right journal -> Right journal++writeSessionJournal :: FilePath -> SessionJournal -> IO ()+writeSessionJournal path journal = do+ let directory = takeDirectory path+ createDirectoryIfMissing True directory+ bracket (openBinaryTempFile directory ".kenshou-session-") cleanup \(temporary, handle) -> do+ hClose handle+ LazyByteString.writeFile temporary (encode journal)+ renameFile temporary path+ where+ cleanup (temporary, _) = do+ exists <- doesFileExist temporary+ if exists then removeFile temporary else pure ()++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest++validWorkPath :: FilePath -> Bool+validWorkPath path = not (null path) && not (isAbsolute path) && all (`notElem` ["..", "."]) (splitDirectories path)++validSliceState :: SliceJournal -> Bool+validSliceState slice = case slice.state of+ SlicePlanned -> empty+ SliceSubmitted -> empty+ SliceSealed -> sealed && slice.fetchedPath == Nothing && slice.entryExitCode == Nothing+ SliceRejected -> slice.rejectionReason /= Nothing && slice.cellOutcome == Nothing && slice.manifestSha256 == Nothing && slice.fetchedPath == Nothing && slice.entryExitCode == Nothing+ SliceFetched -> sealed && slice.fetchedPath /= Nothing && slice.entryExitCode == Nothing+ SliceVerified -> sealed && slice.fetchedPath /= Nothing+ where+ empty = slice.cellOutcome == Nothing && slice.entryExitCode == Nothing && slice.manifestSha256 == Nothing && slice.rejectionReason == Nothing && slice.fetchedPath == Nothing+ sealed = slice.cellOutcome /= Nothing && maybe False validDigest slice.manifestSha256 && slice.rejectionReason == Nothing++validDigest :: Text -> Bool+validDigest digest = Text.length digest == 64 && Text.all (\character -> character `elem` ['0' .. '9'] || character `elem` ['a' .. 'f']) digest
+ src/Kenshou/Remote/Cell/Session/Rebind.hs view
@@ -0,0 +1,86 @@+module Kenshou.Remote.Cell.Session.Rebind+ ( RebindError (..),+ rebindPlannedSlices,+ )+where++import Control.Monad (forM)+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Submission (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseSnapshot, renewLease)+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..))++data RebindError+ = InvalidJournal !Text+ | LeaseUnavailable+ | UnresolvedSlice !RunId !SliceState+ | RemoteMarkerExists !RunId+ deriving stock (Eq, Show)++-- A planned entry may be rebound only when it has no remote claim marker.+-- A submitted entry may be rebound under a different, renewed lease when no+-- status, rejection or result object exists: the new lease fences the old+-- agent's start_run.+rebindPlannedSlices :: ObjectStore -> CellRef -> LeaseHandle -> FilePath -> IO (Either RebindError SessionJournal)+rebindPlannedSlices store ref handle journalPath = do+ decoded <- readSessionJournal journalPath+ case decoded of+ Left failure -> pure (Left (InvalidJournal failure))+ Right journal -> do+ lease <- leaseSnapshot handle+ held <- renewLease store ref handle+ if not held+ then pure (Left LeaseUnavailable)+ else+ if journal.cell /= ref.cellName || journal.controlBucket /= ref.controlBucket.unBucket || journal.leaseMode /= Held+ then pure (Left (InvalidJournal "session cell, control bucket or lease mode is incompatible with a held lease"))+ else case unresolved lease.leaseId journal.leaseId journal.slices of+ Just slice -> pure (Left (UnresolvedSlice slice.cellRun slice.state))+ Nothing -> do+ let planned = filter ((== SlicePlanned) . (.state)) journal.slices+ pending = filter (\slice -> slice.state `elem` [SlicePlanned, SliceSubmitted]) journal.slices+ active <- firstSubmittedStatus (Bucket journal.resultsBucket) journal.slices+ case active of+ Just slice -> pure (Left (UnresolvedSlice slice.cellRun slice.state))+ Nothing -> do+ present <- firstRemoteMarker planned+ case present of+ Just identifier -> pure (Left (RemoteMarkerExists identifier))+ Nothing+ | null pending -> pure (Right journal)+ | otherwise -> do+ rebound <- forM journal.slices \slice ->+ if slice.state `elem` [SlicePlanned, SliceSubmitted]+ then do+ identifier <- newRunId+ let previous = slice.submission+ submission = Submission identifier lease.leaseId previous.payload previous.work previous.env previous.reset previous.limits previous.requires previous.collect previous.labels+ pure (SliceJournal slice.index identifier slice.ordinals slice.runIds slice.reset submission slice.workPath SlicePlanned Nothing Nothing Nothing Nothing Nothing)+ else pure slice+ now <- getCurrentTime+ let updated = journal {leaseId = lease.leaseId, slices = rebound, updatedAt = now}+ writeSessionJournal journalPath updated+ pure (Right updated)+ where+ unresolved _ _ [] = Nothing+ unresolved activeLease priorLease (slice : rest)+ | slice.state `elem` [SliceSealed, SliceFetched] || (slice.state == SliceSubmitted && activeLease == priorLease) = Just slice+ | otherwise = unresolved activeLease priorLease rest+ firstSubmittedStatus _ [] = pure Nothing+ firstSubmittedStatus resultsBucket (slice : rest)+ | slice.state /= SliceSubmitted = firstSubmittedStatus resultsBucket rest+ | otherwise = do+ let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/"+ resultPrefix = "runs/" <> renderRunId slice.cellRun <> "/"+ visible <- or <$> forM ["status.json", "rejected.json"] (fmap isJust . store.statObject ref.controlBucket . ObjectName . (prefix <>))+ results <- store.listObjects resultsBucket resultPrefix+ if visible || not (null results) then pure (Just slice) else firstSubmittedStatus resultsBucket rest+ firstRemoteMarker [] = pure Nothing+ firstRemoteMarker (slice : rest) = do+ let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/"+ found <- or <$> forM ["submission.json", "status.json", "rejected.json"] (fmap isJust . store.statObject ref.controlBucket . ObjectName . (prefix <>))+ if found then pure (Just slice.cellRun) else firstRemoteMarker rest
+ src/Kenshou/Remote/Cell/Session/Resume.hs view
@@ -0,0 +1,128 @@+module Kenshou.Remote.Cell.Session.Resume+ ( ResumeError (..),+ HeldResumeError (..),+ resumeObservedSlices,+ resumeHeldSession,+ )+where++import Data.Aeson (eitherDecode)+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.List.NonEmpty (NonEmpty)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Submission)+import Kenshou.Remote.Cell.Fetch (FetchError, VerifyProblem, fetchCellRun)+import Kenshou.Remote.Cell.Index (deriveCellRunIndex, writeCellRunIndex)+import Kenshou.Remote.Cell.Lease (CellRef (..), LeaseHandle)+import Kenshou.Remote.Cell.Session (SessionTransition (..))+import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Cell.Session.Rebind (RebindError, rebindPlannedSlices)+import Kenshou.Remote.Cell.Session.Runner (SessionRunError, runPendingSlices)+import Kenshou.Remote.Cell.Watch (WatchEvent, WatchSnapshot (..), WatchTerminal (..), newWatchCursor, pollCellRun)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..))+import System.FilePath (takeDirectory)++data ResumeError+ = InvalidJournal !Text+ | ResultFetchFailed !RunId !FetchError+ | ResultVerificationFailed !RunId !(NonEmpty VerifyProblem)+ deriving stock (Eq, Show)++data HeldResumeError+ = ObservationError !ResumeError+ | RebindingError !RebindError+ | ContinuationError !SessionRunError+ deriving stock (Eq, Show)++resumeHeldSession :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> (WatchEvent -> IO ()) -> IO (Either HeldResumeError SessionJournal)+resumeHeldSession store ref resultsBucket handle journalPath emit = do+ observed <- resumeObservedSlices store ref resultsBucket journalPath emit+ case observed of+ Left failure -> pure (Left (ObservationError failure))+ Right observed | not (null observed.slices) && all (\slice -> slice.state `elem` [SliceVerified, SliceRejected]) observed.slices -> pure (Right observed)+ Right _ -> do+ rebound <- rebindPlannedSlices store ref handle journalPath+ case rebound of+ Left failure -> pure (Left (RebindingError failure))+ Right _ -> do+ continued <- runPendingSlices store ref resultsBucket handle journalPath emit+ pure (either (Left . ContinuationError) Right continued)++-- Reconcile observations and collect already sealed results. Planned or still+-- running slices remain untouched for a later lease-aware resume step.+resumeObservedSlices :: ObjectStore -> CellRef -> Bucket -> FilePath -> (WatchEvent -> IO ()) -> IO (Either ResumeError SessionJournal)+resumeObservedSlices store ref resultsBucket journalPath emit = do+ decoded <- readSessionJournal journalPath+ case decoded of+ Left problem -> pure (Left (InvalidJournal problem))+ Right journal+ | journal.cell /= ref.cellName || journal.controlBucket /= ref.controlBucket.unBucket || journal.resultsBucket /= resultsBucket.unBucket -> pure (Left (InvalidJournal "session cell or buckets differ from the requested store"))+ | otherwise -> do+ current <- newIORef journal+ reconcile current journal.slices+ where+ outRoot = takeDirectory (takeDirectory journalPath)+ checkpoint current identifier transition = do+ before <- readIORef current+ now <- getCurrentTime+ changed <- either (ioError . userError . Text.unpack) pure (applyTransition now identifier transition before)+ writeSessionJournal journalPath changed+ writeIORef current changed+ reconcile current [] = Right <$> readIORef current+ reconcile current (slice : rest) = do+ outcome <- case slice.state of+ SlicePlanned -> do+ let name = ObjectName ("cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/submission.json")+ marker <- store.getObject ref.controlBucket name+ case marker of+ Nothing -> pure (Right ())+ Just (bytes, _) -> case eitherDecode bytes :: Either String Submission of+ Right submission | submission == slice.submission -> do+ checkpoint current slice.cellRun SubmissionPublished+ observeSubmitted current slice+ _ -> pure (Left (InvalidJournal "retained planned submission differs from the visible marker"))+ SliceRejected -> pure (Right ())+ SliceVerified -> pure (Right ())+ SliceSubmitted -> observeSubmitted current slice+ SliceSealed -> do+ status <- sealedStatus slice.cellRun+ collect current slice.cellRun status True+ SliceFetched -> do+ status <- sealedStatus slice.cellRun+ collect current slice.cellRun status False+ case outcome of+ Left failure -> pure (Left failure)+ Right () -> reconcile current rest+ observeSubmitted current slice = do+ observed <- observe slice.cellRun+ case observed of+ Nothing -> pure (Right ())+ Just (RunRejected rejected) -> checkpoint current slice.cellRun (SubmissionRejectedByCell rejected) >> pure (Right ())+ Just (RunSealed status) -> do+ checkpoint current slice.cellRun (SubmissionSealed status)+ collect current slice.cellRun (Just status) True+ observe identifier = do+ snapshot <- pollCellRun store ref identifier newWatchCursor+ mapM_ emit snapshot.events+ pure snapshot.terminal+ sealedStatus identifier = do+ terminal <- observe identifier+ pure case terminal of+ Just (RunSealed status) -> Just status+ _ -> Nothing+ collect current identifier status shouldCheckpointFetch = do+ fetched <- fetchCellRun store resultsBucket identifier outRoot+ case fetched of+ Left failure -> pure (Left (ResultFetchFailed identifier failure))+ Right tree -> do+ if shouldCheckpointFetch then checkpoint current identifier (ResultsFetched tree) else pure ()+ verified <- deriveCellRunIndex resultsBucket status tree+ case verified of+ Left problems -> pure (Left (ResultVerificationFailed identifier problems))+ Right index -> do+ _ <- writeCellRunIndex tree index+ checkpoint current identifier (ResultsVerified index)+ pure (Right ())
+ src/Kenshou/Remote/Cell/Session/Runner.hs view
@@ -0,0 +1,120 @@+module Kenshou.Remote.Cell.Session.Runner+ ( SessionRunError (..),+ runPlannedSlices,+ runPendingSlices,+ )+where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Remote.Cell.Docs (Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseSnapshot)+import Kenshou.Remote.Cell.Session (SessionError, SessionTransition, runSubmissionWithTransitions)+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.Watch (WatchEvent)+import Kenshou.Remote.Store (Bucket (..), ObjectStore)+import System.Directory (doesFileExist)+import System.FilePath (takeDirectory, (</>))++data SessionRunError+ = InvalidSession !Text+ | SessionFileExists !FilePath+ | SessionFileMissing !FilePath+ | UnresolvedSlice !RunId !SliceState+ | SliceExecutionFailed !RunId !SessionError+ deriving stock (Eq, Show)++-- Run a new, already prepared journal. The journal and work files are the+-- caller's durable inputs; this function never rewrites a prior session.+runPlannedSlices :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> SessionJournal -> (WatchEvent -> IO ()) -> IO (Either SessionRunError SessionJournal)+runPlannedSlices store ref resultsBucket handle journalPath journal emit = do+ exists <- doesFileExist journalPath+ if exists+ then pure (Left (SessionFileExists journalPath))+ else do+ lease <- leaseSnapshot handle+ case validateSession ref resultsBucket lease journal of+ Left problem -> pure (Left (InvalidSession problem))+ Right ()+ | any ((/= SlicePlanned) . (.state)) journal.slices -> pure (Left (InvalidSession "new session contains a slice that is not planned"))+ | otherwise -> do+ preflight <- checkWorkFiles (takeDirectory journalPath) journal.slices+ case preflight of+ Left problem -> pure (Left (InvalidSession problem))+ Right () -> do+ writeSessionJournal journalPath journal+ executeSlices store ref resultsBucket handle journalPath journal journal.slices emit++-- Continue a journal whose planned slices have already been rebound to this+-- lease. Verified and rejected slices are retained without execution.+runPendingSlices :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> (WatchEvent -> IO ()) -> IO (Either SessionRunError SessionJournal)+runPendingSlices store ref resultsBucket handle journalPath emit = do+ exists <- doesFileExist journalPath+ if not exists+ then pure (Left (SessionFileMissing journalPath))+ else do+ decoded <- readSessionJournal journalPath+ case decoded of+ Left problem -> pure (Left (InvalidSession problem))+ Right journal -> do+ lease <- leaseSnapshot handle+ case validateSession ref resultsBucket lease journal of+ Left problem -> pure (Left (InvalidSession problem))+ Right () -> case filter (\slice -> slice.state `notElem` [SlicePlanned, SliceRejected, SliceVerified]) journal.slices of+ slice : _ -> pure (Left (UnresolvedSlice slice.cellRun slice.state))+ [] -> do+ let planned = filter ((== SlicePlanned) . (.state)) journal.slices+ preflight <- checkWorkFiles (takeDirectory journalPath) planned+ case preflight of+ Left problem -> pure (Left (InvalidSession problem))+ Right () -> executeSlices store ref resultsBucket handle journalPath journal planned emit++validateSession :: CellRef -> Bucket -> Lease -> SessionJournal -> Either Text ()+validateSession ref resultsBucket lease journal = do+ case eitherDecode (encode journal) :: Either String SessionJournal of+ Left failure -> Left (Text.pack failure)+ Right _ -> pure ()+ if journal.cell == ref.cellName && journal.controlBucket == ref.controlBucket.unBucket && journal.resultsBucket == resultsBucket.unBucket && journal.leaseId == lease.leaseId && journal.leaseMode == Held+ then pure ()+ else Left "session cell, bucket, lease or mode differs from the active held lease"++checkWorkFiles :: FilePath -> [SliceJournal] -> IO (Either Text ())+checkWorkFiles _ [] = pure (Right ())+checkWorkFiles directory (slice : rest) = do+ let path = directory </> slice.workPath+ present <- doesFileExist path+ if not present+ then pure (Left ("missing prepared work file: " <> Text.pack slice.workPath))+ else do+ bytes <- LazyByteString.readFile path+ if workObjectFor slice.submission.work.mediaType bytes /= slice.submission.work+ then pure (Left ("prepared work digest or size differs: " <> Text.pack slice.workPath))+ else checkWorkFiles directory rest++executeSlices :: ObjectStore -> CellRef -> Bucket -> LeaseHandle -> FilePath -> SessionJournal -> [SliceJournal] -> (WatchEvent -> IO ()) -> IO (Either SessionRunError SessionJournal)+executeSlices store ref resultsBucket handle journalPath journal slices emit = do+ current <- newIORef journal+ go current slices+ where+ directory = takeDirectory journalPath+ outRoot = takeDirectory directory+ go current [] = Right <$> readIORef current+ go current (slice : rest) = do+ bytes <- LazyByteString.readFile (directory </> slice.workPath)+ let checkpoint :: SessionTransition -> IO ()+ checkpoint transition = do+ before <- readIORef current+ now <- getCurrentTime+ changed <- either (ioError . userError . Text.unpack) pure (applyTransition now slice.cellRun transition before)+ writeSessionJournal journalPath changed+ writeIORef current changed+ result <- runSubmissionWithTransitions store ref resultsBucket handle slice.submission bytes outRoot emit checkpoint+ case result of+ Left failure -> pure (Left (SliceExecutionFailed slice.cellRun failure))+ Right _ -> go current rest
+ src/Kenshou/Remote/Cell/Submit.hs view
@@ -0,0 +1,96 @@+module Kenshou.Remote.Cell.Submit+ ( PublishOutcome (..),+ workObjectFor,+ publishSubmission,+ )+where++import Control.Monad (unless)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text.Encoding qualified as TextEncoding+import Data.Time (addUTCTime)+import Kenshou.Core.Id (renderRunId)+import Kenshou.Remote.Cell.Docs (Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseHeld, leaseSnapshot)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))++data PublishOutcome+ = Submitted+ | LostLease+ | Quarantined+ | WorkAlreadyExists+ | SubmissionAlreadyExists+ deriving stock (Eq, Show)++workObjectFor :: Text -> LazyByteString.ByteString -> WorkObject+workObjectFor mediaType bytes =+ WorkObject+ { sha256 = TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash (LazyByteString.toStrict bytes))),+ bytes = LazyByteString.length bytes,+ mediaType = mediaType+ }++-- The agent increments runsStarted after it claims submission.json. The client+-- only publishes the opaque work object and then the visible submission marker.+publishSubmission :: ObjectStore -> CellRef -> LeaseHandle -> Submission -> LazyByteString.ByteString -> IO PublishOutcome+publishSubmission store ref handle submission workBytes = do+ expected <- leaseSnapshot handle+ unless (expected.cell == ref.cellName && submission.leaseId == expected.leaseId) (ioError (userError "submission lease does not match cell lease handle"))+ case eitherDecode (encode submission) :: Either String Submission of+ Left failure -> ioError (userError ("invalid cell submission: " <> failure))+ Right _ -> pure ()+ let bundle = submission.payload.bundle+ bundleUri = "gs://" <> ref.controlBucket.unBucket <> "/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"+ unless (bundle.uri == bundleUri) (ioError (userError "submission payload bundle is not in the cell control bucket"))+ unless (submission.work == workObjectFor submission.work.mediaType workBytes) (ioError (userError "submission work digest or size does not match its bytes"))+ active <- leaseActive store ref handle+ if not active+ then pure LostLease+ else do+ quarantined <- cellQuarantined store ref+ if quarantined+ then pure Quarantined+ else do+ let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId submission.runId <> "/"+ workResult <- store.putObject ref.controlBucket (ObjectName (prefix <> "work")) submission.work.mediaType DoesNotExist workBytes+ case workResult of+ PreconditionFailed -> pure WorkAlreadyExists+ Written _ -> do+ stillActive <- leaseActive store ref handle+ if not stillActive+ then pure LostLease+ else do+ nowQuarantined <- cellQuarantined store ref+ if nowQuarantined+ then pure Quarantined+ else do+ marker <- store.putObject ref.controlBucket (ObjectName (prefix <> "submission.json")) "application/json" DoesNotExist (encode submission)+ pure case marker of+ Written _ -> Submitted+ PreconditionFailed -> SubmissionAlreadyExists++cellQuarantined :: ObjectStore -> CellRef -> IO Bool+cellQuarantined store ref = do+ status <- store.statObject ref.controlBucket (ObjectName ("cells/" <> ref.cellName <> "/quarantine.json"))+ pure (case status of Nothing -> False; Just _ -> True)++leaseActive :: ObjectStore -> CellRef -> LeaseHandle -> IO Bool+leaseActive store ref handle = do+ held <- leaseHeld handle+ if not held+ then pure False+ else do+ expected <- leaseSnapshot handle+ stored <- store.getObject ref.controlBucket (ObjectName ("cells/" <> ref.cellName <> "/lease.json"))+ case stored of+ Nothing -> pure False+ Just (body, meta) -> case eitherDecode body of+ Left failure -> ioError (userError ("invalid active cell lease: " <> failure))+ Right (current :: Lease) -> do+ now <- store.serverTime+ pure (current.cell == ref.cellName && current.leaseId == expected.leaseId && not current.cancelRequested && addUTCTime (fromIntegral current.ttlSeconds + 30) meta.updated >= now)
+ src/Kenshou/Remote/Cell/Watch.hs view
@@ -0,0 +1,99 @@+module Kenshou.Remote.Cell.Watch+ ( WatchCursor,+ WatchEvent (..),+ WatchTerminal (..),+ WatchSnapshot (..),+ newWatchCursor,+ pollCellRun,+ watchCellRun,+ )+where++import Control.Concurrent (threadDelay)+import Control.Monad (forM, unless)+import Data.Aeson (FromJSON, eitherDecode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (RunId, renderRunId)+import Kenshou.Remote.Cell.Docs (CellPhase (..), CellStatus (..), LogChunks (..), Rejected (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Store (ObjectName (..), ObjectStore (..))++data WatchCursor = WatchCursor+ { lastPhase :: !(Maybe CellPhase),+ stdoutNext :: !Int,+ stderrNext :: !Int+ }+ deriving stock (Eq, Show)++data WatchEvent+ = PhaseChanged !CellPhase+ | StdoutChunk !LazyByteString.ByteString+ | StderrChunk !LazyByteString.ByteString+ deriving stock (Eq, Show)++data WatchTerminal = RunSealed !CellStatus | RunRejected !Rejected+ deriving stock (Eq, Show)++data WatchSnapshot = WatchSnapshot+ { cursor :: !WatchCursor,+ events :: ![WatchEvent],+ terminal :: !(Maybe WatchTerminal)+ }+ deriving stock (Eq, Show)++newWatchCursor :: WatchCursor+newWatchCursor = WatchCursor Nothing 0 0++pollCellRun :: ObjectStore -> CellRef -> RunId -> WatchCursor -> IO WatchSnapshot+pollCellRun store ref identifier cursor = do+ let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId identifier <> "/"+ rejected <- readDocument store ref (ObjectName (prefix <> "rejected.json"))+ case rejected of+ Just record -> do+ unless (record.runId == identifier) (ioError (userError "cell rejection names another run"))+ pure (WatchSnapshot cursor [] (Just (RunRejected record)))+ Nothing -> do+ status <- readDocument store ref (ObjectName (prefix <> "status.json"))+ case status of+ Nothing -> pure (WatchSnapshot cursor [] Nothing)+ Just current -> do+ unless (current.runId == identifier) (ioError (userError "cell status names another run"))+ unless (current.logChunks.stdout >= cursor.stdoutNext && current.logChunks.stderr >= cursor.stderrNext) (ioError (userError "cell log chunk count moved backwards"))+ stdout <- readChunks "stdout" cursor.stdoutNext current.logChunks.stdout StdoutChunk prefix+ stderr <- readChunks "stderr" cursor.stderrNext current.logChunks.stderr StderrChunk prefix+ let changed = if cursor.lastPhase == Just current.phase then [] else [PhaseChanged current.phase]+ next = WatchCursor (Just current.phase) current.logChunks.stdout current.logChunks.stderr+ done = if current.phase == Sealed then Just (RunSealed current) else Nothing+ pure (WatchSnapshot next (changed <> stdout <> stderr) done)+ where+ readChunks :: Text -> Int -> Int -> (LazyByteString.ByteString -> WatchEvent) -> Text -> IO [WatchEvent]+ readChunks stream first end wrap prefix = forM [first .. end - 1] \index -> do+ let object = ObjectName (prefix <> "log/" <> stream <> "." <> showText index)+ stored <- store.getObject ref.controlBucket object+ case stored of+ Nothing -> ioError (userError ("missing cell log chunk " <> show object))+ Just (bytes, _) -> pure (wrap bytes)++watchCellRun :: ObjectStore -> CellRef -> RunId -> (WatchEvent -> IO ()) -> IO WatchTerminal+watchCellRun store ref identifier emit = loop newWatchCursor+ where+ loop cursor = do+ snapshot <- pollCellRun store ref identifier cursor+ mapM_ emit snapshot.events+ case snapshot.terminal of+ Just terminal -> pure terminal+ Nothing -> threadDelay 2000000 >> loop snapshot.cursor++readDocument :: (FromJSON document) => ObjectStore -> CellRef -> ObjectName -> IO (Maybe document)+readDocument store ref name = do+ stored <- store.getObject ref.controlBucket name+ case stored of+ Nothing -> pure Nothing+ Just (bytes, _) -> case eitherDecode bytes of+ Left failure -> ioError (userError ("invalid cell document at " <> show name <> ": " <> failure))+ Right value -> pure (Just value)++showText :: Int -> Text+showText = Text.pack . show
+ src/Kenshou/Remote/Cell/WorkJson.hs view
@@ -0,0 +1,108 @@+module Kenshou.Remote.Cell.WorkJson (decodeWorkPlan, renderPreparedWork) where++import Data.Aeson (FromJSON, Value (..))+import Data.Aeson qualified as Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time.Clock.POSIX (posixSecondsToUTCTime)+import Kenshou.Core.Id (RunId, mkSeed)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Prepare (PreparedRun (..))++data SourceRun = SourceRun+ { ordinal :: !Int,+ runId :: !RunId,+ estimateMinutes :: !Int,+ trial :: !(Maybe TrialInfo),+ spec :: !RunSpec,+ json :: !Value+ }++-- The public plan format is write-only for its provenance members. The typed+-- projection carries only fields used by cell preparation; work rendering+-- copies the original document and replaces selected specs.+decodeWorkPlan :: Value -> Either Text RunPlan+decodeWorkPlan work = do+ (_, entries, planId) <- parseWork work+ selector <- parseSelector "**"+ seed <- mkSeed 1+ let placeholder = Reason (Change (ComponentRef (ComponentId "cell-work") Nothing) Named "internal preparation placeholder") [] selector 0+ planned = [PlannedRun entry.ordinal entry.runId entry.estimateMinutes (placeholder :| []) entry.trial entry.spec | entry <- entries]+ context = PlanContext Nothing "" "" "" (PlanInputs Null) [] []+ pure (RunPlan planId (posixSecondsToUTCTime 0) context (defaultPlanPolicy seed) planned [] (sum (fmap (.estimateMinutes) planned)))++renderPreparedWork :: Value -> [PreparedRun] -> Either Text Value+renderPreparedWork work selected = do+ (fields, entries, _) <- parseWork work+ if unique (fmap (.runId) selected) && unique (fmap (.ordinal) selected)+ then pure ()+ else Left "prepared work contains duplicate run IDs or ordinals"+ let originals = Map.fromList [(entry.runId, entry) | entry <- entries]+ rendered <- traverse (renderEntry originals) selected+ let rows = fmap fst rendered+ minutes = sum (fmap snd rendered)+ pure (Object (KeyMap.insert "runs" (Aeson.toJSON rows) (KeyMap.insert "estimateMinutes" (Aeson.toJSON minutes) fields)))++parseWork :: Value -> Either Text (Aeson.Object, [SourceRun], RunId)+parseWork work = do+ fields <- case work of Object members -> Right members; _ -> Left "run plan is not a JSON object"+ case KeyMap.lookup "schema" fields of+ Just (String "kenshou.run-plan/v1") -> pure ()+ _ -> Left "run plan has an unsupported schema"+ planId <- required "planId" fields+ raw <- case KeyMap.lookup "runs" fields of+ Just (Array rows) -> Right (toList rows)+ _ -> Left "run plan has no runs array"+ entries <- traverse parseEntry raw+ if unique (fmap (.runId) entries) && unique (fmap (.ordinal) entries)+ then Right (fields, entries, planId)+ else Left "run plan contains duplicate run IDs or ordinals"++parseEntry :: Value -> Either Text SourceRun+parseEntry value = case value of+ Object fields -> do+ ordinal <- required "ordinal" fields+ runId <- required "runId" fields+ estimate <- required "estimateMinutes" fields+ spec <- required "spec" fields+ trial <- case KeyMap.lookup "trial" fields of+ Nothing -> Right Nothing+ Just Null -> Right Nothing+ Just (Object members) -> Just <$> (TrialInfo <$> required "group" members <*> required "arm" members <*> required "index" members <*> required "of" members)+ _ -> Left "planned run trial is not an object"+ if ordinal < 0 || estimate <= 0 || maybe False (/= runId) spec.runId+ then Left "planned run has an invalid ordinal, estimate or spec run ID"+ else Right (SourceRun ordinal runId estimate trial spec value)+ _ -> Left "planned run is not an object"++renderEntry :: Map.Map RunId SourceRun -> PreparedRun -> Either Text (Value, Int)+renderEntry originals prepared = case Map.lookup prepared.runId originals of+ Just source+ | source.ordinal /= prepared.ordinal -> Left "prepared run ordinal differs from the source plan"+ | source.spec.scenario /= prepared.spec.scenario -> Left "prepared run scenario differs from the source plan"+ | maybe False (/= prepared.runId) prepared.spec.runId -> Left "prepared spec run ID differs from the source plan"+ | otherwise -> case source.json of+ Object original -> Right (Object (KeyMap.insert "spec" (Aeson.toJSON prepared.spec) original), source.estimateMinutes)+ _ -> Left "prepared run is not an object"+ _ -> Left "prepared run is absent from the source plan"++required :: (FromJSON document) => Text -> Aeson.Object -> Either Text document+required name fields = case KeyMap.lookup (Key.fromText name) fields of+ Nothing -> Left ("run plan is missing " <> name)+ Just value -> case Aeson.fromJSON value of+ Aeson.Error failure -> Left (name <> ": " <> Text.pack failure)+ Aeson.Success result -> Right result++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Pair.hs view
@@ -0,0 +1,93 @@+module Kenshou.Remote.Pair+ ( PairRequest (..),+ PairState (..),+ SliceResult (..),+ pairPlan,+ judgePairs,+ )+where++import Data.List (nub, nubBy)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Word (Word64)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), mkSeed, parseRunId, renderRunId, renderScenarioId)+import Kenshou.Core.RunSpec (ComparisonMembership (..), RunSpec (..))+import Kenshou.Measure.Compare.Ordering (Arm (..), PairedOrdering, TrialSlot (..), pairedSchedule)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))++data PairRequest = PairRequest+ { candidate :: !Text,+ baseline :: !Text,+ pairs :: !Int,+ ordering :: !PairedOrdering,+ seed :: !Word64,+ maxReplacements :: !Int+ }+ deriving stock (Eq, Show)++data PairState = PairValid | PairInvalid !Text+ deriving stock (Eq, Show)++data SliceResult = SliceResult+ { group :: !Text,+ pairIndex :: !Int,+ arm :: !Arm,+ passed :: !Bool+ }+ deriving stock (Eq, Show)++-- A planned benchmark may already have planner-generated trial repeats. The+-- cell schedule replaces those repeats with adjacent, same-seed pairs.+pairPlan :: PairRequest -> RunPlan -> Either Text RunPlan+pairPlan request source = do+ if request.pairs < 3 then Left "paired comparisons need at least three pairs" else pure ()+ if request.maxReplacements < 0 then Left "max replacements cannot be negative" else pure ()+ if Text.null request.baseline || Text.null request.candidate || request.baseline == request.candidate+ then Left "baseline and candidate payload labels must be distinct"+ else pure ()+ if null source.runs then Left "paired comparison plan has no runs" else pure ()+ if any ((/= Benchmark) . (.kind) . (.scenario) . (.spec)) source.runs+ then Left "paired comparisons require benchmark runs only"+ else pure ()+ let configurations = nubBy sameConfiguration source.runs+ slots = pairedSchedule request.ordering (request.pairs + request.maxReplacements) request.seed+ selected = [(configurationIndex, entry, slot) | (configurationIndex, entry) <- zip [0 :: Int ..] configurations, slot <- slots]+ planned <- traverse makeRun (zip [1 :: Int ..] selected)+ pure source {runs = planned, estimateMinutes = sum (fmap (.estimateMinutes) planned)}+ where+ sameConfiguration left right =+ let first = left.spec+ second = right.spec+ in first.scenario == second.scenario+ && first.knobs == second.knobs+ && first.dimensions == second.dimensions+ && first.phases == second.phases+ && first.environment == second.environment++ makeRun (ordinal, (configurationIndex, original, slot)) = do+ identifier <- derivedRunId original.runId configurationIndex slot.position+ pairSeed <- mkSeed (slot.pairSeed `mod` 9007199254740992)+ let groupName = renderScenarioId original.spec.scenario <> "/config-" <> Text.pack (show configurationIndex)+ armName = case slot.arm of Baseline -> request.baseline; Candidate -> request.candidate+ trial = TrialInfo groupName armName slot.pairIndex (request.pairs + request.maxReplacements)+ spec = original.spec {runId = Just identifier, seed = Just pairSeed, comparison = Just (ComparisonMembership groupName armName slot.pairIndex slot.position)}+ pure original {ordinal, runId = identifier, trial = Just trial, spec}++derivedRunId :: RunId -> Int -> Int -> Either Text RunId+derivedRunId original configurationIndex position =+ parseRunId (Text.take 18 (renderRunId original) <> "-a" <> Text.take 3 digest <> "-" <> Text.take 12 (Text.drop 3 digest))+ where+ digest = Text.drop 7 (sha256Hex (TextEncoding.encodeUtf8 (renderRunId original <> ":" <> Text.pack (show configurationIndex) <> ":" <> Text.pack (show position))))++judgePairs :: [SliceResult] -> [((Text, Int), PairState)]+judgePairs results = fmap judge keys+ where+ keys = nub [(result.group, result.pairIndex) | result <- results]+ judge key =+ let members = [result | result <- results, (result.group, result.pairIndex) == key]+ baseline = filter ((== Baseline) . (.arm)) members+ candidate = filter ((== Candidate) . (.arm)) members+ in (key, if length baseline == 1 && length candidate == 1 && all (.passed) members then PairValid else PairInvalid "one or both cell trials failed or are missing")
+ src/Kenshou/Remote/Payload.hs view
@@ -0,0 +1,163 @@+module Kenshou.Remote.Payload+ ( Bundle (..),+ CellPayload (..),+ Harness (..),+ CohortCheck (..),+ PayloadDescriptor (..),+ )+where++import Control.Monad (unless)+import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:), (.=))+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Cohort (CohortIdentity (..), CohortName (..))++data Bundle = Bundle+ { uri :: !Text,+ sha256 :: !Text,+ bytes :: !Int64+ }+ deriving stock (Eq, Show)++data CellPayload = CellPayload+ { bundle :: !Bundle,+ storePath :: !Text,+ narHash :: !Text,+ closurePaths :: ![Text],+ system :: !Text,+ command :: ![Text]+ }+ deriving stock (Eq, Show)++data Harness = Harness+ { revision :: !Text,+ dirty :: !Bool+ }+ deriving stock (Eq, Show)++data CohortCheck = CohortCheck+ { packagesChecked :: !Int+ }+ deriving stock (Eq, Show)++data PayloadDescriptor = PayloadDescriptor+ { cohort :: !Text,+ variant :: !Text,+ flakeAttr :: !Text,+ harness :: !Harness,+ cohortIdentity :: !CohortIdentity,+ cohortCheck :: !CohortCheck,+ cell :: !CellPayload,+ createdAt :: !UTCTime+ }+ deriving stock (Eq, Show)++instance ToJSON Bundle where+ toJSON bundle = object ["uri" .= bundle.uri, "sha256" .= bundle.sha256, "bytes" .= bundle.bytes]++instance FromJSON Bundle where+ parseJSON = withObject "cell payload bundle" \value -> do+ bundle <- Bundle <$> value .: "uri" <*> value .: "sha256" <*> value .: "bytes"+ unless (bundle.bytes > 0 && validSha256 bundle.sha256 && Text.isSuffixOf ("/payloads/sha256/" <> bundle.sha256 <> ".nar.zst") bundle.uri && "gs://" `Text.isPrefixOf` bundle.uri) (fail "invalid content-addressed cell bundle")+ pure bundle++instance ToJSON CellPayload where+ toJSON payload =+ object+ [ "schema" .= ("cell.payload/v1" :: Text),+ "kind" .= ("nix-nar-bundle" :: Text),+ "bundle" .= payload.bundle,+ "storePath" .= payload.storePath,+ "narHash" .= payload.narHash,+ "closurePaths" .= payload.closurePaths,+ "system" .= payload.system,+ "command" .= payload.command+ ]++instance FromJSON CellPayload where+ parseJSON = withObject "cell payload" \value -> do+ schema <- value .: "schema"+ kind <- value .: "kind"+ unless (schema == ("cell.payload/v1" :: Text) && kind == ("nix-nar-bundle" :: Text)) (fail "unsupported cell payload")+ payload <- CellPayload <$> value .: "bundle" <*> value .: "storePath" <*> value .: "narHash" <*> value .: "closurePaths" <*> value .: "system" <*> value .: "command"+ unless (payload.system == "x86_64-linux" && payload.storePath `elem` payload.closurePaths && not (null payload.closurePaths) && unique payload.closurePaths && validStorePath payload.storePath && all validStorePath payload.closurePaths && not (Text.null payload.narHash) && validCommand payload.command) (fail "invalid cell payload")+ pure payload++instance ToJSON Harness where+ toJSON harness = object ["revision" .= harness.revision, "dirty" .= harness.dirty]++instance FromJSON Harness where+ parseJSON = withObject "payload harness" \value -> do+ harness <- Harness <$> value .: "revision" <*> value .: "dirty"+ unless (Text.length harness.revision == 40 && Text.all isHex harness.revision) (fail "invalid harness revision")+ pure harness++instance ToJSON CohortCheck where+ toJSON check = object ["status" .= ("consistent" :: Text), "packagesChecked" .= check.packagesChecked]++instance FromJSON CohortCheck where+ parseJSON = withObject "payload cohort check" \value -> do+ status <- value .: "status"+ unless (status == ("consistent" :: Text)) (fail "payload cohort check is not consistent")+ count <- value .: "packagesChecked"+ unless (count >= (0 :: Int)) (fail "negative checked package count")+ pure (CohortCheck count)++instance ToJSON PayloadDescriptor where+ toJSON descriptor =+ object+ [ "schema" .= ("kenshou.payload/v1" :: Text),+ "cohort" .= descriptor.cohort,+ "variant" .= descriptor.variant,+ "flakeAttr" .= descriptor.flakeAttr,+ "harness" .= descriptor.harness,+ "cohortIdentity" .= descriptor.cohortIdentity,+ "cohortCheck" .= descriptor.cohortCheck,+ "cell" .= descriptor.cell,+ "createdAt" .= descriptor.createdAt+ ]++instance FromJSON PayloadDescriptor where+ parseJSON = withObject "Kenshou payload" \value -> do+ schema <- value .: "schema"+ unless (schema == ("kenshou.payload/v1" :: Text)) (fail "unsupported Kenshou payload schema")+ descriptor <- PayloadDescriptor <$> value .: "cohort" <*> value .: "variant" <*> value .: "flakeAttr" <*> value .: "harness" <*> value .: "cohortIdentity" <*> value .: "cohortCheck" <*> value .: "cell" <*> value .: "createdAt"+ let expectedAttr =+ "packages.x86_64-linux.kenshou-" <> descriptor.cohort <> case descriptor.variant of+ "default" -> ""+ "info-table" -> "-info-table"+ "profiled" -> "-profiled"+ _ -> "-invalid"+ unless (descriptor.variant `elem` ["default", "info-table", "profiled"] && descriptor.cohort == unCohortName descriptor.cohortIdentity.identityCohort && descriptor.flakeAttr == expectedAttr && descriptor.cohortIdentity.identityResolver == Just "nix" && descriptor.cell.command == ["bin/kenshou", "cell", "exec"]) (fail "inconsistent Kenshou payload identity")+ pure descriptor++validSha256 :: Text -> Bool+validSha256 digest = Text.length digest == 64 && Text.all isHex digest++isHex :: Char -> Bool+isHex character = character `elem` ['0' .. '9'] || character `elem` ['a' .. 'f']++validStorePath :: Text -> Bool+validStorePath path = case Text.stripPrefix "/nix/store/" path of+ Just suffix ->+ let (digest, name) = Text.breakOn "-" suffix+ in Text.length digest == 32 && Text.all validNixBase32 digest && not (Text.null (Text.drop 1 name)) && not (Text.any (== '/') name)+ Nothing -> False+ where+ validNixBase32 character = character `elem` ("0123456789abcdfghijklmnpqrsvwxyz" :: String)++validCommand :: [Text] -> Bool+validCommand [] = False+validCommand (first : rest) =+ not (Text.null first)+ && not (Text.isPrefixOf "/" first)+ && not (Text.any (== '\0') first)+ && all (/= "..") (Text.splitOn "/" first)+ && all (not . Text.null) rest++unique :: (Eq value) => [value] -> Bool+unique [] = True+unique (first : rest) = first `notElem` rest && unique rest
+ src/Kenshou/Remote/Payload/Nix.hs view
@@ -0,0 +1,172 @@+module Kenshou.Remote.Payload.Nix+ ( NixError (..),+ NixTools (..),+ ClosureInfo (..),+ BundleInfo (..),+ nixBuild,+ nixEvalJson,+ closureInfo,+ parseClosureInfo,+ exportBundle,+ exportBundleWith,+ )+where++import Control.Exception (IOException, displayException, try)+import Data.Aeson (FromJSON, Value (..), eitherDecodeStrict')+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Int (Int64)+import Data.List (sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Kenshou.Remote.Payload.Publish (digestFile)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.Exit (ExitCode (..))+import System.FilePath (takeDirectory)+import System.IO (hClose, openBinaryTempFile)+import System.Process (StdStream (..), createProcess, proc, readProcessWithExitCode, std_in, std_out, terminateProcess, waitForProcess)++data NixError+ = NixCommandFailed !FilePath ![String] !ExitCode !Text+ | NixCommandIoError !Text+ | NixInvalidOutput !Text+ deriving stock (Eq, Show)++data NixTools = NixTools+ { nixExecutable :: !FilePath,+ nixStoreExecutable :: !FilePath,+ zstdExecutable :: !FilePath+ }+ deriving stock (Eq, Show)++defaultTools :: NixTools+defaultTools = NixTools "nix" "nix-store" "zstd"++data ClosureInfo = ClosureInfo+ { storePath :: !FilePath,+ narHash :: !Text,+ closurePaths :: ![FilePath]+ }+ deriving stock (Eq, Show)++data BundleInfo = BundleInfo+ { file :: !FilePath,+ sha256 :: !Text,+ bytes :: !Int64,+ closure :: !ClosureInfo+ }+ deriving stock (Eq, Show)++nixBuild :: String -> IO (Either NixError FilePath)+nixBuild reference = do+ result <- runCommand defaultTools.nixExecutable ["build", "--no-link", "--print-out-paths", reference]+ pure do+ output <- result+ case lines (Text.unpack output) of+ [path] | "/nix/store/" `Text.isPrefixOf` Text.pack path -> Right path+ _ -> Left (NixInvalidOutput "nix build did not return exactly one store path")++nixEvalJson :: (FromJSON value) => String -> IO (Either NixError value)+nixEvalJson reference = do+ result <- runCommand defaultTools.nixExecutable ["eval", "--json", reference]+ pure do+ output <- result+ firstDecode (eitherDecodeStrict' (TextEncoding.encodeUtf8 output))++closureInfo :: FilePath -> IO (Either NixError ClosureInfo)+closureInfo = closureInfoWith defaultTools++closureInfoWith :: NixTools -> FilePath -> IO (Either NixError ClosureInfo)+closureInfoWith tools path = do+ result <- runCommand tools.nixExecutable ["path-info", "--json", "--recursive", path]+ pure do+ output <- result+ value <- firstDecode (eitherDecodeStrict' (TextEncoding.encodeUtf8 output))+ parseClosureInfo path value++parseClosureInfo :: FilePath -> Value -> Either NixError ClosureInfo+parseClosureInfo path (Object paths) = do+ root <- maybe (Left (NixInvalidOutput "closure metadata omits the requested store path")) Right (KeyMap.lookup (Key.fromText (Text.pack path)) paths)+ hash <- case root of+ Object fields -> case KeyMap.lookup "narHash" fields of+ Just (String value) | "sha256-" `Text.isPrefixOf` value -> Right value+ _ -> Left (NixInvalidOutput "closure metadata omits the root NAR hash")+ _ -> Left (NixInvalidOutput "closure metadata has a malformed root entry")+ let names = sort (map (Text.unpack . Key.toText . fst) (KeyMap.toList paths))+ if null names || any (not . Text.isPrefixOf "/nix/store/" . Text.pack) names+ then Left (NixInvalidOutput "closure metadata contains an invalid store path")+ else Right (ClosureInfo path hash names)+parseClosureInfo _ _ = Left (NixInvalidOutput "closure metadata must be a JSON object")++exportBundle :: FilePath -> FilePath -> IO (Either NixError BundleInfo)+exportBundle = exportBundleWith defaultTools++exportBundleWith :: NixTools -> FilePath -> FilePath -> IO (Either NixError BundleInfo)+exportBundleWith tools path destination = do+ closureResult <- closureInfoWith tools path+ case closureResult of+ Left problem -> pure (Left problem)+ Right closure -> do+ createDirectoryIfMissing True (takeDirectory destination)+ (temporary, handle) <- openBinaryTempFile (takeDirectory destination) ".kenshou-payload-"+ hClose handle+ exportResult <- try (exportToFile tools closure.closurePaths temporary) :: IO (Either IOException (Either NixError ()))+ case exportResult of+ Left problem -> do+ removeIfPresent temporary+ pure (Left (NixCommandIoError (Text.pack (displayException problem))))+ Right (Left problem) -> do+ removeIfPresent temporary+ pure (Left problem)+ Right (Right ()) -> do+ finished <- try @IOException do+ (sha256, size) <- digestFile temporary+ renameFile temporary destination+ pure (BundleInfo destination sha256 size closure)+ case finished of+ Left problem -> do+ removeIfPresent temporary+ pure (Left (NixCommandIoError (Text.pack (displayException problem))))+ Right bundle -> pure (Right bundle)++exportToFile :: NixTools -> [FilePath] -> FilePath -> IO (Either NixError ())+exportToFile tools paths destination = do+ let exportArgs = "--export" : paths+ compressArgs = ["-19", "-T1", "-q", "-f", "-o", destination]+ (_, exportOutput, _, exportProcess) <- createProcess (proc tools.nixStoreExecutable exportArgs) {std_out = CreatePipe}+ case exportOutput of+ Nothing -> pure (Left (NixInvalidOutput "nix-store export did not create a pipe"))+ Just pipe -> do+ compression <- try @IOException (createProcess (proc tools.zstdExecutable compressArgs) {std_in = UseHandle pipe})+ case compression of+ Left problem -> do+ hClose pipe+ terminateProcess exportProcess+ _ <- waitForProcess exportProcess+ pure (Left (NixCommandIoError (Text.pack (displayException problem))))+ Right (_, _, _, compressProcess) -> do+ hClose pipe+ compressExit <- waitForProcess compressProcess+ exportExit <- waitForProcess exportProcess+ pure case (exportExit, compressExit) of+ (ExitSuccess, ExitSuccess) -> Right ()+ (_, failure) | failure /= ExitSuccess -> Left (NixCommandFailed tools.zstdExecutable compressArgs failure "bundle compression failed")+ (failure, _) -> Left (NixCommandFailed tools.nixStoreExecutable exportArgs failure "closure export failed")++runCommand :: FilePath -> [String] -> IO (Either NixError Text)+runCommand executable arguments = do+ result <- try (readProcessWithExitCode executable arguments "") :: IO (Either IOException (ExitCode, String, String))+ pure case result of+ Left problem -> Left (NixCommandIoError (Text.pack (displayException problem)))+ Right (ExitSuccess, output, _) -> Right (Text.pack output)+ Right (failure, _, errors) -> Left (NixCommandFailed executable arguments failure (Text.pack errors))++firstDecode :: Either String value -> Either NixError value+firstDecode = either (Left . NixInvalidOutput . Text.pack) Right++removeIfPresent :: FilePath -> IO ()+removeIfPresent path = do+ exists <- doesFileExist path+ if exists then removeFile path else pure ()
+ src/Kenshou/Remote/Payload/Publish.hs view
@@ -0,0 +1,67 @@+module Kenshou.Remote.Payload.Publish+ ( BundlePublishError (..),+ digestFile,+ publishBundle,+ )+where++import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (eitherDecode, encode)+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import System.IO (IOMode (..), withBinaryFile)++data BundlePublishError+ = InvalidPayloadDescriptor !Text+ | BundleOutsideControlBucket !Text+ | BundleBytesMismatch !Int64 !Int64+ | BundleDigestMismatch !Text !Text+ | ExistingBundleSizeMismatch !Int64 !Int64+ deriving stock (Eq, Show)++-- The caller supplies a finished export file and descriptor. The object name is+-- derived from the checked digest, so a retry can only target the same key.+publishBundle :: ObjectStore -> Bucket -> PayloadDescriptor -> FilePath -> IO (Either BundlePublishError PayloadDescriptor)+publishBundle store bucket descriptor source = case eitherDecode (encode descriptor) :: Either String PayloadDescriptor of+ Left failure -> pure (Left (InvalidPayloadDescriptor (Text.pack failure)))+ Right _ -> do+ let bundle = descriptor.cell.bundle+ expectedUri = "gs://" <> bucket.unBucket <> "/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"+ object = ObjectName ("payloads/sha256/" <> bundle.sha256 <> ".nar.zst")+ if bundle.uri /= expectedUri+ then pure (Left (BundleOutsideControlBucket bundle.uri))+ else do+ (actualDigest, actualBytes) <- digestFile source+ if actualBytes /= bundle.bytes+ then pure (Left (BundleBytesMismatch bundle.bytes actualBytes))+ else+ if actualDigest /= bundle.sha256+ then pure (Left (BundleDigestMismatch bundle.sha256 actualDigest))+ else do+ uploaded <- store.putFile bucket object "application/zstd" DoesNotExist source+ case uploaded of+ Written meta -> pure (checkSize bundle.bytes meta.size)+ PreconditionFailed -> do+ existing <- store.statObject bucket object+ pure case existing of+ Just meta -> checkSize bundle.bytes meta.size+ Nothing -> Left (ExistingBundleSizeMismatch bundle.bytes (-1))+ where+ checkSize expected actual+ | actual == expected = Right descriptor+ | otherwise = Left (ExistingBundleSizeMismatch expected actual)++digestFile :: FilePath -> IO (Text, Int64)+digestFile path = withBinaryFile path ReadMode (go SHA256.init 0)+ where+ go context size handle = do+ chunk <- ByteString.hGetSome handle (1024 * 1024)+ if ByteString.null chunk+ then pure (TextEncoding.decodeUtf8 (Base16.encode (SHA256.finalize context)), size)+ else go (SHA256.update context chunk) (size + fromIntegral (ByteString.length chunk)) handle
+ src/Kenshou/Remote/Payload/Publisher.hs view
@@ -0,0 +1,250 @@+module Kenshou.Remote.Payload.Publisher+ ( PublishOptions (..),+ PreparedPublishOptions (..),+ PublishError (..),+ PublisherDeps (..),+ publishPayload,+ publishPayloadWith,+ publishPreparedPayload,+ publishPreparedPayloadWith,+ validateCohortIdentity,+ )+where++import Control.Exception (IOException, displayException, finally, try)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (eitherDecodeStrict', encode)+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Time (getCurrentTime)+import Kenshou.Core.Cohort (CohortDescriptor (..), CohortIdentity (..), CohortName (..), ComponentSpec (..), checkCohort)+import Kenshou.Perf.Request (immutableRevision)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), CohortCheck (..), Harness (..), PayloadDescriptor (..))+import Kenshou.Remote.Payload.Nix (BundleInfo (..), ClosureInfo (..), NixError (..), exportBundle, nixBuild, nixEvalJson)+import Kenshou.Remote.Payload.Publish (BundlePublishError, publishBundle)+import Kenshou.Remote.Store (Bucket (..), ObjectStore)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.Exit (ExitCode (..))+import System.FilePath (takeDirectory, (</>))+import System.IO (hClose, openBinaryTempFile)+import System.Process (readProcessWithExitCode)++data PublishOptions = PublishOptions+ { root :: !FilePath,+ cohort :: !Text,+ variant :: !Text,+ bucket :: !Bucket,+ out :: !FilePath,+ allowDirty :: !Bool+ }+ deriving stock (Eq, Show)++data PreparedPublishOptions = PreparedPublishOptions+ { harnessRoot :: !FilePath,+ harnessCommit :: !Text,+ generatedFlake :: !FilePath,+ generatedDescriptor :: !FilePath,+ selectedCohort :: !Text,+ selectedVariant :: !Text,+ controlBucket :: !Bucket,+ output :: !FilePath+ }+ deriving stock (Eq, Show)++data PublishError+ = PublishInvalidSelection !Text+ | PublishDirtyWorktree+ | PublishDescriptorError !Text+ | PublishCohortMismatch !Text+ | PublishNixError !NixError+ | PublishBundleError !BundlePublishError+ | PublishIoError !Text+ deriving stock (Eq, Show)++data PublisherDeps = PublisherDeps+ { readHarness :: FilePath -> IO (Either PublishError Harness),+ evalIdentity :: String -> IO (Either NixError CohortIdentity),+ build :: String -> IO (Either NixError FilePath),+ export :: FilePath -> FilePath -> IO (Either NixError BundleInfo)+ }++defaultDeps :: PublisherDeps+defaultDeps = PublisherDeps gitHarness nixEvalJson nixBuild exportBundle++publishPayload :: ObjectStore -> PublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPayload = publishPayloadWith defaultDeps++publishPayloadWith :: PublisherDeps -> ObjectStore -> PublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPayloadWith deps store options = case flakeAttribute options.cohort options.variant of+ Left problem -> pure (Left (PublishInvalidSelection problem))+ Right attribute ->+ publishAt+ deps+ store+ options+ attribute+ (options.root <> "#" <> Text.unpack attribute)+ (options.root </> "cohort" </> Text.unpack options.cohort <> ".json")++publishPreparedPayload :: ObjectStore -> PreparedPublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPreparedPayload = publishPreparedPayloadWith defaultDeps++publishPreparedPayloadWith :: PublisherDeps -> ObjectStore -> PreparedPublishOptions -> IO (Either PublishError PayloadDescriptor)+publishPreparedPayloadWith deps store prepared+ | prepared.selectedVariant `notElem` ["default", "info-table", "profiled"] = pure (Left (PublishInvalidSelection "invalid prepared variant"))+ | not ("perf-" `Text.isPrefixOf` prepared.selectedCohort && Text.all (\c -> c `elem` ['a' .. 'z'] || c `elem` ['0' .. '9'] || c == '-') prepared.selectedCohort) = pure (Left (PublishInvalidSelection "prepared cohort must use its generated perf name"))+ | not (immutableRevision prepared.harnessCommit) = pure (Left (PublishInvalidSelection "prepared harness commit must be immutable"))+ | otherwise = do+ harness <- deps.readHarness prepared.harnessRoot+ case harness of+ Left problem -> pure (Left problem)+ Right actual+ | actual.dirty -> pure (Left PublishDirtyWorktree)+ | actual.revision /= prepared.harnessCommit -> pure (Left (PublishDescriptorError "prepared harness commit differs from the clean build root"))+ | otherwise ->+ publishAt+ deps+ store+ (PublishOptions prepared.harnessRoot prepared.selectedCohort prepared.selectedVariant prepared.controlBucket prepared.output False)+ (preparedAttribute prepared)+ (prepared.generatedFlake <> "#" <> Text.unpack (preparedAttribute prepared))+ prepared.generatedDescriptor++preparedAttribute :: PreparedPublishOptions -> Text+preparedAttribute prepared =+ "packages.x86_64-linux.kenshou-" <> prepared.selectedCohort <> case prepared.selectedVariant of+ "default" -> ""+ other -> "-" <> other++publishAt :: PublisherDeps -> ObjectStore -> PublishOptions -> Text -> String -> FilePath -> IO (Either PublishError PayloadDescriptor)+publishAt deps store options attribute flake descriptorPath = do+ harnessResult <- deps.readHarness options.root+ case harnessResult of+ Left problem -> pure (Left problem)+ Right harness+ | harness.dirty && not options.allowDirty -> pure (Left PublishDirtyWorktree)+ | otherwise -> do+ descriptorResult <- readDescriptor descriptorPath+ case descriptorResult of+ Left problem -> pure (Left problem)+ Right (descriptor, _) | descriptor.descriptorName /= CohortName options.cohort -> pure (Left (PublishDescriptorError "selected descriptor names another cohort"))+ Right (descriptor, descriptorSha) -> do+ identityResult <- deps.evalIdentity (flake <> ".cohortIdentity")+ case identityResult of+ Left problem -> pure (Left (PublishNixError problem))+ Right identity -> case validateCohortIdentity descriptor descriptorSha identity of+ Left problem -> pure (Left (PublishCohortMismatch problem))+ Right checked -> do+ built <- deps.build flake+ case built of+ Left problem -> pure (Left (PublishNixError problem))+ Right storePath -> withBundleFile options.out \bundlePath -> do+ exported <- deps.export storePath bundlePath+ case exported of+ Left problem -> pure (Left (PublishNixError problem))+ Right bundle | bundle.closure.storePath /= storePath -> pure (Left (PublishNixError (NixInvalidOutput "exported closure root differs from the built store path")))+ Right bundle -> do+ now <- getCurrentTime+ let digest = bundle.sha256+ object = "payloads/sha256/" <> digest <> ".nar.zst"+ payload =+ PayloadDescriptor+ options.cohort+ options.variant+ attribute+ harness+ identity+ (CohortCheck checked)+ ( CellPayload+ (Bundle ("gs://" <> options.bucket.unBucket <> "/" <> object) digest bundle.bytes)+ (Text.pack storePath)+ bundle.closure.narHash+ (map Text.pack bundle.closure.closurePaths)+ "x86_64-linux"+ ["bin/kenshou", "cell", "exec"]+ )+ now+ uploaded <- publishBundle store options.bucket payload bundle.file+ case uploaded of+ Left problem -> pure (Left (PublishBundleError problem))+ Right published -> do+ written <- writeDescriptor options.out published+ pure (published <$ written)++validateCohortIdentity :: CohortDescriptor -> Text -> CohortIdentity -> Either Text Int+validateCohortIdentity descriptor descriptorSha identity+ | identity.identityDescriptorSha256 /= descriptorSha = Left "Nix identity descriptor digest differs from the selected descriptor"+ | identity.identityResolver /= Just "nix" = Left "payload identity must use the Nix resolver"+ | identity.identityOs /= "linux" || identity.identityArch /= "x86_64" = Left "payload identity must target x86_64-linux"+ | identity.identityIndexState /= Just descriptor.descriptorIndexState = Left "Nix identity index state differs from the selected descriptor"+ | not (null mismatches) = Left (Text.pack (show mismatches))+ | otherwise = Right (sum (map (length . (.componentPackages)) descriptor.descriptorComponents))+ where+ mismatches = checkCohort descriptor identity++flakeAttribute :: Text -> Text -> Either Text Text+flakeAttribute cohort variant+ | cohort `notElem` ["released", "head"] = Left "--cohort must be released or head"+ | variant `notElem` ["default", "info-table", "profiled"] = Left "--variant must be default, info-table or profiled"+ | otherwise = Right ("packages.x86_64-linux.kenshou-" <> cohort <> if variant == "default" then "" else "-" <> variant)++readDescriptor :: FilePath -> IO (Either PublishError (CohortDescriptor, Text))+readDescriptor path = do+ contents <- try @IOException (ByteString.readFile path)+ pure case contents of+ Left problem -> Left (PublishIoError (Text.pack (displayException problem)))+ Right bytes -> case eitherDecodeStrict' bytes of+ Left problem -> Left (PublishDescriptorError (Text.pack problem))+ Right descriptor -> Right (descriptor, TextEncoding.decodeUtf8 (Base16.encode (SHA256.hash bytes)))++gitHarness :: FilePath -> IO (Either PublishError Harness)+gitHarness path = do+ revision <- git ["-C", path, "rev-parse", "HEAD"]+ status <- git ["-C", path, "status", "--porcelain"]+ pure do+ full <- Text.strip <$> revision+ changed <- status+ if Text.length full /= 40 || not (Text.all isHex full)+ then Left (PublishDescriptorError "git did not return a full commit revision")+ else Right (Harness full (not (Text.null changed)))+ where+ isHex character = character `elem` ['0' .. '9'] || character `elem` ['a' .. 'f']++git :: [String] -> IO (Either PublishError Text)+git arguments = do+ result <- try @IOException (readProcessWithExitCode "git" arguments "")+ pure case result of+ Left problem -> Left (PublishIoError (Text.pack (displayException problem)))+ Right (ExitSuccess, output, _) -> Right (Text.pack output)+ Right (_, _, errors) -> Left (PublishIoError (Text.pack errors))++withBundleFile :: FilePath -> (FilePath -> IO result) -> IO result+withBundleFile destination operation = do+ let directory = takeDirectory destination+ createDirectoryIfMissing True directory+ (temporary, handle) <- openBinaryTempFile directory ".kenshou-export-"+ hClose handle+ operation temporary `finally` removeIfPresent temporary++writeDescriptor :: FilePath -> PayloadDescriptor -> IO (Either PublishError ())+writeDescriptor destination descriptor = do+ let directory = takeDirectory destination+ createDirectoryIfMissing True directory+ (temporary, handle) <- openBinaryTempFile directory ".kenshou-descriptor-"+ result <- try @IOException do+ LazyByteString.hPut handle (encode descriptor) `finally` hClose handle+ renameFile temporary destination+ case result of+ Left problem -> do+ removeIfPresent temporary+ pure (Left (PublishIoError (Text.pack (displayException problem))))+ Right () -> pure (Right ())++removeIfPresent :: FilePath -> IO ()+removeIfPresent path = do+ exists <- doesFileExist path+ if exists then removeFile path else pure ()
+ src/Kenshou/Remote/Perf/Attempt.hs view
@@ -0,0 +1,283 @@+module Kenshou.Remote.Perf.Attempt+ ( TrialExecutor,+ executeAttempt,+ verifyAttempt,+ verifyAttemptInputs,+ )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseEither)+import Data.List (find, nub, sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (renderRunId, renderScenarioId)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.RunSpec (ComparisonMembership (..), RunSpec (..))+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility (VaryingAxis)+import Kenshou.Measure.Compare.Policy (Policy)+import Kenshou.Measure.Perf+import Kenshou.Perf.Plan (PerfProfile (..), PlannedProfile (..))+import Kenshou.Perf.Result+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Remote.Perf.Journal (LeaseIdentity)+import Kenshou.Remote.Perf.Runner (ComparisonHooks (..))+import System.Directory (canonicalizePath, createDirectoryIfMissing, doesFileExist, renameFile)+import System.FilePath (isAbsolute, makeRelative, splitDirectories, (</>))++-- The adapter persists cell work before submission. Calling it again for the+-- same entry reconciles the existing low-level journal under the SAME lease.+-- It returns a verified nested run directory, retaining the outer cell tree.+type TrialExecutor = PlannedRun -> IO (Either Text FilePath)++data Observation = Observation PlannedRun (Maybe FilePath) [AdmissionProblem]++executeAttempt :: LeaseIdentity -> PlannedProfile -> RunPlan -> Policy -> NonEmpty VaryingAxis -> FilePath -> ComparisonHooks -> TrialExecutor -> IO ConfigurationResult+executeAttempt lease entry schedule policy axes directory callbacks execute = do+ createDirectoryIfMissing True directory+ let inputs = attemptInputs lease entry schedule+ immutable (directory </> "attempt.json") inputs+ validateSchedule entry schedule+ paths <- loadPaths directory+ validatePaths directory schedule paths+ failures <- loadFailures directory schedule paths+ atomicPaths directory paths+ collect paths failures [] [0 .. entry.profile.minimumPairs - 1] >>= continue entry.profile.minimumPairs+ where+ collect paths failures observations [] = pure (paths, failures, observations)+ collect paths failures observations (index : rest) = do+ let runs = filter (maybe False ((== index) . (.index)) . (.trial)) schedule.runs+ (changed, rejected, appended) <- runEntries paths failures runs+ collect changed rejected (observations <> appended) rest+ runEntries paths failures [] = pure (paths, failures, [])+ runEntries paths failures (run : rest) = do+ let identifier = renderRunId run.runId+ located <- case (Map.lookup identifier paths, Map.lookup identifier failures) of+ (Just relative, _) -> pure (Right (directory </> relative))+ (_, Just reason) -> pure (Left reason)+ _ -> execute run+ (changed, rejected, observation) <- case located of+ Left reason -> do+ let rejected = Map.insert identifier reason failures+ atomicJSON (directory </> "trial-failures.json") rejected+ pure (paths, rejected, Observation run Nothing [InvalidPerformanceEvidence reason])+ Right path -> do+ relative <- relativePath directory path+ let changed = Map.insert (renderRunId run.runId) relative paths+ atomicPaths directory changed+ observation <- admitTrial entry.profile run path+ mapM_ callbacks.recordRuntimeFailure (runtimeProblems observation)+ pure (changed, failures, observation)+ (final, finalFailures, appended) <- runEntries changed rejected rest+ pure (final, finalFailures, observation : appended)+ continue next (paths, failures, observations)+ | length (validPairs observations) >= entry.profile.minimumPairs = finish directory entry policy axes observations+ | next >= entry.profile.minimumPairs + entry.profile.replacementPairs = finish directory entry policy axes observations+ | otherwise = do+ callbacks.reserveReplacement next+ updated <- collect paths failures observations [next]+ continue (next + 1) updated++-- Replay uses only retained paths and rechecks every trial, including failures.+-- It never submits work or overwrites the original comparison bytes.+verifyAttempt :: PlannedProfile -> RunPlan -> Policy -> NonEmpty VaryingAxis -> FilePath -> ConfigurationResult -> IO Bool+verifyAttempt entry schedule policy axes directory retained = do+ validateSchedule entry schedule+ paths <- loadPaths directory+ validatePaths directory schedule paths+ failures <- loadFailures directory schedule paths+ observations <-+ traverse+ ( \run -> case Map.lookup (renderRunId run.runId) paths of+ Nothing -> pure (Observation run Nothing [InvalidPerformanceEvidence (Map.findWithDefault "trial has no retained result" (renderRunId run.runId) failures)])+ Just relative -> admitTrial entry.profile run (directory </> relative)+ )+ [run | run <- schedule.runs, Map.member (renderRunId run.runId) paths || Map.member (renderRunId run.runId) failures]+ recomputed <- finish directory entry policy axes observations+ pure (same recomputed retained)+ where+ same left right =+ left.profile == right.profile+ && left.workloadSha256 == right.workloadSha256+ && left.performanceVerdict == right.performanceVerdict+ && left.infrastructureState == right.infrastructureState+ && all (`elem` right.runtimeFailures) left.runtimeFailures+ && left.validPairs == right.validPairs+ && left.attempts == right.attempts+ && case (left.comparison, right.comparison) of+ (Just a, Just b) -> semanticComparison a == semanticComparison b+ (Nothing, Nothing) -> True+ _ -> False++verifyAttemptInputs :: LeaseIdentity -> PlannedProfile -> RunPlan -> FilePath -> IO ()+verifyAttemptInputs lease entry schedule directory = do+ validateSchedule entry schedule+ header <- eitherDecodeFileStrict' (directory </> "attempt.json") >>= either (ioError . userError) pure+ unless (header == attemptInputs lease entry schedule) (ioError (userError "retained attempt header differs from its journaled lease or pinned workload"))+ paths <- loadPaths directory+ validatePaths directory schedule paths+ _ <- loadFailures directory schedule paths+ pure ()++attemptInputs :: LeaseIdentity -> PlannedProfile -> RunPlan -> Value+attemptInputs lease entry schedule = object ["schema" .= ("kenshou.perf-attempt/v1" :: Text), "lease" .= lease, "workloadSha256" .= entry.workloadSha256, "plan" .= schedule]++admitTrial :: PerfProfile -> PlannedRun -> FilePath -> IO Observation+admitTrial profile run directory = do+ admitted <- verifyPerformanceRun profile directory+ result <- eitherDecodeFileStrict' (directory </> "run-result.json")+ spec <- eitherDecodeFileStrict' (directory </> "run-spec.json") :: IO (Either String RunSpec)+ let binding = do+ document <- either (Left . Text.pack) Right result+ effective <- either (Left . Text.pack) Right spec+ either (Left . Text.pack) Right (parseEither (withObject "runtime result" (.: "runId")) document) >>= \identifier ->+ unless (identifier == run.runId) (Left "retained result has a different run ID")+ unless+ ( effective.runId == Just run.runId+ && effective.scenario == run.spec.scenario+ && effective.seed == run.spec.seed+ && effective.scenarioRevision == Just profile.scenarioRevision+ && effective.phases == Just zeroPhases+ && effective.timeoutSeconds == Just profile.timeoutSeconds+ && knobs (toJSON effective) == knobs (toJSON run.spec)+ && Map.fromList effective.dimensions == Map.fromList run.spec.dimensions+ )+ (Left "retained runtime inputs differ from the pinned trial")+ problems = either id (const []) admitted <> either (\reason -> [InvalidPerformanceEvidence reason]) (const []) binding+ pure (Observation run (Just directory) problems)+ where+ knobs (Object fields) = KeyMap.lookup "knobs" fields+ knobs _ = Nothing++runtimeProblems :: Observation -> [Text]+runtimeProblems (Observation _ _ problems) = [reason | RuntimeWorkFailure reason <- problems]++validPairs :: [Observation] -> [Int]+validPairs observations = filter valid (sort (nub [trial.index | Observation run _ _ <- observations, Just trial <- [run.trial]]))+ where+ valid index = case [observation | observation@(Observation run _ _) <- observations, maybe False ((== index) . (.index)) run.trial] of+ [Observation a (Just _) ap, Observation b (Just _) bp] ->+ fmap (.arm) a.trial /= fmap (.arm) b.trial && all runtimeOnly (ap <> bp)+ _ -> False+ runtimeOnly (RuntimeWorkFailure _) = True+ runtimeOnly _ = False++finish :: FilePath -> PlannedProfile -> Policy -> NonEmpty VaryingAxis -> [Observation] -> IO ConfigurationResult+finish root entry policy axes observations = do+ let chosen = take entry.profile.minimumPairs (validPairs observations)+ path index arm = case find (\(Observation run _ _) -> maybe False (\trial -> trial.index == index && trial.arm == arm) run.trial) observations of+ Just (Observation _ (Just directory) _) -> directory+ _ -> error "valid pair lacks a retained arm"+ failures = nub (concatMap runtimeProblems observations)+ attempts = [object ["runId" .= run.runId, "path" .= fmap (makeRelative root) directory, "problems" .= map (Text.replace (Text.pack root) "." . Text.pack . show) problems] | Observation run directory problems <- observations]+ make verdict infrastructure comparison = ConfigurationResult entry.profile.name entry.workloadSha256 verdict infrastructure failures (length chosen) entry.profile.minimumPairs attempts comparison+ if length chosen < entry.profile.minimumPairs+ then pure (make "inconclusive" "failed" Nothing)+ else do+ compared <- compareRuns policy axes (map (`path` "baseline") chosen) (map (`path` "candidate") chosen)+ case compared of+ Left (CompareError _) -> pure (make "inconclusive" "failed" Nothing)+ Right comparison ->+ pure+ ( make+ (case comparison.verdict of VerdictPass -> "pass"; VerdictRegression -> "regression"; _ -> "inconclusive")+ (if comparison.verdict == VerdictInfrastructureFailure || length chosen < entry.profile.minimumPairs then "failed" else "healthy")+ (Just (toJSON comparison))+ )++validateSchedule :: PlannedProfile -> RunPlan -> IO ()+validateSchedule entry schedule = do+ let runs = schedule.runs+ identifiers = map (.runId) runs+ expected = entry.profile.minimumPairs + entry.profile.replacementPairs+ trials = [(trial.index, trial.arm) | run <- runs, Just trial <- [run.trial]]+ ordering = [(index, arm) | index <- [0 .. expected - 1], arm <- if even index then ["baseline", "candidate"] else ["candidate", "baseline"]]+ seeds = [nub [run.spec.seed | run <- runs, maybe False ((== index) . (.index)) run.trial] | index <- [0 .. expected - 1]]+ unless+ ( length runs == 2 * expected+ && nub identifiers == identifiers+ && trials == ordering+ && map (.ordinal) runs == [1 .. 2 * expected]+ && all ((== 1) . length) seeds+ && all (\run -> run.spec.runId == Just run.runId && maybe False ((== expected) . (.of_)) run.trial) runs+ && all memberMatches runs+ )+ (ioError (userError "attempt must contain the full fixed paired schedule"))+ unless (all matches runs) (ioError (userError "attempt workload differs from its pinned profile"))+ where+ memberMatches run = case (run.trial, run.spec.comparison) of+ (Just trial, Just membership) -> membership.group == trial.group && membership.arm == trial.arm && membership.trial == trial.index && membership.position == run.ordinal - 1+ _ -> False+ matches run =+ renderScenarioId run.spec.scenario == entry.profile.scenario+ && run.spec.scenarioRevision == Just entry.profile.scenarioRevision+ && run.spec.seed /= Nothing+ && run.spec.phases == Just zeroPhases+ && run.spec.timeoutSeconds == Just entry.profile.timeoutSeconds+ && Map.fromList run.spec.dimensions == entry.profile.dimensions+ && case toJSON run.spec of+ Object fields -> KeyMap.lookup "knobs" fields == Just (toJSON entry.profile.knobs)+ _ -> False++validatePaths :: FilePath -> RunPlan -> Map Text FilePath -> IO ()+validatePaths directory schedule paths = do+ unless+ (all (`elem` map (renderRunId . (.runId)) schedule.runs) (Map.keys paths))+ (ioError (userError "retained trial path names an unplanned run"))+ mapM_+ ( \relative -> do+ actual <- relativePath directory (directory </> relative)+ unless (actual == relative) (ioError (userError "retained trial path contains a directory alias"))+ )+ (Map.elems paths)++loadPaths :: FilePath -> IO (Map Text FilePath)+loadPaths directory = do+ let path = directory </> "trial-paths.json"+ exists <- doesFileExist path+ paths <- if exists then eitherDecodeFileStrict' path >>= either (ioError . userError) pure else pure Map.empty+ unless (all safe (Map.elems paths)) (ioError (userError "unsafe retained trial path"))+ pure paths+ where+ safe path = not (null path) && not (isAbsolute path) && all (`notElem` ["..", ".", ""]) (splitDirectories path)++loadFailures :: FilePath -> RunPlan -> Map Text FilePath -> IO (Map Text Text)+loadFailures directory schedule paths = do+ let path = directory </> "trial-failures.json"+ exists <- doesFileExist path+ failures <- if exists then eitherDecodeFileStrict' path >>= either (ioError . userError) pure else pure Map.empty+ unless (all (`elem` map (renderRunId . (.runId)) schedule.runs) (Map.keys failures) && all (`Map.notMember` paths) (Map.keys failures) && all (not . Text.null) (Map.elems failures)) (ioError (userError "retained trial failure has an unknown, conflicting or empty identity"))+ pure failures++relativePath :: FilePath -> FilePath -> IO FilePath+relativePath root path = do+ absoluteRoot <- canonicalizePath root+ absolutePath <- canonicalizePath path+ let relative = makeRelative absoluteRoot absolutePath+ unless (relative /= absolutePath && ".." `notElem` splitDirectories relative) (ioError (userError "trial evidence escaped its attempt directory"))+ pure relative++atomicPaths :: FilePath -> Map Text FilePath -> IO ()+atomicPaths directory paths = do+ atomicJSON (directory </> "trial-paths.json") paths++atomicJSON :: (ToJSON document) => FilePath -> document -> IO ()+atomicJSON destination document = do+ let temporary = destination <> ".next"+ encodeFile temporary document+ renameFile temporary destination++immutable :: FilePath -> Value -> IO ()+immutable path document = do+ exists <- doesFileExist path+ if exists+ then do+ retained <- eitherDecodeFileStrict' path >>= either (ioError . userError) pure+ unless (retained == document) (ioError (userError "attempt lease, workload or schedule changed"))+ else atomicJSON path document
+ src/Kenshou/Remote/Perf/Cell.hs view
@@ -0,0 +1,167 @@+module Kenshou.Remote.Perf.Cell+ ( CellAdapter (..),+ executeCellTrial,+ verifyCellTrial,+ retainedRuntimeProblems,+ )+where++import Control.Concurrent (threadDelay)+import Control.Exception (IOException, catch, displayException)+import Control.Monad (unless)+import Data.Aeson (Value, eitherDecodeFileStrict', encode)+import Data.ByteString.Lazy qualified as Bytes+import Data.Map.Strict (Map)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (Registry)+import Kenshou.Core.Id (renderRunId)+import Kenshou.Core.RunSpec (EnvironmentSpec (..), RunSpec (..))+import Kenshou.Measure.Perf (AdmissionProblem (..), runtimeResultProblems, verifyPerformanceRun)+import Kenshou.Perf.Plan (PerfProfile)+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))+import Kenshou.Remote.Cell.Docs (CellBuckets (..), CellDescriptor (..), CellImages (..), CellNodes (..), CellOutcome (..), Requirements (..), ResetBlock (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..), EvidenceLink (..), RunLink (..), deriveCellRunIndex)+import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseSnapshot, withHeartbeat)+import Kenshou.Remote.Cell.Prepare (cellMachineProfile)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities, RoutingRule)+import Kenshou.Remote.Cell.Session.Build+import Kenshou.Remote.Cell.Session.Journal+import Kenshou.Remote.Cell.Session.Resume+import Kenshou.Remote.Cell.Session.Runner+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.Watch (WatchEvent)+import Kenshou.Remote.Payload (PayloadDescriptor)+import Kenshou.Remote.Perf.Conditions (validateCellConditions, validateCellObservations)+import Kenshou.Remote.Store (Bucket (..), ObjectStore)+import System.Directory (createDirectoryIfMissing, doesFileExist)+import System.FilePath (takeDirectory, (</>))++data CellAdapter = CellAdapter+ { registry :: Registry,+ store :: ObjectStore,+ ref :: CellRef,+ descriptor :: CellDescriptor,+ healthPolicy :: Value,+ capabilities :: Maybe CellCapabilities,+ rules :: [RoutingRule],+ payloads :: Map Text PayloadDescriptor,+ options :: BuildOptions,+ storeLabel :: Text,+ emit :: WatchEvent -> IO ()+ }++-- Each trial has a separate low-level cell journal. Its work digest is checked+-- against the immutable paired schedule before any new or resumed submission.+executeCellTrial :: CellAdapter -> LeaseHandle -> RunPlan -> FilePath -> PerfProfile -> (Text -> IO ()) -> PlannedRun -> IO (Either Text FilePath)+executeCellTrial adapter handle schedule attempt profile observeRuntime run = do+ lease <- leaseSnapshot handle+ let directory = attempt </> "trials" </> Text.unpack (renderRunId run.runId)+ journalPath = directory </> "session.json"+ bytes = encode schedule {runs = [run], estimateMinutes = run.estimateMinutes}+ expected = (workObjectFor "application/json" bytes).sha256+ results = Bucket adapter.descriptor.buckets.results+ exists <- doesFileExist journalPath+ result <- withHeartbeat adapter.store adapter.ref handle \_ ->+ if exists+ then do+ saved <- readSessionJournal journalPath+ case saved of+ Left problem -> pure (Left problem)+ Right journal+ | journal.leaseId /= lease.leaseId || journal.planSha256 /= expected || journal.payloads /= adapter.payloads ->+ pure (Left "retained cell journal differs from the pinned trial, payloads or lease")+ Right _ -> continue results journalPath+ else do+ built <- buildSession adapter.registry adapter.descriptor adapter.capabilities adapter.rules adapter.payloads adapter.options adapter.storeLabel lease.leaseId bytes+ case built of+ Left problem -> pure (Left problem)+ Right session -> do+ createDirectoryIfMissing True directory+ Bytes.writeFile (directory </> "plan.json") bytes+ mapM_+ ( \(relative, content) -> do+ createDirectoryIfMissing True (takeDirectory (directory </> relative))+ Bytes.writeFile (directory </> relative) content+ )+ session.workFiles+ either (Left . Text.pack . show) Right <$> runPlannedSlices adapter.store adapter.ref results handle journalPath session.journal adapter.emit+ case result of+ Left problem -> pure (Left problem)+ Right journal -> do+ problems <- retainedRuntimeProblems profile run directory journal+ mapM_ observeRuntime problems+ verifyCellTrial adapter.healthPolicy adapter.descriptor run directory journal+ where+ continue results journalPath = do+ resumed <- resumeHeldSession adapter.store adapter.ref results handle journalPath adapter.emit+ case resumed of+ Left (ContinuationError (UnresolvedSlice _ SliceSubmitted)) -> threadDelay 2000000 >> continue results journalPath+ Left failure -> pure (Left (Text.pack (show failure)))+ Right journal -> pure (Right journal)++-- A bad outer health gate cannot erase a sealed nested runtime assertion or+-- failed/incomplete steady work. Reconstruct these before rejecting the pair.+retainedRuntimeProblems :: PerfProfile -> PlannedRun -> FilePath -> SessionJournal -> IO [Text]+retainedRuntimeProblems profile run directory journal = case journal.slices of+ [slice] | slice.state `elem` [SliceFetched, SliceVerified] -> do+ let tree = takeDirectory directory </> Text.unpack (renderRunId slice.cellRun) </> "tree"+ verified <- deriveCellRunIndex (Bucket journal.resultsBucket) Nothing tree+ case verified of+ Right index | index.cellRun == slice.cellRun && index.leaseId == journal.leaseId && index.cell == journal.cell && slice.manifestSha256 == Just index.cellManifest.sha256 && slice.cellOutcome == Just index.cellOutcome && any (\link -> link.runId == run.runId && link.path == "output/" <> renderRunId run.runId) index.runs -> do+ let nested = tree </> "output" </> Text.unpack (renderRunId run.runId)+ result <- eitherDecodeFileStrict' (nested </> "run-result.json")+ measured <- verifyPerformanceRun profile nested+ pure ([reason | RuntimeWorkFailure reason <- either (const []) runtimeResultProblems result <> either id (const []) measured])+ _ -> pure []+ _ -> pure []++-- Recheck the outer cell seal and health even when a low-level journal already+-- says verified. An exit-one runtime result is still returned for independent+-- runtime classification; failed cell health cannot become a valid trial.+verifyCellTrial :: Value -> CellDescriptor -> PlannedRun -> FilePath -> SessionJournal -> IO (Either Text FilePath)+verifyCellTrial healthPolicy descriptor run directory journal =+ catch (verifyTrial healthPolicy descriptor run directory journal) (\(problem :: IOException) -> pure (Left (Text.pack (displayException problem))))++verifyTrial :: Value -> CellDescriptor -> PlannedRun -> FilePath -> SessionJournal -> IO (Either Text FilePath)+verifyTrial healthPolicy descriptor run directory journal = case journal.slices of+ [slice] | slice.state == SliceVerified && slice.runIds == [run.runId] -> do+ let tree = takeDirectory directory </> Text.unpack (renderRunId slice.cellRun) </> "tree"+ verified <- deriveCellRunIndex (Bucket journal.resultsBucket) Nothing tree+ let evidencePresent = either (const False) (\index -> all (`elem` map (.kind) index.evidence) ["cell-health", "cell-reset-evidence"]) verified+ conditions <-+ if evidencePresent+ then do+ health <- eitherDecodeFileStrict' (tree </> "cell/health.json")+ reset <- eitherDecodeFileStrict' (tree </> "cell/reset-evidence.json")+ interval <- eitherDecodeFileStrict' (tree </> "cell/execution-window.json")+ let roles = [("driver", (case descriptor.instances.drivers of driver : _ -> driver; [] -> ""), descriptor.images.driver), ("postgres", descriptor.instances.postgres, descriptor.images.postgres), ("monitoring", descriptor.instances.monitoring, descriptor.images.monitoring)] <> case (slice.submission.reset.broker, descriptor.instances.broker) of (Just _, Just broker) -> [("broker", broker, descriptor.images.broker)]; _ -> []+ raw <- traverse (\(role, machine, image) -> do value <- eitherDecodeFileStrict' (tree </> "cell/observations" </> Text.unpack role <> ".json"); pure (role, machine, image, value)) roles+ let machines =+ descriptor.instances.drivers+ <> [descriptor.instances.postgres, descriptor.instances.monitoring]+ <> case (slice.submission.reset.broker, descriptor.instances.broker) of (Just _, Just broker) -> [broker]; _ -> []+ pure do+ healthDocument <- either (Left . Text.pack) Right health+ resetDocument <- either (Left . Text.pack) Right reset+ validateCellConditions slice.cellRun descriptor.name machines healthDocument resetDocument+ executionWindow <- either (Left . Text.pack) Right interval+ windows <- traverse (\(role, machine, image, value) -> (role,machine,image,) <$> either (Left . Text.pack) Right value) raw+ validateCellObservations slice.cellRun slice.submission.requires.minAgentVersion healthPolicy healthDocument executionWindow windows+ else pure (Left "outer cell is missing sealed health or reset evidence")+ effective <- eitherDecodeFileStrict' (tree </> "output" </> Text.unpack (renderRunId run.runId) </> "run-spec.json") :: IO (Either String RunSpec)+ pure do+ index <- either (Left . Text.pack . show) Right verified+ conditions+ spec <- either (Left . Text.pack) Right effective+ unless (spec.environment.machineProfile == Just (cellMachineProfile descriptor)) (Left "effective machine profile differs from the frozen cell descriptor")+ unless+ ( index.cellOutcome == Completed+ && index.leaseId == journal.leaseId+ && index.cell == journal.cell+ && index.cellRun == slice.cellRun+ && slice.manifestSha256 == Just index.cellManifest.sha256+ )+ (Left "outer cell identity, seal or health differs from its journal")+ pure (tree </> "output" </> Text.unpack (renderRunId run.runId))+ _ -> pure (Left "cell trial is incomplete, rejected or has unexpected runs")
+ src/Kenshou/Remote/Perf/Cleanup.hs view
@@ -0,0 +1,52 @@+module Kenshou.Remote.Perf.Cleanup (ownedWorkSettled, waitForOwnedWork) where++import Control.Concurrent (threadDelay)+import Control.Monad (unless)+import Data.Aeson (eitherDecode)+import Kenshou.Core.Id (renderRunId)+import Kenshou.Remote.Cell.Docs (CellPhase (Sealed), CellStatus (..), Rejected (..), Submission (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..), SliceState (..))+import Kenshou.Remote.Perf.Journal (LeaseIdentity (..))+import Kenshou.Remote.Store (ObjectName (..), ObjectStore (..))+import Kenshou.Remote.Store qualified as Store++-- Caller bounds this wait with the original cleanup deadline. Do not clear a+-- lease while an ambiguous or active submitted unit still belongs to it.+waitForOwnedWork :: ObjectStore -> CellRef -> LeaseIdentity -> [SessionJournal] -> IO ()+waitForOwnedWork store ref identity journals = do+ settled <- ownedWorkSettled store ref identity journals+ unless settled (threadDelay 1000000 >> waitForOwnedWork store ref identity journals)++ownedWorkSettled :: ObjectStore -> CellRef -> LeaseIdentity -> [SessionJournal] -> IO Bool+ownedWorkSettled store ref identity journals = do+ unless (ref.cellName == identity.cell) (ioError (userError "owned cleanup reference names another cell"))+ unless (all ((== refBucket) . (.controlBucket)) owned) (ioError (userError "owned cleanup journal names another control bucket"))+ and <$> traverse inspect slices+ where+ Store.Bucket refBucket = ref.controlBucket+ owned = [journal | journal <- journals, journal.cell == identity.cell, journal.leaseId == identity.leaseId]+ slices = concatMap (.slices) owned+ inspect slice = do+ let prefix = "cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/"+ get name = store.getObject ref.controlBucket (ObjectName (prefix <> name))+ marker <- get "submission.json"+ case marker of+ Nothing -> pure (slice.state == SlicePlanned)+ Just (bytes, _) -> do+ submission <- either (ioError . userError) pure (eitherDecode bytes)+ unless ((submission :: Submission).leaseId == identity.leaseId && submission == slice.submission) (ioError (userError "owned cleanup submission differs from its retained journal"))+ rejected <- get "rejected.json"+ case rejected of+ Just (body, _) -> do+ record <- either (ioError . userError) pure (eitherDecode body)+ unless ((record :: Rejected).runId == slice.cellRun) (ioError (userError "owned cleanup rejection names another run"))+ pure True+ Nothing -> do+ status <- get "status.json"+ case status of+ Nothing -> pure False+ Just (body, _) -> do+ record <- either (ioError . userError) pure (eitherDecode body)+ unless ((record :: CellStatus).runId == slice.cellRun) (ioError (userError "owned cleanup status names another run"))+ pure (record.phase == Sealed && record.manifestSha256 /= Nothing && record.outcome /= Nothing)
+ src/Kenshou/Remote/Perf/Conditions.hs view
@@ -0,0 +1,180 @@+module Kenshou.Remote.Perf.Conditions (validateCellConditions, validateCellObservations, validateHealthPolicy) where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser, parseEither)+import Data.List (nub, sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, diffUTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Perf.Request (strictObject)++data Sample = Sample {at :: UTCTime, total :: Integer, idle :: Integer, steal :: Integer, disk :: Double}++-- Canonical algorithm: mori://shinzui/load-testing-infra, project-relative+-- nixos/pkgs/cell-agent/src/src/health.rs and runner.rs (artifact URIs pending).+-- The immutable execution interval supplies the actual payload start; nested+-- runtime timestamps and mutable status timestamps cannot substitute for it.+validateCellObservations :: RunId -> Text -> Value -> Value -> Value -> [(Text, Text, Text, Value)] -> Either Text ()+validateCellObservations runId agentVersion policy health interval windows = either (Left . Text.pack) Right do+ let roles = [role | (role, _, _, _) <- windows]+ machines = [machine | (_, machine, _, _) <- windows]+ unless (sort roles `elem` [sort ["driver", "postgres", "monitoring"], sort ["driver", "postgres", "monitoring", "broker"]] && nub machines == machines) (Left "raw observation role coverage is missing or duplicated")+ parseEither replay interval+ where+ replay = withObject "execution window" \v -> do+ strictObject ["schema", "runId", "resetStarted", "runStarted", "runFinished"] v+ schema <- v .: "schema"+ identifier <- v .: "runId"+ resetStarted <- v .: "resetStarted"+ started <- v .: "runStarted"+ finished <- v .: "runFinished"+ unless (schema == ("cell.execution-window/v1" :: Text) && identifier == runId && resetStarted <= addUTCTime (-10) started && started < finished) (fail "invalid sealed execution interval")+ (healthStart, healthEnd, gates) <- withObject "health report" (\h -> (,,) <$> h .: "windowStart" <*> h .: "windowEnd" <*> h .: "gates") health+ unless (healthStart == resetStarted && healthEnd == finished) (fail "health and execution intervals differ")+ limits <- healthLimits policy+ mapM_ (window resetStarted started finished limits gates) windows+ window resetStarted started finished (busyLimit, stealLimit, startLimit, runLimit) gates (role, machine, image, document) =+ withObject+ "raw machine observations"+ ( \v -> do+ strictObject ["schema", "machine", "image", "agentVersion", "systemClosure", "from", "to", "samples", "maintenanceEvents", "samplingErrors"] v+ schema <- v .: "schema"+ actualMachine <- v .: "machine"+ actualImage <- v .: "image"+ actualVersion <- v .: "agentVersion"+ closure <- v .: "systemClosure"+ from <- v .: "from"+ to <- v .: "to"+ maintenance <- v .: "maintenanceEvents" :: Parser [Value]+ errors <- v .: "samplingErrors" :: Parser [Value]+ unless (schema == ("cell.observations/v1" :: Text) && actualMachine == machine && actualImage == image && actualVersion == agentVersion && "/nix/store/" `Text.isPrefixOf` closure && Text.length closure > 11 && abs (diffUTCTime from resetStarted) <= 0.001 && abs (diffUTCTime to finished) <= 0.001 && null maintenance && null errors) (fail "raw machine identity, window, maintenance or sampling errors invalidate the trial")+ samples <- v .: "samples" >>= traverse sample+ unless (not (null samples) && all (\s -> s.at <= to) samples && and [a.at < b.at && diffUTCTime b.at a.at <= 2 && a.total < b.total && a.idle <= b.idle && a.steal <= b.steal | (a, b) <- zip samples (drop 1 samples)]) (fail "raw samples are missing, unordered, interrupted or have invalid counters")+ let settle = filter (\s -> s.at >= addUTCTime (-10) started && s.at <= started) samples+ running = filter (\s -> s.at >= started && s.at <= finished) samples+ (firstSettle, lastSettle) <- endpoints settle+ (firstRun, lastRun) <- endpoints running+ unless (diffUTCTime lastSettle.at firstSettle.at >= 8 && diffUTCTime firstRun.at started <= 2 && diffUTCTime finished lastRun.at <= 2) (fail "raw settle or run sample coverage is incomplete")+ busy <- fraction True firstSettle lastSettle+ steals <- traverse (\(prior, latest) -> fraction False (rolling prior latest) latest) [(take i running, latest) | (i, latest) <- zip [1 ..] (drop 1 running)]+ let settleBusy = if role `elem` ["monitoring", "broker"] then 0 else busy+ runSteal = if role == "monitoring" then 0 else maximum (0 : steals)+ startFree = firstRun.disk+ runFree = minimum (map (.disk) running)+ unless (settleBusy <= busyLimit && runSteal <= stealLimit && startFree >= startLimit && runFree >= runLimit) (fail "raw observations violate the frozen health policy")+ checkGate gates machine "background-load" (object ["settleCpuPercent" .= settleBusy, "runStealCpuPercent" .= runSteal]) (object ["settleCpuMaxPercent" .= busyLimit, "runStealCpuMaxPercent" .= stealLimit]) finished+ checkGate gates machine "storage-pressure" (object ["startFreePercent" .= startFree, "runFreePercent" .= runFree, "outputExceeded" .= False]) (object ["startFreeMinPercent" .= startLimit, "runFreeMinPercent" .= runLimit, "outputExceeded" .= False]) finished+ )+ document+ sample = withObject "health sample" \v -> do+ strictObject ["at", "cpuTotalTicks", "cpuIdleTicks", "cpuStealTicks", "diskFreePercent"] v+ value <- Sample <$> v .: "at" <*> v .: "cpuTotalTicks" <*> v .: "cpuIdleTicks" <*> v .: "cpuStealTicks" <*> v .: "diskFreePercent"+ unless (value.total >= 0 && value.idle >= 0 && value.steal >= 0 && value.idle + value.steal <= value.total && validPercent value.disk) (fail "invalid health counters or disk percentage")+ pure value+ endpoints [] = fail "missing health samples"+ endpoints (first : rest) = pure (first, foldl (\_ next -> next) first rest)+ rolling prior latest = case reverse (filter (\s -> diffUTCTime latest.at s.at >= 30) prior) of+ first : _ -> first+ [] -> case prior of first : _ -> first; [] -> latest+ fraction countIdle first latest = do+ let total = latest.total - first.total+ counter = if countIdle then latest.idle - first.idle else latest.steal - first.steal+ unless (total > 0 && counter >= 0 && counter <= total) (fail "CPU counters did not advance consistently")+ pure (100 * fromIntegral (if countIdle then total - counter else counter) / fromIntegral total :: Double)+ checkGate :: [Value] -> Text -> Text -> Value -> Value -> UTCTime -> Parser ()+ checkGate gates machine name observed threshold finished = do+ matching <- filter (\(m, n, _, _, _) -> m == machine && n == name) <$> traverse (withObject "gate" \v -> (,,,,) <$> v .: "machine" <*> v .: "name" <*> v .: "observed" <*> v .: "threshold" <*> v .: "at") gates+ case matching of+ [(_, _, actual, actualThreshold, at)] | near actual observed && actualThreshold == threshold && at == finished -> pure ()+ _ -> fail "reported health gate differs from raw observations or frozen policy"+ near (Number a) (Number b) = abs (realToFrac a - realToFrac b :: Double) <= 0.000001+ near (Object a) (Object b) = a == b || (length a == length b && and [maybe False (near value) (KeyMap.lookup key b) | (key, value) <- KeyMap.toList a])+ near a b = a == b++validateHealthPolicy :: Value -> Either Text ()+validateHealthPolicy = either (Left . Text.pack) (const (Right ())) . parseEither healthLimits++healthLimits :: Value -> Parser (Double, Double, Double, Double)+healthLimits = withObject "cell policy" \v -> do+ schema <- v .: "schema"+ unless (schema == ("cell.policy/v1" :: Text)) (fail "unsupported cell health policy")+ limits <- v .: "health" >>= withObject "health limits" (\h -> (,,,) <$> h .: "settleCpuMaxPercent" <*> h .: "runStealCpuMaxPercent" <*> h .: "startFreeDiskMinPercent" <*> h .: "runFreeDiskMinPercent")+ let (a, b, c, d) = limits+ unless (all validPercent [a, b, c, d]) (fail "invalid frozen health thresholds")+ pure limits++validPercent :: Double -> Bool+validPercent value = not (isNaN value || isInfinite value) && value >= 0 && value <= 100++-- Canonical owner: mori://shinzui/load-testing-infra, artifact URI pending;+-- project-relative nixos/pkgs/cell-agent/src/src/health.rs. Check the reported+-- observations against their thresholds, rather than trusting passed alone.+validateCellConditions :: RunId -> Text -> [Text] -> Value -> Value -> Either Text ()+validateCellConditions runId cell machines health reset = either (Left . Text.pack) Right do+ parseEither healthDocument health+ parseEither resetDocument reset+ where+ names = ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"]+ healthDocument = withObject "cell health" \v -> do+ schema <- v .: "schema"+ identifier <- v .: "runId"+ passed <- v .: "passed"+ start <- v .: "windowStart" :: Parser UTCTime+ end <- v .: "windowEnd" :: Parser UTCTime+ unless (schema == ("cell.health/v1" :: Text) && identifier == runId && passed && start <= end && not (null machines)) (fail "cell health identity, window or outcome is invalid")+ gates <- v .: "gates" >>= traverse (gate start end)+ unless (sort gates == sort [(machine, name) | machine <- machines, name <- names]) (fail "cell health gate coverage is missing, duplicated or unexpected")+ gate start end = withObject "cell health gate" \v -> do+ machine <- v .: "machine"+ name <- v .: "name"+ status <- v .: "status"+ at <- v .: "at" :: Parser UTCTime+ observed <- v .: "observed"+ threshold <- v .: "threshold"+ unless (status == ("ok" :: Text) && at >= start && at <= end) (fail "cell health gate tripped or lies outside its window")+ valid <- case name :: Text of+ "host-maintenance" -> pure (observed == String "NONE" && threshold == String "NONE")+ "incomplete-reset" -> pure (observed == Bool True && threshold == Bool True)+ "version-mismatch" -> pure (observed == Bool True && threshold == Bool True)+ "background-load" -> do+ busy <- percent "settleCpuPercent" observed+ steal <- percent "runStealCpuPercent" observed+ busyLimit <- percent "settleCpuMaxPercent" threshold+ stealLimit <- percent "runStealCpuMaxPercent" threshold+ pure (busy <= busyLimit && steal <= stealLimit)+ "storage-pressure" -> do+ initial <- percent "startFreePercent" observed+ final <- percent "runFreePercent" observed+ initialLimit <- percent "startFreeMinPercent" threshold+ finalLimit <- percent "runFreeMinPercent" threshold+ exceeded <- withObject "disk observation" (.: "outputExceeded") observed+ expectedExceeded <- withObject "disk threshold" (.: "outputExceeded") threshold+ pure (initial >= initialLimit && final >= finalLimit && not exceeded && not expectedExceeded)+ _ -> fail "unknown cell health gate"+ unless valid (fail "cell health observation violates its threshold")+ pure (machine, name)+ percent key = withObject "health percentage" \v -> do+ value <- v .: key :: Parser Double+ unless (not (isNaN value || isInfinite value) && value >= 0 && value <= 100) (fail "invalid cell health percentage")+ pure value+ resetDocument = withObject "cell reset" \v -> do+ schema <- v .: "schema"+ identifier <- v .: "runId"+ owner <- v .: "cell"+ cache <- v .: "cachePolicy"+ verified <- v .: "verified"+ start <- v .: "startedAt" :: Parser UTCTime+ end <- v .: "finishedAt" :: Parser UTCTime+ steps <- v .: "steps" :: Parser [Value]+ unless (schema == ("cell.reset-evidence/v1" :: Text) && identifier == runId && owner == cell && cache == ("cold" :: Text) && verified && start <= end && not (null steps)) (fail "cell reset identity, cache policy, window or verification is invalid")+ mapM_+ ( withObject "reset step" \step -> do+ status <- step .: "status"+ from <- step .: "startedAt" :: Parser UTCTime+ to <- step .: "finishedAt" :: Parser UTCTime+ unless (status == ("ok" :: Text) && from >= start && to <= end && from <= to) (fail "cell reset step failed or lies outside its window")+ )+ steps
+ src/Kenshou/Remote/Perf/Journal.hs view
@@ -0,0 +1,287 @@+module Kenshou.Remote.Perf.Journal+ ( PerfSession (..),+ SessionEvent (..),+ SessionAction (..),+ LeaseIdentity (..),+ JournalState (..),+ initialSession,+ journalState,+ appendEvent,+ readPerfSession,+ writePerfSession,+ validateSessionInputs,+ )+where++import Control.Exception (bracket)+import Control.Monad (foldM, unless)+import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString.Lazy qualified as ByteString+import Data.List (find, nub)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, diffUTCTime)+import Kenshou.Core.Id (RunId)+import Kenshou.Perf.Contracts (PerfContracts)+import Kenshou.Perf.Plan+import Kenshou.Perf.Prepared qualified as Prepared+import Kenshou.Perf.Request (PerfRequest (..), strictObject)+import Kenshou.Perf.Result (ConfigurationResult (..))+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile, renameFile)+import System.FilePath (takeDirectory)+import System.IO (hClose, openBinaryTempFile)++data LeaseIdentity = LeaseIdentity {cell :: Text, leaseId :: RunId}+ deriving stock (Eq, Show)++data SessionAction+ = AdmissionReserved Int+ | LeaseClaimProposed Text LeaseIdentity+ | LeaseClaimWithdrawn+ | LeaseClaimReleased+ | LeaseAcquired LeaseIdentity+ | LeaseLost Text+ | LeaseReleased+ | ComparisonStarted Text Int+ | ReplacementReserved Text+ | ReplacementScheduled Text Int Int+ | RuntimeBlocked Text Text+ | ComparisonCompleted ConfigurationResult+ deriving stock (Eq, Show)++data SessionEvent = SessionEvent {at :: UTCTime, action :: SessionAction}+ deriving stock (Eq, Show)++-- Immutable header plus an append-only transition history. State is derived+-- from that history, so resume cannot silently reset allowances or a deadline.+data PerfSession = PerfSession+ { sessionId :: RunId,+ preparedSha256 :: Text,+ plan :: PerfPlan,+ evidenceBaseUri :: Text,+ createdAt :: UTCTime,+ events :: [SessionEvent],+ executionInputs :: Maybe Value+ }+ deriving stock (Eq, Show)++data JournalState = JournalState+ { admissionSpentSeconds :: Int,+ executionDeadline :: Maybe UTCTime,+ cleanupDeadline :: Maybe UTCTime,+ lease :: Maybe LeaseIdentity,+ pendingClaim :: Maybe (Text, LeaseIdentity, UTCTime),+ activeComparison :: Maybe (Text, Int),+ attempts :: Map Text Int,+ replacements :: Map Text Int,+ replacementIdentities :: [(Text, Int, Int)],+ completed :: Map Text ConfigurationResult,+ runtimeBlockers :: Map Text [Text],+ lastAt :: UTCTime+ }+ deriving stock (Eq, Show)++initialSession :: RunId -> Text -> PerfPlan -> Text -> UTCTime -> Either Text PerfSession+initialSession identifier digest plan uri now = do+ let (bucket, path) = Text.breakOn "/" (Text.drop 5 uri)+ unless (validDigest digest && "gs://" `Text.isPrefixOf` uri && not (Text.null bucket) && Text.length path > 1 && not (Text.any (`elem` ['\n', '\r', ' ', '?', '#']) uri)) (Left "invalid prepared digest or durable evidence destination")+ case fromJSON (toJSON plan) :: Result PerfPlan of+ Error problem -> Left (Text.pack problem)+ Success _ -> pure ()+ pure (PerfSession identifier digest plan uri now [] Nothing)++validateSessionInputs :: PerfContracts -> Prepared.PerfPrepared -> PerfSession -> Either Text ()+validateSessionInputs contracts prepared session = do+ Prepared.validatePrepared contracts prepared+ unless+ (session.preparedSha256 == Prepared.preparedDigest prepared && toJSON session.plan == toJSON prepared.plan)+ (Left "session inputs differ from the verified immutable preparation")+ _ <- journalState session+ pure ()++journalState :: PerfSession -> Either Text JournalState+journalState session = foldM (advance session) initial session.events+ where+ initial = JournalState 0 Nothing Nothing Nothing Nothing Nothing Map.empty Map.empty [] Map.empty Map.empty session.createdAt++appendEvent :: SessionEvent -> PerfSession -> Either Text PerfSession+appendEvent event session = do+ before <- journalState session+ _ <- advance session before event+ pure session {events = session.events <> [event]}++advance :: PerfSession -> JournalState -> SessionEvent -> Either Text JournalState+advance session state event = do+ unless (event.at >= state.lastAt) (Left "performance journal clock moved backwards")+ changed <- case event.action of+ AdmissionReserved seconds -> do+ unless (state.executionDeadline == Nothing && state.lease == Nothing && seconds > 0 && state.admissionSpentSeconds + seconds <= session.plan.request.waitSeconds) (Left "initial admission allowance exhausted or already measured")+ pure state {admissionSpentSeconds = state.admissionSpentSeconds + seconds}+ LeaseClaimProposed name identity -> do+ _ <- selected name+ measurementTime+ unless (state.lease == Nothing && state.pendingClaim == Nothing && identity.cell `elem` session.plan.request.cells && Map.notMember name state.completed) (Left "unexpected or overlapping proposed cell claim")+ pure state {pendingClaim = Just (name, identity, event.at)}+ LeaseClaimWithdrawn -> do+ unless (state.lease == Nothing && state.pendingClaim /= Nothing) (Left "no proposed cell claim to withdraw")+ pure state {pendingClaim = Nothing}+ LeaseClaimReleased -> case state.pendingClaim of+ Just (_, _, proposedAt) | state.lease == Nothing -> do+ let deadline = maybe (addUTCTime (fromIntegral (session.plan.request.budgetMinutes * 60)) proposedAt) id state.executionDeadline+ pure state {pendingClaim = Nothing, executionDeadline = Just deadline, cleanupDeadline = Just (addUTCTime 60 deadline)}+ _ -> Left "no proposed owned cell claim to release"+ LeaseAcquired identity -> do+ unless (state.lease == Nothing && identity.cell `elem` session.plan.request.cells) (Left "unexpected or overlapping cell lease")+ measurementTime+ proposedAt <- case state.pendingClaim of+ Just (_, proposed, at) | proposed == identity -> Right at+ Just _ -> Left "confirmed lease differs from its proposed identity"+ Nothing -> Right event.at -- retained pre-claim-journal sessions+ let deadline = maybe (addUTCTime (fromIntegral (session.plan.request.budgetMinutes * 60)) proposedAt) id state.executionDeadline+ unless (event.at <= deadline) (Left "confirmation exceeds the original proposed-claim deadline")+ pure state {lease = Just identity, pendingClaim = Nothing, executionDeadline = Just deadline, cleanupDeadline = Just (addUTCTime 60 deadline)}+ LeaseLost reason -> do+ unless (state.lease /= Nothing && not (Text.null reason)) (Left "cannot lose an absent lease without a diagnosis")+ -- Retained low-level journals belong to the abandoned attempt. The next+ -- comparison starts a full schedule and cannot pool those old pairs.+ pure state {lease = Nothing, activeComparison = Nothing}+ LeaseReleased -> do+ unless (state.lease /= Nothing && state.activeComparison == Nothing) (Left "cannot release a lease with unresolved active comparison")+ pure state {lease = Nothing}+ ComparisonStarted name index -> do+ entry <- selected name+ measurementTime+ let count = Map.findWithDefault 0 name state.attempts+ remaining = maybe 0 (floor . (`diffUTCTime` event.at)) state.executionDeadline :: Integer+ fixedCost = 2 * fromIntegral entry.profile.minimumPairs * fromIntegral (entry.profile.trialSeconds + entry.profile.resetSeconds + entry.profile.sealSeconds)+ unless (state.lease /= Nothing && state.activeComparison == Nothing && Map.notMember name state.completed && index == count && count <= entry.profile.replacementPairs && fixedCost <= remaining) (Left "comparison has no lease, is already complete, or exceeds persistent attempt/deadline budget")+ pure state {activeComparison = Just (name, index), attempts = Map.insert name (count + 1) state.attempts}+ ReplacementReserved name -> do+ entry <- selected name+ measurementTime+ let spent = Map.findWithDefault 0 name state.replacements+ unless (maybe False ((== name) . fst) state.activeComparison && spent < entry.profile.replacementPairs) (Left "infrastructure replacement allowance exhausted or wrong active configuration")+ pure state {replacements = Map.insert name (spent + 1) state.replacements}+ ReplacementScheduled name attempt pair -> do+ entry <- selected name+ measurementTime+ let spent = Map.findWithDefault 0 name state.replacements+ unless (state.activeComparison == Just (name, attempt) && pair >= entry.profile.minimumPairs && pair < entry.profile.minimumPairs + entry.profile.replacementPairs && spent < entry.profile.replacementPairs && (name, attempt, pair) `notElem` state.replacementIdentities) (Left "replacement identity is invalid, duplicated or exceeds its persistent allowance")+ pure state {replacements = Map.insert name (spent + 1) state.replacements, replacementIdentities = state.replacementIdentities <> [(name, attempt, pair)]}+ RuntimeBlocked name reason -> do+ _ <- selected name+ -- Recovery may discover a sealed observation after the client lost its+ -- lease or finished a comparison. It still belongs to the started profile.+ unless (Map.findWithDefault 0 name state.attempts > 0 && not (Text.null reason)) (Left "runtime blocker must name a previously started configuration and diagnosis")+ let blockers = nub (Map.findWithDefault [] name state.runtimeBlockers <> [reason])+ pure state {runtimeBlockers = Map.insert name blockers state.runtimeBlockers, completed = Map.adjust (\result -> result {runtimeFailures = nub (result.runtimeFailures <> [reason])}) name state.completed}+ ComparisonCompleted result -> do+ entry <- selected result.profile+ unless (maybe False ((== result.profile) . fst) state.activeComparison && result.workloadSha256 == entry.workloadSha256 && result.requiredPairs == entry.profile.minimumPairs && Map.notMember result.profile state.completed) (Left "completion differs from active pinned configuration")+ -- Sealing/retrieval may finish in cleanup grace; result assembly still+ -- reports deadline exhaustion and cannot admit more measurements.+ unless (maybe False (event.at <=) state.cleanupDeadline) (Left "comparison completion exceeded cleanup grace")+ case fromJSON (toJSON result) :: Result ConfigurationResult of+ Error problem -> Left (Text.pack problem)+ Success _ -> pure ()+ let retained = result {runtimeFailures = nub (Map.findWithDefault [] result.profile state.runtimeBlockers <> result.runtimeFailures)}+ pure state {activeComparison = Nothing, completed = Map.insert result.profile retained state.completed, runtimeBlockers = Map.insert result.profile retained.runtimeFailures state.runtimeBlockers}+ pure changed {lastAt = event.at}+ where+ selected name = maybe (Left "journal names an unselected configuration") Right (find ((== name) . (.name) . (.profile)) session.plan.selected)+ measurementTime = unless (maybe True (event.at <) state.executionDeadline) (Left "original performance deadline exhausted")++validDigest :: Text -> Bool+validDigest digest = Text.length digest == 71 && "sha256:" `Text.isPrefixOf` digest && Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) (Text.drop 7 digest)++readPerfSession :: FilePath -> IO (Either Text PerfSession)+readPerfSession path = either (Left . Text.pack) Right <$> eitherDecodeFileStrict' path++-- The caller holds the session's exclusive orchestration lock and writes this+-- transition before its corresponding side effect. Atomic replacement alone+-- does not serialize two clients.+writePerfSession :: FilePath -> PerfSession -> IO ()+writePerfSession path session = do+ either (ioError . userError . Text.unpack) (const (pure ())) (journalState session)+ present <- doesFileExist path+ if present+ then do+ previous <- readPerfSession path >>= either (ioError . userError . Text.unpack) pure+ unless (toJSON previous {events = []} == toJSON session {events = []}) (ioError (userError "immutable performance session header differs"))+ unless (toJSON previous.events == toJSON (take (length previous.events) session.events)) (ioError (userError "performance journal history is not an append-only continuation"))+ else pure ()+ let directory = takeDirectory path+ createDirectoryIfMissing True directory+ bracket (openBinaryTempFile directory ".kenshou-perf-session-") cleanup \(temporary, handle) -> do+ hClose handle+ ByteString.writeFile temporary (encode session)+ renameFile temporary path+ where+ cleanup (temporary, _) = do+ exists <- doesFileExist temporary+ if exists then removeFile temporary else pure ()++instance ToJSON LeaseIdentity where+ toJSON identity = object ["cell" .= identity.cell, "leaseId" .= identity.leaseId]++instance FromJSON LeaseIdentity where+ parseJSON = withObject "PerformanceLease" \v -> do+ strictObject ["cell", "leaseId"] v+ LeaseIdentity <$> v .: "cell" <*> v .: "leaseId"++instance ToJSON SessionAction where+ toJSON = \case+ AdmissionReserved seconds -> object ["kind" .= ("admission-reserved" :: Text), "seconds" .= seconds]+ LeaseClaimProposed name lease -> object ["kind" .= ("lease-claim-proposed" :: Text), "profile" .= name, "lease" .= lease]+ LeaseClaimWithdrawn -> object ["kind" .= ("lease-claim-withdrawn" :: Text)]+ LeaseClaimReleased -> object ["kind" .= ("lease-claim-released" :: Text)]+ LeaseAcquired lease -> object ["kind" .= ("lease-acquired" :: Text), "lease" .= lease]+ LeaseLost reason -> object ["kind" .= ("lease-lost" :: Text), "reason" .= reason]+ LeaseReleased -> object ["kind" .= ("lease-released" :: Text)]+ ComparisonStarted name index -> object ["kind" .= ("comparison-started" :: Text), "profile" .= name, "index" .= index]+ ReplacementReserved name -> object ["kind" .= ("replacement-reserved" :: Text), "profile" .= name]+ ReplacementScheduled name attempt pair -> object ["kind" .= ("replacement-scheduled" :: Text), "profile" .= name, "attempt" .= attempt, "pair" .= pair]+ RuntimeBlocked name reason -> object ["kind" .= ("runtime-blocked" :: Text), "profile" .= name, "reason" .= reason]+ ComparisonCompleted result -> object ["kind" .= ("comparison-completed" :: Text), "result" .= result]++instance FromJSON SessionAction where+ parseJSON = withObject "PerformanceAction" \v -> do+ kind <- v .: "kind" :: Parser Text+ case kind of+ "admission-reserved" -> strictObject ["kind", "seconds"] v >> AdmissionReserved <$> v .: "seconds"+ "lease-claim-proposed" -> strictObject ["kind", "profile", "lease"] v >> LeaseClaimProposed <$> v .: "profile" <*> v .: "lease"+ "lease-claim-withdrawn" -> strictObject ["kind"] v >> pure LeaseClaimWithdrawn+ "lease-claim-released" -> strictObject ["kind"] v >> pure LeaseClaimReleased+ "lease-acquired" -> strictObject ["kind", "lease"] v >> LeaseAcquired <$> v .: "lease"+ "lease-lost" -> strictObject ["kind", "reason"] v >> LeaseLost <$> v .: "reason"+ "lease-released" -> strictObject ["kind"] v >> pure LeaseReleased+ "comparison-started" -> strictObject ["kind", "profile", "index"] v >> ComparisonStarted <$> v .: "profile" <*> v .: "index"+ "replacement-reserved" -> strictObject ["kind", "profile"] v >> ReplacementReserved <$> v .: "profile"+ "replacement-scheduled" -> strictObject ["kind", "profile", "attempt", "pair"] v >> ReplacementScheduled <$> v .: "profile" <*> v .: "attempt" <*> v .: "pair"+ "runtime-blocked" -> strictObject ["kind", "profile", "reason"] v >> RuntimeBlocked <$> v .: "profile" <*> v .: "reason"+ "comparison-completed" -> strictObject ["kind", "result"] v >> ComparisonCompleted <$> v .: "result"+ _ -> fail "unknown performance journal action"++instance ToJSON SessionEvent where+ toJSON event = object ["at" .= event.at, "action" .= event.action]++instance FromJSON SessionEvent where+ parseJSON = withObject "PerformanceEvent" \v -> do+ strictObject ["at", "action"] v+ SessionEvent <$> v .: "at" <*> v .: "action"++instance ToJSON PerfSession where+ toJSON session = object ["schema" .= ("kenshou.perf-session/v1" :: Text), "sessionId" .= session.sessionId, "preparedSha256" .= session.preparedSha256, "plan" .= session.plan, "evidenceBaseUri" .= session.evidenceBaseUri, "createdAt" .= session.createdAt, "events" .= session.events, "executionInputs" .= session.executionInputs]++instance FromJSON PerfSession where+ parseJSON = withObject "PerfSession" \v -> do+ strictObject ["schema", "sessionId", "preparedSha256", "plan", "evidenceBaseUri", "createdAt", "events", "executionInputs"] v+ schema <- v .: "schema"+ unless (schema == ("kenshou.perf-session/v1" :: Text)) (fail "unsupported performance session schema")+ session <- PerfSession <$> v .: "sessionId" <*> v .: "preparedSha256" <*> v .: "plan" <*> v .: "evidenceBaseUri" <*> v .: "createdAt" <*> v .: "events" <*> v .:? "executionInputs"+ _ <- either (fail . Text.unpack) pure (initialSession session.sessionId session.preparedSha256 session.plan session.evidenceBaseUri session.createdAt)+ _ <- either (fail . Text.unpack) pure (journalState session)+ pure session
+ src/Kenshou/Remote/Perf/Lock.hs view
@@ -0,0 +1,33 @@+module Kenshou.Remote.Perf.Lock (withPerformanceLock) where++import Control.Concurrent.MVar+import Control.Exception (bracket, bracket_)+import Data.Map.Strict qualified as Map+import System.Directory (canonicalizePath)+import System.FilePath ((</>))+import System.IO (SeekMode (AbsoluteSeek))+import System.IO.Unsafe (unsafePerformIO)+import System.Posix.IO (LockRequest (WriteLock), OpenFileFlags (creat), OpenMode (ReadWrite), closeFd, defaultFileFlags, openFd, setLock)++-- POSIX record locks serialize processes, while the MVar closes their same-+-- process loophole. Neither lock leaves stale ownership after client death.+-- Canonical dependency owner: mori://haskell/unix (artifact URI pending),+-- project-relative System/Posix/IO/Common.hsc.+withPerformanceLock :: FilePath -> IO a -> IO a+withPerformanceLock directory action = do+ canonical <- canonicalizePath directory+ lock <- modifyMVar locks \known -> case Map.lookup canonical known of+ Just existing -> pure (known, existing)+ Nothing -> do+ created <- newMVar ()+ pure (Map.insert canonical created known, created)+ bracket_ (claim lock) (putMVar lock ()) $+ bracket (openFd (canonical </> ".client.lock") ReadWrite defaultFileFlags {creat = Just 0o600}) closeFd \fd -> do+ setLock fd (WriteLock, AbsoluteSeek, 0, 0)+ action+ where+ claim lock = tryTakeMVar lock >>= maybe (ioError (userError "performance session is already controlled by another client")) pure++{-# NOINLINE locks #-}+locks :: MVar (Map.Map FilePath (MVar ()))+locks = unsafePerformIO (newMVar Map.empty)
+ src/Kenshou/Remote/Perf/Publication.hs view
@@ -0,0 +1,95 @@+module Kenshou.Remote.Perf.Publication (publishEvidence, verifyPublication, retrieveEvidence) where++import Control.Exception (IOException, bracket, displayException, try)+import Control.Monad (unless)+import Data.Aeson (eitherDecodeFileStrict')+import Data.List (nub)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..))+import Kenshou.Perf.Artifacts (safePath, verifyPerfManifest)+import Kenshou.Remote.Payload.Publish (digestFile)+import Kenshou.Remote.Store+import System.Directory (createDirectoryIfMissing, getTemporaryDirectory, removeFile)+import System.FilePath (takeDirectory, (</>))+import System.IO (hClose, openBinaryTempFile)++-- Manifest is published last. Every object is create-only, and even existing+-- objects are retrieved and hashed. Successful upload metadata is insufficient.+publishEvidence :: ObjectStore -> Bucket -> Text -> FilePath -> IO (Either Text Text)+publishEvidence store bucket prefix directory = protected do+ validatePrefix prefix+ checked <- verifyPerfManifest directory >>= require+ checked `seq` pure ()+ files <- publicationFiles directory+ mapM_ publish files+ require =<< verifyPublication store bucket prefix directory+ pure ("gs://" <> bucket.unBucket <> "/" <> prefix <> "/manifest.json")+ where+ publish (relative, _, _, media) = do+ result <- store.putFile bucket (object relative) media DoesNotExist (directory </> relative)+ case result of+ Written _ -> pure ()+ PreconditionFailed -> pure ()+ object relative = ObjectName (prefix <> "/" <> Text.pack relative)++verifyPublication :: ObjectStore -> Bucket -> Text -> FilePath -> IO (Either Text ())+verifyPublication store bucket prefix directory = protected do+ validatePrefix prefix+ verifyPerfManifest directory >>= require+ files <- publicationFiles directory+ temporaryDirectory <- getTemporaryDirectory+ bracket (openBinaryTempFile temporaryDirectory "kenshou-perf-retrieval-") (removeFile . fst) \(temporary, handle) -> do+ hClose handle+ mapM_ (check temporary) files+ where+ check temporary (relative, expected, bytes, _) = do+ present <- store.downloadTo bucket (ObjectName (prefix <> "/" <> Text.pack relative)) temporary+ unless (present /= Nothing) (ioError (userError ("published artifact is missing: " <> relative)))+ (actual, actualBytes) <- digestFile temporary+ unless (actual == expected && toInteger actualBytes == bytes) (ioError (userError ("published artifact differs: " <> relative)))++-- Retrieve into a fresh caller-owned directory. Validate the complete inventory+-- before allowing any remote name to become a local output path.+retrieveEvidence :: ObjectStore -> Bucket -> Text -> Text -> FilePath -> IO (Either Text ())+retrieveEvidence store bucket prefix expectedManifest directory = protected do+ validatePrefix prefix+ createDirectoryIfMissing True directory+ fetch "manifest.json"+ (digest, _) <- digestFile (directory </> "manifest.json")+ unless ("sha256:" <> digest == expectedManifest) (ioError (userError "retrieved performance manifest differs from its receipt"))+ manifest <- eitherDecodeFileStrict' (directory </> "manifest.json") >>= either (ioError . userError) pure+ let paths = map (.path) (manifest :: Manifest).files+ unless (all safePath paths && "manifest.json" `notElem` paths && nub paths == paths && all ((>= 0) . (.bytes)) manifest.files) (ioError (userError "unsafe calibration report inventory"))+ mapM_ fetch paths+ verifyPerfManifest directory >>= require+ where+ fetch relative = do+ let path = directory </> relative+ createDirectoryIfMissing True (takeDirectory path)+ present <- store.downloadTo bucket (ObjectName (prefix <> "/" <> Text.pack relative)) path+ unless (present /= Nothing) (ioError (userError ("missing calibration report artifact: " <> relative)))++publicationFiles :: FilePath -> IO [(FilePath, Text, Integer, Text)]+publicationFiles directory = do+ manifest <- eitherDecodeFileStrict' (directory </> "manifest.json") >>= either (ioError . userError) pure+ (digest, bytes) <- digestFile (directory </> "manifest.json")+ pure+ ( [(file.path, Text.drop 7 file.sha256, file.bytes, file.mediaType) | file <- (manifest :: Manifest).files]+ <> [("manifest.json", digest, toInteger bytes, "application/json")]+ )++validatePrefix :: Text -> IO ()+validatePrefix prefix =+ unless+ ( not (Text.null prefix)+ && all (\part -> not (Text.null part) && part `notElem` [".", ".."]) (Text.splitOn "/" prefix)+ && not (Text.any (`elem` ['\\', '\n', '\r', '?', '#']) prefix)+ )+ (ioError (userError "unsafe performance publication prefix"))++require :: Either Text a -> IO a+require = either (ioError . userError . Text.unpack) pure++protected :: IO a -> IO (Either Text a)+protected action = either (Left . Text.pack . displayException) Right <$> try @IOException action
+ src/Kenshou/Remote/Perf/Runner.hs view
@@ -0,0 +1,226 @@+module Kenshou.Remote.Perf.Runner+ ( RunnerHooks (..),+ ComparisonHooks (..),+ Admission (..),+ AdmissionHooks (..),+ runPerformance,+ )+where++import Control.Exception (IOException, mask_, onException, try)+import Control.Monad (unless, when)+import Data.Aeson (toJSON)+import Data.IORef+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, addUTCTime, diffUTCTime)+import Kenshou.Perf.Contracts (PerfContracts)+import Kenshou.Perf.Plan (PerfPlan (..), PerfProfile (..), PlannedProfile (..))+import Kenshou.Perf.Prepared qualified as Prepared+import Kenshou.Perf.Request (PerfRequest (..))+import Kenshou.Perf.Result+import Kenshou.Remote.Perf.Journal+import System.Timeout (timeout)++data Admission = Admitted LeaseIdentity | Occupied | AdmissionFailed Text+ deriving stock (Eq, Show)++-- Replacement and runtime observations are persisted before another trial can+-- be dispatched. The cell adapter also persists its low-level work/submission+-- journal before effects and reconciles that journal when resuming an attempt.+data ComparisonHooks = ComparisonHooks+ { reserveReplacement :: Int -> IO (),+ recordRuntimeFailure :: Text -> IO ()+ }++-- Persist the proposed ID before CAS; withdraw only on a definite non-claim.+-- An unresolved proposal is reconciled read-only on resume, never replaced.+data AdmissionHooks = AdmissionHooks {proposeClaim :: LeaseIdentity -> IO (), withdrawClaim :: IO ()}++data RunnerHooks = RunnerHooks+ { -- | The suite's performance contracts, against which the preparation is+ -- re-validated before any work resumes.+ contracts :: PerfContracts,+ clock :: IO UTCTime,+ admit :: PlannedProfile -> Int -> Maybe (Text, LeaseIdentity, UTCTime) -> AdmissionHooks -> IO Admission,+ ownsLease :: LeaseIdentity -> IO Bool,+ canRunConfiguration :: LeaseIdentity -> PlannedProfile -> IO Bool,+ compareConfiguration :: LeaseIdentity -> PlannedProfile -> Int -> Bool -> ComparisonHooks -> IO ConfigurationResult,+ verifyCompleted :: ConfigurationResult -> IO Bool,+ cancelOwnedWork :: LeaseIdentity -> IO (),+ releaseOwnedLease :: LeaseIdentity -> IO Bool,+ recoverRuntimeFailures :: LeaseIdentity -> IO [(Text, Text)],+ reportMetadata :: PerfSession -> PerfResult -> IO PerfResult,+ publishAndVerify :: PerfSession -> PerfResult -> IO Bool+ }++-- Caller holds the session's exclusive orchestration lock. All inputs have+-- already passed manifest and payload retrieval verification. Admission hooks+-- include their bounded waiting; measurement callbacks must never pool pairs+-- across different lease identities or extend the supplied session deadline.+runPerformance :: RunnerHooks -> Prepared.PerfPrepared -> FilePath -> IO PerfResult+runPerformance hooks prepared path = do+ loaded <- readPerfSession path >>= require+ require (validateSessionInputs hooks.contracts prepared loaded)+ retained <- require (journalState loaded)+ mapM_ verify (Map.elems retained.completed)+ current <- newIORef loaded+ let checkpoint action = mask_ do+ now <- hooks.clock+ before <- readIORef current+ changed <- require (appendEvent (SessionEvent now action) before)+ writePerfSession path changed+ writeIORef current changed+ state = readIORef current >>= require . journalState+ cleanup = do+ saved <- state+ let proposed = case saved.pendingClaim of Just (_, identity, _) -> Just identity; Nothing -> Nothing+ case (case saved.lease of Just identity -> Just identity; Nothing -> proposed) of+ Nothing -> pure ()+ Just identity -> do+ now <- hooks.clock+ let deadline = case saved.pendingClaim of Just (_, _, at) -> Just (addUTCTime (fromIntegral (prepared.plan.request.budgetMinutes * 60 + 60)) at); Nothing -> saved.cleanupDeadline+ micros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` now)) deadline+ released <- timeout micros (hooks.cancelOwnedWork identity >> hooks.releaseOwnedLease identity >>= \released -> when released (recover identity) >> pure released)+ case saved.lease of+ Just _ -> checkpoint (LeaseLost "client interrupted; owned cleanup requested")+ Nothing -> when (released == Just True) (checkpoint LeaseClaimReleased)+ recover identity = do+ faults <- hooks.recoverRuntimeFailures identity+ mapM_ (\(profile, reason) -> do observed <- state; unless (reason `elem` Map.findWithDefault [] profile observed.runtimeBlockers) (checkpoint (RuntimeBlocked profile reason))) faults+ bounded :: String -> IO value -> IO value+ bounded label action = do+ observed <- state+ started <- hooks.clock+ let micros = maybe 10000000 (min 10000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` started)) observed.executionDeadline+ timeout micros action >>= maybe (ioError (userError (label <> " exceeded its bounded transport allowance"))) pure+ execute = do+ saved <- state+ now <- hooks.clock+ session <- readIORef current+ case saved.lease of+ Just identity -> do+ owned <- bounded "lease ownership check" (hooks.ownsLease identity)+ if not owned+ then do+ released <- bounded "lost lease reconciliation" (hooks.releaseOwnedLease identity)+ unless released (ioError (userError "lost lease still has unsettled owned work"))+ bounded "sealed lost-lease observations" (recover identity)+ checkpoint (LeaseLost "lease ownership lost; owned work reconciled")+ execute+ else case saved.activeComparison of+ Just (name, index) -> do+ entry <- require (selected session name)+ runOne identity entry index True+ Nothing -> case filter (\entry -> Map.notMember entry.profile.name saved.completed) session.plan.selected of+ [] -> release identity+ entry : _ -> do+ let index = Map.findWithDefault 0 entry.profile.name saved.attempts+ compatible <- bounded "configuration compatibility check" (hooks.canRunConfiguration identity entry)+ if not compatible+ then release identity >> execute+ else case appendEvent (SessionEvent now (ComparisonStarted entry.profile.name index)) session of+ Left _ -> release identity+ Right _ -> checkpoint (ComparisonStarted entry.profile.name index) >> runOne identity entry index False+ Nothing -> case filter (\entry -> Map.notMember entry.profile.name saved.completed) session.plan.selected of+ [] -> pure ()+ entry : _ -> do+ let admissionLeft = session.plan.request.waitSeconds - saved.admissionSpentSeconds+ measured = saved.executionDeadline /= Nothing+ enough = maybe True (\deadline -> cost entry <= floor (diffUTCTime deadline now)) saved.executionDeadline+ initialAdmissionAt = case [event.at | event <- session.events, AdmissionReserved _ <- [event.action]] of first : _ -> first; [] -> now+ wallLeft = max 0 (session.plan.request.waitSeconds - ceiling (diffUTCTime now initialAdmissionAt))+ seconds = if measured then maybe 1 (min 10 . max 0 . floor . (`diffUTCTime` now)) saved.executionDeadline else case saved.pendingClaim of Just _ -> 10; Nothing -> min 10 (min admissionLeft wallLeft)+ if not enough || seconds <= 0+ then case saved.pendingClaim of Nothing -> pure (); Just _ -> ioError (userError "unresolved cell claim exceeds the original session budget")+ else do+ unless (measured || saved.pendingClaim /= Nothing) (checkpoint (AdmissionReserved seconds))+ let callbacks = AdmissionHooks (checkpoint . LeaseClaimProposed entry.profile.name) (checkpoint LeaseClaimWithdrawn)+ granted <- timeout (seconds * 1000000) (hooks.admit entry seconds saved.pendingClaim callbacks)+ receivedAt <- hooks.clock+ observed <- state+ admission <- case granted of+ Nothing | observed.pendingClaim == Nothing -> pure Occupied+ Nothing -> ioError (userError "cell admission transport exceeded its reserved allowance; proposed claim retained")+ Just response -> do+ when (receivedAt > addUTCTime (fromIntegral seconds) now) (ioError (userError "cell admission returned after its reserved allowance"))+ pure response+ case admission of+ Admitted identity -> checkpoint (LeaseAcquired identity) >> execute+ Occupied -> do+ unless (observed.pendingClaim == Nothing) (ioError (userError "occupied admission left an unresolved claim"))+ if measured then pure () else execute+ AdmissionFailed problem -> ioError (userError (Text.unpack problem))+ runOne identity entry index resuming = do+ saved <- state+ now <- hooks.clock+ let micros = maybe 0 (max 0 . floor . (* 1000000) . (`diffUTCTime` now)) saved.executionDeadline+ reserve pair = do+ session <- readIORef current+ let action = ReplacementScheduled entry.profile.name index pair+ unless (action `elem` map (.action) session.events) (checkpoint action)+ blocked reason = do+ observed <- state+ unless (reason `elem` Map.findWithDefault [] entry.profile.name observed.runtimeBlockers) (checkpoint (RuntimeBlocked entry.profile.name reason))+ callbacks = ComparisonHooks reserve blocked+ result <- timeout micros (hooks.compareConfiguration identity entry index resuming callbacks)+ case result of+ Nothing -> do+ cleanupAt <- hooks.clock+ let cleanupMicros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` cleanupAt)) saved.cleanupDeadline+ _ <- timeout cleanupMicros (hooks.cancelOwnedWork identity)+ checkpoint (ComparisonCompleted (ConfigurationResult entry.profile.name entry.workloadSha256 "inconclusive" "pending" [] 0 entry.profile.minimumPairs [] Nothing))+ release identity+ Just comparison -> checkpoint (ComparisonCompleted comparison) >> execute+ release identity = do+ saved <- state+ now <- hooks.clock+ let micros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` now)) saved.cleanupDeadline+ released <- timeout micros (hooks.releaseOwnedLease identity >>= \released -> when released (recover identity) >> pure released)+ if released == Just True then checkpoint LeaseReleased else ioError (userError "owned lease release failed or exceeded cleanup grace")+ execution <- try @IOException (execute `onException` cleanup)+ session <- readIORef current+ saved <- require (journalState session)+ let unfinished entry =+ ConfigurationResult+ entry.profile.name+ entry.workloadSha256+ "not-run"+ (either (const "failed") (const "pending") execution)+ (Map.findWithDefault [] entry.profile.name saved.runtimeBlockers)+ 0+ entry.profile.minimumPairs+ (map toJSON session.events)+ Nothing+ completed = [Map.findWithDefault (unfinished entry) entry.profile.name saved.completed | entry <- session.plan.selected]+ preliminary <- require (aggregateResult session.plan session.preparedSha256 (either (const False) (const True) execution) completed)+ publicationStarted <- hooks.clock+ let publicationMicros =+ maybe+ (session.plan.request.budgetMinutes * 60000000)+ (min (session.plan.request.budgetMinutes * 60000000) . max 0 . floor . (* 1000000) . (`diffUTCTime` publicationStarted))+ saved.executionDeadline+ publication <-+ try @IOException+ ( timeout publicationMicros do+ reported <- hooks.reportMetadata session preliminary+ require (validateResult reported)+ unless (reported {stageSeconds = Map.empty, artifactReferences = []} == preliminary) (ioError (userError "report metadata changed a performance decision"))+ published <- hooks.publishAndVerify session reported+ pure (reported, published)+ )+ let (reported, published) = either (const (preliminary, False)) (maybe (preliminary, False) id) publication+ verifiedAt <- hooks.clock+ let withinBudget = maybe True (verifiedAt <=) saved.executionDeadline+ final <- require (aggregateResult session.plan session.preparedSha256 (published && withinBudget && either (const False) (const True) execution) completed)+ pure final {stageSeconds = reported.stageSeconds, artifactReferences = reported.artifactReferences}+ where+ verify result = hooks.verifyCompleted result >>= \valid -> unless valid (ioError (userError "retained completed configuration failed evidence verification"))+ cost entry = 2 * entry.profile.minimumPairs * (entry.profile.trialSeconds + entry.profile.resetSeconds + entry.profile.sealSeconds)+ selected session name = case filter ((== name) . (.name) . (.profile)) session.plan.selected of+ [entry] -> Right entry+ _ -> Left "active comparison has no pinned profile"++require :: Either Text a -> IO a+require = either (ioError . userError . Text.unpack) pure
+ src/Kenshou/Remote/Selftest.hs view
@@ -0,0 +1,136 @@+module Kenshou.Remote.Selftest (bundle) where++import Control.Exception (SomeAsyncException, SomeException, bracket, fromException, throwIO, try)+import Data.Aeson (object, (.=))+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Context (RunContext (..), SummarySection (..), putSummary, requirePostgres)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), noDimensions, postgresDimensions)+import Kenshou.Core.Env (PostgresRequirement (..), noEnvironment)+import Kenshou.Core.Env qualified as Env+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot (..), PostgresEnv (..), PostgresMode (..))+import Kenshou.Core.Id (parseScenarioId, selftestLayer)+import Kenshou.Core.Knob (Allowed (..), KnobName, KnobSpec (..), KnobType (..), KnobValue (..), knobText, mkKnobName)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.RunSpec (EnvironmentSpec (..), SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Scenario (..), ScenarioReport, Tier (..), failedWith, passed)+import Kenshou.Core.Selftest.Schema (selftestSchema)+import Network.HTTP.Client (Request (..), RequestBody (..), httpLbs, newManager, parseRequest, responseStatus, responseTimeoutMicro)+import Network.HTTP.Client.TLS (tlsManagerSettings)+import Network.HTTP.Types.Status (statusCode)+import Network.Socket (SocketType (Stream), addrAddress, addrFamily, addrSocketType, close, connect, defaultHints, defaultProtocol, getAddrInfo, socket)+import System.Environment (lookupEnv)+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)+import System.Timeout (timeout)++bundle :: LayerBundle+bundle = mkBundle selftestLayer [cellEnvironment] []++cellEnvironment :: Scenario+cellEnvironment =+ Scenario+ { id = either (error . show) id (parseScenarioId "selftest/remote/correctness/cell-environment"),+ revision = 2,+ summary = "Checks the environment supplied to a local or cell run.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs =+ [ KnobSpec expectPlacement "Expected execution placement" KnobText (VText "any") (OneOf (VText "any" :| [VText "local", VText "cell"])) [],+ KnobSpec otlpEndpoint "Optional OTLP HTTP endpoint to check" KnobText (VText "") AnyValue [],+ KnobSpec tcpEndpoint "Optional host:port to check for TCP reachability" KnobText (VText "") AnyValue []+ ],+ dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) noDimensions,+ phases = zeroPhases,+ requires = noEnvironment {Env.postgres = Just (PostgresRequirement [selftestSchema] [] False)},+ knownDefect = Nothing,+ run = runCellEnvironment+ }++runCellEnvironment :: RunContext -> IO ScenarioReport+runCellEnvironment context = do+ cellId <- lookupEnv "CELL_RUN_ID"+ let postgres = requirePostgres context+ cell = context.environmentSpec.placement == RunOnCell+ requestedPlacement = knobText context.knobs expectPlacement+ placementOk = (requestedPlacement == "any" || requestedPlacement == if cell then "cell" else "local") && (cell == maybe False (not . null) cellId)+ modeOk = postgres.mode == if cell then PgExternal else PgEphemeral+ major = postgres.snapshot.serverVersionNum `div` 10000+ versionOk = major == case context.dimensions.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> 0+ durabilityOk = Map.lookup "fsync" postgres.snapshot.settings == Just (if context.dimensions.pgDurability == Just PgDurable then "on" else "off")+ fresh <- (== Right "0") <$> psql postgres.connectionString "SELECT count(*) FROM kenshou_selftest.roundtrip"+ otlp <- reachableOtlp (knobText context.knobs otlpEndpoint)+ tcp <- reachableTcp (knobText context.knobs tcpEndpoint)+ let checks =+ [ ("placement-as-expected", placementOk),+ ("postgres-mode-as-expected", modeOk),+ ("pg-version-honoured", versionOk),+ ("pg-durability-honoured", durabilityOk),+ ("database-is-fresh", fresh),+ ("otlp-reachable", otlp),+ ("tcp-endpoint-reachable", tcp)+ ]+ failures = [label | (label, False) <- checks]+ putSummary context Verdicts "cell-environment" (object ["checks" .= Map.fromList checks])+ pure (if null failures then passed else failedWith failures "cell environment differs from the declared run specification")++psql :: Text -> Text -> IO (Either Text Text)+psql connection query = do+ (code, output, errors) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+ pure case code of+ ExitSuccess -> Right (Text.strip (Text.pack output))+ _ -> Left (Text.strip (Text.pack errors))++reachableOtlp :: Text -> IO Bool+reachableOtlp endpoint+ | Text.null endpoint = pure True+ | otherwise = do+ result <- try @SomeException do+ request <- parseRequest (Text.unpack (Text.dropWhileEnd (== '/') endpoint <> "/v1/traces"))+ manager <- newManager tlsManagerSettings+ response <-+ httpLbs+ request+ { method = "POST",+ requestHeaders = [("Content-Type", "application/json")],+ requestBody = RequestBodyLBS "{\"resourceSpans\":[]}",+ responseTimeout = responseTimeoutMicro 2000000+ }+ manager+ pure (statusCode (responseStatus response) `elem` [200, 202, 204])+ recovered result++-- | Whether the first of a comma-separated list of host:port addresses accepts a+-- TCP connection within two seconds; an empty list is trivially reachable.+reachableTcp :: Text -> IO Bool+reachableTcp endpoints+ | Text.null endpoints = pure True+ | otherwise = do+ let firstAddress = Text.takeWhile (/= ',') endpoints+ (host, colonPort) = Text.breakOnEnd ":" firstAddress+ if Text.null host || Text.null colonPort+ then pure False+ else do+ result <- try @SomeException $ timeout 2000000 do+ addresses <- getAddrInfo (Just defaultHints {addrSocketType = Stream}) (Just (Text.unpack (Text.dropEnd 1 host))) (Just (Text.unpack colonPort))+ case addresses of+ [] -> pure False+ address : _ -> bracket (socket (addrFamily address) Stream defaultProtocol) close (\connection -> connect connection (addrAddress address) >> pure True)+ recovered (fmap (maybe False id) result)++recovered :: Either SomeException Bool -> IO Bool+recovered (Right available) = pure available+recovered (Left failure)+ | Just asynchronous <- fromException failure :: Maybe SomeAsyncException = throwIO asynchronous+ | otherwise = pure False++expectPlacement, otlpEndpoint, tcpEndpoint :: KnobName+expectPlacement = knobName "remote.expect-placement"+otlpEndpoint = knobName "remote.otlp-endpoint"+tcpEndpoint = knobName "remote.tcp-endpoint"++knobName :: Text -> KnobName+knobName = either (error . show) id . mkKnobName
+ src/Kenshou/Remote/Store.hs view
@@ -0,0 +1,43 @@+module Kenshou.Remote.Store+ ( Bucket (..),+ ObjectName (..),+ Precondition (..),+ ObjectMeta (..),+ PutOutcome (..),+ ObjectStore (..),+ )+where++import Data.ByteString.Lazy (ByteString)+import Data.Int (Int64)+import Data.Text (Text)+import Data.Time (UTCTime)++newtype Bucket = Bucket {unBucket :: Text} deriving stock (Eq, Ord, Show)++newtype ObjectName = ObjectName {unObjectName :: Text} deriving stock (Eq, Ord, Show)++data Precondition = NoPrecondition | DoesNotExist | GenerationIs !Int64+ deriving stock (Eq, Show)++data ObjectMeta = ObjectMeta+ { generation :: !Int64,+ size :: !Int64,+ updated :: !UTCTime,+ contentType :: !Text+ }+ deriving stock (Eq, Show)++data PutOutcome = Written !ObjectMeta | PreconditionFailed+ deriving stock (Eq, Show)++data ObjectStore = ObjectStore+ { getObject :: Bucket -> ObjectName -> IO (Maybe (ByteString, ObjectMeta)),+ statObject :: Bucket -> ObjectName -> IO (Maybe ObjectMeta),+ putObject :: Bucket -> ObjectName -> Text -> Precondition -> ByteString -> IO PutOutcome,+ putFile :: Bucket -> ObjectName -> Text -> Precondition -> FilePath -> IO PutOutcome,+ downloadTo :: Bucket -> ObjectName -> FilePath -> IO (Maybe ObjectMeta),+ deleteObject :: Bucket -> ObjectName -> Precondition -> IO Bool,+ listObjects :: Bucket -> Text -> IO [(ObjectName, ObjectMeta)],+ serverTime :: IO UTCTime+ }
+ src/Kenshou/Remote/Store/File.hs view
@@ -0,0 +1,187 @@+module Kenshou.Remote.Store.File (newFileStore) where++import Control.Concurrent.MVar (MVar, modifyMVar, newMVar)+import Control.Exception (bracket)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (FromJSON (..), ToJSON, eitherDecodeFileStrict', encodeFile, withObject, (.!=), (.:), (.:?))+import Data.ByteString qualified as ByteString+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.List (sortOn)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import GHC.Generics (Generic)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import System.Directory (copyFile, createDirectoryIfMissing, doesFileExist, getFileSize, listDirectory, renameFile)+import System.FilePath (takeDirectory, takeExtension, (</>))+import System.IO (SeekMode (AbsoluteSeek))+import System.IO.Unsafe (unsafePerformIO)+import System.Posix.IO (LockRequest (..), OpenFileFlags (creat), OpenMode (ReadWrite), closeFd, defaultFileFlags, openFd, waitToSetLock)++data StoredMeta = StoredMeta+ { name :: !Text,+ generation :: !Int64,+ size :: !Int64,+ updated :: !UTCTime,+ present :: !Bool,+ contentType :: !Text+ }+ deriving stock (Generic)+ deriving anyclass (ToJSON)++instance FromJSON StoredMeta where+ parseJSON = withObject "cell object metadata" \value ->+ StoredMeta+ <$> value .: "name"+ <*> value .: "generation"+ <*> value .: "size"+ <*> value .: "updated"+ <*> value .: "present"+ <*> (value .:? "contentType" .!= "application/octet-stream")++newFileStore :: FilePath -> IO ObjectStore+newFileStore root = do+ let localLock = processLock+ let stat bucket object = withObjectLock localLock root bucket object \bucketRoot hash -> fmap public <$> readMeta bucketRoot object hash+ get bucket object = withObjectLock localLock root bucket object \bucketRoot hash -> do+ current <- readMeta bucketRoot object hash+ case current of+ Nothing -> pure Nothing+ Just meta -> do+ bytes <- ByteString.readFile (dataPath bucketRoot hash meta.generation)+ pure (Just (LazyByteString.fromStrict bytes, public meta))+ write bucket object media pre writeData = withObjectLock localLock root bucket object \bucketRoot hash -> do+ current <- readStoredMeta bucketRoot object hash+ if not (preconditionHolds current pre)+ then pure PreconditionFailed+ else do+ now <- getCurrentTime+ let next = maybe 1 ((+ 1) . (.generation)) current+ destination = dataPath bucketRoot hash next+ createDirectoryIfMissing True (takeDirectory destination)+ bytes <- writeData destination+ let meta = StoredMeta object.unObjectName next bytes now True media+ writeMeta bucketRoot hash meta+ pure (Written (public meta))+ delete bucket object pre = withObjectLock localLock root bucket object \bucketRoot hash -> do+ current <- readStoredMeta bucketRoot object hash+ if not (preconditionHolds current pre)+ then pure False+ else case current of+ Nothing -> pure True+ Just prior -> do+ now <- getCurrentTime+ writeMeta bucketRoot hash prior {generation = prior.generation + 1, updated = now, present = False}+ pure True+ list bucket prefix = withBucketLock localLock root bucket \bucketRoot -> do+ let metaDirectory = bucketRoot </> "meta"+ createDirectoryIfMissing True metaDirectory+ files <- filter ((== ".json") . takeExtension) <$> listDirectory metaDirectory+ records <- mapM (readMetaFile . (metaDirectory </>)) files+ pure (sortOn fst [(ObjectName meta.name, public meta) | meta <- records, meta.present, prefix `Text.isPrefixOf` meta.name])+ pure+ ObjectStore+ { getObject = get,+ statObject = stat,+ putObject = \bucket object media pre bytes -> write bucket object media pre \destination -> do+ LazyByteString.writeFile destination bytes+ pure (LazyByteString.length bytes),+ putFile = \bucket object media pre source -> write bucket object media pre \destination -> do+ copyFile source destination+ fromIntegral <$> getFileSize destination,+ downloadTo = \bucket object destination -> withObjectLock localLock root bucket object \bucketRoot hash -> do+ current <- readMeta bucketRoot object hash+ case current of+ Nothing -> pure Nothing+ Just meta -> do+ createDirectoryIfMissing True (takeDirectory destination)+ copyFile (dataPath bucketRoot hash meta.generation) destination+ pure (Just (public meta)),+ deleteObject = delete,+ listObjects = list,+ serverTime = getCurrentTime+ }++-- POSIX record locks are per process, so separate handles in one process also+-- share a local mutex before taking the cross-process lock.+{-# NOINLINE processLock #-}+processLock :: MVar ()+processLock = unsafePerformIO (newMVar ())++withObjectLock :: MVar () -> FilePath -> Bucket -> ObjectName -> (FilePath -> FilePath -> IO a) -> IO a+withObjectLock localLock root bucket object action = withBucketLock localLock root bucket \bucketRoot -> do+ validateName object+ action bucketRoot (key object)++withBucketLock :: MVar () -> FilePath -> Bucket -> (FilePath -> IO a) -> IO a+withBucketLock localLock root bucket action = do+ bucketRoot <- bucketPath root bucket+ createDirectoryIfMissing True bucketRoot+ modifyMVar localLock \state -> do+ result <- bracket (openFd (bucketRoot </> ".lock") ReadWrite defaultFileFlags {creat = Just 0o600}) closeFd \fd -> do+ waitToSetLock fd (WriteLock, AbsoluteSeek, 0, 0)+ action bucketRoot+ pure (state, result)++bucketPath :: FilePath -> Bucket -> IO FilePath+bucketPath root bucket+ | not (Text.null bucket.unBucket) && Text.all valid bucket.unBucket = pure (root </> "buckets" </> Text.unpack bucket.unBucket)+ | otherwise = ioError (userError "invalid cell bucket name")+ where+ valid character = ('a' <= character && character <= 'z') || ('A' <= character && character <= 'Z') || ('0' <= character && character <= '9') || character `elem` ['-', '_']++validateName :: ObjectName -> IO ()+validateName object+ | Text.null value || Text.any (== '\\') value || any invalid (Text.splitOn "/" value) = ioError (userError "invalid cell object name")+ | otherwise = pure ()+ where+ value = object.unObjectName+ invalid part = Text.null part || part == "." || part == ".."++key :: ObjectName -> FilePath+key = Text.unpack . Text.decodeUtf8 . Base16.encode . SHA256.hash . Text.encodeUtf8 . (.unObjectName)++metaPath :: FilePath -> FilePath -> FilePath+metaPath bucketRoot hash = bucketRoot </> "meta" </> hash <> ".json"++dataPath :: FilePath -> FilePath -> Int64 -> FilePath+dataPath bucketRoot hash generation = bucketRoot </> "objects" </> hash </> show generation++readStoredMeta :: FilePath -> ObjectName -> FilePath -> IO (Maybe StoredMeta)+readStoredMeta bucketRoot object hash = do+ let path = metaPath bucketRoot hash+ exists <- doesFileExist path+ if not exists+ then pure Nothing+ else do+ meta <- readMetaFile path+ if meta.name == object.unObjectName+ then pure (Just meta)+ else ioError (userError "cell object metadata key collision")++readMeta :: FilePath -> ObjectName -> FilePath -> IO (Maybe StoredMeta)+readMeta bucketRoot object hash = do+ meta <- readStoredMeta bucketRoot object hash+ pure (meta >>= \record -> if record.present then Just record else Nothing)++readMetaFile :: FilePath -> IO StoredMeta+readMetaFile path = either (ioError . userError) pure =<< eitherDecodeFileStrict' path++writeMeta :: FilePath -> FilePath -> StoredMeta -> IO ()+writeMeta bucketRoot hash meta = do+ let destination = metaPath bucketRoot hash+ temporary = destination <> ".writing"+ createDirectoryIfMissing True (takeDirectory destination)+ encodeFile temporary meta+ renameFile temporary destination++public :: StoredMeta -> ObjectMeta+public meta = ObjectMeta meta.generation meta.size meta.updated meta.contentType++preconditionHolds :: Maybe StoredMeta -> Precondition -> Bool+preconditionHolds _ NoPrecondition = True+preconditionHolds current DoesNotExist = maybe True (not . (.present)) current+preconditionHolds current (GenerationIs expected) = maybe False (\meta -> meta.present && meta.generation == expected) current
+ src/Kenshou/Remote/Store/Gcs.hs view
@@ -0,0 +1,409 @@+module Kenshou.Remote.Store.Gcs+ ( TokenProvider (..),+ newTokenProvider,+ newTokenProviderWith,+ newGcsStore,+ newGcsStoreAt,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.MVar (modifyMVar, modifyMVar_, newMVar)+import Control.Exception (bracket, try)+import Control.Monad (unless)+import Data.Aeson (FromJSON (..), decode, eitherDecode, withObject, (.:), (.:?))+import Data.ByteString qualified as ByteString+import Data.ByteString.Char8 qualified as ByteString.Char8+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Int (Int64)+import Data.Maybe (fromMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as TextEncoding+import Data.Time.Clock (addUTCTime, getCurrentTime)+import Data.Time.Format (defaultTimeLocale, parseTimeM)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Network.HTTP.Client (BodyReader, HttpException, Request (..), RequestBody (..), Response, brRead, httpLbs, newManager, parseRequest, responseBody, responseHeaders, responseStatus, responseTimeoutMicro, withResponse)+import Network.HTTP.Client.TLS (tlsManagerSettings)+import Network.HTTP.Types.Status (statusCode)+import Numeric (showHex)+import System.Directory (createDirectoryIfMissing, doesFileExist, getFileSize, removeFile, renameFile)+import System.Environment (lookupEnv)+import System.Exit (ExitCode (ExitSuccess))+import System.FilePath (takeDirectory)+import System.IO (Handle, IOMode (..), SeekMode (AbsoluteSeek), hClose, hIsClosed, hSeek, openBinaryTempFile, withBinaryFile)+import System.Process (readProcessWithExitCode)+import Text.Read (readMaybe)++data TokenProvider = TokenProvider+ { controlBucket :: !Bucket,+ accessToken :: IO Text,+ invalidateToken :: IO ()+ }++newTokenProvider :: Bucket -> IO TokenProvider+newTokenProvider bucket = do+ manager <- newManager tlsManagerSettings+ request <- parseRequest "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"+ newTokenProviderWith+ bucket+ (fmap Text.pack <$> lookupEnv "KENSHOU_GCS_TOKEN")+ ( do+ let metadataRequest = request {requestHeaders = [("Metadata-Flavor", "Google")], responseTimeout = responseTimeoutMicro 2000000}+ response <- try (httpLbs metadataRequest manager) :: IO (Either HttpException (Response LazyByteString.ByteString))+ pure case response of+ Right value | statusCode (responseStatus value) == 200 -> (\token -> token.tokenResponse) <$> (decode (responseBody value) :: Maybe MetadataToken)+ _ -> Nothing+ )+ ( do+ (exit, output, _) <- readProcessWithExitCode "gcloud" ["auth", "print-access-token"] ""+ unless (exit == ExitSuccess) (ioError (userError "gcloud access-token command failed"))+ pure (Text.strip (Text.pack output))+ )++-- | Injection seam for the three credential sources; only CLI tokens are cached.+newTokenProviderWith :: Bucket -> IO (Maybe Text) -> IO (Maybe Text) -> IO Text -> IO TokenProvider+newTokenProviderWith bucket environment metadata cli = do+ cache <- newMVar Nothing+ let choose = do+ environment >>= \case+ Just token | validToken token -> pure token+ Just _ -> ioError (userError "invalid GCS environment token")+ Nothing -> modifyMVar cache \cached -> do+ now <- getCurrentTime+ case cached of+ Just (expiry, token) | now < expiry -> pure (cached, token)+ _ ->+ metadata >>= \case+ Just token | validToken token -> pure (Nothing, token)+ Just _ -> ioError (userError "invalid GCS metadata token")+ Nothing -> do+ token <- cli+ unless (validToken token) (ioError (userError "invalid gcloud access token"))+ pure (Just (addUTCTime (45 * 60) now, token), token)+ pure (TokenProvider bucket choose (modifyMVar_ cache (const (pure Nothing))))++newtype MetadataToken = MetadataToken {tokenResponse :: Text}++instance FromJSON MetadataToken where+ parseJSON = withObject "GCE access token" \value -> MetadataToken <$> value .: "access_token"++data GcsObject = GcsObject+ { objectName :: !Text,+ objectMeta :: !ObjectMeta+ }++instance FromJSON GcsObject where+ parseJSON = withObject "GCS object" \value -> do+ name <- value .: "name"+ generationText <- value .: "generation"+ sizeText <- value .: "size"+ generation <- maybe (fail "invalid GCS generation") pure (readMaybe (Text.unpack generationText))+ size <- maybe (fail "invalid GCS size") pure (readMaybe (Text.unpack sizeText))+ updated <- value .: "updated"+ contentType <- fromMaybe "application/octet-stream" <$> value .:? "contentType"+ pure (GcsObject name (ObjectMeta generation size updated contentType))++data GcsPage = GcsPage+ { items :: ![GcsObject],+ nextPageToken :: !(Maybe Text)+ }++instance FromJSON GcsPage where+ parseJSON = withObject "GCS objects page" \value -> GcsPage <$> (fromMaybe [] <$> value .:? "items") <*> value .:? "nextPageToken"++newGcsStore :: TokenProvider -> IO ObjectStore+newGcsStore = newGcsStoreAt "https://storage.googleapis.com"++-- | The endpoint seam keeps the HTTP protocol testable against a local server.+newGcsStoreAt :: Text -> TokenProvider -> IO ObjectStore+newGcsStoreAt endpoint provider = do+ unless ("http://" `Text.isPrefixOf` endpoint || "https://" `Text.isPrefixOf` endpoint) (ioError (userError "invalid GCS endpoint"))+ manager <- newManager tlsManagerSettings+ let base = Text.dropWhileEnd (== '/') endpoint+ runRequest verb url body headers = do+ initial <- parseRequest (Text.unpack url)+ snd+ <$> withAuthorizedRetry+ provider+ ( \token -> do+ response <-+ retryResponse $+ httpLbs+ initial+ { method = verb,+ requestHeaders = ("Authorization", "Bearer " <> TextEncoding.encodeUtf8 token) : headers,+ requestBody = body+ }+ manager+ pure (statusCode (responseStatus response), response)+ )+ metadata bucket object = do+ url <- objectUrl base bucket object+ response <- runRequest "GET" url (RequestBodyBS ByteString.empty) []+ case statusCode (responseStatus response) of+ 404 -> pure Nothing+ 200 -> Just . (\item -> item.objectMeta) <$> decodeObject object (responseBody response)+ code -> badStatus "reading GCS metadata" code+ get bucket object = do+ current <- metadata bucket object+ case current of+ Nothing -> pure Nothing+ Just meta -> do+ url <- objectUrl base bucket object+ response <- runRequest "GET" (url <> "?alt=media&generation=" <> tshow meta.generation) (RequestBodyBS ByteString.empty) []+ case statusCode (responseStatus response) of+ 200 -> do+ let bytes = responseBody response+ unless (LazyByteString.length bytes == meta.size) (ioError (userError "GCS media length differs from metadata"))+ pure (Just (bytes, meta))+ code -> badStatus "reading GCS media" code+ upload bucket object media pre body = do+ url <- uploadUrl base bucket object pre "media"+ response <- runRequest "POST" url body [("Content-Type", TextEncoding.encodeUtf8 media)]+ putResponse object response+ download bucket object destination = do+ current <- metadata bucket object+ case current of+ Nothing -> pure Nothing+ Just meta -> do+ url <- objectUrl base bucket object+ initial <- parseRequest (Text.unpack (url <> "?alt=media&generation=" <> tshow meta.generation))+ createDirectoryIfMissing True (takeDirectory destination)+ bracket+ (openBinaryTempFile (takeDirectory destination) ".kenshou-gcs-download")+ ( \(temporary, handle) -> do+ closed <- hIsClosed handle+ unless closed (hClose handle)+ exists <- doesFileExist temporary+ if exists then removeFile temporary else pure ()+ )+ \(temporary, handle) -> do+ (code, bytes) <-+ withAuthorizedRetry+ provider+ ( \token -> retryStatus $ withResponse initial {requestHeaders = [("Authorization", "Bearer " <> TextEncoding.encodeUtf8 token)]} manager \response -> do+ let code = statusCode (responseStatus response)+ if code == 200+ then (\count -> (code, count)) <$> copyBody (responseBody response) handle+ else pure (code, 0)+ )+ unless (code == 200) (badStatus "downloading GCS media" code)+ unless (bytes == meta.size) (ioError (userError "GCS downloaded length differs from metadata"))+ hClose handle+ renameFile temporary destination+ pure (Just meta)+ putLargeFile bucket object media pre source size = do+ url <- uploadUrl base bucket object pre "resumable"+ response <- runRequest "POST" url (RequestBodyBS ByteString.empty) [("X-Upload-Content-Type", TextEncoding.encodeUtf8 media), ("X-Upload-Content-Length", ByteString.Char8.pack (show size))]+ case statusCode (responseStatus response) of+ 412 -> pure PreconditionFailed+ code | code >= 200 && code < 300 -> do+ location <- maybe (ioError (userError "GCS resumable upload omitted Location")) (pure . TextEncoding.decodeUtf8) (lookup "Location" (responseHeaders response))+ initial <- parseRequest (Text.unpack location)+ let sendAuthorized method range body =+ snd+ <$> withAuthorizedRetry+ provider+ ( \token -> do+ chunkResponse <- httpLbs initial {method, requestHeaders = [("Content-Range", range), ("Authorization", "Bearer " <> TextEncoding.encodeUtf8 token), ("Content-Type", TextEncoding.encodeUtf8 media)], requestBody = body} manager+ pure (statusCode (responseStatus chunkResponse), chunkResponse)+ )+ sendChunk offset failures = do+ let lengthBytes = min (8 * 1024 * 1024) (size - offset)+ lastByte = offset + lengthBytes - 1+ range = "bytes " <> tshow offset <> "-" <> tshow lastByte <> "/" <> tshow size+ result <- sendAuthorized "PUT" (TextEncoding.encodeUtf8 range) (streamFileChunk source offset lengthBytes)+ case statusCode (responseStatus result) of+ 200 -> putResponse object result+ 201 -> putResponse object result+ 412 -> pure PreconditionFailed+ 308 -> continueFrom offset failures result+ transient | transient == 429 || transient >= 500 && transient <= 599 -> do+ statusResult <- retryResponse $ sendAuthorized "PUT" ("bytes */" <> ByteString.Char8.pack (show size)) (RequestBodyBS ByteString.empty)+ case statusCode (responseStatus statusResult) of+ 200 -> putResponse object statusResult+ 201 -> putResponse object statusResult+ 308 -> continueFrom offset failures statusResult+ other -> badStatus "querying GCS resumable upload" other+ other -> badStatus "sending GCS resumable chunk" other+ continueFrom offset failures result = do+ next <- case lookup "Range" (responseHeaders result) of+ Nothing -> pure 0+ Just value -> maybe (ioError (userError "invalid GCS resumable Range")) pure (parseRangeNext value)+ unless (next <= size) (ioError (userError "GCS resumable Range exceeds file size"))+ let stalled = if next <= offset then failures + 1 else 0+ unless (stalled < 4) (ioError (userError "GCS resumable upload made no progress"))+ unless (next < size) (ioError (userError "GCS resumable upload lacks final metadata"))+ sendChunk next stalled+ sendChunk 0 (0 :: Int)+ code -> badStatus "starting GCS resumable upload" code+ list bucket prefix = do+ listUrl <- bucketUrl base bucket+ let loop token accumulated = do+ let parameters = "?prefix=" <> encodeComponent prefix <> "&maxResults=1000" <> maybe "" ("&pageToken=" <>) (encodeComponent <$> token)+ response <- runRequest "GET" (listUrl <> parameters) (RequestBodyBS ByteString.empty) []+ case statusCode (responseStatus response) of+ 200 -> do+ page <- decodeJson "GCS object list" (responseBody response) :: IO GcsPage+ let entries = [(ObjectName item.objectName, item.objectMeta) | item <- page.items]+ maybe (pure (accumulated <> entries)) (\next -> loop (Just next) (accumulated <> entries)) page.nextPageToken+ code -> badStatus "listing GCS objects" code+ loop Nothing []+ clock = do+ url <- bucketUrl base provider.controlBucket+ response <- runRequest "GET" (url <> "?maxResults=1") (RequestBodyBS ByteString.empty) []+ case statusCode (responseStatus response) of+ 200 -> do+ date <- maybe (ioError (userError "GCS response omitted Date")) pure (lookup "Date" (responseHeaders response))+ maybe (ioError (userError "invalid GCS Date header")) pure (parseTimeM True defaultTimeLocale "%a, %d %b %Y %H:%M:%S GMT" (ByteString.Char8.unpack date))+ code -> badStatus "reading GCS server time" code+ pure+ ObjectStore+ { getObject = get,+ statObject = metadata,+ putObject = \bucket object media pre bytes -> upload bucket object media pre (RequestBodyLBS bytes),+ putFile = \bucket object media pre source -> do+ size <- getFileSize source+ if size <= 8 * 1024 * 1024+ then do+ bytes <- LazyByteString.readFile source+ upload bucket object media pre (RequestBodyLBS bytes)+ else putLargeFile bucket object media pre source (fromIntegral size),+ downloadTo = download,+ deleteObject = \bucket object pre -> do+ url <- objectUrl base bucket object+ response <- runRequest "DELETE" (url <> preconditionQuery pre) (RequestBodyBS ByteString.empty) []+ case statusCode (responseStatus response) of+ 204 -> pure True+ 200 -> pure True+ 404 -> pure False+ 412 -> pure False+ code -> badStatus "deleting GCS object" code,+ listObjects = list,+ serverTime = clock+ }++putResponse :: ObjectName -> Response LazyByteString.ByteString -> IO PutOutcome+putResponse object response = case statusCode (responseStatus response) of+ 412 -> pure PreconditionFailed+ code | code == 200 || code == 201 -> (\item -> Written item.objectMeta) <$> decodeObject object (responseBody response)+ code -> badStatus "uploading GCS object" code++retryResponse :: IO (Response body) -> IO (Response body)+retryResponse action = retryStatus ((\response -> (statusCode (responseStatus response), response)) <$> action) >>= pure . snd++retryStatus :: IO (Int, value) -> IO (Int, value)+retryStatus action = go (0 :: Int)+ where+ go attempts = do+ result@(code, _) <- action+ if attempts < 4 && (code == 429 || code >= 500 && code <= 599)+ then threadDelay (100000 * (2 ^ attempts)) >> go (attempts + 1)+ else pure result++-- A gcloud token can already be near expiry when first returned. Refresh once+-- on authorization failure instead of assuming a fixed lifetime in our cache.+withAuthorizedRetry :: TokenProvider -> (Text -> IO (Int, value)) -> IO (Int, value)+withAuthorizedRetry provider action = do+ token <- provider.accessToken+ unless (validToken token) (ioError (userError "invalid GCS token"))+ result@(code, _) <- action token+ if code == 401+ then do+ provider.invalidateToken+ refreshed <- provider.accessToken+ unless (validToken refreshed) (ioError (userError "invalid GCS token"))+ action refreshed+ else pure result++decodeObject :: ObjectName -> LazyByteString.ByteString -> IO GcsObject+decodeObject object body = do+ result <- decodeJson "GCS object" body+ unless (result.objectName == object.unObjectName) (ioError (userError "GCS returned a different object name"))+ pure result++decodeJson :: (FromJSON value) => Text -> LazyByteString.ByteString -> IO value+decodeJson label bytes = either (ioError . userError . ((Text.unpack label <> ": ") <>)) pure (eitherDecode bytes)++badStatus :: Text -> Int -> IO value+badStatus operation code = ioError (userError (Text.unpack operation <> ": HTTP " <> show code))++bucketUrl :: Text -> Bucket -> IO Text+bucketUrl endpoint bucket = do+ unless (not (Text.null bucket.unBucket) && Text.all validBucketCharacter bucket.unBucket) (ioError (userError "invalid GCS bucket name"))+ pure (endpoint <> "/storage/v1/b/" <> bucket.unBucket <> "/o")++objectUrl :: Text -> Bucket -> ObjectName -> IO Text+objectUrl endpoint bucket object = do+ validateName object+ (<> "/" <> encodeComponent object.unObjectName) <$> bucketUrl endpoint bucket++uploadUrl :: Text -> Bucket -> ObjectName -> Precondition -> Text -> IO Text+uploadUrl endpoint bucket object pre uploadType = do+ validateName object+ root <- bucketUrl endpoint bucket+ pure (Text.replace "/storage/v1/" "/upload/storage/v1/" root <> "?uploadType=" <> uploadType <> "&name=" <> encodeComponent object.unObjectName <> preconditionSuffix pre)++preconditionQuery :: Precondition -> Text+preconditionQuery NoPrecondition = ""+preconditionQuery pre = "?" <> Text.drop 1 (preconditionSuffix pre)++preconditionSuffix :: Precondition -> Text+preconditionSuffix NoPrecondition = ""+preconditionSuffix DoesNotExist = "&ifGenerationMatch=0"+preconditionSuffix (GenerationIs generation) = "&ifGenerationMatch=" <> tshow generation++validateName :: ObjectName -> IO ()+validateName object = unless (not (Text.null value) && all validPart (Text.splitOn "/" value) && not (Text.any (== '\\') value)) (ioError (userError "invalid GCS object name"))+ where+ value = object.unObjectName+ validPart part = not (Text.null part) && part /= "." && part /= ".."++validBucketCharacter :: Char -> Bool+validBucketCharacter character = ('a' <= character && character <= 'z') || ('A' <= character && character <= 'Z') || ('0' <= character && character <= '9') || character `elem` ['.', '_', '-']++encodeComponent :: Text -> Text+encodeComponent = Text.concatMap encodeByte . TextEncoding.decodeLatin1 . TextEncoding.encodeUtf8+ where+ encodeByte character+ | ('a' <= character && character <= 'z') || ('A' <= character && character <= 'Z') || ('0' <= character && character <= '9') || character `elem` ['-', '_', '.', '~'] = Text.singleton character+ | otherwise = let raw = showHex (fromEnum character) "" in "%" <> Text.toUpper (Text.justifyRight 2 '0' (Text.pack raw))++validToken :: Text -> Bool+validToken token = not (Text.null (Text.strip token)) && not (Text.any (`elem` ['\r', '\n']) token)++tshow :: (Show value) => value -> Text+tshow = Text.pack . show++streamFileChunk :: FilePath -> Int64 -> Int64 -> RequestBody+streamFileChunk source offset size = RequestBodyStream size \consume -> withBinaryFile source ReadMode \handle -> do+ hSeek handle AbsoluteSeek (fromIntegral offset)+ remaining <- newIORef size+ consume do+ left <- readIORef remaining+ if left == 0+ then pure ByteString.empty+ else do+ chunk <- ByteString.hGetSome handle (fromIntegral (min left (1024 * 1024)))+ whenEmpty chunk+ writeIORef remaining (left - fromIntegral (ByteString.length chunk))+ pure chunk+ where+ whenEmpty chunk = unless (not (ByteString.null chunk)) (ioError (userError "GCS upload source shrank during transfer"))++parseRangeNext :: ByteString.ByteString -> Maybe Int64+parseRangeNext value = do+ suffix <- ByteString.Char8.stripPrefix "bytes=0-" value+ lastByte <- readMaybe (ByteString.Char8.unpack suffix)+ if lastByte >= 0 then Just (lastByte + 1) else Nothing++copyBody :: BodyReader -> Handle -> IO Int64+copyBody reader handle = go 0+ where+ go total = do+ chunk <- brRead reader+ if ByteString.null chunk+ then pure total+ else ByteString.hPut handle chunk >> go (total + fromIntegral (ByteString.length chunk))
+ test/DebugSpec.hs view
@@ -0,0 +1,26 @@+module DebugSpec (spec) where++import Data.Aeson (eitherDecodeFileStrict')+import Kenshou.Remote.Cell.Debug (remoteCommand, resolveDebugRole)+import Kenshou.Remote.Cell.Docs (CellDescriptor)+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)+import Test.Hspec++spec :: Spec+spec = describe "cell debug target" do+ it "selects only instances named by the verified descriptor" do+ decoded <- eitherDecodeFileStrict' "test/golden/cell/cell.descriptor.v1.json"+ descriptor <- either (fail . ("descriptor fixture: " <>)) pure (decoded :: Either String CellDescriptor)+ resolveDebugRole descriptor "driver" `shouldBe` Right "cell-alpha-driver-0"+ resolveDebugRole descriptor "driver-0" `shouldBe` Right "cell-alpha-driver-0"+ resolveDebugRole descriptor "postgres" `shouldBe` Right "cell-alpha-postgres"+ resolveDebugRole descriptor "monitoring" `shouldBe` Right "cell-alpha-monitoring"+ resolveDebugRole descriptor "driver-1" `shouldBe` Left "cell descriptor has no driver at that index"+ resolveDebugRole descriptor "broker" `shouldBe` Left "role must be postgres, monitoring, driver, or driver-N"++ it "preserves spaces and apostrophes in remote command arguments" do+ (code, output, failure) <- readProcessWithExitCode "/bin/sh" ["-c", remoteCommand ["printf", "%s", "two words", "it's"]] ""+ code `shouldBe` ExitSuccess+ failure `shouldBe` ""+ output `shouldBe` "two wordsit's"
+ test/DocsSpec.hs view
@@ -0,0 +1,98 @@+module DocsSpec (spec) where++import Data.Aeson (ToJSON, Value (..), eitherDecode, encode, object, toJSON, (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isLeft, isRight)+import Data.Text (Text)+import Kenshou.Remote.Cell.Docs (Artifact, CellDescriptor, CellEnvironment, CellManifest (..), CellRunResult, CellStatus, Rejected, Submission)+import Test.Hspec++spec :: Spec+spec = describe "cell owner documents" do+ it "decodes status, rejection, run result and manifest fixtures without changing their JSON" do+ checkFixture "cell.status.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellStatus)+ checkFixture "cell.rejected.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String Rejected)+ checkFixture "cell.run-result.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellRunResult)+ checkFixture "cell.artifact-manifest.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellManifest)++ it "decodes the owner descriptor and environment fixtures without changing their JSON" do+ checkFixture "cell.descriptor.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellDescriptor)+ checkFixture "cell.environment.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String CellEnvironment)++ it "decodes the generic owner submission, including its non-Kenshou payload" do+ checkFixture "cell.submission.v1.json" (eitherDecode :: LazyByteString.ByteString -> Either String Submission)++ it "ignores future status fields while enforcing sealed terminal fields" do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.status.v1.json"+ case eitherDecode bytes :: Either String Value of+ Right (Object fields) -> do+ let extended = Object (KeyMap.insert "futureField" (Bool True) fields)+ (eitherDecode (encode extended) :: Either String CellStatus) `shouldSatisfy` isRight+ _ -> expectationFailure "invalid status fixture"+ let incomplete = "{\"schema\":\"cell.status/v1\",\"runId\":\"0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9\",\"phase\":\"sealed\",\"updatedAt\":\"2026-09-25T12:00:00Z\",\"logChunks\":{\"stdout\":0,\"stderr\":0},\"outcome\":\"completed\"}"+ (eitherDecode incomplete :: Either String CellStatus) `shouldSatisfy` isLeft++ it "accepts a nonterminal status without a verdict" do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.status.v1.json"+ case eitherDecode bytes :: Either String Value of+ Right (Object fields) -> do+ let running = Object (KeyMap.insert "phase" (String "running") (KeyMap.delete "manifestSha256" (KeyMap.delete "outcome" fields)))+ case eitherDecode (encode running) :: Either String CellStatus of+ Left failure -> expectationFailure failure+ Right status -> toJSON status `shouldBe` running+ _ -> expectationFailure "invalid status fixture"++ it "preserves optional broker, OTLP, clock and fault-hook environment fields" do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.environment.v1.json"+ case eitherDecode bytes :: Either String Value of+ Right (Object fields) -> do+ let endpoint = object ["grpc" .= ("http://10.0.0.4:4317" :: String), "http" .= ("http://10.0.0.4:4318" :: String)]+ broker = object ["bootstrapServers" .= ("10.0.0.5:9092" :: String), "adminUrl" .= ("http://10.0.0.5:9644" :: String), "implementation" .= ("redpanda" :: String), "version" .= ("25.1.0" :: String)]+ extended = Object (KeyMap.insert "faultHook" (String "/run/cell/fault-hook") (KeyMap.insert "clock" (object ["skewBoundMicros" .= (250 :: Int)]) (KeyMap.insert "otlp" (object ["null" .= endpoint, "file" .= endpoint]) (KeyMap.insert "broker" broker fields))))+ case eitherDecode (encode extended) :: Either String CellEnvironment of+ Left failure -> expectationFailure failure+ Right environment -> toJSON environment `shouldBe` extended+ _ -> expectationFailure "invalid environment fixture"++ it "round-trips the generic resources block of a cell environment" do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.environment.v1.json"+ case eitherDecode bytes :: Either String Value of+ Right (Object fields) -> do+ let extended = Object (KeyMap.insert "resources" (object ["object-store" .= object ["endpoint" .= ("http://10.0.0.6:9000" :: String)]]) fields)+ case eitherDecode (encode extended) :: Either String CellEnvironment of+ Left failure -> expectationFailure failure+ Right environment -> toJSON environment `shouldBe` extended+ _ -> expectationFailure "invalid environment fixture"++ it "keeps a submission's resource resets beside the legacy broker reset" do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+ case eitherDecode bytes :: Either String Value of+ Right (Object fields) | Just (Object reset) <- KeyMap.lookup "reset" fields -> do+ let resets = KeyMap.insert "resources" (object ["object-store" .= object ["empty" .= True]]) (KeyMap.insert "broker" (object ["wipe" .= True]) reset)+ extended = Object (KeyMap.insert "reset" (Object resets) fields)+ case eitherDecode (encode extended) :: Either String Submission of+ Left failure -> expectationFailure failure+ Right submission -> toJSON submission `shouldBe` extended+ _ -> expectationFailure "invalid submission fixture"++ it "rejects an artifact path that escapes the fetched tree" do+ let escaped = "{\"path\":\"../outside\",\"sha256\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"bytes\":1,\"mediaType\":\"application/json\"}"+ (eitherDecode escaped :: Either String Artifact) `shouldSatisfy` isLeft++ it "rejects ambiguous artifact paths and duplicate manifest entries" do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.artifact-manifest.v1.json"+ let badPaths = ["a//b", "a/./b", "a/../b", "a\\b", "/a/b"] :: [Text]+ mapM_ (\path -> (eitherDecode (encode (object ["path" .= path, "sha256" .= ("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" :: String), "bytes" .= (1 :: Int), "mediaType" .= ("application/json" :: String)])) :: Either String Artifact) `shouldSatisfy` isLeft) badPaths+ case eitherDecode bytes :: Either String CellManifest of+ Right manifest -> do+ let duplicate = manifest {artifacts = manifest.artifacts <> take 1 manifest.artifacts}+ (eitherDecode (encode duplicate) :: Either String CellManifest) `shouldSatisfy` isLeft+ _ -> expectationFailure "invalid cell manifest fixture"++checkFixture :: (ToJSON document) => FilePath -> (LazyByteString.ByteString -> Either String document) -> IO ()+checkFixture name decoder = do+ bytes <- LazyByteString.readFile ("test/golden/cell/" <> name)+ case decoder bytes of+ Left failure -> expectationFailure failure+ Right document -> Right (toJSON document) `shouldBe` (eitherDecode bytes :: Either String Value)
+ test/ExecSpec.hs view
@@ -0,0 +1,208 @@+module ExecSpec (execSpec) where++import Control.Exception (bracket)+import Data.Aeson (Value (..), eitherDecodeFileStrict', object, (.=))+import Data.Aeson qualified as Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Bundle (LayerBundle (..), Registry, lookupScenario, mkBundle, mkRegistry)+import Kenshou.Core.Bundle qualified as Bundle+import Kenshou.Core.Dimension (allTelemetryArms)+import Kenshou.Core.Env (EnvRequirements (..))+import Kenshou.Core.Id (mkSeed, newRunId, parseScenarioId, renderRunId, selftestLayer)+import Kenshou.Core.Knob (Allowed (..), KnobSpec (..), KnobType (..), KnobValue (..), RawKnob (..), mkKnobName)+import Kenshou.Core.Resource (CellBinding (..), ProvisionedResource (..), ResourceProvider (..))+import Kenshou.Core.Resource.Types (legacyKafkaName)+import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..), minimalRunSpec)+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Core.Selftest qualified as Selftest+import Kenshou.Core.Selftest.Schema (selftestMigrationSet)+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (..), RunPlan (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Docs (CellBroker (..), CellEnvironment (..), CellPostgres (..), OtlpEndpoint (..), OtlpSinks (..))+import Kenshou.Remote.Cell.Exec (cellExec, resolveOnCell, resolveWorkJson)+import Kenshou.Remote.Cell.Prepare (OtlpSink (..))+import Kenshou.Remote.Selftest qualified as RemoteSelftest+import System.Environment (lookupEnv, setEnv, unsetEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++execSpec :: Spec+execSpec = describe "cell-side run-plan resolution" do+ it "binds durable PostgreSQL to the cell connection without changing run identity" do+ (plan, environment) <- fixtures+ let durable = withDimensions [("pg.durability", "durable"), ("pg.version", "18")] plan+ case resolveOnCell registry environment NullSink durable of+ Left problem -> expectationFailure (show problem)+ Right resolved -> do+ resolved.planId `shouldBe` durable.planId+ fmap (.runId) resolved.runs `shouldBe` fmap (.runId) durable.runs+ resolveWorkJson registry environment NullSink (Aeson.toJSON durable) `shouldBe` Right (Aeson.toJSON resolved)+ case resolved.runs of+ [run] -> do+ run.spec.environment.placement `shouldBe` RunOnCell+ run.spec.environment.postgres `shouldBe` Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))+ _ -> expectationFailure "expected one resolved run"++ it "refuses a non-durable or wrong-major PostgreSQL run" do+ (plan, environment) <- fixtures+ resolveOnCell registry environment NullSink plan `shouldSatisfy` isLeft+ let pg = environment.postgres+ pg17 = CellPostgres 17 pg.host pg.port pg.database pg.user pg.connectionString+ cell17 = CellEnvironment environment.cell environment.runId environment.leaseId pg17 environment.broker environment.otlp environment.victoriaMetricsUrl environment.drivers environment.clockSkewBoundMicros environment.faultHook environment.resources+ durable = withDimensions [("pg.durability", "durable"), ("pg.version", "18")] plan+ resolveOnCell registry cell17 NullSink durable `shouldSatisfy` isLeft++ it "binds the cell broker and selected OTLP endpoint for a tracing run" do+ (plan, environment) <- fixtures+ let broker = CellBroker "10.0.0.5:9092" "http://10.0.0.5:9644" "redpanda" "v25"+ sinks = OtlpSinks (OtlpEndpoint "http://10.0.0.4:4317" "http://10.0.0.4:4318") (OtlpEndpoint "http://10.0.0.4:5317" "http://10.0.0.4:5318")+ cell = withEndpoints environment (Just broker) (Just sinks)+ tracing = withDimensions [("pg.durability", "durable"), ("pg.version", "18"), ("telemetry.tracing", "sdk-otlp")] plan+ endpoint = either (error . Text.unpack) id (mkKnobName "otel.endpoint")+ case resolveOnCell tracingRegistry cell FileSink tracing of+ Left problem -> expectationFailure (show problem)+ Right resolved -> case resolved.runs of+ [run] -> do+ Map.lookup legacyKafkaName run.spec.environment.resources `shouldBe` Just (object ["backend" .= ("external" :: Text), "brokers" .= (["10.0.0.5:9092"] :: [Text]), "lanes" .= (0 :: Int)])+ lookup endpoint run.spec.knobs `shouldBe` Just (RawText "http://10.0.0.4:5318")+ _ -> expectationFailure "expected one tracing run"+ resolveOnCell tracingRegistry environment FileSink tracing `shouldSatisfy` isLeft+ resolveOnCell tracingRegistry (withEndpoints environment (Just broker) Nothing) FileSink tracing `shouldSatisfy` isLeft++ it "fills the remote environment probe from the cell descriptor" do+ (plan, environment) <- fixtures+ let scenario = either (error . Text.unpack) id (parseScenarioId "selftest/remote/correctness/cell-environment")+ broker = CellBroker "10.0.0.5:9092" "http://10.0.0.5:9644" "redpanda" "v25"+ sinks = OtlpSinks (OtlpEndpoint "http://10.0.0.4:4317" "http://10.0.0.4:4318") (OtlpEndpoint "http://10.0.0.4:5317" "http://10.0.0.4:5318")+ cell = withEndpoints environment (Just broker) (Just sinks)+ change entry =+ let old = entry.spec+ spec = RunSpec old.runId scenario old.scenarioRevision old.knobs old.dimensions old.seed old.phases old.timeoutSeconds old.environment old.cohortExpectation old.comparison old.labels+ in PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial spec+ changed = RunPlan plan.planId plan.createdAt plan.context plan.policy (fmap change plan.runs) plan.skipped plan.estimateMinutes+ durable = withDimensions [("pg.durability", "durable"), ("pg.version", "18")] changed+ name text = either (error . Text.unpack) id (mkKnobName text)+ case resolveOnCell remoteRegistry cell FileSink durable of+ Left problem -> expectationFailure (show problem)+ Right resolved -> case resolved.runs of+ [run] -> do+ lookup (name "remote.expect-placement") run.spec.knobs `shouldBe` Nothing+ lookup (name "remote.otlp-endpoint") run.spec.knobs `shouldBe` Just (RawText "http://10.0.0.4:5318")+ lookup (name "remote.tcp-endpoint") run.spec.knobs `shouldBe` Nothing+ _ -> expectationFailure "expected one remote probe run"++ it "leaves a secondary driver idle without requiring the work or environment" $+ withSystemTempDirectory "kenshou-cell-exec" \root ->+ withEnv "CELL_DRIVER_INDEX" (Just "1") $+ withEnv "CELL_DRIVER_COUNT" (Just "2") do+ let out = root </> "output"+ cellExec registry (root </> "missing-work.json") out `shouldReturn` ExitSuccess+ idle <- eitherDecodeFileStrict' (out </> "idle-driver.json")+ idle `shouldBe` Right (object ["schema" .= ("kenshou.cell-idle-driver/v1" :: Text), "index" .= (1 :: Int), "count" .= (2 :: Int)])++ it "records a cell adapter failure when the owner environment is missing" $+ withSystemTempDirectory "kenshou-cell-exec" \root ->+ withEnv "CELL_DRIVER_INDEX" (Just "0") $+ withEnv "CELL_DRIVER_COUNT" (Just "1") $+ withEnv "CELL_ENV_FILE" Nothing do+ let out = root </> "output"+ cellExec registry (root </> "missing-work.json") out `shouldReturn` ExitFailure 4+ failure <- eitherDecodeFileStrict' (out </> "kenshou-cell" </> "adapter-error.json")+ failure `shouldBe` Right (object ["schema" .= ("kenshou.cell-adapter-error/v1" :: Text), "reason" .= ("cell-environment-missing" :: Text), "detail" .= ("user error (CELL_ENV_FILE is required by the cell payload)" :: Text)])++ it "reports missing wrapper identity before reading cell work" $+ withSystemTempDirectory "kenshou-cell-exec" \root -> do+ (_, environment) <- fixtures+ withEnv "CELL_DRIVER_INDEX" (Just "0") $+ withEnv "CELL_DRIVER_COUNT" (Just "1") $+ withEnv "CELL_ENV_FILE" (Just "test/golden/cell/cell.environment.v1.json") $+ withEnv "CELL_RUN_ID" (Just (Text.unpack (renderRunId environment.runId))) $+ withEnv "CELL_SCRATCH_DIR" (Just (root </> "scratch")) $+ withEnv "KENSHOU_OTLP_SINK" (Just "null") $+ withEnv "KENSHOU_COHORT_IDENTITY" Nothing do+ let out = root </> "output"+ cellExec registry (root </> "missing-work.json") out `shouldReturn` ExitFailure 4+ failure <- eitherDecodeFileStrict' (out </> "kenshou-cell" </> "adapter-error.json")+ case (failure :: Either String Value) of+ Right (Object fields) -> KeyMap.lookup "reason" fields `shouldBe` Just (String "payload-identity-missing")+ _ -> expectationFailure (show failure)++fixtures :: IO (RunPlan, CellEnvironment)+fixtures = do+ planId <- newRunId+ runId <- newRunId+ now <- getCurrentTime+ let scenario = either (error . Text.unpack) id (parseScenarioId "selftest/kernel/correctness/postgres-roundtrip")+ selector = either (error . Text.unpack) id (parseSelector "selftest/kernel/**")+ reason = Reason (Change (ComponentRef (ComponentId "kernel") Nothing) Named "cell test") [] selector 0+ base = minimalRunSpec scenario+ spec = RunSpec (Just runId) scenario base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+ entry = PlannedRun 0 runId 1 (reason :| []) Nothing spec+ planContext = PlanContext Nothing "fixture" "released" "sha256:fixture" (PlanInputs Null) [] []+ seed = either (error . Text.unpack) id (mkSeed 7)+ plan = RunPlan planId now planContext (defaultPlanPolicy seed) [entry] [] 1+ environment <- eitherDecodeFileStrict' "test/golden/cell/cell.environment.v1.json" >>= either fail pure+ pure (plan, environment)++registry :: Registry+registry = either (error . show) id (mkRegistry [Selftest.bundle])++remoteRegistry :: Registry+remoteRegistry = either (error . show) id (mkRegistry [Selftest.bundle, RemoteSelftest.bundle])++tracingRegistry :: Registry+tracingRegistry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [enhanced] []) {migrations = [selftestMigrationSet], Bundle.resources = [brokerProvider]}])+ where+ scenarioId = either (error . Text.unpack) id (parseScenarioId "selftest/kernel/correctness/postgres-roundtrip")+ original = maybe (error "missing PostgreSQL selftest") id (lookupScenario registry scenarioId)+ endpoint = either (error . Text.unpack) id (mkKnobName "otel.endpoint")+ knob = KnobSpec endpoint "OTLP endpoint" KnobText (VText "") AnyValue []+ requirements = EnvRequirements original.requires.postgres original.requires.extraPostgres [legacyKafkaName]+ enhanced = Scenario original.id original.revision original.summary original.tier original.placement (knob : original.knobs) (allTelemetryArms original.dimensions) original.phases requirements original.knownDefect original.run++withEndpoints :: CellEnvironment -> Maybe CellBroker -> Maybe OtlpSinks -> CellEnvironment+withEndpoints environment broker otlp =+ CellEnvironment environment.cell environment.runId environment.leaseId environment.postgres broker otlp environment.victoriaMetricsUrl environment.drivers environment.clockSkewBoundMicros environment.faultHook environment.resources++withDimensions :: [(Text, Text)] -> RunPlan -> RunPlan+withDimensions dimensions plan =+ let runs = fmap update plan.runs+ in RunPlan plan.planId plan.createdAt plan.context plan.policy runs plan.skipped plan.estimateMinutes+ where+ update entry =+ let spec = entry.spec+ changed = RunSpec spec.runId spec.scenario spec.scenarioRevision spec.knobs dimensions spec.seed spec.phases spec.timeoutSeconds spec.environment spec.cohortExpectation spec.comparison spec.labels+ in PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial changed++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft _ = False++withEnv :: String -> Maybe String -> IO result -> IO result+withEnv name value operation = bracket (lookupEnv name) (restore name) (const (apply name value >> operation))+ where+ apply key Nothing = unsetEnv key+ apply key (Just setting) = setEnv key setting+ restore key prior = apply key prior++-- A provider for the resource the legacy cell broker binds; the cell supplies the+-- broker, so the local bracket only passes the value through.+brokerProvider :: ResourceProvider+brokerProvider =+ ResourceProvider+ { name = legacyKafkaName,+ summary = "Cell broker",+ normalize = Right . maybe Null id,+ withLocal = \_ value action -> action (ProvisionedResource value Null),+ cell = Just (CellBinding "broker" (Just (object ["wipe" .= True])))+ }
+ test/FetchSpec.hs view
@@ -0,0 +1,258 @@+module FetchSpec (spec, completeTreeWith, completeTreeWithNested) where++import Data.Aeson (Value, eitherDecode, encode, object, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, newRunId, parseScenarioId, renderRunId)+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..))+import Kenshou.Core.Outcome qualified as Outcome+import Kenshou.Core.RunSpec (minimalRunSpec)+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellPhase (..), CellRunResult (..), CellStatus (..), LogChunks (..), ManifestPayload (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Fetch (FetchError (..), VerifyProblem (..), effectiveOutcome, fetchCellRun, verifyCellRun, verifyCellRunWithStatus, verifyCellTree)+import Kenshou.Remote.Cell.Index (CellRunIndex (..), RunLink (..), deriveCellRunIndex, writeCellRunIndex)+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (createDirectoryIfMissing, doesFileExist, removeFile)+import System.FilePath (takeDirectory, (</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "sealed cell tree fetch" do+ it "round-trips the derived cell-run index fixture" do+ bytes <- LazyByteString.readFile "test/golden/cell-run.json"+ case eitherDecode bytes :: Either String CellRunIndex of+ Left failure -> expectationFailure failure+ Right index -> eitherDecode (encode index) `shouldBe` (eitherDecode bytes :: Either String Value)++ it "maps cell failures onto effective nested-run outcomes" do+ effectiveOutcome Completed True Outcome.Failed `shouldBe` Outcome.Failed+ effectiveOutcome InfrastructureFailure True Outcome.Passed `shouldBe` Outcome.InfrastructureFailure+ effectiveOutcome Cancelled True Outcome.Passed `shouldBe` Outcome.Passed+ effectiveOutcome Cancelled False Outcome.Passed `shouldBe` Outcome.Errored+ effectiveOutcome TimedOut False Outcome.Inconclusive `shouldBe` Outcome.Errored++ it "requires a seal, verifies bytes, and repairs a damaged local artifact on replay" $ withSystemTempDirectory "kenshou-fetch" \root -> do+ store <- newFileStore (root </> "store")+ identifier <- newRunId+ fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Left Unsealed+ manifest <- publishFixture store identifier+ fetched <- fetchCellRun store bucket identifier (root </> "out")+ tree <- case fetched of+ Right path -> pure path+ Left failure -> expectationFailure (show failure) >> error "unreachable"+ verifyCellTree tree `shouldReturn` Right manifest+ LazyByteString.writeFile (tree </> "cell" </> "result.json") "bad"+ damaged <- verifyCellTree tree+ damaged `shouldSatisfy` hasDigestMismatch+ fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Right tree+ verifyCellTree tree `shouldReturn` Right manifest+ LazyByteString.writeFile (tree </> "unexpected.txt") "extra"+ withExtra <- verifyCellTree tree+ withExtra `shouldSatisfy` hasExtra+ removeFile (tree </> "unexpected.txt")+ removeFile (tree </> "cell" </> "result.json")+ missing <- verifyCellTree tree+ missing `shouldSatisfy` hasMissing++ it "refuses an unlisted result object and a corrupt published artifact" $ withSystemTempDirectory "kenshou-fetch" \root -> do+ store <- newFileStore (root </> "store")+ identifier <- newRunId+ _ <- publishFixture store identifier+ put store identifier "stray.txt" "extra"+ result <- fetchCellRun store bucket identifier (root </> "out")+ result `shouldSatisfy` \case+ Left (ObjectSetMismatch _ [unexpected]) -> unexpected == prefix identifier <> "stray.txt"+ _ -> False+ store.deleteObject bucket (ObjectName (prefix identifier <> "stray.txt")) NoPrecondition `shouldReturn` True+ put store identifier "cell/result.json" (LazyByteString.replicate (LazyByteString.length resultBytes) 120)+ fetchCellRun store bucket identifier (root </> "out") `shouldReturn` Left (ObjectCorrupt "cell/result.json")+ doesFileExist (root </> "out" </> Text.unpack (renderRunId identifier) </> "tree" </> "cell" </> "result.json") `shouldReturn` False++ it "cross-checks the submission, cell result and nested Kenshou manifest" $ withSystemTempDirectory "kenshou-verify" \root -> do+ (tree, manifest) <- completeTree root False False False+ verifyCellRun tree `shouldReturn` Right manifest++ it "accepts cell support directories without nested run manifests" $ withSystemTempDirectory "kenshou-verify" \root -> do+ (tree, manifest) <- completeTree root False False False+ let supporting = [("output/kenshou-cell/context.json", "{}"), ("output/specs/0001.json", "{}")]+ expanded = manifest {artifacts = manifest.artifacts <> [artifactFor name bytes | (name, bytes) <- supporting]}+ mapM_+ ( \(name, bytes) -> do+ let destination = tree </> Text.unpack name+ createDirectoryIfMissing True (takeDirectory destination)+ LazyByteString.writeFile destination bytes+ )+ supporting+ LazyByteString.writeFile (tree </> "manifest.json") (encode expanded)+ verifyCellRun tree `shouldReturn` Right expanded++ it "rejects a payload identity mismatch after all file digests pass" $ withSystemTempDirectory "kenshou-verify" \root -> do+ (tree, _) <- completeTree root True False False+ result <- verifyCellRun tree+ result `shouldSatisfy` \case+ Left problems -> any isPayloadMismatch problems+ Right _ -> False++ it "requires a nested manifest and a matching nested run identifier" $ withSystemTempDirectory "kenshou-verify" \root -> do+ (tree, _) <- completeTree (root </> "wrong-id") False True False+ mismatch <- verifyCellRun tree+ mismatch `shouldSatisfy` \case+ Left problems -> any isRunResultMismatch problems+ Right _ -> False+ (otherTree, manifest) <- completeTree (root </> "missing") False False False+ case [artifact.path | artifact <- manifest.artifacts, "/manifest.json" `Text.isSuffixOf` artifact.path] of+ [nestedPath] -> do+ removeFile (otherTree </> Text.unpack nestedPath)+ let withoutNested = manifest {artifacts = filter ((/= nestedPath) . (.path)) manifest.artifacts}+ LazyByteString.writeFile (otherTree </> "manifest.json") (encode withoutNested)+ absent <- verifyCellRun otherTree+ absent `shouldSatisfy` \case+ Left problems -> any isNestedManifestProblem problems+ Right _ -> False+ _ -> expectationFailure "expected one nested manifest"++ it "rejects an unsafe nested path before calling the kernel verifier" $ withSystemTempDirectory "kenshou-verify" \root -> do+ (tree, _) <- completeTree root False False True+ result <- verifyCellRun tree+ result `shouldSatisfy` \case+ Left problems -> any isNestedManifestProblem problems+ Right _ -> False++ it "requires the final status to agree with the fetched seal" $ withSystemTempDirectory "kenshou-verify" \root -> do+ (tree, manifest) <- completeTree root False False False+ bytes <- LazyByteString.readFile (tree </> "manifest.json")+ let sealed = CellStatus manifest.runId Sealed (Just manifest.leaseSequence) manifest.sealedAt Nothing (LogChunks 0 0) (Just manifest.outcome) (Just (digestOf bytes)) (Just [])+ verifyCellRunWithStatus sealed tree `shouldReturn` Right manifest+ let wrong = CellStatus sealed.runId sealed.phase sealed.leaseSequence sealed.updatedAt sealed.phaseStartedAt sealed.logChunks sealed.outcome (Just (Text.replicate 64 "f")) sealed.reasons+ result <- verifyCellRunWithStatus wrong tree+ result `shouldSatisfy` \case+ Left problems -> StatusDigestMismatch `elem` problems+ Right _ -> False++ it "derives a durable index beside a verified cell tree" $ withSystemTempDirectory "kenshou-index" \root -> do+ (tree, manifest) <- completeTree root False False False+ derived <- deriveCellRunIndex (Bucket "results") Nothing tree+ case derived of+ Left failure -> expectationFailure (show failure)+ Right index -> do+ index.cellRun `shouldBe` manifest.runId+ index.resultsBaseUri `shouldBe` ("gs://results/runs/" <> renderRunId manifest.runId)+ index.dataBaseUri `shouldBe` (index.resultsBaseUri <> "/output")+ length index.runs `shouldBe` 1+ map (.recordedOutcome) index.runs `shouldBe` [Outcome.Passed]+ map (.effectiveOutcome) index.runs `shouldBe` [Outcome.Passed]+ destination <- writeCellRunIndex tree index+ destination `shouldBe` root </> "cell-run.json"+ stored <- LazyByteString.readFile destination+ eitherDecode stored `shouldBe` Right index++publishFixture :: ObjectStore -> RunId -> IO CellManifest+publishFixture store identifier = do+ lease <- newRunId+ now <- getCurrentTime+ let payload = ManifestPayload "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-fixture"+ work = workObjectFor "application/json" resultBytes+ artifact = Artifact "cell/result.json" work.sha256 work.bytes "application/json"+ manifest = CellManifest identifier "alpha" lease 1 now "0.1.0" payload Completed 3600 [artifact]+ put store identifier "cell/result.json" resultBytes+ put store identifier "manifest.json" (encode manifest)+ pure manifest++put :: ObjectStore -> RunId -> Text -> LazyByteString.ByteString -> IO ()+put store identifier suffix bytes = do+ _ <- store.putObject bucket (ObjectName (prefix identifier <> suffix)) "application/json" NoPrecondition bytes+ pure ()++prefix :: RunId -> Text+prefix identifier = "runs/" <> renderRunId identifier <> "/"++bucket :: Bucket+bucket = Bucket "results"++resultBytes :: LazyByteString.ByteString+resultBytes = "{\"schema\":\"cell.run-result/v1\"}"++completeTree :: FilePath -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)+completeTree root wrongPayload wrongRunId unsafeNested = do+ identifier <- newRunId+ lease <- newRunId+ completeTreeWith root identifier lease wrongPayload wrongRunId unsafeNested++completeTreeWith :: FilePath -> RunId -> RunId -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)+completeTreeWith root identifier lease wrongPayload wrongRunId unsafeNested = do+ nestedId <- newRunId+ completeTreeWithNested root identifier lease nestedId wrongPayload wrongRunId unsafeNested++completeTreeWithNested :: FilePath -> RunId -> RunId -> RunId -> Bool -> Bool -> Bool -> IO (FilePath, CellManifest)+completeTreeWithNested root identifier lease nestedId wrongPayload wrongRunId unsafeNested = do+ otherId <- newRunId+ now <- getCurrentTime+ scenario <- either (ioError . userError . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/always-pass")+ source <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+ fixture <- either (ioError . userError) pure (eitherDecode source :: Either String Submission)+ let workBytes = encode (object ["schema" .= ("kenshou.run-plan/v1" :: Text), "planId" .= identifier, "runs" .= [object ["ordinal" .= (1 :: Int), "runId" .= nestedId, "estimateMinutes" .= (1 :: Int), "spec" .= minimalRunSpec scenario]]])+ workInfo = workObjectFor "application/json" workBytes+ submitted = Submission identifier lease fixture.payload workInfo fixture.env fixture.reset fixture.limits fixture.requires Nothing fixture.labels+ payloadDigest = fixture.payload.bundle.sha256+ manifestDigest = if wrongPayload then Text.replicate 64 "b" else payloadDigest+ manifestPayload = ManifestPayload manifestDigest fixture.payload.storePath+ cellResult = CellRunResult identifier "alpha" lease 1 Completed (Just 0) Nothing []+ runBytes = encode (object ["schema" .= ("kenshou.run-result/v1" :: Text), "runId" .= (if wrongRunId then otherId else nestedId), "scenario" .= ("selftest/kernel/correctness/always-pass" :: Text), "outcome" .= Outcome.Passed, "fingerprint" .= object ["cell" .= object ["payload" .= object ["bundleSha256" .= payloadDigest]]]])+ nestedFile = ManifestFile (if unsafeNested then "../escape" else "run-result.json") ("sha256:" <> digestOf runBytes) (fromIntegral (LazyByteString.length runBytes)) "application/json"+ nestedManifest = Manifest nestedId now [nestedFile]+ files =+ [ ("submission/work", workBytes),+ ("submission/submission.json", encode submitted),+ ("cell/result.json", encode cellResult),+ ("output/" <> renderRunId nestedId <> "/run-result.json", runBytes),+ ("output/" <> renderRunId nestedId <> "/manifest.json", encode nestedManifest)+ ]+ artifacts = [artifactFor name bytes | (name, bytes) <- files]+ manifest = CellManifest identifier "alpha" lease 1 now "0.1.0" manifestPayload Completed 3600 artifacts+ tree = root </> "tree"+ mapM_+ ( \(name, bytes) -> do+ let destination = tree </> Text.unpack name+ createDirectoryIfMissing True (takeDirectory destination)+ LazyByteString.writeFile destination bytes+ )+ files+ LazyByteString.writeFile (tree </> "manifest.json") (encode manifest)+ pure (tree, manifest)++artifactFor :: Text -> LazyByteString.ByteString -> Artifact+artifactFor name bytes = let info = workObjectFor "application/json" bytes in Artifact name info.sha256 info.bytes "application/json"++digestOf :: LazyByteString.ByteString -> Text+digestOf bytes = (workObjectFor "application/json" bytes).sha256++isPayloadMismatch :: VerifyProblem -> Bool+isPayloadMismatch (PayloadMismatch _) = True+isPayloadMismatch _ = False++isRunResultMismatch :: VerifyProblem -> Bool+isRunResultMismatch (RunResultMismatch _) = True+isRunResultMismatch _ = False++isNestedManifestProblem :: VerifyProblem -> Bool+isNestedManifestProblem (NestedManifestProblem _) = True+isNestedManifestProblem _ = False++hasDigestMismatch :: Either (NonEmpty VerifyProblem) CellManifest -> Bool+hasDigestMismatch (Left problems) = DigestMismatch "cell/result.json" `elem` toList problems+hasDigestMismatch _ = False++hasExtra :: Either (NonEmpty VerifyProblem) CellManifest -> Bool+hasExtra (Left problems) = Extra "unexpected.txt" `elem` toList problems+hasExtra _ = False++hasMissing :: Either (NonEmpty VerifyProblem) CellManifest -> Bool+hasMissing (Left problems) = Missing "cell/result.json" `elem` toList problems+hasMissing _ = False
+ test/GcsSpec.hs view
@@ -0,0 +1,256 @@+module GcsSpec (spec) where++import Control.Concurrent (forkIO, killThread)+import Control.Exception (bracket, finally)+import Control.Monad (forever, unless, void)+import Data.ByteString qualified as ByteString+import Data.ByteString.Char8 qualified as Char8+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.List (sort)+import Data.Text qualified as Text+import Data.Time (UTCTime (..), fromGregorian, secondsToDiffTime)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.Gcs (TokenProvider (..), newGcsStoreAt, newTokenProviderWith)+import Network.Socket (Family (AF_INET), SockAddr (SockAddrInet), SocketType (Stream), accept, bind, close, defaultProtocol, getSocketName, listen, socket, socketToHandle, tupleToHostAddress)+import System.Directory (doesFileExist)+import System.FilePath ((</>))+import System.IO (BufferMode (NoBuffering), Handle, IOMode (ReadWriteMode), hClose, hSetBuffering)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++data MockResponse = MockResponse !Int ![(ByteString.ByteString, ByteString.ByteString)] !LazyByteString.ByteString++spec :: Spec+spec = describe "GCS cell object store" do+ it "prefers the explicit token, then VM metadata, and caches gcloud tokens" do+ calls <- newIORef (0 :: Int)+ let cli = modifyIORef' calls (+ 1) >> pure "cli-token"+ explicit <- newTokenProviderWith (Bucket "control") (pure (Just "environment-token")) (pure (Just "metadata-token")) cli+ explicit.accessToken `shouldReturn` "environment-token"+ machine <- newTokenProviderWith (Bucket "control") (pure Nothing) (pure (Just "metadata-token")) cli+ machine.accessToken `shouldReturn` "metadata-token"+ laptop <- newTokenProviderWith (Bucket "control") (pure Nothing) (pure Nothing) cli+ laptop.accessToken `shouldReturn` "cli-token"+ laptop.accessToken `shouldReturn` "cli-token"+ readIORef calls `shouldReturn` 1++ it "encodes the object name and create-only precondition, and maps HTTP 412" do+ seen <- newIORef []+ withMock+ ( \_ target headers -> do+ modifyIORef' seen (<> [(target, lookup "Authorization" headers)])+ pure (MockResponse 412 [] "")+ )+ \port -> do+ store <- testStore port+ store.putObject (Bucket "control") (ObjectName "cells/alpha/lease space.json") "application/json" DoesNotExist "claim" `shouldReturn` PreconditionFailed+ readIORef seen `shouldReturn` [("/upload/storage/v1/b/control/o?uploadType=media&name=cells%2Falpha%2Flease%20space.json&ifGenerationMatch=0", Just "Bearer test-token")]++ it "pins media reads and downloads to the metadata generation" $ withSystemTempDirectory "kenshou-gcs" \root -> do+ seen <- newIORef []+ let object = ObjectName "cells/alpha/work"+ destination = root </> "nested" </> "work"+ metaPath = "/storage/v1/b/control/o/cells%2Falpha%2Fwork"+ withMock+ ( \_ target _ -> do+ modifyIORef' seen (<> [target])+ pure+ if "alt=media" `ByteString.isInfixOf` target+ then MockResponse 200 [] "hello"+ else MockResponse 200 [] objectMetadata+ )+ \port -> do+ store <- testStore port+ fmap fst <$> store.getObject (Bucket "control") object `shouldReturn` Just "hello"+ result <- store.downloadTo (Bucket "control") object destination+ result `shouldSatisfy` maybe False (const True)+ LazyByteString.readFile destination `shouldReturn` "hello"+ readIORef seen `shouldReturn` [metaPath, metaPath <> "?alt=media&generation=7", metaPath, metaPath <> "?alt=media&generation=7"]++ it "does not leave a truncated download as the destination" $ withSystemTempDirectory "kenshou-gcs" \root -> do+ let destination = root </> "work"+ withMock+ ( \_ target _ ->+ pure+ if "alt=media" `ByteString.isInfixOf` target+ then MockResponse 200 [] "short"+ else MockResponse 200 [] "{\"name\":\"cells/alpha/work\",\"generation\":\"7\",\"size\":\"6\",\"updated\":\"2026-09-27T00:00:00Z\"}"+ )+ \port -> do+ store <- testStore port+ store.downloadTo (Bucket "control") (ObjectName "cells/alpha/work") destination `shouldThrow` anyIOException+ doesFileExist destination `shouldReturn` False++ it "paginates listings and reads server time from the Date header" do+ withMock+ ( \_ target _ ->+ pure+ if target == "/storage/v1/b/control/o?maxResults=1"+ then MockResponse 200 [("Date", "Sun, 27 Sep 2026 12:34:56 GMT")] "{}"+ else+ if target == "/storage/v1/b/control/o?prefix=cells%2F&maxResults=1000"+ then MockResponse 200 [] "{\"items\":[{\"name\":\"cells/a\",\"generation\":\"1\",\"size\":\"0\",\"updated\":\"2026-09-27T00:00:00Z\"}],\"nextPageToken\":\"page 2\"}"+ else MockResponse 200 [] "{\"items\":[{\"name\":\"cells/b\",\"generation\":\"2\",\"size\":\"0\",\"updated\":\"2026-09-27T00:00:00Z\"}]}"+ )+ \port -> do+ store <- testStore port+ objects <- store.listObjects (Bucket "control") "cells/"+ sort (map fst objects) `shouldBe` [ObjectName "cells/a", ObjectName "cells/b"]+ store.serverTime `shouldReturn` UTCTime (fromGregorian 2026 9 27) (secondsToDiffTime (12 * 3600 + 34 * 60 + 56))++ it "retries a transient metadata HTTP 503" do+ calls <- newIORef (0 :: Int)+ withMock+ ( \_ _ _ -> do+ modifyIORef' calls (+ 1)+ count <- readIORef calls+ pure if count == 1 then MockResponse 503 [] "" else MockResponse 200 [] objectMetadata+ )+ \port -> do+ store <- testStore port+ store.statObject (Bucket "control") (ObjectName "cells/alpha/work") `shouldSatisfyIO` maybe False (const True)+ readIORef calls `shouldReturn` 2++ it "refreshes a cached gcloud token after HTTP 401" do+ calls <- newIORef (0 :: Int)+ seen <- newIORef []+ let cli = do+ modifyIORef' calls (+ 1)+ count <- readIORef calls+ pure (if count == 1 then "stale-token" else "fresh-token")+ withMock+ ( \_ _ headers -> do+ let authorization = lookup "Authorization" headers+ modifyIORef' seen (<> [authorization])+ pure if authorization == Just "Bearer stale-token" then MockResponse 401 [] "" else MockResponse 200 [] objectMetadata+ )+ \port -> do+ provider <- newTokenProviderWith (Bucket "control") (pure Nothing) (pure Nothing) cli+ store <- newGcsStoreAt ("http://127.0.0.1:" <> Text.pack (show port)) provider+ store.statObject (Bucket "control") (ObjectName "cells/alpha/work") `shouldSatisfyIO` maybe False (const True)+ store.statObject (Bucket "control") (ObjectName "cells/alpha/work") `shouldSatisfyIO` maybe False (const True)+ readIORef calls `shouldReturn` 2+ readIORef seen `shouldReturn` [Just "Bearer stale-token", Just "Bearer fresh-token", Just "Bearer fresh-token"]++ it "sends a large file in 8 MiB resumable chunks" $ withSystemTempDirectory "kenshou-gcs" \root -> do+ let source = root </> "payload.nar.zst"+ size = 8 * 1024 * 1024 + 5+ LazyByteString.writeFile source (LazyByteString.replicate size 120)+ ranges <- newIORef []+ withMock+ ( \port target headers ->+ if "uploadType=resumable" `ByteString.isInfixOf` target+ then pure (MockResponse 200 [("Location", Char8.pack ("http://127.0.0.1:" <> show port <> "/upload-session"))] "")+ else do+ modifyIORef' ranges (<> [lookup "Content-Range" headers])+ pure+ if lookup "Content-Range" headers == Just "bytes 0-8388607/8388613"+ then MockResponse 308 [("Range", "bytes=0-8388607")] ""+ else MockResponse 201 [] "{\"name\":\"payloads/sha256/abc.nar.zst\",\"generation\":\"9\",\"size\":\"8388613\",\"updated\":\"2026-09-27T00:00:00Z\"}"+ )+ \port -> do+ store <- testStore port+ result <- store.putFile (Bucket "control") (ObjectName "payloads/sha256/abc.nar.zst") "application/zstd" DoesNotExist source+ result `shouldSatisfy` \case+ Written meta -> meta.size == size+ _ -> False+ readIORef ranges `shouldReturn` [Just "bytes 0-8388607/8388613", Just "bytes 8388608-8388612/8388613"]++ it "queries resumable progress after HTTP 503 and resends an unpersisted chunk" $ withSystemTempDirectory "kenshou-gcs" \root -> do+ let source = root </> "payload.nar.zst"+ LazyByteString.writeFile source (LazyByteString.replicate (8 * 1024 * 1024 + 1) 120)+ ranges <- newIORef []+ withMock+ ( \port target headers ->+ if "uploadType=resumable" `ByteString.isInfixOf` target+ then pure (MockResponse 200 [("Location", Char8.pack ("http://127.0.0.1:" <> show port <> "/upload-session"))] "")+ else do+ let range = lookup "Content-Range" headers+ modifyIORef' ranges (<> [range])+ history <- readIORef ranges+ pure case range of+ Just "bytes */8388609" -> MockResponse 308 [] ""+ Just "bytes 0-8388607/8388609" | length history == 1 -> MockResponse 503 [] ""+ Just "bytes 0-8388607/8388609" -> MockResponse 308 [("Range", "bytes=0-8388607")] ""+ _ -> MockResponse 201 [] "{\"name\":\"payloads/sha256/abc.nar.zst\",\"generation\":\"9\",\"size\":\"8388609\",\"updated\":\"2026-09-27T00:00:00Z\"}"+ )+ \port -> do+ store <- testStore port+ result <- store.putFile (Bucket "control") (ObjectName "payloads/sha256/abc.nar.zst") "application/zstd" DoesNotExist source+ result `shouldSatisfy` \case+ Written _ -> True+ _ -> False+ readIORef ranges `shouldReturn` [Just "bytes 0-8388607/8388609", Just "bytes */8388609", Just "bytes 0-8388607/8388609", Just "bytes 8388608-8388608/8388609"]++testStore :: Int -> IO ObjectStore+testStore port = newGcsStoreAt ("http://127.0.0.1:" <> Text.pack (show port)) (TokenProvider (Bucket "control") (pure "test-token") (pure ()))++objectMetadata :: LazyByteString.ByteString+objectMetadata = "{\"name\":\"cells/alpha/work\",\"generation\":\"7\",\"size\":\"5\",\"updated\":\"2026-09-27T00:00:00Z\",\"contentType\":\"application/octet-stream\"}"++withMock :: (Int -> ByteString.ByteString -> [(ByteString.ByteString, ByteString.ByteString)] -> IO MockResponse) -> (Int -> IO value) -> IO value+withMock respond action = bracket open close \listener -> do+ port <-+ getSocketName listener >>= \case+ SockAddrInet number _ -> pure (fromIntegral number)+ _ -> error "expected IPv4 listener"+ thread <-+ forkIO+ ( forever do+ (connection, _) <- accept listener+ void (forkIO (bracket (socketToHandle connection ReadWriteMode) hClose (serve (respond port))))+ )+ action port `finally` killThread thread+ where+ open = do+ listener <- socket AF_INET Stream defaultProtocol+ bind listener (SockAddrInet 0 (tupleToHostAddress (127, 0, 0, 1)))+ listen listener 16+ pure listener++serve :: (ByteString.ByteString -> [(ByteString.ByteString, ByteString.ByteString)] -> IO MockResponse) -> Handle -> IO ()+serve handler connection = do+ hSetBuffering connection NoBuffering+ requestLine <- stripCr <$> Char8.hGetLine connection+ requestHeaders <- readHeaders connection+ let lengthBytes = maybe 0 (read . Char8.unpack) (lookup "Content-Length" requestHeaders)+ consumeBody connection lengthBytes+ MockResponse status headers body <- handler (requestTarget requestLine) requestHeaders+ let statusText = case status of+ 200 -> "OK"+ 201 -> "Created"+ 204 -> "No Content"+ 412 -> "Precondition Failed"+ _ -> "Error"+ responseHeaders = headers <> [("Content-Length", Char8.pack (show (LazyByteString.length body))), ("Connection", "close")]+ Char8.hPutStr connection ("HTTP/1.1 " <> Char8.pack (show status) <> " " <> statusText <> "\r\n")+ mapM_ (\(name, value) -> Char8.hPutStr connection (name <> ": " <> value <> "\r\n")) responseHeaders+ Char8.hPutStr connection "\r\n"+ LazyByteString.hPut connection body++requestTarget :: ByteString.ByteString -> ByteString.ByteString+requestTarget line = case Char8.words line of+ _ : target : _ -> target+ _ -> error "invalid mock request line"++readHeaders :: Handle -> IO [(ByteString.ByteString, ByteString.ByteString)]+readHeaders connection = do+ line <- stripCr <$> Char8.hGetLine connection+ if ByteString.null line+ then pure []+ else do+ let (name, rest) = Char8.break (== ':') line+ ((name, Char8.dropWhile (== ' ') (ByteString.drop 1 rest)) :) <$> readHeaders connection++stripCr :: ByteString.ByteString -> ByteString.ByteString+stripCr = Char8.dropWhileEnd (== '\r')++consumeBody :: Handle -> Int -> IO ()+consumeBody connection remaining = unless (remaining == 0) do+ chunk <- ByteString.hGetSome connection (min remaining (64 * 1024))+ unless (not (ByteString.null chunk)) (error "mock request body ended early")+ consumeBody connection (remaining - ByteString.length chunk)++shouldSatisfyIO :: (Show value) => IO value -> (value -> Bool) -> IO ()+shouldSatisfyIO action predicate = action >>= (`shouldSatisfy` predicate)
+ test/HealthSpec.hs view
@@ -0,0 +1,50 @@+module HealthSpec (spec) where++import Control.Concurrent (threadDelay)+import Data.Aeson (Value (..), eitherDecodeFileStrict', encode)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Kenshou.Core.Id (parseRunId)+import Kenshou.Remote.Cell.Health (mapCellHealth, withCellHealthFile)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell health mapping" do+ it "maps a tripped maintenance gate into a hard run notice" do+ document <- ownerHealth+ identifier <- either (error . show) pure (parseRunId "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9")+ case mapCellHealth identifier document of+ Left problem -> expectationFailure (show problem)+ Right [Object fields] -> do+ KeyMap.lookup "schema" fields `shouldBe` Just (String "kenshou.health-notice/v1")+ KeyMap.lookup "source" fields `shouldBe` Just (String "cell-health:host-maintenance:cell-alpha-postgres")+ KeyMap.lookup "severity" fields `shouldBe` Just (String "hard")+ Right notices -> expectationFailure ("expected one notice: " <> show notices)++ it "rejects another cell run's health document" do+ document <- ownerHealth+ identifier <- either (error . show) pure (parseRunId "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011")+ mapCellHealth identifier document `shouldSatisfy` isLeft++ it "reads new complete lines while a run executes and does not duplicate notices" $+ withSystemTempDirectory "kenshou-cell-health" \root -> do+ identifier <- either (error . show) pure (parseRunId "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9")+ let source = root </> "cell-health.jsonl"+ destination = root </> "health-notices.jsonl"+ LazyByteString.writeFile destination ""+ line <- encode <$> ownerHealth+ withCellHealthFile (Just source) destination identifier do+ LazyByteString.writeFile source (line <> "\n" <> line <> "\n")+ threadDelay 1500000+ notices <- LazyByteString.readFile destination+ length (filter (not . LazyByteString.null) (LazyByteString.split 10 notices)) `shouldBe` 1++ownerHealth :: IO Value+ownerHealth = eitherDecodeFileStrict' "test/golden/cell/cell.health.v1.json" >>= either fail pure++isLeft :: Either Text [Value] -> Bool+isLeft (Left _) = True+isLeft _ = False
+ test/LeaseSpec.hs view
@@ -0,0 +1,198 @@+module LeaseSpec (spec) where++import Control.Concurrent (forkIO, threadDelay)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Monad (replicateM)+import Data.Aeson (Value, eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isRight)+import Data.IORef (atomicModifyIORef', newIORef)+import Data.Maybe (isJust)+import Data.Time (addUTCTime, getCurrentTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Remote.Cell.Control (CellSnapshot (..), readCellSnapshot)+import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), Quarantine (..), acquireLease, acquireLeaseWithId, leaseHeld, leaseSnapshot, reattachLease, releaseLease, renewLease, requestCancel, resizeLease, withHeartbeat)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell lease protocol" do+ it "reconciles a caller-retained claim without another write or reviving an expired lease" $ withSystemTempDirectory "kenshou-retained-claim" \root -> do+ store <- newFileStore root+ identifier <- newRunId+ handle <- acquireLeaseWithId store cellRef request identifier >>= expectAcquired+ original <- store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json")+ recovered <- acquireLeaseWithId store cellRef request identifier >>= expectAcquired+ (leaseSnapshot recovered >>= pure . (.leaseId)) `shouldReturn` identifier+ store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` original+ now <- store.serverTime+ let future = store {serverTime = pure (addUTCTime 1000 now)}+ expired <- acquireLeaseWithId future cellRef request identifier+ case expired of Busy lease -> lease.leaseId `shouldBe` identifier; _ -> expectationFailure "expired identity was revived"+ store.getObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` original+ releaseLease store cellRef handle `shouldReturn` True++ it "gives exactly one contender a create-only lease" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ replies <- replicateM 12 newEmptyMVar+ mapM_ (\reply -> forkIO (acquireLease store cellRef request >>= putMVar reply)) replies+ outcomes <- mapM takeMVar replies+ length [() | Acquired _ <- outcomes] `shouldBe` 1+ length [() | Busy _ <- outcomes] `shouldBe` 11++ it "renews with the latest generation, reattaches by id, and releases" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ first <- leaseSnapshot handle+ renewLease store cellRef handle `shouldReturn` True+ renewed <- leaseSnapshot handle+ renewed.heartbeatAt `shouldSatisfy` (>= first.heartbeatAt)+ reattached <- reattachLease store cellRef first.leaseId+ isJust reattached `shouldBe` True+ releaseLease store cellRef handle `shouldReturn` True+ leaseHeld handle `shouldReturn` False+ store.statObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing++ it "retains the lease when the cell agent advances runsStarted" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ lease <- leaseSnapshot handle+ let objectName = ObjectName "cells/alpha/lease.json"+ meta <- store.statObject cellRef.controlBucket objectName >>= maybe (expectationFailure "missing lease" >> error "unreachable") pure+ updated <- store.putObject cellRef.controlBucket objectName "application/json" (GenerationIs meta.generation) (encode (lease {runsStarted = 1}))+ updated `shouldSatisfy` \case Written _ -> True; _ -> False+ renewLease store cellRef handle `shouldReturn` True+ refreshed <- leaseSnapshot handle+ refreshed.runsStarted `shouldBe` 1+ releaseLease store cellRef handle `shouldReturn` True++ it "takes over only after server time exceeds updated plus TTL and grace" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ old <- acquireLease store cellRef shortRequest >>= expectAcquired+ let later = store {serverTime = addUTCTime 100 <$> store.serverTime}+ replacement <- acquireLease later cellRef request >>= expectAcquired+ leaseHeld replacement `shouldReturn` True+ renewLease store cellRef old `shouldReturn` False+ releaseLease later cellRef replacement `shouldReturn` True++ it "fences a detached TTL extension against a stale lease generation" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ resizeLease store cellRef handle 900 `shouldReturn` True+ snapshot <- leaseSnapshot handle+ snapshot.ttlSeconds `shouldBe` 900+ stale <- reattachLease store cellRef snapshot.leaseId+ case stale of+ Nothing -> expectationFailure "extended lease disappeared"+ Just old -> do+ renewLease store cellRef handle `shouldReturn` True+ resizeLease store cellRef old 1200 `shouldReturn` False+ leaseHeld old `shouldReturn` False+ releaseLease store cellRef handle `shouldReturn` True++ it "releases a newly won lease when quarantine exists" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ now <- getCurrentTime+ let record = Quarantine "alpha" "reset-failed" Nothing now+ _ <- store.putObject cellRef.controlBucket (ObjectName "cells/alpha/quarantine.json") "application/json" DoesNotExist (encode record)+ outcome <- acquireLease store cellRef request+ case outcome of+ Quarantined observed -> observed `shouldBe` record+ _ -> expectationFailure "expected quarantined cell"+ store.statObject cellRef.controlBucket (ObjectName "cells/alpha/lease.json") `shouldReturn` Nothing++ it "keeps a heartbeat and allows release after its thread stops" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef shortRequest >>= expectAcquired+ withHeartbeat store cellRef handle \stillHeld -> do+ threadDelay 1200000+ stillHeld `shouldReturn` True+ releaseLease store cellRef handle `shouldReturn` True++ it "retries one transient heartbeat write failure while the lease remains valid" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef shortRequest >>= expectAcquired+ initial <- leaseSnapshot handle+ failNext <- newIORef True+ let flaky =+ store+ { putObject = \bucket object mediaType precondition bytes -> do+ shouldFail <-+ if object == ObjectName "cells/alpha/lease.json"+ then atomicModifyIORef' failNext (\failPending -> (False, failPending))+ else pure False+ if shouldFail+ then ioError (userError "transient heartbeat write failure")+ else store.putObject bucket object mediaType precondition bytes+ }+ withHeartbeat flaky cellRef handle \stillHeld -> do+ threadDelay 1700000+ stillHeld `shouldReturn` True+ renewed <- leaseSnapshot handle+ renewed.heartbeatAt `shouldSatisfy` (> initial.heartbeatAt)+ releaseLease store cellRef handle `shouldReturn` True++ it "cancels only the expected lease unless force is explicit" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ current <- leaseSnapshot handle+ otherId <- newRunId+ requestCancel store cellRef Nothing False `shouldThrow` anyIOException+ requestCancel store cellRef (Just otherId) False `shouldReturn` False+ requestCancel store cellRef (Just current.leaseId) False `shouldReturn` True+ renewLease store cellRef handle `shouldReturn` False+ reattached <- reattachLease store cellRef current.leaseId+ case reattached of+ Just fresh -> releaseLease store cellRef fresh `shouldReturn` True+ Nothing -> expectationFailure "cancelled lease could not be reattached for release"+ second <- acquireLease store cellRef request >>= expectAcquired+ requestCancel store cellRef Nothing True `shouldReturn` True+ leaseHeld second `shouldReturn` True+ renewLease store cellRef second `shouldReturn` False++ it "round-trips the cell lease document" $ withSystemTempDirectory "kenshou-lease" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ lease <- leaseSnapshot handle+ eitherDecode (encode lease) `shouldBe` Right lease++ it "decodes the cell owner's lease and quarantine fixtures and preserves their JSON" do+ leaseBytes <- LazyByteString.readFile "test/golden/cell/cell.lease.v1.json"+ quarantineBytes <- LazyByteString.readFile "test/golden/cell/cell.quarantine.v1.json"+ let decodedLease = eitherDecode leaseBytes :: Either String Lease+ decodedQuarantine = eitherDecode quarantineBytes :: Either String Quarantine+ decodedLease `shouldSatisfy` isRight+ decodedQuarantine `shouldSatisfy` isRight+ fmap (eitherDecode . encode) decodedLease `shouldBe` Right (eitherDecode leaseBytes :: Either String Value)+ fmap (eitherDecode . encode) decodedQuarantine `shouldBe` Right (eitherDecode quarantineBytes :: Either String Value)++ it "reads owner control objects only for an allowed project and matching bucket" $ withSystemTempDirectory "kenshou-cell-control" \root -> do+ store <- newFileStore root+ descriptor <- LazyByteString.readFile "test/golden/cell/cell.descriptor.v1.json"+ let ref = CellRef "alpha" (Bucket "tan-nb-exp-cells-control")+ _ <- store.putObject ref.controlBucket (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor+ snapshot <- readCellSnapshot store ref ["tan-nb-exp"]+ case snapshot of+ Right observed -> do+ observed.lease `shouldBe` Nothing+ observed.quarantine `shouldBe` Nothing+ Left problem -> expectationFailure (show problem)+ readCellSnapshot store ref ["another-project"] `shouldReturn` Left "cell project is outside KENSHOU_GCP_ALLOWED_PROJECTS: tan-nb-exp"+ let wrongBucket = CellRef "alpha" (Bucket "other-control")+ _ <- store.putObject wrongBucket.controlBucket (ObjectName "cells/alpha/descriptor.json") "application/json" DoesNotExist descriptor+ readCellSnapshot store wrongBucket ["tan-nb-exp"] `shouldReturn` Left "cell descriptor control bucket differs from selected bucket"++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")++request :: LeaseRequest+request = LeaseRequest "tester@workstation" "verification" 120++shortRequest :: LeaseRequest+shortRequest = LeaseRequest "tester@workstation" "verification" 1++expectAcquired :: AcquireOutcome -> IO LeaseHandle+expectAcquired (Acquired handle) = pure handle+expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"
+ test/Main.hs view
@@ -0,0 +1,200 @@+module Main (main) where++import Control.Concurrent (forkIO)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Monad (replicateM)+import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy.Char8 qualified as LazyByteString+import Data.Either (isLeft)+import Data.Text qualified as Text+import Data.Time (UTCTime (..), fromGregorian, secondsToDiffTime)+import DebugSpec qualified+import DocsSpec qualified+import ExecSpec qualified+import FetchSpec qualified+import GcsSpec qualified+import HealthSpec qualified+import Kenshou.Core.Cohort (CohortIdentity (..), CohortName (..), PlanHash (..))+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), CohortCheck (..), Harness (..), PayloadDescriptor (..))+import Kenshou.Remote.Perf.Lock (withPerformanceLock)+import Kenshou.Remote.Store (Bucket (..), ObjectMeta (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.File (newFileStore)+import LeaseSpec qualified+import PairSpec qualified+import ParitySpec qualified+import PayloadSpec qualified+import PerfAttemptSpec qualified+import PerfCellSpec qualified+import PerfCleanupSpec qualified+import PerfConditionsSpec qualified+import PerfJournalSpec qualified+import PerfLockSpec qualified+import PerfPublicationSpec qualified+import PerfRunnerSpec qualified+import PrepareSpec qualified+import SessionSpec qualified+import SubmitSpec qualified+import System.Directory (getFileSize)+import System.Environment (getArgs, getExecutablePath)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (proc, readCreateProcessWithExitCode)+import Test.Hspec+import WatchSpec qualified++main :: IO ()+main =+ getArgs >>= \case+ ["--perf-lock", root] -> withPerformanceLock root (putStrLn "locked")+ ["--claim", root] -> do+ store <- newFileStore root+ result <- store.putObject (Bucket "control") (ObjectName "cells/alpha/lease.json") "application/json" DoesNotExist "claim"+ putStrLn case result of+ Written _ -> "written"+ PreconditionFailed -> "precondition-failed"+ _ -> hspec tests++tests :: Spec+tests = do+ DebugSpec.spec+ DocsSpec.spec+ ExecSpec.execSpec+ FetchSpec.spec+ GcsSpec.spec+ HealthSpec.spec+ LeaseSpec.spec+ PairSpec.pairSpec+ ParitySpec.spec+ PayloadSpec.spec+ PerfJournalSpec.spec+ PerfRunnerSpec.spec+ PerfPublicationSpec.spec+ PerfConditionsSpec.spec+ PerfAttemptSpec.spec+ PerfCleanupSpec.spec+ PerfCellSpec.spec+ PerfLockSpec.spec+ PrepareSpec.spec+ SessionSpec.spec+ SubmitSpec.spec+ WatchSpec.spec+ describe "Kenshou payload descriptor" do+ it "round-trips the complete cell payload and Nix cohort identity" do+ eitherDecode (encode examplePayload) `shouldBe` Right examplePayload++ it "rejects a bundle URI that disagrees with its content digest" do+ let badBundle = examplePayload.cell.bundle {uri = "gs://control/payloads/sha256/other.nar.zst"}+ bad = examplePayload {cell = examplePayload.cell {bundle = badBundle}}+ (eitherDecode (encode bad) :: Either String PayloadDescriptor) `shouldSatisfy` isLeft++ it "requires the Kenshou entry point even though the cell protocol accepts other commands" do+ let generic = examplePayload {cell = examplePayload.cell {command = ["bin/cell-fixture-hello"]}}+ (eitherDecode (encode generic) :: Either String PayloadDescriptor) `shouldSatisfy` isLeft++ describe "file cell object store" do+ it "fences stale generations and preserves tombstones" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+ store <- newFileStore root+ let bucket = Bucket "control"+ object = ObjectName "cells/alpha/lease.json"+ first <- store.putObject bucket object "application/json" DoesNotExist "first"+ firstMeta <- expectWritten first+ firstMeta.generation `shouldBe` 1+ stale <- store.putObject bucket object "application/json" DoesNotExist "stale"+ stale `shouldBe` PreconditionFailed+ next <- store.putObject bucket object "application/json" (GenerationIs firstMeta.generation) "second"+ secondMeta <- expectWritten next+ secondMeta.generation `shouldBe` 2+ store.deleteObject bucket object (GenerationIs firstMeta.generation) `shouldReturn` False+ store.deleteObject bucket object (GenerationIs secondMeta.generation) `shouldReturn` True+ store.statObject bucket object `shouldReturn` Nothing+ afterDelete <- store.putObject bucket object "application/json" DoesNotExist "third"+ thirdMeta <- expectWritten afterDelete+ thirdMeta.generation `shouldBe` 4+ stored <- store.getObject bucket object+ fmap fst stored `shouldBe` Just "third"+ store.listObjects bucket "cells/alpha/" `shouldReturn` [(object, thirdMeta)]++ it "admits exactly one concurrent create-only claim" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+ store <- newFileStore root+ let bucket = Bucket "control"+ object = ObjectName "cells/alpha/lease.json"+ outcomes <- replicateM 20 newEmptyMVar+ mapM_ (\reply -> forkIO (store.putObject bucket object "application/json" DoesNotExist "claim" >>= putMVar reply)) outcomes+ results <- mapM takeMVar outcomes+ length [() | Written _ <- results] `shouldBe` 1+ length [() | PreconditionFailed <- results] `shouldBe` 19++ it "fences independent store handles in one process" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+ stores <- replicateM 12 (newFileStore root)+ replies <- replicateM 12 newEmptyMVar+ let bucket = Bucket "control"+ object = ObjectName "cells/alpha/lease.json"+ mapM_ (\(store, reply) -> forkIO (store.putObject bucket object "application/json" DoesNotExist "claim" >>= putMVar reply)) (zip stores replies)+ results <- mapM takeMVar replies+ length [() | Written _ <- results] `shouldBe` 1+ length [() | PreconditionFailed <- results] `shouldBe` 11++ it "fences claims made by separate processes" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+ executable <- getExecutablePath+ replies <- replicateM 4 newEmptyMVar+ mapM_ (\reply -> forkIO (readCreateProcessWithExitCode (proc executable ["--claim", root]) "" >>= putMVar reply)) replies+ results <- mapM takeMVar replies+ [exit | (exit, _, _) <- results] `shouldBe` replicate 4 ExitSuccess+ length [() | (_, output, _) <- results, output == "written\n"] `shouldBe` 1+ length [() | (_, output, _) <- results, output == "precondition-failed\n"] `shouldBe` 3++ it "streams file publication and download under one object name" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+ store <- newFileStore (root </> "store")+ let bucket = Bucket "control"+ object = ObjectName "payloads/sha256/abc.nar.zst"+ source = root </> "source"+ fetched = root </> "fetched"+ bytes = LazyByteString.replicate (2 * 1024 * 1024) 'a'+ LazyByteString.writeFile source bytes+ result <- store.putFile bucket object "application/zstd" DoesNotExist source+ meta <- expectWritten result+ meta.size `shouldBe` LazyByteString.length bytes+ store.downloadTo bucket object fetched `shouldReturn` Just meta+ getFileSize fetched `shouldReturn` fromIntegral (LazyByteString.length bytes)++ it "rejects names that could escape a bucket" $ withSystemTempDirectory "kenshou-cell-store" \root -> do+ store <- newFileStore root+ store.putObject (Bucket "control") (ObjectName "../lease") "application/json" DoesNotExist "bad" `shouldThrow` anyIOException++expectWritten :: PutOutcome -> IO ObjectMeta+expectWritten (Written meta) = pure meta+expectWritten PreconditionFailed = expectationFailure "write was unexpectedly rejected" >> error "unreachable"++examplePayload :: PayloadDescriptor+examplePayload =+ PayloadDescriptor+ { cohort = "released",+ variant = "default",+ flakeAttr = "packages.x86_64-linux.kenshou-released",+ harness = Harness (Text.replicate 40 "a") False,+ cohortIdentity =+ CohortIdentity+ { identityCohort = CohortName "released",+ identityCompiler = "ghc-9.12.4",+ identityCabalVersion = "nix",+ identityOs = "linux",+ identityArch = "x86_64",+ identityIndexState = Nothing,+ identityPlanHash = PlanHash "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ identityDescriptorSha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ identityComponents = [],+ identityResolver = Just "nix"+ },+ cohortCheck = CohortCheck 0,+ cell =+ CellPayload+ { bundle = Bundle "gs://control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" 1,+ storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released",+ narHash = "sha256-example",+ closurePaths = ["/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released"],+ system = "x86_64-linux",+ command = ["bin/kenshou", "cell", "exec"]+ },+ createdAt = UTCTime (fromGregorian 2026 9 27) (secondsToDiffTime 0)+ }
+ test/PairSpec.hs view
@@ -0,0 +1,59 @@+module PairSpec (pairSpec) where++import Data.Aeson (Value, eitherDecodeFileStrict', toJSON)+import Data.List (nub)+import Data.Text qualified as Text+import Kenshou.Core.Id (parseScenarioId)+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Measure.Compare.Ordering (Arm (..), PairedOrdering (..))+import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Pair (PairRequest (..), PairState (..), SliceResult (..), judgePairs, pairPlan)+import Test.Hspec++pairSpec :: Spec+pairSpec = describe "cell paired schedule" do+ it "interleaves fresh same-seed benchmark arms and reserves replacement pairs" do+ source <- fixture+ let request = PairRequest "candidate" "baseline" 3 ABBA 7 1+ paired <- case pairPlan request source of+ Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"+ Right result -> pure result+ length paired.runs `shouldBe` 8+ paired.estimateMinutes `shouldBe` 8+ fmap (fmap (.arm) . (.trial)) paired.runs `shouldBe` fmap Just ["baseline", "candidate", "candidate", "baseline", "baseline", "candidate", "candidate", "baseline"]+ length (nub (fmap (.runId) paired.runs)) `shouldBe` 8+ fmap (.spec.seed) (take 2 paired.runs) `shouldSatisfy` \case [Just first, Just second] -> first == second; _ -> False+ fmap (.spec.seed) (take 2 (drop 2 paired.runs)) `shouldSatisfy` \case [Just first, Just second] -> first == second; _ -> False+ fmap (.comparison) (fmap (.spec) paired.runs) `shouldSatisfy` all (/= Nothing)+ pairPlan request source `shouldBe` Right paired+ decodeWorkPlan (toJSON paired) `shouldBe` Right paired++ it "keeps one failed arm out of a valid pair" do+ let result index arm passed = SliceResult "group" index arm passed+ judgePairs [result 0 Baseline True, result 0 Candidate True, result 1 Candidate False, result 1 Baseline True]+ `shouldBe` [(("group", 0), PairValid), (("group", 1), PairInvalid "one or both cell trials failed or are missing")]++ it "rejects a correctness plan or fewer than three pairs" do+ source <- sourceFixture+ pairPlan (PairRequest "candidate" "baseline" 3 ABBA 7 0) source `shouldBe` Left "paired comparisons require benchmark runs only"+ benchmark <- fixture+ pairPlan (PairRequest "candidate" "baseline" 2 ABBA 7 0) benchmark `shouldBe` Left "paired comparisons need at least three pairs"++fixture :: IO RunPlan+fixture = do+ source <- sourceFixture+ scenario <- case parseScenarioId "selftest/measure/benchmark/sleep-service" of+ Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"+ Right identifier -> pure identifier+ pure source {runs = [entry {spec = entry.spec {scenario = scenario}} | entry <- source.runs]}++sourceFixture :: IO RunPlan+sourceFixture = do+ decoded <- eitherDecodeFileStrict' "../kenshou-core/test/golden/run-plan.minimal.json" :: IO (Either String Value)+ document <- case decoded of+ Left problem -> expectationFailure problem >> error "unreachable"+ Right value -> pure value+ case decodeWorkPlan document of+ Left problem -> expectationFailure (Text.unpack problem) >> error "unreachable"+ Right plan -> pure plan
+ test/ParitySpec.hs view
@@ -0,0 +1,82 @@+module ParitySpec (spec) where++import Data.Aeson (Value, encode, object, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text qualified as Text+import Kenshou.Remote.Cell.Parity (ParityDifference (..), ParityOptions (..), ParityReport (..), compareForParity)+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "local and cell parity" do+ it "names placement differences while preserving scenario and verdict equality" $ withSystemTempDirectory "kenshou-parity" \root -> do+ let localDir = root </> "local"+ cellDir = root </> "cell"+ writeRun localDir (result "local" "passed" 7) (runSpec "local" 7)+ writeRun cellDir (result "cell" "passed" 7) (runSpec "cell" 7)+ report <- compareForParity (ParityOptions []) localDir cellDir+ report.unexpected `shouldBe` []+ let intentionalPaths = fmap (.path) report.intentional+ mapM_ (\path -> intentionalPaths `shouldContain` [path]) ["result.fingerprint.placement", "spec.environment.placement"]+ mapM_ (\path -> report.equal `shouldContain` [path]) ["result.outcome", "spec.seed"]+ report.localRunResultSha256 `shouldNotBe` report.cellRunResultSha256++ it "rejects changed outcomes, seeds and verdict counts" $ withSystemTempDirectory "kenshou-parity" \root -> do+ let localDir = root </> "local"+ cellDir = root </> "cell"+ writeRun localDir (result "local" "passed" 7) (runSpec "local" 7)+ writeRun cellDir (result "cell" "failed" 8) (runSpec "cell" 8)+ writeVerdict localDir 3+ writeVerdict cellDir 2+ report <- compareForParity (ParityOptions []) localDir cellDir+ let unexpectedPaths = fmap (.path) report.unexpected+ mapM_ (\path -> unexpectedPaths `shouldContain` [path]) ["result.outcome", "spec.seed", "verdicts.ledger.json.counts.checked"]++ it "allows only a named volatile verdict field" $ withSystemTempDirectory "kenshou-parity" \root -> do+ let localDir = root </> "local"+ cellDir = root </> "cell"+ writeRun localDir (result "local" "passed" 7) (runSpec "local" 7)+ writeRun cellDir (result "cell" "passed" 7) (runSpec "cell" 7)+ writeVerdict localDir 3+ writeVerdict cellDir 2+ report <- compareForParity (ParityOptions ["verdicts.ledger.json.counts.checked"]) localDir cellDir+ report.unexpected `shouldBe` []+ fmap (.path) report.intentional `shouldContain` ["verdicts.ledger.json.counts.checked"]++writeRun :: FilePath -> Value -> Value -> IO ()+writeRun directory runResult specValue = do+ createDirectoryIfMissing True directory+ LazyByteString.writeFile (directory </> "run-result.json") (encode runResult)+ LazyByteString.writeFile (directory </> "run-spec.json") (encode specValue)++writeVerdict :: FilePath -> Int -> IO ()+writeVerdict directory count = do+ let verdictDir = directory </> "verdicts"+ createDirectoryIfMissing True verdictDir+ LazyByteString.writeFile (verdictDir </> "ledger.json") (encode (object ["status" .= ("passed" :: Text.Text), "counts" .= object ["checked" .= count]]))++result :: Text.Text -> Text.Text -> Int -> Value+result placement outcome seed =+ object+ [ "schema" .= ("kenshou.run-result/v1" :: Text.Text),+ "runId" .= placement,+ "scenario" .= ("selftest/kernel/correctness/postgres-roundtrip" :: Text.Text),+ "outcome" .= outcome,+ "seed" .= seed,+ "cohort" .= object ["components" .= [object ["name" .= ("synthetic-store" :: Text.Text), "version" .= ("0.8.0.1" :: Text.Text)]], "os" .= placement],+ "fingerprint" .= object ["placement" .= placement]+ ]++runSpec :: Text.Text -> Int -> Value+runSpec placement seed =+ object+ [ "runId" .= placement,+ "scenario" .= ("selftest/kernel/correctness/postgres-roundtrip" :: Text.Text),+ "seed" .= seed,+ "knobs" .= object ["selftest.events" .= (3 :: Int)],+ "dimensions" .= object ["pg.version" .= ("18" :: Text.Text)],+ "environment" .= object ["placement" .= placement],+ "labels" .= (if placement == "cell" then object ["postgresPlacement" .= ("cell-server" :: Text.Text)] else object [])+ ]
+ test/PayloadSpec.hs view
@@ -0,0 +1,212 @@+module PayloadSpec (spec) where++import Data.Aeson (eitherDecode, eitherDecodeStrict', encodeFile, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort (CohortDescriptor (..), CohortIdentity (..), CohortName (..), ComponentSpec (..), PackagePin (..), PackageSource (..), PlanHash (..), ResolvedComponent (..), ResolvedPackage (..))+import Kenshou.Remote.Cell.Docs (WorkObject (..))+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), CohortCheck (..), Harness (..), PayloadDescriptor (..))+import Kenshou.Remote.Payload.Nix (BundleInfo (..), ClosureInfo (..), NixError (..), NixTools (..), exportBundleWith, parseClosureInfo)+import Kenshou.Remote.Payload.Publish (BundlePublishError (..), publishBundle)+import Kenshou.Remote.Payload.Publisher (PreparedPublishOptions (..), PublishError (..), PublishOptions (..), PublisherDeps (..), publishPayloadWith, publishPreparedPayloadWith)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (Permissions (..), doesFileExist, getPermissions, listDirectory, setPermissions)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "content-addressed payload publication" do+ it "checks a Nix identity before build, then publishes its exported bundle and descriptor" $ withSystemTempDirectory "kenshou-publisher" \root -> do+ identity <- releasedIdentity+ store <- newFileStore (root </> "store")+ builds <- newIORef (0 :: Int)+ let output = root </> "payload.json"+ options = PublishOptions "../fixtures/consumer-baseline" "released" "default" (Bucket "control") output False+ storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+ work = workObjectFor "application/octet-stream" bundleBytes+ exported destination = do+ LazyByteString.writeFile destination bundleBytes+ pure (Right (BundleInfo destination work.sha256 work.bytes (ClosureInfo storePath "sha256-root" [storePath])))+ deps selected =+ PublisherDeps+ (\_ -> pure (Right (Harness (Text.replicate 40 "a") False)))+ (\_ -> pure (Right selected))+ (\_ -> modifyIORef' builds (+ 1) >> pure (Right storePath))+ (\_ -> exported)+ let stale = identity {identityDescriptorSha256 = Text.replicate 64 "0"}+ rejected <- publishPayloadWith (deps stale) store options+ rejected `shouldBe` Left (PublishCohortMismatch "Nix identity descriptor digest differs from the selected descriptor")+ readIORef builds `shouldReturn` 0+ store.listObjects (Bucket "control") "payloads/sha256/" `shouldReturn` []+ accepted <- publishPayloadWith (deps identity) store options+ descriptor <- case accepted of+ Left problem -> expectationFailure (show problem) >> error "unreachable"+ Right payload -> pure payload+ descriptor.cohortCheck.packagesChecked `shouldBe` 42+ descriptor.cell.bundle.sha256 `shouldBe` work.sha256+ readIORef builds `shouldReturn` 1+ stored <- store.getObject (Bucket "control") (objectFor descriptor)+ fmap fst stored `shouldBe` Just bundleBytes+ saved <- eitherDecode <$> LazyByteString.readFile output+ saved `shouldBe` Right descriptor++ it "publishes generated performance inputs while verifying a separate clean harness" $ withSystemTempDirectory "kenshou-prepared-publisher" \root -> do+ bytes <- ByteString.readFile "../fixtures/consumer-baseline/cohort/released.json"+ original <- either (ioError . userError) pure (eitherDecodeStrict' bytes :: Either String CohortDescriptor)+ let generated = original {descriptorName = CohortName "perf-fixture-aa"}+ descriptorPath = root </> "cohort.json"+ harness = root </> "clean-harness"+ flake = "path:" <> root <> "/generated"+ commit = Text.replicate 40 "a"+ storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+ work = workObjectFor "application/octet-stream" bundleBytes+ encodeFile descriptorPath generated+ descriptorBytes <- ByteString.readFile descriptorPath+ identity <- releasedIdentity+ let resolved = identity {identityCohort = generated.descriptorName, identityDescriptorSha256 = Text.drop 7 (sha256Hex descriptorBytes)}+ store <- newFileStore (root </> "store")+ builds <- newIORef (0 :: Int)+ let options = PreparedPublishOptions harness commit flake descriptorPath "perf-fixture-aa" "default" (Bucket "control") (root </> "payload.json")+ deps dirty =+ PublisherDeps+ (\path -> (path `shouldBe` harness) >> pure (Right (Harness commit dirty)))+ (\reference -> (reference `shouldBe` flake <> "#packages.x86_64-linux.kenshou-perf-fixture-aa.cohortIdentity") >> pure (Right resolved))+ (\_ -> modifyIORef' builds (+ 1) >> pure (Right storePath))+ (\_ destination -> LazyByteString.writeFile destination bundleBytes >> pure (Right (BundleInfo destination work.sha256 work.bytes (ClosureInfo storePath "sha256-root" [storePath]))))+ publishPreparedPayloadWith (deps True) store options `shouldReturn` Left PublishDirtyWorktree+ wrong <- publishPreparedPayloadWith (deps False) store options {harnessCommit = Text.replicate 40 "b"}+ wrong `shouldSatisfy` \case Left (PublishDescriptorError _) -> True; _ -> False+ readIORef builds `shouldReturn` 0+ accepted <- publishPreparedPayloadWith (deps False) store options+ accepted `shouldSatisfy` \case Right payload -> payload.cohort == "perf-fixture-aa" && payload.harness == Harness commit False; _ -> False+ readIORef builds `shouldReturn` 1++ it "reads the root NAR hash and complete closure from Nix metadata" do+ let root = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+ dependency = "/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-dependency"+ metadata =+ object+ [ Key.fromText (Text.pack root) .= object ["narHash" .= ("sha256-root" :: Text.Text)],+ Key.fromText (Text.pack dependency) .= object ["narHash" .= ("sha256-dependency" :: Text.Text)]+ ]+ parseClosureInfo root metadata `shouldBe` Right (ClosureInfo root "sha256-root" [root, dependency])+ parseClosureInfo dependency (object [Key.fromText (Text.pack root) .= object ["narHash" .= ("sha256-root" :: Text.Text)]])+ `shouldBe` Left (NixInvalidOutput "closure metadata omits the requested store path")++ it "streams a closure export through compression and removes a failed temporary bundle" $ withSystemTempDirectory "kenshou-nix-export" \root -> do+ let storePath = "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou"+ destination = root </> "bundle.nar.zst"+ tools = NixTools (root </> "nix") (root </> "nix-store") (root </> "zstd")+ exported = "exported bytes"+ work = workObjectFor "application/octet-stream" exported+ writeTool tools.nixExecutable (unlines ["#!/bin/sh", "printf '%s' '{\"" <> storePath <> "\":{\"narHash\":\"sha256-root\"}}'"])+ writeTool tools.nixStoreExecutable (unlines ["#!/bin/sh", "printf 'exported bytes'"])+ writeTool tools.zstdExecutable (unlines ["#!/bin/sh", "cat > \"$6\""])+ exportBundleWith tools storePath destination+ `shouldReturn` Right (BundleInfo destination work.sha256 work.bytes (ClosureInfo storePath "sha256-root" [storePath]))+ LazyByteString.readFile destination `shouldReturn` exported+ writeTool tools.zstdExecutable (unlines ["#!/bin/sh", "exit 7"])+ let failedDestination = root </> "failed.nar.zst"+ result <- exportBundleWith tools storePath failedDestination+ result `shouldSatisfy` \case+ Left (NixCommandFailed executable _ _ _) -> executable == tools.zstdExecutable+ _ -> False+ doesFileExist failedDestination `shouldReturn` False+ entries <- listDirectory root+ filter (Text.isPrefixOf ".kenshou-payload-" . Text.pack) entries `shouldBe` []++ it "publishes a checked bundle once and accepts an identical retry" $ withSystemTempDirectory "kenshou-payload" \root -> do+ store <- newFileStore (root </> "store")+ let source = root </> "bundle.nar.zst"+ LazyByteString.writeFile source bundleBytes+ descriptor <- fixtureFor bundleBytes+ publishBundle store (Bucket "control") descriptor source `shouldReturn` Right descriptor+ publishBundle store (Bucket "control") descriptor source `shouldReturn` Right descriptor+ objects <- store.listObjects (Bucket "control") "payloads/sha256/"+ length objects `shouldBe` 1+ stored <- store.getObject (Bucket "control") (objectFor descriptor)+ fmap fst stored `shouldBe` Just bundleBytes++ it "refuses a digest mismatch before creating an object" $ withSystemTempDirectory "kenshou-payload" \root -> do+ store <- newFileStore (root </> "store")+ let source = root </> "bundle.nar.zst"+ LazyByteString.writeFile source bundleBytes+ descriptor <- fixtureFor bundleBytes+ let cell = descriptor.cell+ original = cell.bundle+ wrongDigest = Text.replicate 64 "b"+ wrong = descriptor {cell = cell {bundle = original {sha256 = wrongDigest, uri = "gs://control/payloads/sha256/" <> wrongDigest <> ".nar.zst"}}}+ result <- publishBundle store (Bucket "control") wrong source+ result `shouldBe` Left (BundleDigestMismatch wrongDigest descriptor.cell.bundle.sha256)+ store.listObjects (Bucket "control") "payloads/sha256/" `shouldReturn` []++ it "refuses an incorrect declared byte count before creating an object" $ withSystemTempDirectory "kenshou-payload" \root -> do+ store <- newFileStore (root </> "store")+ let source = root </> "bundle.nar.zst"+ LazyByteString.writeFile source bundleBytes+ descriptor <- fixtureFor bundleBytes+ let cell = descriptor.cell+ original = cell.bundle+ wrong = descriptor {cell = cell {bundle = original {bytes = original.bytes + 1}}}+ publishBundle store (Bucket "control") wrong source `shouldReturn` Left (BundleBytesMismatch (original.bytes + 1) original.bytes)+ store.listObjects (Bucket "control") "payloads/sha256/" `shouldReturn` []++ it "rejects a wrong control bucket and an existing object of another size" $ withSystemTempDirectory "kenshou-payload" \root -> do+ store <- newFileStore (root </> "store")+ let source = root </> "bundle.nar.zst"+ LazyByteString.writeFile source bundleBytes+ descriptor <- fixtureFor bundleBytes+ publishBundle store (Bucket "other") descriptor source `shouldReturn` Left (BundleOutsideControlBucket descriptor.cell.bundle.uri)+ _ <- store.putObject (Bucket "control") (objectFor descriptor) "application/zstd" DoesNotExist "x"+ publishBundle store (Bucket "control") descriptor source `shouldReturn` Left (ExistingBundleSizeMismatch descriptor.cell.bundle.bytes 1)++fixtureFor :: LazyByteString.ByteString -> IO PayloadDescriptor+fixtureFor bytes = do+ encoded <- LazyByteString.readFile "test/golden/payload.json"+ fixture <- either (ioError . userError) pure (eitherDecode encoded :: Either String PayloadDescriptor)+ let work = workObjectFor "application/octet-stream" bytes+ bundle = Bundle ("gs://control/payloads/sha256/" <> work.sha256 <> ".nar.zst") work.sha256 work.bytes+ cell = fixture.cell+ pure fixture {cell = cell {bundle}}++objectFor :: PayloadDescriptor -> ObjectName+objectFor descriptor = ObjectName ("payloads/sha256/" <> descriptor.cell.bundle.sha256 <> ".nar.zst")++bundleBytes :: LazyByteString.ByteString+bundleBytes = "bundle-bytes"++writeTool :: FilePath -> String -> IO ()+writeTool path contents = do+ writeFile path contents+ permissions <- getPermissions path+ setPermissions path permissions {executable = True}++releasedIdentity :: IO CohortIdentity+releasedIdentity = do+ contents <- ByteString.readFile "../fixtures/consumer-baseline/cohort/released.json"+ descriptor <- either (ioError . userError) pure (eitherDecodeStrict' contents :: Either String CohortDescriptor)+ let resolved component =+ ResolvedComponent+ component.componentId+ component.componentMoriUri+ [ResolvedPackage pin.pinName pin.pinVersion (FromHackage Nothing) | pin <- component.componentPackages]+ pure+ ( CohortIdentity+ (CohortName "released")+ "ghc-9.12.4"+ "nix"+ "linux"+ "x86_64"+ (Just descriptor.descriptorIndexState)+ (PlanHash ("sha256:" <> Text.replicate 64 "a"))+ "5c08f30a78a7d36ab707cde925813987356bdc46d0c3626b2882814db685cf56"+ (map resolved descriptor.descriptorComponents)+ (Just "nix")+ )
+ test/PerfAttemptSpec.hs view
@@ -0,0 +1,203 @@+module PerfAttemptSpec (spec) where++import Data.Aeson+import Data.ByteString qualified as Bytes+import Data.IORef+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text qualified as Text+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Data.Word (Word64)+import Kenshou.Core.Id (mkSeed, newRunId, renderRunId)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Measure.Clock (Origin (..))+import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..))+import Kenshou.Measure.Compare.Ordering (PairedOrdering (..))+import Kenshou.Measure.Compare.Policy qualified as Policy+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (encodeHistogram)+import Kenshou.Measure.Samples+import Kenshou.Measure.Summary (summarizeRunDir)+import Kenshou.Perf.Plan+import Kenshou.Perf.Result+import Kenshou.Perf.Result qualified as Configuration (ConfigurationResult (..))+import Kenshou.Plan.RunPlan (PlannedRun, RunPlan (..), TrialInfo (..))+import Kenshou.Plan.RunPlan qualified as Plan (PlannedRun (..))+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Pair+import Kenshou.Remote.Perf.Attempt+import Kenshou.Remote.Perf.Journal+import Kenshou.Remote.Perf.Runner+import System.Directory (createDirectoryIfMissing, renameDirectory)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance paired attempts" do+ it "reconstructs five raw pairs, preserves an exit-zero defect and replays portable paths" $ withSystemTempDirectory "perf-raw-pairs" \directory -> do+ (entry, schedule, policy) <- fixture+ identifier <- newRunId+ calls <- newIORef (0 :: Int)+ blockers <- newIORef ([] :: [Text.Text])+ let attempt = directory </> "original"+ moved = directory </> "relocated"+ callbacks = ComparisonHooks (\_ -> expectationFailure "stable five-pair control must not replace pairs") (\reason -> modifyIORef' blockers (<> [reason]))+ execute run = do+ modifyIORef' calls (+ 1)+ let path = attempt </> "retained" </> Text.unpack (renderRunId run.runId)+ writeRawRun run (run.ordinal == 2) 1000000 path+ pure (Right path)+ result <- executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) attempt callbacks execute+ result.performanceVerdict `shouldBe` "pass"+ result.infrastructureState `shouldBe` "healthy"+ result.validPairs `shouldBe` 5+ result.runtimeFailures `shouldBe` ["runtime assertions failed (including nonblocking known defects)"]+ readIORef calls `shouldReturn` 10+ readIORef blockers `shouldReturn` result.runtimeFailures+ verifyAttempt entry schedule policy (VaryControl :| []) attempt result `shouldReturn` True+ encodeFile (attempt </> "comparison.json") result.comparison+ originalBytes <- Bytes.readFile (attempt </> "comparison.json")+ renameDirectory attempt moved+ verifyAttempt entry schedule policy (VaryControl :| []) moved result `shouldReturn` True+ Bytes.readFile (moved </> "comparison.json") `shouldReturn` originalBytes+ plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+ aggregate <- either (fail . show) pure (aggregateResult (plan {selected = [entry]} :: PerfPlan) ("sha256:" <> Text.replicate 64 "a") True [result])+ aggregate.exitCode `shouldBe` 1+ it "replaces an invalid raw pair once and detects a genuine retained latency regression" $ withSystemTempDirectory "perf-raw-regression" \directory -> do+ (entry, schedule, policy) <- fixture+ identifier <- newRunId+ calls <- newIORef (0 :: Int)+ replacements <- newIORef (0 :: Int)+ let callbacks = ComparisonHooks (\_ -> modifyIORef' replacements (+ 1)) (\_ -> pure ())+ execute run = do+ modifyIORef' calls (+ 1)+ let path = directory </> "retained" </> Text.unpack (renderRunId run.runId)+ latency = if maybe False ((== "candidate") . (.arm)) run.trial then 3000000 else 1000000+ writeRawRun run False latency path+ if run.ordinal == 1 then Bytes.appendFile (path </> "samples/publish-to-finalize.raw") "invalid" else pure ()+ pure (Right path)+ result <- executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) directory callbacks execute+ result.performanceVerdict `shouldBe` "regression"+ result.infrastructureState `shouldBe` "healthy"+ result.validPairs `shouldBe` 5+ readIORef calls `shouldReturn` 12+ readIORef replacements `shouldReturn` 1+ verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True+ it "bounds replacements and reuses retained invalid paths without resubmitting trials" $ withSystemTempDirectory "perf-attempt" \directory -> do+ (entry, schedule, policy) <- fixture+ identifier <- newRunId+ calls <- newIORef (0 :: Int)+ replacements <- newIORef (0 :: Int)+ let lease = LeaseIdentity "alpha" identifier+ callbacks = ComparisonHooks (\_ -> modifyIORef' replacements (+ 1)) (\_ -> pure ())+ execute run = do+ modifyIORef' calls (+ 1)+ let path = directory </> "retained" </> Text.unpack (renderRunId run.runId)+ createDirectoryIfMissing True path+ encodeFile (path </> "run-result.json") (object [])+ encodeFile (path </> "run-spec.json") (object [])+ pure (Right path)+ result <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+ result.infrastructureState `shouldBe` "failed"+ result.validPairs `shouldBe` 0+ readIORef calls `shouldReturn` 14+ readIORef replacements `shouldReturn` 2+ verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True+ verifyAttempt entry schedule policy (VaryControl :| []) directory result {Configuration.attempts = []} `shouldReturn` False+ _ <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+ readIORef calls `shouldReturn` 14+ another <- newRunId+ verifyAttemptInputs lease entry schedule directory+ verifyAttemptInputs (LeaseIdentity "alpha" another) entry schedule directory `shouldThrow` anyIOException+ executeAttempt (LeaseIdentity "alpha" another) entry schedule policy (VaryControl :| []) directory callbacks execute `shouldThrow` anyIOException+ it "checkpoints terminal executor failures and replays them without new work" $ withSystemTempDirectory "perf-attempt-failures" \directory -> do+ (entry, schedule, policy) <- fixture+ identifier <- newRunId+ calls <- newIORef (0 :: Int)+ let callbacks = ComparisonHooks (\_ -> pure ()) (\_ -> pure ())+ execute _ = modifyIORef' calls (+ 1) >> pure (Left "sealed outer health failed")+ lease = LeaseIdentity "alpha" identifier+ result <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+ readIORef calls `shouldReturn` 14+ repeated <- executeAttempt lease entry schedule policy (VaryControl :| []) directory callbacks execute+ repeated `shouldBe` result+ readIORef calls `shouldReturn` 14+ verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldReturn` True+ encodeFile (directory </> "trial-failures.json") (Map.singleton ("unplanned" :: Text.Text) ("invalid" :: Text.Text))+ verifyAttempt entry schedule policy (VaryControl :| []) directory result `shouldThrow` anyIOException+ it "rejects an incomplete schedule and a forged retained path before dispatch" $ withSystemTempDirectory "perf-attempt-rejection" \directory -> do+ (entry, schedule, policy) <- fixture+ identifier <- newRunId+ let callbacks = ComparisonHooks (\_ -> pure ()) (\_ -> pure ())+ execute _ = expectationFailure "must not dispatch an invalid attempt" >> pure (Left "invalid")+ executeAttempt (LeaseIdentity "alpha" identifier) entry schedule {runs = drop 1 schedule.runs} policy (VaryControl :| []) (directory </> "incomplete") callbacks execute `shouldThrow` anyIOException+ createDirectoryIfMissing True (directory </> "forged")+ encodeFile (directory </> "forged/trial-paths.json") (Map.singleton ("unplanned" :: Text.Text) ("../escape" :: FilePath))+ executeAttempt (LeaseIdentity "alpha" identifier) entry schedule policy (VaryControl :| []) (directory </> "forged") callbacks execute `shouldThrow` anyIOException++fixture :: IO (PlannedProfile, RunPlan, Policy.Policy)+fixture = do+ plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+ entry <- case (plan :: PerfPlan).selected of value : _ -> pure value; _ -> fail "empty profile fixture"+ identifier <- newRunId+ seed <- either (fail . show) pure (mkSeed 37)+ source <-+ either+ (fail . show)+ pure+ ( decodeWorkPlan+ ( object+ [ "schema" .= ("kenshou.run-plan/v1" :: Text.Text),+ "planId" .= identifier,+ "runs"+ .= [ object+ [ "ordinal" .= (0 :: Int),+ "runId" .= identifier,+ "estimateMinutes" .= (1 :: Int),+ "spec"+ .= object+ [ "schema" .= ("kenshou.run-spec/v1" :: Text.Text),+ "runId" .= identifier,+ "scenario" .= entry.profile.scenario,+ "scenarioRevision" .= entry.profile.scenarioRevision,+ "knobs" .= entry.profile.knobs,+ "dimensions" .= entry.profile.dimensions,+ "seed" .= seed,+ "phases" .= zeroPhases,+ "timeoutSeconds" .= entry.profile.timeoutSeconds+ ]+ ]+ ]+ ]+ )+ )+ schedule <- either (fail . show) pure (pairPlan (PairRequest "candidate" "baseline" 5 ABBA 37 2) source)+ policy <- Bytes.readFile "../policies/default.json" >>= either (fail . show) pure . Policy.decodePolicy+ pure (entry, schedule, policy)++-- Synthetic protocol evidence only; this is not workload calibration.+writeRawRun :: PlannedRun -> Bool -> Word64 -> FilePath -> IO ()+writeRawRun run knownFailure latency directory = do+ createDirectoryIfMissing True (directory </> "samples")+ createDirectoryIfMissing True (directory </> "series")+ Bytes.writeFile (directory </> "series/load.csv") "mono_ns,wall,phase\n1000,0,steady\n5000001000,0,drain\n"+ encodeFile (directory </> "run-spec.json") run.spec+ mutable <- Histogram.newHistogram Histogram.defaultHistogramConfig+ mapM_ (const (Histogram.recordValue mutable latency)) [1 :: Int .. 1000]+ histogram <- Histogram.freeze mutable+ let base = directory </> "samples/publish-to-finalize"+ Bytes.writeFile (base <> ".hist") (encodeHistogram histogram)+ Bytes.writeFile (base <> ".service.hist") (encodeHistogram histogram)+ writer <- openSampleWriter (base <> ".raw") (SampleHeader (Origin 0 0) MonotonicClock "publish-to-finalize" "synthetic")+ writeSampleBlock writer 0 [SampleRecord start start (start + latency) 0 1 | index <- [0 .. 999], let start = 1000 + index * 5000000]+ closeSampleWriter writer+ encodeFile (base <> ".meta.json") (object ["operation" .= ("publish-to-finalize" :: Text.Text), "latencyBasis" .= ("intended-start" :: Text.Text), "rawSamples" .= ("full" :: Text.Text), "phaseCounts" .= object ["steady" .= object ["successes" .= (1000 :: Int), "failures" .= (0 :: Int), "units" .= (1000 :: Int)]]])+ measurements <- summarizeRunDir directory >>= either (fail . show) pure+ let origin = UTCTime (fromGregorian 2026 10 10) (secondsToDiffTime 0)+ encodeFile+ (directory </> "run-result.json")+ (object ["runId" .= run.runId, "outcome" .= (if knownFailure then "failed" else "passed" :: Text.Text), "exitCode" .= (0 :: Int), "blocking" .= False, "compatibility" .= object ["inputs" .= object ["cohortPlanHash" .= ("synthetic-control" :: Text.Text)]], "fingerprint" .= object ["machineProfile" .= ("synthetic" :: Text.Text)], "timings" .= object ["startedAt" .= addUTCTime (fromIntegral run.ordinal) origin], "summaries" .= object ["measurements" .= object ["measurements" .= measurements]]])+ manifest <- writeManifest directory run.runId Map.empty+ encodeFile (directory </> "manifest.json") manifest
+ test/PerfCellSpec.hs view
@@ -0,0 +1,57 @@+module PerfCellSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as Lazy+import Data.List qualified as List+import Data.Map.Strict qualified as Map+import Data.Text qualified as Text+import FetchSpec (completeTreeWithNested)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Perf.Plan+import Kenshou.Plan.RunPlan qualified as Plan+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (Completed), WorkObject (..))+import Kenshou.Remote.Cell.Docs qualified as Submission (Submission (..))+import Kenshou.Remote.Cell.Session.Journal+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan)+import Kenshou.Remote.Perf.Cell+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance sealed runtime observations" do+ it "retains an exit-zero runtime defect even before the outer trial becomes admissible" $ withSystemTempDirectory "perf-cell-defect" \root -> do+ plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+ entry <- case (plan :: PerfPlan).selected of first : _ -> pure first; _ -> fail "empty plan fixture"+ journal <- eitherDecodeFileStrict' "test/golden/cell-session.json" >>= either fail pure+ slice <- case (journal :: SessionJournal).slices of [first] -> pure first; _ -> fail "expected one slice"+ identifier <- newRunId+ cellRun <- newRunId+ schedule <- either (fail . Text.unpack) pure (decodeWorkPlan (object ["schema" .= ("kenshou.run-plan/v1" :: Text.Text), "planId" .= identifier, "runs" .= [object ["ordinal" .= (1 :: Int), "runId" .= identifier, "estimateMinutes" .= (1 :: Int), "spec" .= object ["schema" .= ("kenshou.run-spec/v1" :: Text.Text), "runId" .= identifier, "scenario" .= entry.profile.scenario]]]]))+ run <- case schedule.runs of [first] -> pure first; _ -> fail "expected one run"+ (tree, outer) <- completeTreeWithNested (root </> "trials" </> Text.unpack (renderRunId cellRun)) cellRun journal.leaseId identifier False False False+ let nested = tree </> "output" </> Text.unpack (renderRunId identifier)+ original <- eitherDecodeFileStrict' (nested </> "run-result.json") >>= either fail pure+ case original of+ Object fields -> encodeFile (nested </> "run-result.json") (Object (KeyMap.insert "outcome" (String "failed") (KeyMap.insert "exitCode" (Number 0) (KeyMap.insert "blocking" (Bool False) fields))))+ _ -> fail "expected a result object"+ manifest <- writeManifest nested identifier Map.empty+ encodeFile (nested </> "manifest.json") manifest+ artifacts <-+ traverse+ ( \artifact -> do+ bytes <- Lazy.readFile (tree </> Text.unpack artifact.path)+ let work = workObjectFor "application/json" bytes+ pure (Artifact artifact.path work.sha256 work.bytes artifact.mediaType)+ )+ outer.artifacts+ let updated = outer {artifacts}+ encodeFile (tree </> "manifest.json") updated+ bytes <- Lazy.readFile (tree </> "manifest.json")+ let retained = journal {slices = [slice {cellRun, runIds = [identifier], submission = slice.submission {Submission.runId = cellRun}, state = SliceFetched, cellOutcome = Just Completed, manifestSha256 = Just (workObjectFor "application/json" bytes).sha256, fetchedPath = Just "tree"}]}+ trial = root </> "trials" </> Text.unpack (renderRunId identifier)+ problems <- retainedRuntimeProblems entry.profile (run :: Plan.PlannedRun) trial retained+ List.nub problems `shouldBe` ["runtime assertions failed (including nonblocking known defects)"]
+ test/PerfCleanupSpec.hs view
@@ -0,0 +1,82 @@+module PerfCleanupSpec (spec) where++import Data.Aeson (ToJSON, eitherDecode, encode)+import Data.ByteString.Lazy qualified as Lazy+import Data.Maybe (isJust, isNothing)+import Data.Text qualified as Text+import Data.Time (addUTCTime)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs+import Kenshou.Remote.Cell.Docs qualified as Submission (Submission (..))+import Kenshou.Remote.Cell.Lease qualified as Lease+import Kenshou.Remote.Cell.Session.Journal+import Kenshou.Remote.Perf.Cleanup+import Kenshou.Remote.Perf.Journal (LeaseIdentity (..))+import Kenshou.Remote.Store+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance owned cleanup" do+ it "accepts an unpublished planned slice but retains an ambiguous submitted slice" $ fixture \store ref identity journal -> do+ ownedWorkSettled store ref identity [journal] `shouldReturn` True+ let submitted = journal {slices = map (\slice -> slice {state = SliceSubmitted}) journal.slices}+ ownedWorkSettled store ref identity [submitted] `shouldReturn` False+ it "waits for a submitted run to seal with both outcome and digest" $ fixture \store ref identity journal -> do+ slice <- onlySlice journal+ let status = CellStatus slice.cellRun Running Nothing journal.createdAt Nothing (LogChunks 0 0) Nothing Nothing Nothing+ put store ref slice "submission.json" slice.submission+ put store ref slice "status.json" status+ ownedWorkSettled store ref identity [journal] `shouldReturn` False+ put store ref slice "status.json" (status {phase = Sealed})+ ownedWorkSettled store ref identity [journal] `shouldThrow` anyIOException+ put store ref slice "status.json" (status {phase = Sealed, outcome = Just Cancelled, manifestSha256 = Just (Text.replicate 64 "a")})+ ownedWorkSettled store ref identity [journal] `shouldReturn` True+ it "accepts owner rejection and rejects a marker naming another lease" $ fixture \store ref identity journal -> do+ slice <- onlySlice journal+ put store ref slice "submission.json" slice.submission+ put store ref slice "rejected.json" (Rejected slice.cellRun "cancelled lease" journal.createdAt)+ ownedWorkSettled store ref identity [journal] `shouldReturn` True+ other <- newRunId+ put store ref slice "submission.json" (slice.submission {Submission.leaseId = other})+ ownedWorkSettled store ref identity [journal] `shouldThrow` anyIOException+ it "does not inspect another owner's work and rejects a mismatched control reference" $ fixture \store ref identity journal -> do+ other <- newRunId+ ownedWorkSettled store ref (LeaseIdentity identity.cell other) [journal] `shouldReturn` True+ ownedWorkSettled store (ref {Lease.controlBucket = Bucket "wrong"}) identity [journal] `shouldThrow` anyIOException+ ownedWorkSettled store (ref {Lease.cellName = "beta"}) identity [journal] `shouldThrow` anyIOException+ it "reattaches live work but treats expired and cancelled leases as lost" $ withSystemTempDirectory "perf-live-lease" \root -> do+ store <- newFileStore root+ let ref = Lease.CellRef "alpha" (Bucket "control")+ acquired <- Lease.acquireLease store ref (Lease.LeaseRequest "test" "recovery" 120)+ handle <- case acquired of+ Lease.Acquired value -> pure value+ _ -> expectationFailure "lease was not acquired" >> error "unreachable"+ lease <- Lease.leaseSnapshot handle+ isJust <$> Lease.reattachActiveLease store ref lease.leaseId `shouldReturn` True+ let later = store {serverTime = addUTCTime 151 <$> store.serverTime}+ isNothing <$> Lease.reattachActiveLease later ref lease.leaseId `shouldReturn` True+ isJust <$> Lease.reattachLease later ref lease.leaseId `shouldReturn` True+ Lease.requestCancel store ref (Just lease.leaseId) False `shouldReturn` True+ isNothing <$> Lease.reattachActiveLease store ref lease.leaseId `shouldReturn` True+ isJust <$> Lease.reattachLease store ref lease.leaseId `shouldReturn` True++fixture :: (ObjectStore -> Lease.CellRef -> LeaseIdentity -> SessionJournal -> IO ()) -> IO ()+fixture action = withSystemTempDirectory "perf-cleanup" \root -> do+ store <- newFileStore root+ bytes <- Lazy.readFile "test/golden/cell-session.json"+ journal <- either (ioError . userError) pure (eitherDecode bytes)+ action store (Lease.CellRef journal.cell (Bucket journal.controlBucket)) (LeaseIdentity journal.cell journal.leaseId) journal++onlySlice :: SessionJournal -> IO SliceJournal+onlySlice journal = case journal.slices of+ [slice] -> pure slice+ _ -> expectationFailure "cleanup fixture must have one slice" >> error "unreachable"++put :: (ToJSON value) => ObjectStore -> Lease.CellRef -> SliceJournal -> Text.Text -> value -> IO ()+put store ref slice name value = do+ let path = ObjectName ("cells/" <> ref.cellName <> "/submissions/" <> renderRunId slice.cellRun <> "/" <> name)+ current <- store.statObject ref.controlBucket path+ _ <- store.putObject ref.controlBucket path "application/json" (maybe DoesNotExist (GenerationIs . (.generation)) current) (encode value)+ pure ()
+ test/PerfConditionsSpec.hs view
@@ -0,0 +1,137 @@+module PerfConditionsSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Either (isLeft)+import Data.Text (Text)+import Data.Time (UTCTime, addUTCTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Remote.Perf.Conditions+import Test.Hspec++spec :: Spec+spec = describe "performance cell conditions" do+ it "requires all five passing gates for each machine and a verified cold reset" do+ identifier <- newRunId+ let health = report identifier (concatMap gates machines)+ reset = resetReport identifier+ validateCellConditions identifier "alpha" machines health reset `shouldBe` Right ()+ validateCellConditions identifier "alpha" machines (report identifier []) reset `shouldSatisfy` isLeft+ validateCellConditions identifier "alpha" machines (report identifier (concatMap gates (drop 1 machines))) reset `shouldSatisfy` isLeft+ validateCellConditions identifier "alpha" machines (report identifier (concatMap gates machines <> gates "driver")) reset `shouldSatisfy` isLeft+ it "rejects a lying passed flag, tripped gate and pressure beyond the reported threshold" do+ identifier <- newRunId+ let health values = report identifier (values <> concatMap gates (drop 1 machines))+ reset = resetReport identifier+ entries = gates "driver"+ tripped = map (change "status" (String "tripped")) entries+ overloaded = case entries of+ first : load : rest -> first : change "observed" (object ["settleCpuPercent" .= (95 :: Int), "runStealCpuPercent" .= (0 :: Int)]) load : rest+ _ -> entries+ validateCellConditions identifier "alpha" machines (health tripped) reset `shouldSatisfy` isLeft+ validateCellConditions identifier "alpha" machines (health overloaded) reset `shouldSatisfy` isLeft+ validateCellConditions identifier "alpha" machines (change "passed" (Bool False) (health entries)) reset `shouldSatisfy` isLeft+ it "rejects wrong run identity, warm reset, failed reset and inverted windows" do+ identifier <- newRunId+ other <- newRunId+ let health = report identifier (concatMap gates machines)+ reset = resetReport identifier+ validateCellConditions other "alpha" machines health reset `shouldSatisfy` isLeft+ mapM_+ (\changed -> validateCellConditions identifier "alpha" machines health changed `shouldSatisfy` isLeft)+ [change "cachePolicy" (String "warm") reset, change "verified" (Bool False) reset, change "finishedAt" (String "2026-10-10T23:00:00Z") reset]+ it "reconstructs health from sealed raw samples and rejects scalar laundering" do+ identifier <- newRunId+ let health = rawReport identifier+ windows = rawWindows+ check = validateCellObservations identifier "0.1.0" policy health (executionWindow identifier)+ check windows `shouldBe` Right ()+ let rounded = case windows of (role, machine, image, value) : rest -> (role, machine, image, change "from" (toJSON (addUTCTime (-0.0005) (at 0))) (change "to" (toJSON (addUTCTime (-0.0005) (at 60))) value)) : rest; [] -> []+ -- Role HTTP requests retain millisecond bounds, while the driver has the+ -- original timestamp. Samples must still fall inside the retained window.+ check (case rounded of (role, machine, image, value) : rest -> (role, machine, image, change "samples" (toJSON (map healthSample [0 .. 59])) value) : rest; [] -> []) `shouldBe` Right ()+ check [] `shouldSatisfy` isLeft+ check (drop 1 windows) `shouldSatisfy` isLeft+ let corrupt update = case windows of (role, machine, image, value) : rest -> (role, machine, image, update value) : rest; [] -> []+ check (corrupt (change "image" (String "different-image"))) `shouldSatisfy` isLeft+ check (corrupt (change "agentVersion" (String "stale-agent"))) `shouldSatisfy` isLeft+ check (corrupt (change "maintenanceEvents" (toJSON [object ["at" .= at 20, "value" .= ("maintenance" :: Text)]]))) `shouldSatisfy` isLeft+ check (corrupt (change "samples" (toJSON (filter (\i -> i < 20 || i > 25) [0 .. 60 :: Int] >>= \i -> [healthSample i])))) `shouldSatisfy` isLeft+ check (corrupt (change "samples" (toJSON [change "cpuIdleTicks" (toJSON (900 + i * 50)) (healthSample i) | i <- [0 .. 60 :: Int]]))) `shouldSatisfy` isLeft+ let alteredGates = [change "threshold" (object ["settleCpuMaxPercent" .= (100 :: Int), "runStealCpuMaxPercent" .= (100 :: Int)]) g | g <- gates "driver"] <> concatMap gates (drop 1 machines)+ validateCellObservations identifier "0.1.0" policy (report identifier alteredGates) (executionWindow identifier) windows `shouldSatisfy` isLeft+ it "rejects missing windows, stale sampling, counter reversals and changed execution intervals" do+ identifier <- newRunId+ let check interval windows = validateCellObservations identifier "0.1.0" policy (rawReport identifier) interval windows+ corrupt samples = case rawWindows of (role, machine, image, value) : rest -> (role, machine, image, change "samples" (toJSON samples) value) : rest; [] -> []+ check (executionWindow identifier) (corrupt ([] :: [Value])) `shouldSatisfy` isLeft+ check (executionWindow identifier) (corrupt (map healthSample [1 .. 59])) `shouldBe` Right ()+ check (executionWindow identifier) (corrupt (map healthSample [5 .. 55])) `shouldSatisfy` isLeft+ check (executionWindow identifier) (corrupt (map healthSample [0 .. 30] <> map healthSample [29 .. 60])) `shouldSatisfy` isLeft+ check (change "runFinished" (toJSON (at 59)) (executionWindow identifier)) rawWindows `shouldSatisfy` isLeft++policy :: Value+policy = object ["schema" .= ("cell.policy/v1" :: Text), "health" .= object ["settleCpuMaxPercent" .= (5 :: Int), "runStealCpuMaxPercent" .= (5 :: Int), "startFreeDiskMinPercent" .= (20 :: Int), "runFreeDiskMinPercent" .= (10 :: Int)]]++at :: Int -> UTCTime+at seconds = addUTCTime (fromIntegral seconds) (read "2026-10-11 00:00:00 UTC")++executionWindow :: (ToJSON a) => a -> Value+executionWindow identifier = object ["schema" .= ("cell.execution-window/v1" :: Text), "runId" .= identifier, "resetStarted" .= at 0, "runStarted" .= at 10, "runFinished" .= at 60]++rawReport :: (ToJSON a) => a -> Value+rawReport identifier = report identifier [change "at" (toJSON (at 60)) (if machine == "monitoring" && jsonName gate == Just (String "background-load") then change "observed" (object ["settleCpuPercent" .= (0 :: Int), "runStealCpuPercent" .= (0 :: Int)]) gate else gate) | machine <- machines, gate <- gates machine]+ where+ jsonName (Object value) = KeyMap.lookup "name" value+ jsonName _ = Nothing++rawWindows :: [(Text, Text, Text, Value)]+rawWindows = [(role, role, "image-" <> role, object ["schema" .= ("cell.observations/v1" :: Text), "machine" .= role, "image" .= ("image-" <> role), "agentVersion" .= ("0.1.0" :: Text), "systemClosure" .= ("/nix/store/synthetic-system-identity" :: Text), "from" .= at 0, "to" .= at 60, "samples" .= map healthSample [0 .. 60], "maintenanceEvents" .= ([] :: [Value]), "samplingErrors" .= ([] :: [Value])]) | role <- machines]++healthSample :: Int -> Value+healthSample i = object ["at" .= at i, "cpuTotalTicks" .= (1000 + i * 100), "cpuIdleTicks" .= (990 + i * 99), "cpuStealTicks" .= (0 :: Int), "diskFreePercent" .= (if i <= 10 then 80 else 75 :: Int)]++machines :: [Text]+machines = ["driver", "postgres", "monitoring"]++report :: (ToJSON a) => a -> [Value] -> Value+report identifier entries =+ object+ [ "schema" .= ("cell.health/v1" :: Text),+ "runId" .= identifier,+ "passed" .= True,+ "windowStart" .= ("2026-10-11T00:00:00Z" :: Text),+ "windowEnd" .= ("2026-10-11T00:01:00Z" :: Text),+ "gates" .= entries+ ]++gates :: Text -> [Value]+gates machine =+ [ gate "host-maintenance" (String "NONE") (String "NONE"),+ gate "background-load" (object ["settleCpuPercent" .= (1 :: Int), "runStealCpuPercent" .= (0 :: Int)]) (object ["settleCpuMaxPercent" .= (5 :: Int), "runStealCpuMaxPercent" .= (5 :: Int)]),+ gate "storage-pressure" (object ["startFreePercent" .= (80 :: Int), "runFreePercent" .= (75 :: Int), "outputExceeded" .= False]) (object ["startFreeMinPercent" .= (20 :: Int), "runFreeMinPercent" .= (10 :: Int), "outputExceeded" .= False]),+ gate "incomplete-reset" (Bool True) (Bool True),+ gate "version-mismatch" (Bool True) (Bool True)+ ]+ where+ gate name observed threshold = object ["name" .= (name :: Text), "machine" .= machine, "status" .= ("ok" :: Text), "at" .= ("2026-10-11T00:00:30Z" :: Text), "observed" .= observed, "threshold" .= threshold]++resetReport :: (ToJSON a) => a -> Value+resetReport identifier =+ object+ [ "schema" .= ("cell.reset-evidence/v1" :: Text),+ "runId" .= identifier,+ "cell" .= ("alpha" :: Text),+ "cachePolicy" .= ("cold" :: Text),+ "verified" .= True,+ "startedAt" .= start,+ "finishedAt" .= end,+ "steps" .= [object ["name" .= ("stop-driver-processes" :: Text), "status" .= ("ok" :: Text), "startedAt" .= start, "finishedAt" .= end]]+ ]+ where+ start = "2026-10-11T00:00:00Z" :: Text+ end = "2026-10-11T00:00:05Z" :: Text++change :: Key -> Value -> Value -> Value+change key value (Object fields) = Object (KeyMap.insert key value fields)+change _ _ value = value
+ test/PerfJournalSpec.hs view
@@ -0,0 +1,153 @@+module PerfJournalSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as MapKey+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Perf.Plan+import Kenshou.Perf.Request qualified as Request+import Kenshou.Perf.Result (ConfigurationResult (..))+import Kenshou.Remote.Perf.Journal+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance session budgets" do+ it "replays the synthetic interruption fixture without extending its deadline" do+ session <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-remote/test/golden/perf-session.fixture.json" >>= either fail pure+ state <- right (journalState session)+ state.executionDeadline `shouldBe` Just (time 1810)+ state.admissionSpentSeconds `shouldBe` 30+ it "retains proposed claims and anchors confirmation or owned release to their original time" do+ session <- fixture+ profile <- firstProfile session+ identifier <- newRunId+ other <- newRunId+ let identity = LeaseIdentity "alpha" identifier+ proposed <- apply 10 (LeaseClaimProposed profile identity) session+ restored <- right (eitherDecode (encode proposed) :: Either String PerfSession)+ state <- right (journalState restored)+ state.pendingClaim `shouldBe` Just (profile, identity, time 10)+ appendEvent (SessionEvent (time 11) (LeaseClaimProposed profile identity)) restored `shouldSatisfy` isLeft+ appendEvent (SessionEvent (time 11) (LeaseAcquired (LeaseIdentity "alpha" other))) restored `shouldSatisfy` isLeft+ confirmed <- apply 20 (LeaseAcquired identity) restored >>= right . journalState+ confirmed.executionDeadline `shouldBe` Just (time 1810)+ confirmed.pendingClaim `shouldBe` Nothing+ withdrawn <- apply 21 LeaseClaimWithdrawn restored >>= right . journalState+ withdrawn.executionDeadline `shouldBe` Nothing+ released <- apply 21 LeaseClaimReleased restored+ next <- apply 30 (LeaseAcquired (LeaseIdentity "alpha" other)) released >>= right . journalState+ next.executionDeadline `shouldBe` Just (time 1810)+ appendEvent (SessionEvent (time 1811) (LeaseAcquired identity)) restored `shouldSatisfy` isLeft+ it "keeps spent admission allowance through serialization and repeated interruption" do+ session <- fixture+ waited <- apply 1 (AdmissionReserved 60) session+ restored <- right (eitherDecode (encode waited) :: Either String PerfSession)+ appendEvent (SessionEvent (time 2) (AdmissionReserved 31)) restored `shouldSatisfy` isLeft+ final <- apply 2 (AdmissionReserved 30) restored >>= right . journalState+ final.admissionSpentSeconds `shouldBe` 90+ it "retains its first deadline and refuses a fresh near-deadline comparison after lease loss" do+ session <- fixture+ firstLease <- newRunId+ nextLease <- newRunId+ acquired <- apply 10 (LeaseAcquired (LeaseIdentity "alpha" firstLease)) session+ name <- firstProfile acquired+ started <- apply 11 (ComparisonStarted name 0) acquired+ lost <- apply 1790 (LeaseLost "client lease expired") started+ restored <- right (eitherDecode (encode lost) :: Either String PerfSession)+ reacquired <- apply 1791 (LeaseAcquired (LeaseIdentity "alpha" nextLease)) restored+ state <- right (journalState reacquired)+ state.executionDeadline `shouldBe` Just (time 1810)+ state.cleanupDeadline `shouldBe` Just (time 1870)+ appendEvent (SessionEvent (time 1792) (ComparisonStarted name 1)) reacquired `shouldSatisfy` isLeft+ appendEvent (SessionEvent (time 1810) (ReplacementReserved name)) reacquired `shouldSatisfy` isLeft+ it "starts a separate attempt after loss and preserves replacement counts and runtime blockers" do+ session <- fixture+ lease <- newRunId+ acquired <- apply 1 (LeaseAcquired (LeaseIdentity "alpha" lease)) session+ name <- firstProfile acquired+ started <- apply 2 (ComparisonStarted name 0) acquired+ invalid <- apply 3 (ReplacementReserved name) started+ blocked <- apply 4 (RuntimeBlocked name "known defect reproduced with exit zero") invalid+ lost <- apply 5 (LeaseLost "reset health failure") blocked+ nextLease <- newRunId+ continued <- apply 6 (LeaseAcquired (LeaseIdentity "alpha" nextLease)) lost >>= apply 7 (ComparisonStarted name 1)+ state <- right (journalState continued)+ Map.lookup name state.attempts `shouldBe` Just 2+ Map.lookup name state.replacements `shouldBe` Just 1+ state.activeComparison `shouldBe` Just (name, 1)+ twice <- apply 8 (ReplacementReserved name) continued+ appendEvent (SessionEvent (time 9) (ReplacementReserved name)) twice `shouldSatisfy` isLeft+ entry <- selectedProfile twice+ let comparison = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= ("pass" :: Text), "pairCount" .= (5 :: Int)]+ result = ConfigurationResult name entry.workloadSha256 "pass" "healthy" [] 5 5 [] (Just comparison)+ finished <- apply 10 (ComparisonCompleted result) twice+ final <- right (journalState finished)+ fmap (.runtimeFailures) (Map.lookup name final.completed) `shouldBe` Just ["known defect reproduced with exit zero"]+ appendEvent (SessionEvent (time 11) (ComparisonStarted name 2)) finished `shouldSatisfy` isLeft+ it "rejects overlapping leases, backward clocks, unknown profiles and reset deadlines" do+ session <- fixture+ lease <- newRunId+ acquired <- apply 10 (LeaseAcquired (LeaseIdentity "alpha" lease)) session+ appendEvent (SessionEvent (time 11) (LeaseAcquired (LeaseIdentity "alpha" lease))) acquired `shouldSatisfy` isLeft+ appendEvent (SessionEvent (time 9) LeaseReleased) acquired `shouldSatisfy` isLeft+ appendEvent (SessionEvent (time 11) (ComparisonStarted "unselected" 0)) acquired `shouldSatisfy` isLeft+ appendEvent (SessionEvent (time 11) (AdmissionReserved 1)) acquired `shouldSatisfy` isLeft+ it "recovers a sealed blocker after lease loss or comparison completion without duplicating it" do+ session <- fixture+ name <- firstProfile session+ appendEvent (SessionEvent (time 1) (RuntimeBlocked name "sealed defect")) session `shouldSatisfy` isLeft+ identifier <- newRunId+ acquired <- apply 1 (LeaseAcquired (LeaseIdentity "alpha" identifier)) session+ started <- apply 2 (ComparisonStarted name 0) acquired+ lost <- apply 3 (LeaseLost "interrupted before runtime checkpoint") started+ recovered <- apply 4 (RuntimeBlocked name "sealed defect") lost+ state <- right (journalState recovered)+ Map.lookup name state.runtimeBlockers `shouldBe` Just ["sealed defect"]+ entry <- selectedProfile started+ let comparison = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= ("pass" :: Text), "pairCount" .= (5 :: Int)]+ result = ConfigurationResult name entry.workloadSha256 "pass" "healthy" [] 5 5 [] (Just comparison)+ finished <- apply 3 (ComparisonCompleted result) started+ late <- apply 4 (RuntimeBlocked name "sealed defect") finished >>= apply 5 (RuntimeBlocked name "sealed defect")+ final <- right (journalState late)+ fmap (.runtimeFailures) (Map.lookup name final.completed) `shouldBe` Just ["sealed defect"]+ it "persists the transition journal atomically and refuses changed request purpose" $ withSystemTempDirectory "perf-journal" \directory -> do+ session <- fixture+ writePerfSession (directory </> "session.json") session+ readPerfSession (directory </> "session.json") `shouldReturn` Right session+ changed <- apply 1 (AdmissionReserved 10) session+ writePerfSession (directory </> "session.json") changed+ writePerfSession (directory </> "session.json") session `shouldThrow` anyIOException+ writePerfSession (directory </> "session.json") changed {evidenceBaseUri = "gs://different-evidence/performance"} `shouldThrow` anyIOException+ let mutate (Object document) = Object (MapKey.insert "schema" (String "unknown") document)+ mutate value = value+ (eitherDecode (encode (mutate (toJSON session))) :: Either String PerfSession) `shouldSatisfy` isLeft+ let pinned = session.plan+ (eitherDecode (encode session {plan = pinned {request = pinned.request {Request.purpose = "gating"}}}) :: Either String PerfSession) `shouldSatisfy` isLeft++fixture :: IO PerfSession+fixture = do+ plan <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either fail pure+ identifier <- newRunId+ right (initialSession identifier ("sha256:" <> "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") plan "gs://fixture-evidence/performance" (time 0))++time :: Integer -> UTCTime+time seconds = addUTCTime (fromInteger seconds) (UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0))++apply :: Integer -> SessionAction -> PerfSession -> IO PerfSession+apply seconds action = right . appendEvent (SessionEvent (time seconds) action)++selectedProfile :: PerfSession -> IO PlannedProfile+selectedProfile session = case session.plan.selected of+ entry : _ -> pure entry+ [] -> fail "fixture must select a profile"++firstProfile :: PerfSession -> IO Text+firstProfile session = (.name) . (.profile) <$> selectedProfile session++right :: (Show e) => Either e a -> IO a+right = either (fail . show) pure
+ test/PerfLockSpec.hs view
@@ -0,0 +1,31 @@+module PerfLockSpec (spec) where++import Control.Exception (IOException, try)+import Kenshou.Remote.Perf.Lock+import System.Directory (createDirectory, createDirectoryLink)+import System.Environment (getExecutablePath)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (proc, readCreateProcessWithExitCode)+import Test.Hspec++spec :: Spec+spec = describe "performance client locking" do+ it "fences another process and releases ownership after a client exception" $ withSystemTempDirectory "perf-lock" \directory -> do+ executable <- getExecutablePath+ withPerformanceLock directory do+ (code, _, _) <- readCreateProcessWithExitCode (proc executable ["--perf-lock", directory]) ""+ code `shouldBe` ExitFailure 1+ _ <- try @IOException (withPerformanceLock directory (ioError (userError "interrupted client")))+ (code, output, _) <- readCreateProcessWithExitCode (proc executable ["--perf-lock", directory]) ""+ code `shouldBe` ExitSuccess+ output `shouldBe` "locked\n"+ it "fences two clients in one process, including directory aliases" $ withSystemTempDirectory "perf-lock-alias" \root -> do+ let target = root </> "target"+ alias = root </> "alias"+ createDirectory target+ createDirectoryLink target alias+ withPerformanceLock target $+ withPerformanceLock alias (pure ()) `shouldThrow` anyIOException+ withPerformanceLock target (pure ()) `shouldReturn` ()
+ test/PerfPublicationSpec.hs view
@@ -0,0 +1,60 @@+module PerfPublicationSpec (spec) where++import Data.Aeson (encode, object, (.=))+import Data.ByteString.Lazy.Char8 qualified as Bytes+import Data.Either (isLeft)+import Data.Text (Text)+import Data.Time (getCurrentTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (newRunId)+import Kenshou.Perf.Artifacts (sealPerfDirectory)+import Kenshou.Remote.Perf.Publication+import Kenshou.Remote.Store+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (createDirectory, doesPathExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance evidence publication" do+ it "retrieves every artifact, permits an identical retry and rejects a changed existing object" $ withSystemTempDirectory "perf-publication" \root -> do+ let directory = root </> "evidence"+ bucket = Bucket "evidence"+ prefix = "performance/fixture"+ createDirectory directory+ Bytes.writeFile (directory </> "result.json") "retained result"+ sealPerfDirectory directory >>= either (fail . show) pure+ store <- newFileStore (root </> "store")+ published <- publishEvidence store bucket prefix directory+ published `shouldBe` Right "gs://evidence/performance/fixture/manifest.json"+ publishEvidence store bucket prefix directory `shouldReturn` published+ verifyPublication store bucket prefix directory `shouldReturn` Right ()+ digest <- sha256Hex . Bytes.toStrict <$> Bytes.readFile (directory </> "manifest.json")+ retrieveEvidence store bucket prefix digest (root </> "retrieved") `shouldReturn` Right ()+ Bytes.readFile (root </> "retrieved/result.json") `shouldReturn` "retained result"+ retrieveEvidence store bucket prefix "sha256:wrong" (root </> "wrong") >>= (`shouldSatisfy` isLeft)+ current <- store.statObject bucket (ObjectName (prefix <> "/result.json")) >>= maybe (fail "missing published result") pure+ _ <- store.putObject bucket (ObjectName (prefix <> "/result.json")) "application/json" (GenerationIs current.generation) "changed result"+ verifyPublication store bucket prefix directory >>= (`shouldSatisfy` isLeft)+ publishEvidence store bucket prefix directory >>= (`shouldSatisfy` isLeft)+ it "rejects local tampering and unsafe destinations before uploading any object" $ withSystemTempDirectory "perf-publication-rejection" \root -> do+ let directory = root </> "evidence"+ bucket = Bucket "evidence"+ createDirectory directory+ Bytes.writeFile (directory </> "result.json") "sealed"+ sealPerfDirectory directory >>= either (fail . show) pure+ store <- newFileStore (root </> "store")+ publishEvidence store bucket "performance/../escape" directory >>= (`shouldSatisfy` isLeft)+ store.listObjects bucket "" `shouldReturn` []+ Bytes.appendFile (directory </> "result.json") "tampered"+ publishEvidence store bucket "performance/fixture" directory >>= (`shouldSatisfy` isLeft)+ store.listObjects bucket "" `shouldReturn` []+ it "rejects a pinned manifest containing an escaping path before creating its target" $ withSystemTempDirectory "perf-retrieval-escape" \root -> do+ store <- newFileStore (root </> "store")+ identifier <- newRunId+ now <- getCurrentTime+ let manifest = encode (object ["schema" .= ("kenshou.artifact-manifest/v1" :: Text), "algorithm" .= ("sha256" :: Text), "runId" .= identifier, "createdAt" .= now, "files" .= [object ["path" .= ("../escape" :: Text), "sha256" .= ("sha256:wrong" :: Text), "bytes" .= (1 :: Int), "mediaType" .= ("text/plain" :: Text)]]])+ _ <- store.putObject (Bucket "evidence") (ObjectName "performance/fixture/manifest.json") "application/json" DoesNotExist manifest+ retrieveEvidence store (Bucket "evidence") "performance/fixture" (sha256Hex (Bytes.toStrict manifest)) (root </> "retrieved") >>= (`shouldSatisfy` isLeft)+ doesPathExist (root </> "escape") `shouldReturn` False
+ test/PerfRunnerSpec.hs view
@@ -0,0 +1,310 @@+module PerfRunnerSpec (spec) where++import Control.Concurrent (threadDelay)+import Data.Aeson+import Data.IORef+import Data.Map.Strict qualified as Map+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)+import Kenshou.Perf.Plan+import Kenshou.Perf.Prepared qualified as Prepared+import Kenshou.Perf.Result+import Kenshou.Perf.Result qualified as Result+import Kenshou.Remote.Cell.Lease qualified as Lease+import Kenshou.Remote.Perf.Journal+import Kenshou.Remote.Perf.Runner+import Kenshou.Remote.Store (Bucket (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance runner orchestration" do+ it "persists a claim before CAS and reconciles a timed-out accepted claim without redispatch" $ withSystemTempDirectory "perf-admission-ambiguous" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ store <- newFileStore (directory </> "store")+ identifier <- newRunId+ writes <- newIORef (0 :: Int)+ let identity = LeaseIdentity "alpha" identifier+ ref = Lease.CellRef "alpha" (Bucket "control")+ stalled =+ hooks+ { admit = \entry _ pending callbacks -> do+ pending `shouldBe` Nothing+ callbacks.proposeClaim identity+ state <- readPerfSession path >>= right >>= right . journalState+ state.pendingClaim `shouldBe` Just (entry.profile.name, identity, now)+ claimed <- Lease.acquireLeaseWithId store ref (Lease.LeaseRequest "fixture" "ambiguous transport control" 180) identifier+ case claimed of Lease.Acquired _ -> pure (); _ -> fail "fixture claim failed"+ modifyIORef' writes (+ 1)+ threadDelay 60000000+ pure (Admitted identity),+ releaseOwnedLease = \_ -> pure False+ }+ failed <- runPerformance stalled prepared path+ failed.exitCode `shouldBe` 4+ retained <- readPerfSession path >>= right >>= right . journalState+ retained.pendingClaim `shouldSatisfy` isJust+ readIORef calls `shouldReturn` []+ let resumed =+ hooks+ { admit = \_ _ pending _ -> do+ case pending of Just (_, proposed, _) -> proposed `shouldBe` identity; _ -> fail "lost retained proposal"+ active <- Lease.reattachActiveLease store ref identifier+ isJust active `shouldBe` True+ pure (Admitted identity),+ ownsLease = \_ -> isJust <$> Lease.reattachActiveLease store ref identifier,+ releaseOwnedLease = \_ -> Lease.reattachLease store ref identifier >>= maybe (pure False) (Lease.releaseLease store ref)+ }+ recovered <- runPerformance resumed prepared path+ recovered.exitCode `shouldBe` 0+ readIORef writes `shouldReturn` 1+ final <- readPerfSession path >>= right >>= right . journalState+ final.pendingClaim `shouldBe` Nothing+ final.executionDeadline `shouldBe` Just (addUTCTime 1800 now)+ it "pins the deadline when cleanup confirms release of an unacknowledged claim" $ withSystemTempDirectory "perf-admission-release" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ identifier <- newRunId+ clockRef <- newIORef now+ let late =+ hooks+ { clock = readIORef clockRef,+ admit = \_ _ _ callbacks -> do+ callbacks.proposeClaim (LeaseIdentity "alpha" identifier)+ writeIORef clockRef (addUTCTime 11 now)+ pure (Admitted (LeaseIdentity "alpha" identifier))+ }+ result <- runPerformance late prepared path+ result.exitCode `shouldBe` 4+ state <- readPerfSession path >>= right >>= right . journalState+ state.pendingClaim `shouldBe` Nothing+ state.executionDeadline `shouldBe` Just (addUTCTime 1800 now)+ readIORef calls `shouldReturn` []+ it "collects sealed runtime failures under a lost lease before starting a fresh attempt" $ withSystemTempDirectory "perf-lost-seal" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ first <- newRunId+ session <- readPerfSession path >>= right+ entry <- case prepared.plan.selected of value : _ -> pure value; _ -> fail "empty fixture"+ acquired <- right (appendEvent (SessionEvent now (LeaseAcquired (LeaseIdentity "alpha" first))) session)+ started <- right (appendEvent (SessionEvent now (ComparisonStarted entry.profile.name 0)) acquired)+ writePerfSession path started+ let recovered =+ hooks+ { ownsLease = \identity -> pure (identity.leaseId /= first),+ recoverRuntimeFailures = \identity -> pure [(entry.profile.name, "sealed exit-zero defect") | identity.leaseId == first]+ }+ result <- runPerformance recovered prepared path+ result.exitCode `shouldBe` 1+ result.blockingRuntimeFailures `shouldBe` ["sealed exit-zero defect"]+ readIORef calls >>= \executed -> map (\(_, attempt, _) -> attempt) executed `shouldBe` [1, 0]+ it "executes both pinned configurations and reuses completed evidence on resume" $ withSystemTempDirectory "perf-runner" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ result <- runPerformance hooks prepared path+ result.exitCode `shouldBe` 0+ length result.configurations `shouldBe` 2+ result.gatingEligible `shouldBe` False+ resumed <- runPerformance hooks prepared path+ resumed `shouldBe` result+ readIORef calls >>= \executed -> length executed `shouldBe` 2+ it "persists runtime blockers and retries a whole configuration after interruption" $ withSystemTempDirectory "perf-runner" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ let interrupted =+ hooks+ { compareConfiguration = \_ _ _ _ callbacks -> do+ callbacks.reserveReplacement 5+ callbacks.recordRuntimeFailure "known defect with exit zero"+ ioError (userError "simulated lost connection")+ }+ failed <- runPerformance interrupted prepared path+ failed.exitCode `shouldBe` 4+ restored <- readPerfSession path >>= right+ interruptedState <- right (journalState restored)+ Map.elems interruptedState.replacements `shouldBe` [1]+ resumed <- runPerformance hooks prepared path+ resumed.exitCode `shouldBe` 1+ resumed.blockingRuntimeFailures `shouldBe` ["known defect with exit zero"]+ resumedState <- readPerfSession path >>= right >>= right . journalState+ resumedState.executionDeadline `shouldBe` interruptedState.executionDeadline+ Map.elems resumedState.replacements `shouldBe` [1]+ readIORef calls >>= \executed -> map (\(_, index, _) -> index) executed `shouldBe` [1, 0]+ it "returns infrastructure failure when durable publication throws or rejects verification" $ withSystemTempDirectory "perf-runner" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ result <- runPerformance hooks {publishAndVerify = \_ _ -> ioError (userError "storage unavailable")} prepared path+ result.exitCode `shouldBe` 4+ recovered <- runPerformance hooks prepared path+ recovered.exitCode `shouldBe` 0+ invalid <- runPerformance hooks {publishAndVerify = \_ _ -> pure False} prepared path+ invalid.exitCode `shouldBe` 4+ it "rejects a report decorator that changes a performance decision" $ withSystemTempDirectory "perf-runner-report" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ published <- newIORef False+ result <- runPerformance hooks {reportMetadata = \_ report -> pure report {Result.changeCoverage = "complete"}, publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path+ result.exitCode `shouldBe` 4+ readIORef published `shouldReturn` False+ it "rejects changed immutable preparation before any admission" $ withSystemTempDirectory "perf-runner" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ runPerformance hooks prepared {Prepared.harnessRevision = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} path `shouldThrow` anyIOException+ readIORef calls `shouldReturn` []+ it "does not extend cleanup or publication time when a finished session is resumed" $ withSystemTempDirectory "perf-runner-deadline" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ original <- runPerformance hooks prepared path+ original.exitCode `shouldBe` 0+ published <- newIORef False+ result <- runPerformance hooks {clock = pure (addUTCTime 1870 now), publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path+ result.exitCode `shouldBe` 4+ readIORef published `shouldReturn` False+ it "does not publish during cleanup grace after the execution deadline" $ withSystemTempDirectory "perf-publication-grace" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ runPerformance hooks prepared path >>= \result -> result.exitCode `shouldBe` 0+ published <- newIORef False+ result <- runPerformance hooks {clock = pure (addUTCTime 1830 now), publishAndVerify = \_ _ -> writeIORef published True >> pure True} prepared path+ result.exitCode `shouldBe` 4+ readIORef published `shouldReturn` False+ it "rejects publication that completes beyond the original execution deadline" $ withSystemTempDirectory "perf-publication-late" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ current <- newIORef now+ result <- runPerformance hooks {clock = readIORef current, publishAndVerify = \_ _ -> writeIORef current (addUTCTime 1801 now) >> pure True} prepared path+ result.exitCode `shouldBe` 4+ result.evidenceVerified `shouldBe` False+ it "reserves an identified replacement once when the same callback is reconciled" $ withSystemTempDirectory "perf-runner-reservation" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ let reconciled =+ hooks+ { compareConfiguration = \lease entry index resuming callbacks -> do+ callbacks.reserveReplacement entry.profile.minimumPairs+ callbacks.reserveReplacement entry.profile.minimumPairs+ hooks.compareConfiguration lease entry index resuming callbacks+ }+ result <- runPerformance reconciled prepared path+ result.exitCode `shouldBe` 0+ state <- readPerfSession path >>= right >>= right . journalState+ Map.elems state.replacements `shouldBe` [1, 1]+ length state.replacementIdentities `shouldBe` 2+ it "does not restart cancellation or release grace for an expired interrupted session" $ withSystemTempDirectory "perf-expired-cleanup" \directory -> do+ (prepared, path) <- fixture directory+ session <- readPerfSession path >>= right+ identifier <- newRunId+ acquired <- right (appendEvent (SessionEvent now (LeaseAcquired (LeaseIdentity "alpha" identifier))) session)+ active <- right (appendEvent (SessionEvent now (ComparisonStarted (firstProfile prepared) 0)) acquired)+ writePerfSession path active+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ cleanupCalls <- newIORef (0 :: Int)+ hooks <- fixtureHooks calls+ result <- runPerformance hooks {clock = pure (addUTCTime 1870 now), cancelOwnedWork = \_ -> modifyIORef' cleanupCalls (+ 1), releaseOwnedLease = \_ -> modifyIORef' cleanupCalls (+ 1) >> pure True} prepared path+ result.exitCode `shouldBe` 4+ readIORef calls `shouldReturn` []+ readIORef cleanupCalls `shouldReturn` 0+ it "releases an incompatible lease before admitting the next configuration" $ withSystemTempDirectory "perf-runner-cells" \directory -> do+ (prepared, path) <- fixture directory+ calls <- newIORef ([] :: [(Text, Int, Bool)])+ hooks <- fixtureHooks calls+ admissions <- newIORef (0 :: Int)+ releases <- newIORef (0 :: Int)+ first <- newRunId+ second <- newRunId+ let changed =+ hooks+ { admit = \_ _ _ _ -> do+ count <- atomicModifyIORef' admissions (\n -> (n + 1, n))+ pure (Admitted (LeaseIdentity "alpha" (if count == 0 then first else second))),+ canRunConfiguration = \lease entry -> pure (lease.leaseId /= first || entry.profile.name == firstProfile prepared),+ releaseOwnedLease = \_ -> modifyIORef' releases (+ 1) >> pure True+ }+ result <- runPerformance changed prepared path+ result.exitCode `shouldBe` 0+ readIORef admissions `shouldReturn` 2+ readIORef releases `shouldReturn` 2+ state <- readPerfSession path >>= right >>= right . journalState+ state.executionDeadline `shouldBe` Just (addUTCTime 1800 now)++fixture :: FilePath -> IO (Prepared.PerfPrepared, FilePath)+fixture directory = do+ prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= either fail pure+ identifier <- newRunId+ session <- right (initialSession identifier (Prepared.preparedDigest prepared) prepared.plan "gs://fixture-evidence/performance" now)+ let path = directory </> "session.json"+ writePerfSession path session+ pure (prepared, path)++fixtureHooks :: IORef [(Text, Int, Bool)] -> IO RunnerHooks+fixtureHooks calls = do+ contracts <- fixtureContracts+ lease <- newRunId+ pure+ RunnerHooks+ { contracts,+ clock = pure now,+ admit = \_ _ _ _ -> pure (Admitted (LeaseIdentity "alpha" lease)),+ ownsLease = \_ -> pure True,+ canRunConfiguration = \_ _ -> pure True,+ compareConfiguration = \_ entry index resuming _ -> do+ modifyIORef' calls (<> [(entry.profile.name, index, resuming)])+ let comparison = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= ("pass" :: Text), "pairCount" .= entry.profile.minimumPairs]+ pure (ConfigurationResult entry.profile.name entry.workloadSha256 "pass" "healthy" [] entry.profile.minimumPairs entry.profile.minimumPairs [] (Just comparison)),+ verifyCompleted = \_ -> pure True,+ cancelOwnedWork = \_ -> pure (),+ releaseOwnedLease = \_ -> pure True,+ recoverRuntimeFailures = \_ -> pure [],+ reportMetadata = \_ result -> pure result,+ publishAndVerify = \_ _ -> pure True+ }++now :: UTCTime+now = UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0)++right :: (Show e) => Either e a -> IO a+right = either (fail . show) pure++-- The captured preparation pins the first consumer's reviewed benchmark revisions;+-- its contracts are data in the consumer-baseline bridge.+fixtureContracts :: IO PerfContracts+fixtureContracts = do+ document <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/perf-contracts.json" >>= either fail pure+ let entries = [(entry.scenario, entry.revision) | entry <- (document :: ContractsDocument).reviewed]+ countKnob <- either (fail . show) pure (mkKnobName document.countKnob)+ pure noPerfContracts {reviewedPhaseContracts = entries, steadyCountKnob = Just countKnob}++data ContractsDocument = ContractsDocument {reviewed :: [ReviewedContract], countKnob :: Text}++data ReviewedContract = ReviewedContract {scenario :: Text, revision :: Int}++instance FromJSON ContractsDocument where+ parseJSON = withObject "PerfContracts" \value -> ContractsDocument <$> value .: "reviewedPhaseContracts" <*> value .: "steadyCountKnob"++instance FromJSON ReviewedContract where+ parseJSON = withObject "ReviewedPhaseContract" \value -> ReviewedContract <$> value .: "scenario" <*> value .: "revision"++-- The first profile the captured preparation selects.+firstProfile :: Prepared.PerfPrepared -> Text+firstProfile prepared = case prepared.plan.selected of+ entry : _ -> entry.profile.name+ [] -> error "the captured preparation selects no profile"
+ test/PrepareSpec.hs view
@@ -0,0 +1,441 @@+module PrepareSpec (spec) where++import Data.Aeson (Value (..), eitherDecode, eitherDecodeFileStrict', encode, toJSON)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (toList)+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Bundle (LayerBundle (..), lookupScenario, mkBundle, mkRegistry, registryRoutingRules)+import Kenshou.Core.Bundle qualified as Bundle+import Kenshou.Core.Dimension (PgDurability (..), PgVersion (..), noDimensions, postgresDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (Kind (..), mkSeed, newRunId, parseScenarioId, renderKind, selftestLayer)+import Kenshou.Core.Knob (RawKnob (..), mkKnobName)+import Kenshou.Core.Resource (CellBinding (..), ProvisionedResource (..), ResourceProvider (..), mkResourceName)+import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), minimalRunSpec)+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Core.Selftest qualified as Selftest+import Kenshou.Core.Selftest.Schema (selftestMigrationSet)+import Kenshou.Plan.Change (Change (..), ChangeSource (..), Reason (..))+import Kenshou.Plan.Components (ComponentId (..), ComponentRef (..))+import Kenshou.Plan.Policy (defaultPlanPolicy)+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlannedRun (PlannedRun), RunPlan (..))+import Kenshou.Plan.Selector (parseSelector)+import Kenshou.Remote.Cell.Docs (CachePolicy (..), CellBuckets (..), CellDescriptor (..), CellImages (..), CollectOptions (..), Limits (..), PgReset (..), ResetBlock (..), Submission (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Prepare (Granularity (..), OtlpSink (..), PrepareOptions (..), Prepared (..), PreparedRun (..), RejectReason (..), Routed (..), Slice (..), SubmissionInputs (..), prepareForCell, routePlan, slicePlan, sliceRuns, submissionFor)+import Kenshou.Remote.Cell.Prepare qualified as Prepare+import Kenshou.Remote.Cell.RouteJson (RouteDocuments (..), routeWorkJson)+import Kenshou.Remote.Cell.RouteRules (CellCapabilities (..), RoutingRule (..), RuleCondition (..), descriptorDigest, loadCellCapabilities, matchingRules)+import Kenshou.Remote.Cell.Session.Build (BuildOptions (..), BuiltSession (..), buildSession)+import Kenshou.Remote.Cell.Session.Journal (SessionJournal (..), SliceJournal (..))+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Cell.WorkJson (decodeWorkPlan, renderPreparedWork)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))+import Kenshou.Remote.Store (Bucket (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell run slicing" do+ it "invalidates machine profiles when cell images change and preserves equivalent hardware profiles" do+ (_, descriptor, _) <- preparationFixture+ let original = Prepare.cellMachineProfile descriptor+ Prepare.cellMachineProfile descriptor {name = "same-shape-other-cell"} `shouldBe` original+ Prepare.cellMachineProfile descriptor {images = descriptor.images {driver = "different-driver-image"}} `shouldNotBe` original+ Prepare.cellMachineProfile descriptor {images = descriptor.images {postgres = "different-postgres-image"}} `shouldNotBe` original++ it "prepares durable PostgreSQL work and records reset settings and payload cohort" do+ (plan, descriptor, payload) <- preparationFixture+ let registry = either (error . show) id (mkRegistry [Selftest.bundle])+ payloads = Map.singleton "default" payload+ options = PrepareOptions False False [] Cold+ rejected = prepareForCell registry descriptor Nothing [] payloads options plan+ accepted = prepareForCell registry descriptor Nothing [] payloads (options {coerceDurable = True}) plan+ fmap snd rejected.rejected `shouldBe` [DurabilityNotDurable]+ case accepted.accepted of+ [run] -> do+ run.spec.environment.postgres `shouldBe` Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))+ case run.reset.postgres of+ Just reset -> do+ Map.lookup "fsync" reset.settings `shouldBe` Just "on"+ Map.lookup "synchronous_commit" reset.settings `shouldBe` Just "on"+ Nothing -> expectationFailure "expected PostgreSQL reset"+ run.spec.cohortExpectation `shouldSatisfy` (/= Nothing)+ accepted.warnings `shouldSatisfy` (not . null)+ _ -> expectationFailure "expected one prepared run"+ fmap snd (prepareForCell registry (descriptor {postgresMajor = 17}) Nothing [] payloads (options {coerceDurable = True}) plan).rejected `shouldBe` [PgVersionMismatch 18 17]+ fmap snd (prepareForCell registry descriptor Nothing [] payloads (options {coerceDurable = True, pgSettings = [("fsync", "off")]}) plan).rejected `shouldBe` [ConflictingPostgresSetting "fsync"]++ it "keeps image-owned preloaded libraries out of PostgreSQL reset settings" do+ (plan, descriptor, payload) <- preparationFixture+ let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+ scenarioId = case plan.runs of+ [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario+ _ -> error "expected one planned run"+ original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)+ requirement = maybe (error "missing PostgreSQL requirement") id original.requires.postgres+ required = PostgresRequirement requirement.schemas [("shared_preload_libraries", "'pg_stat_statements'")] requirement.needsServerControl+ withPreload = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements (Just required) original.requires.extraPostgres original.requires.resources) original.knownDefect original.run+ registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [withPreload] []) {migrations = [selftestMigrationSet]}])+ prepared = prepareForCell registry descriptor Nothing [] (Map.singleton "default" payload) (PrepareOptions True False [] Cold) plan+ case prepared.accepted of+ [run] -> case run.reset.postgres of+ Just reset -> Map.member "shared_preload_libraries" reset.settings `shouldBe` False+ Nothing -> expectationFailure "expected PostgreSQL reset"+ _ -> expectationFailure "expected one prepared run"++ it "keeps the owner PostgreSQL reset path for a worker with no database requirement" do+ (plan, descriptor, payload) <- preparationFixture+ worker <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseScenarioId "selftest/kernel/concurrency/worker-echo")+ let registry = either (error . show) id (mkRegistry [Selftest.bundle])+ payloads = Map.singleton "default" payload+ workerPlan = case plan.runs of+ [PlannedRun ordinal identifier estimate reasons trial _] ->+ let base = minimalRunSpec worker+ workerSpec = RunSpec (Just identifier) worker base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+ in plan {runs = [PlannedRun ordinal identifier estimate reasons trial workerSpec]}+ _ -> error "expected one planned run"+ case (prepareForCell registry descriptor Nothing [] payloads (PrepareOptions False False [] Warm) workerPlan).accepted of+ [preparedRun] -> do+ preparedRun.spec.environment.postgres `shouldBe` Nothing+ preparedRun.reset.postgres `shouldBe` Just (PgReset descriptor.postgresMajor [] Map.empty)+ _ -> expectationFailure "expected one worker run"++ it "rejects a resource the cell cannot bind or does not report" do+ (plan, descriptor, payload) <- preparationFixture+ now <- getCurrentTime+ let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+ scenarioId = case plan.runs of+ [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario+ _ -> error "expected one planned run"+ original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)+ objectStore = either (error . Text.unpack) id (mkResourceName "object-store")+ needsStore = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements original.requires.postgres original.requires.extraPostgres [objectStore]) original.knownDefect original.run+ provider binding = ResourceProvider objectStore "Object store" (Right . maybe Null id) (\_ value action -> action (ProvisionedResource value Null)) binding+ registryWith binding = either (error . show) id (mkRegistry [(mkBundle selftestLayer [needsStore] []) {migrations = [selftestMigrationSet], Bundle.resources = [provider binding]}])+ payloads = Map.singleton "default" payload+ options = PrepareOptions True False [] Cold+ binding = Just (CellBinding "object-store" Nothing)+ reporting = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "object-store" (Bool True))+ fmap snd (prepareForCell (registryWith Nothing) descriptor Nothing [] payloads options plan).rejected `shouldBe` [ResourceUnavailable objectStore]+ fmap snd (prepareForCell (registryWith binding) descriptor Nothing [] payloads options plan).rejected `shouldBe` [ResourceUnavailable objectStore]+ length (prepareForCell (registryWith binding) descriptor (Just reporting) [] payloads options plan).accepted `shouldBe` 1++ it "keeps server-control work off the cell server unless the driver is explicitly selected" do+ (plan, descriptor, payload) <- preparationFixture+ let originalRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+ scenarioId = case plan.runs of+ [PlannedRun _ _ _ _ _ runSpec] -> runSpec.scenario+ _ -> error "expected one planned run"+ original = maybe (error "missing selftest") id (lookupScenario originalRegistry scenarioId)+ requirement = maybe (error "missing PostgreSQL requirement") id original.requires.postgres+ controlled = Scenario original.id original.revision original.summary original.tier original.placement original.knobs original.dimensions original.phases (EnvRequirements (Just (requirement {needsServerControl = True})) original.requires.extraPostgres original.requires.resources) original.knownDefect original.run+ registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [controlled] []) {migrations = [selftestMigrationSet]}])+ payloads = Map.singleton "default" payload+ options = PrepareOptions False False [] Cold+ fmap snd (prepareForCell registry descriptor Nothing [] payloads options plan).rejected `shouldBe` [NeedsServerControl "primary"]+ case (prepareForCell registry descriptor Nothing [] payloads (options {ephemeralOnDriver = True}) plan).accepted of+ [run] -> do+ run.spec.environment.postgres `shouldBe` Just (PostgresEphemeral [])+ run.reset.postgres `shouldBe` Just (PgReset descriptor.postgresMajor [] Map.empty)+ _ -> expectationFailure "expected one driver-local prepared run"+ let routed = routePlan registry ((descriptor, Nothing) :| []) [] payloads options plan+ fmap (.planId) routed.local `shouldBe` Just plan.planId+ Map.null routed.perCell `shouldBe` True++ it "routes PostgreSQL majors to matching cells while preserving plan order and run IDs" do+ (plan, descriptor18, payload) <- preparationFixture+ secondId <- newRunId+ let original = case plan.runs of+ [PlannedRun _ _ _ originalReasons originalTrial originalSpec] -> (originalReasons, originalTrial, originalSpec)+ _ -> error "expected one planned run"+ (reasons, trial, firstSpec) = original+ secondSpec = RunSpec (Just secondId) firstSpec.scenario firstSpec.scenarioRevision firstSpec.knobs [("pg.version", "17")] firstSpec.seed firstSpec.phases firstSpec.timeoutSeconds firstSpec.environment firstSpec.cohortExpectation firstSpec.comparison firstSpec.labels+ second = PlannedRun 1 secondId 1 reasons trial secondSpec+ twoRunPlan = RunPlan plan.planId plan.createdAt plan.context plan.policy (plan.runs <> [second]) plan.skipped 2+ descriptor17 = descriptor18 {name = "beta", postgresMajor = 17}+ baseRegistry = either (error . show) id (mkRegistry [Selftest.bundle])+ scenario = maybe (error "missing selftest") id (lookupScenario baseRegistry firstSpec.scenario)+ bothMajors = Scenario scenario.id scenario.revision scenario.summary scenario.tier scenario.placement scenario.knobs (postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) noDimensions) scenario.phases (EnvRequirements (Just (PostgresRequirement [] [] False)) [] []) scenario.knownDefect scenario.run+ registry = either (error . show) id (mkRegistry [(mkBundle selftestLayer [bothMajors] []) {migrations = [selftestMigrationSet]}])+ routed = routePlan registry ((descriptor17, Nothing) :| [(descriptor18, Nothing)]) [] (Map.singleton "default" payload) (PrepareOptions True False [] Cold) twoRunPlan+ runIds selected = fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) selected.runs+ fmap runIds (Map.lookup "alpha" routed.perCell) `shouldBe` Just [case plan.runs of [PlannedRun _ identifier _ _ _ _] -> identifier; _ -> error "expected one planned run"]+ fmap runIds (Map.lookup "beta" routed.perCell) `shouldBe` Just [secondId]+ routed.local `shouldBe` Nothing+ routed.unroutable `shouldBe` []++ it "routes a public JSON plan while preserving its provenance and reporting refusals" do+ (plan, descriptor, payload) <- preparationFixture+ let registry = either (error . show) id (mkRegistry [Selftest.bundle])+ input = toJSON plan+ route = routeWorkJson registry ((descriptor, Nothing) :| []) [] (Just (Map.singleton "default" payload)) (PrepareOptions True False [] Cold) input+ documents <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure route+ documents.routeComplete `shouldBe` True+ documents.local `shouldBe` Nothing+ case (input, Map.lookup descriptor.name documents.perCell) of+ (Object source, Just (Object selected)) -> do+ KeyMap.lookup "planId" selected `shouldBe` KeyMap.lookup "planId" source+ KeyMap.lookup "createdAt" selected `shouldBe` KeyMap.lookup "createdAt" source+ KeyMap.lookup "policy" selected `shouldBe` KeyMap.lookup "policy" source+ KeyMap.lookup "changes" selected `shouldBe` KeyMap.lookup "changes" source+ KeyMap.lookup "runs" selected `shouldNotBe` KeyMap.lookup "runs" source+ _ -> expectationFailure "expected a routed plan object"+ case input of+ Object source -> do+ let originalRun = case KeyMap.lookup "runs" source of+ Just (Array runs) -> case toList runs of+ [run] -> run+ _ -> error "expected one run"+ _ -> error "missing runs"+ duplicate = Object (KeyMap.insert "runs" (toJSON [originalRun, originalRun]) source)+ routeWorkJson registry ((descriptor, Nothing) :| []) [] (Just (Map.singleton "default" payload)) (PrepareOptions True False [] Cold) duplicate `shouldSatisfy` isLeft+ _ -> expectationFailure "expected a plan object"+ withoutPayload <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (routeWorkJson registry ((descriptor, Nothing) :| []) [] Nothing (PrepareOptions True False [] Cold) input)+ withoutPayload.routeComplete `shouldBe` True+ case Map.lookup descriptor.name withoutPayload.perCell of+ Just (Object selected) -> case KeyMap.lookup "runs" selected of+ Just (Array rows) -> case toList rows of+ [Object row] -> case KeyMap.lookup "spec" row of+ Just (Object selectedSpec) -> KeyMap.lookup "cohortExpectation" selectedSpec `shouldBe` Just Null+ _ -> expectationFailure "expected a selected spec"+ _ -> expectationFailure "expected one selected run"+ _ -> expectationFailure "expected runs"+ _ -> expectationFailure "expected a routed plan"++ it "renders prepared slices from the original plan with reset settings intact" do+ (plan, descriptor, payload) <- preparationFixture+ let input = toJSON plan+ registry = either (error . show) id (mkRegistry [Selftest.bundle])+ options = PrepareOptions True False [] Cold+ decoded <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (decodeWorkPlan input)+ decoded.planId `shouldBe` plan.planId+ let selectedWork = prepareForCell registry descriptor Nothing [] (Map.singleton "default" payload) options decoded+ case selectedWork.accepted of+ [selected] -> do+ rendered <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (renderPreparedWork input [selected])+ case (input, rendered) of+ (Object original, Object slice) -> do+ KeyMap.lookup "policy" slice `shouldBe` KeyMap.lookup "policy" original+ KeyMap.lookup "changes" slice `shouldBe` KeyMap.lookup "changes" original+ KeyMap.lookup "runs" slice `shouldNotBe` KeyMap.lookup "runs" original+ _ -> expectationFailure "expected a rendered slice"+ renderPreparedWork input [selected {ordinal = 99}] `shouldSatisfy` isLeft+ _ -> expectationFailure "expected a prepared run"++ it "builds a durable session and refuses incompatible work before publication" do+ (plan, descriptor, payload) <- preparationFixture+ lease <- newRunId+ let registry = either (error . show) id (mkRegistry [Selftest.bundle])+ payloads = Map.singleton "default" payload+ matchingDescriptor = descriptor {buckets = descriptor.buckets {control = "control"}}+ options coerce = BuildOptions (PrepareOptions coerce False [] Cold) GranularityAuto False NullSink Nothing (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0"+ source = encode plan+ rejected <- buildSession registry matchingDescriptor Nothing [] payloads (options False) "file:fixture" lease source+ rejected `shouldSatisfy` isLeft+ built <- buildSession registry matchingDescriptor Nothing [] payloads (options True) "file:fixture" lease source+ case built of+ Left problem -> expectationFailure (Text.unpack problem)+ Right session -> do+ session.journal.leaseId `shouldBe` lease+ length session.journal.slices `shouldBe` 1+ length session.workFiles `shouldBe` 1+ case (session.journal.slices, session.workFiles) of+ ([slice], [(_, bytes)]) -> do+ slice.runIds `shouldBe` fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) plan.runs+ case eitherDecode bytes of+ Right (Object rendered) -> KeyMap.lookup "policy" rendered `shouldBe` case toJSON plan of Object original -> KeyMap.lookup "policy" original; _ -> Nothing+ _ -> expectationFailure "expected a work plan"+ _ -> expectationFailure "expected one slice and work file"++ it "uses cached denials and warns when a matching capability has not been probed" do+ (plan, descriptor, payload) <- preparationFixture+ now <- getCurrentTime+ selector <- either (error . Text.unpack) pure (parseSelector "selftest/kernel/**")+ let registry = either (error . show) id (mkRegistry [Selftest.bundle])+ rule = RoutingRule (ScenarioMatches selector) "postgres.pg_partman" "requires extension"+ options = PrepareOptions True False [] Cold+ payloads = Map.singleton "default" payload+ unprobed = prepareForCell registry descriptor Nothing [rule] payloads options plan+ deniedCache = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "postgres.pg_partman" (Bool False))+ denied = prepareForCell registry descriptor (Just deniedCache) [rule] payloads options plan+ stale = prepareForCell registry (descriptor {postgresMajor = 17}) (Just deniedCache) [rule] payloads options plan+ length unprobed.accepted `shouldBe` 1+ unprobed.warnings `shouldSatisfy` any (Text.isInfixOf "kenshou cell probe")+ fmap snd denied.rejected `shouldBe` [MissingCapability "postgres.pg_partman"]+ fmap snd stale.rejected `shouldBe` [PgVersionMismatch 18 17]++ it "reads the routing rules registered bundles contribute" do+ scenario <- either (error . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/always-pass")+ knob <- either (error . Text.unpack) pure (mkKnobName "synthetic.queue-kind")+ let rule = RoutingRule (KnobEquals "synthetic.queue-kind" "partitioned") "postgres.pg_partman" "Partitioned queues need pg_partman on the cell PostgreSQL server."+ registry = either (error . show) id (mkRegistry [Selftest.bundle, (mkBundle selftestLayer [] []) {routingRules = [rule]}])+ base = minimalRunSpec scenario+ withKnob = RunSpec base.runId base.scenario base.scenarioRevision [(knob, RawText "partitioned")] base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+ registryRoutingRules registry `shouldBe` [rule]+ fmap (.requires) (matchingRules (registryRoutingRules registry) withKnob) `shouldBe` ["postgres.pg_partman"]+ matchingRules (registryRoutingRules registry) base `shouldBe` []++ it "round-trips a probed capability cache" do+ cache <- eitherDecodeFileStrict' "test/golden/cell-capabilities.json" >>= either fail pure+ eitherDecode (encode cache) `shouldBe` Right (cache :: CellCapabilities)++ it "loads only a valid cache for the current cell descriptor" $+ withSystemTempDirectory "kenshou-capabilities" \directory -> do+ (plan, descriptor, _) <- preparationFixture+ now <- getCurrentTime+ let path = directory </> Text.unpack descriptor.name <> ".capabilities.json"+ cache = CellCapabilities descriptor.name (descriptorDigest descriptor) plan.planId now (Map.singleton "postgres.pg_partman" (Bool False))+ loadCellCapabilities directory descriptor `shouldReturn` Right Nothing+ LazyByteString.writeFile path (encode cache)+ loadCellCapabilities directory descriptor `shouldReturn` Right (Just cache)+ loadCellCapabilities directory (descriptor {postgresMajor = 17}) `shouldReturn` Right Nothing+ LazyByteString.writeFile path "invalid JSON"+ malformed <- loadCellCapabilities directory descriptor+ malformed `shouldSatisfy` isLeft++ it "shares adjacent correctness resets and isolates benchmark and soak runs" do+ first <- prepared 0 Correctness "default" cold+ second <- prepared 1 Concurrency "default" cold+ benchmark <- prepared 2 Benchmark "default" cold+ third <- prepared 3 Correctness "default" cold+ soak <- prepared 4 Soak "default" cold+ fourth <- prepared 5 Correctness "default" cold+ slices <- expectSlices (sliceRuns GranularityAuto [first, second, benchmark, third, soak, fourth])+ fmap (NonEmpty.length . (.entries)) slices `shouldBe` [2, 1, 1, 1, 1]+ fmap (.index) slices `shouldBe` [0, 1, 2, 3, 4]+ fmap (.wallClockSeconds) slices `shouldBe` [320, 310, 310, 910, 310]++ it "leaves bounded time for full-soak drain and diagnosis without extending benchmark limits" do+ soak <- prepared 0 Soak "default" cold+ benchmark <- prepared 1 Benchmark "default" cold+ slices <- expectSlices (sliceRuns GranularityRun [soak {Prepare.timeoutSeconds = 14400}, benchmark {Prepare.timeoutSeconds = 600}])+ fmap (.wallClockSeconds) slices `shouldBe` [15300, 900]++ it "budgets finalization for each soak in a grouped slice and rejects overflow" do+ first <- prepared 0 Soak "default" cold+ second <- prepared 1 Soak "default" cold+ slices <- expectSlices (sliceRuns GranularityPlan [first {Prepare.timeoutSeconds = 14400}, second {Prepare.timeoutSeconds = 14400}])+ fmap (.wallClockSeconds) slices `shouldBe` [30300]+ sliceRuns GranularityRun [first {Prepare.timeoutSeconds = maxBound}] `shouldSatisfy` isLeft++ it "starts a new automatic slice when the payload or reset changes" do+ first <- prepared 0 Correctness "default" cold+ second <- prepared 1 Correctness "candidate" cold+ third <- prepared 2 Correctness "candidate" warm+ fourth <- prepared 3 Correctness "candidate" warm+ slices <- expectSlices (sliceRuns GranularityAuto [first, second, third, fourth])+ fmap (NonEmpty.length . (.entries)) slices `shouldBe` [1, 1, 2]+ fmap (.payloadLabel) slices `shouldBe` ["default", "candidate", "candidate"]++ it "requires a homogeneous plan slice and refuses duplicate identities" do+ first <- prepared 0 Correctness "default" cold+ second <- prepared 1 Correctness "default" cold+ runSlices <- expectSlices (sliceRuns GranularityRun [first, second])+ fmap (NonEmpty.length . (.entries)) runSlices `shouldBe` [1, 1]+ planSlices <- expectSlices (sliceRuns GranularityPlan [first, second])+ fmap (NonEmpty.length . (.entries)) planSlices `shouldBe` [2]+ sliceRuns GranularityPlan [first, PreparedRun second.ordinal second.runId second.spec second.payloadLabel warm second.timeoutSeconds] `shouldBe` Left "plan granularity requires one payload and reset for every run"+ sliceRuns GranularityRun [first, first {ordinal = 1}] `shouldSatisfy` isLeft++ it "keeps the original plan metadata while replacing only selected run specs" do+ first <- prepared 0 Correctness "default" cold+ second <- prepared 1 Correctness "default" cold+ planId <- newRunId+ now <- getCurrentTime+ selector <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseSelector "selftest/**")+ seed <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (mkSeed 1)+ let reason = Reason (Change (ComponentRef (ComponentId "remote") Nothing) Everything "fixture") [] selector 0+ planned run = PlannedRun run.ordinal run.runId 1 (reason :| []) Nothing run.spec+ planContext = PlanContext Nothing "graph" "released" "plan-hash" (PlanInputs Null) [] []+ original = RunPlan planId now planContext (defaultPlanPolicy seed) [planned first, planned second] [] 2+ changed = PreparedRun first.ordinal first.runId (first.spec {RunSpec.timeoutSeconds = Just 42}) first.payloadLabel first.reset first.timeoutSeconds+ slices <- expectSlices (sliceRuns GranularityAuto [changed, second])+ case slices of+ [slice] -> do+ sliced <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (slicePlan original slice)+ sliced.planId `shouldBe` original.planId+ fmap (\(PlannedRun _ identifier _ _ _ _) -> identifier) sliced.runs `shouldBe` [first.runId, second.runId]+ fmap (\(PlannedRun _ _ _ _ _ runSpec) -> runSpec.timeoutSeconds) sliced.runs `shouldBe` [Just 42, Nothing]+ sliced.estimateMinutes `shouldBe` 2+ slicePlan (original {runs = [planned first]}) slice `shouldSatisfy` isLeft+ _ -> expectationFailure "expected one slice"++ it "builds a cell submission with payload transport, limits and session labels" do+ planned <- prepared 0 Correctness "default" cold+ slices <- expectSlices (sliceRuns GranularityRun [planned])+ slice <- case slices of+ [value] -> pure value+ _ -> expectationFailure "expected one slice" >> error "unreachable"+ bytes <- LazyByteString.readFile "test/golden/payload.json"+ payload <- either (\problem -> expectationFailure problem >> error "unreachable") pure (eitherDecode bytes :: Either String PayloadDescriptor)+ cellRun <- newRunId+ lease <- newRunId+ session <- newRunId+ plan <- newRunId+ let inputs = SubmissionInputs cellRun lease session plan FileSink (Just "-N2 -T") (24 * 1024 * 1024 * 1024) (20 * 1024 * 1024 * 1024) "0.1.0" Nothing+ ref = CellRef "alpha" (Bucket "control")+ work = workObjectFor "application/json" "{}"+ submission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref inputs payload slice work)+ submission.runId `shouldBe` cellRun+ submission.payload `shouldBe` payload.cell+ submission.limits.wallClockSeconds `shouldBe` 310+ Map.lookup "KENSHOU_PAYLOAD_BUNDLE_SHA256" submission.env `shouldBe` Just payload.cell.bundle.sha256+ Map.lookup "KENSHOU_OTLP_SINK" submission.env `shouldBe` Just "file"+ fmap (.traces) submission.collect `shouldBe` Just True+ fmap (.profiles) submission.collect `shouldBe` Just False+ Map.lookup "GHCRTS" submission.env `shouldBe` Just "-N2 -T"+ Map.lookup "payload" submission.labels `shouldBe` Just "default"+ defaultSubmission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref (inputs {otlpSink = NullSink, rtsOptions = Nothing}) payload slice work)+ Map.lookup "KENSHOU_OTLP_SINK" defaultSubmission.env `shouldBe` Just "null"+ defaultSubmission.collect `shouldBe` Nothing+ Map.lookup "GHCRTS" defaultSubmission.env `shouldBe` Nothing+ profileSubmission <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (submissionFor ref (inputs {otlpSink = NullSink, rtsOptions = Just "-p"}) payload slice work)+ profileSubmission.collect `shouldBe` Just (CollectOptions False True)+ eitherDecode (encode profileSubmission) `shouldBe` Right profileSubmission+ submissionFor (CellRef "alpha" (Bucket "other")) inputs payload slice work `shouldBe` Left "payload bundle is outside the cell control bucket"+ submissionFor ref inputs payload slice (workObjectFor "text/plain" "{}") `shouldBe` Left "cell work must be a nonempty JSON run plan"++prepared :: Int -> Kind -> Text -> ResetBlock -> IO PreparedRun+prepared ordinal kind label reset = do+ identifier <- newRunId+ scenario <- either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure (parseScenarioId ("selftest/remote/" <> renderKind kind <> "/slice"))+ pure (PreparedRun ordinal identifier (minimalRunSpec scenario) label reset 10)++expectSlices :: Either Text [Slice] -> IO [Slice]+expectSlices = either (\problem -> expectationFailure (Text.unpack problem) >> error "unreachable") pure++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft _ = False++cold :: ResetBlock+cold = ResetBlock Cold Nothing Nothing Map.empty++warm :: ResetBlock+warm = ResetBlock Warm Nothing Nothing Map.empty++preparationFixture :: IO (RunPlan, CellDescriptor, PayloadDescriptor)+preparationFixture = do+ planId <- newRunId+ runId <- newRunId+ now <- getCurrentTime+ selector <- either (error . Text.unpack) pure (parseSelector "selftest/kernel/**")+ scenario <- either (error . Text.unpack) pure (parseScenarioId "selftest/kernel/correctness/postgres-roundtrip")+ seed <- either (error . Text.unpack) pure (mkSeed 7)+ descriptor <- eitherDecodeFileStrict' "test/golden/cell/cell.descriptor.v1.json" >>= either fail pure+ payload <- eitherDecodeFileStrict' "test/golden/payload.json" >>= either fail pure+ let reason = Reason (Change (ComponentRef (ComponentId "kernel") Nothing) Named "cell prepare") [] selector 0+ base = minimalRunSpec scenario+ runSpec = RunSpec (Just runId) scenario base.scenarioRevision base.knobs base.dimensions base.seed base.phases base.timeoutSeconds base.environment base.cohortExpectation base.comparison base.labels+ run = PlannedRun 0 runId 1 (reason :| []) Nothing runSpec+ planContext = PlanContext Nothing "fixture" "released" "sha256:fixture" (PlanInputs Null) [] []+ pure (RunPlan planId now planContext (defaultPlanPolicy seed) [run] [] 1, descriptor, payload)
+ test/SessionSpec.hs view
@@ -0,0 +1,365 @@+module SessionSpec (spec) where++import Control.Concurrent (forkIO, threadDelay)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, takeMVar)+import Control.Exception (SomeException, try)+import Control.Monad (forM_)+import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isLeft)+import Data.IORef (newIORef, readIORef, writeIORef)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import FetchSpec (completeTreeWithNested)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Artifact (..), CellManifest (..), CellOutcome (..), CellPhase (..), CellStatus (..), LogChunks (..), Rejected (..), Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..))+import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), acquireLease, leaseSnapshot, releaseLease, requestCancel)+import Kenshou.Remote.Cell.Session (SessionError (..), runSubmission)+import Kenshou.Remote.Cell.Session qualified as Session+import Kenshou.Remote.Cell.Session.Journal (LeaseMode (..), SessionJournal (..), SliceJournal (..), SliceState (..), applyTransition, readSessionJournal, writeSessionJournal)+import Kenshou.Remote.Cell.Session.Rebind (RebindError (RemoteMarkerExists), rebindPlannedSlices)+import Kenshou.Remote.Cell.Session.Rebind qualified as Rebind+import Kenshou.Remote.Cell.Session.Resume (HeldResumeError (..), resumeHeldSession, resumeObservedSlices)+import Kenshou.Remote.Cell.Session.Runner (SessionRunError (..), runPendingSlices, runPlannedSlices)+import Kenshou.Remote.Cell.Submit (workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.Directory (createDirectoryIfMissing, doesFileExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Timeout (timeout)+import Test.Hspec++spec :: Spec+spec = describe "one leased cell submission" do+ it "round-trips a durable journal and rejects skipped transitions" do+ journal <- readSessionJournal "test/golden/cell-session.json" >>= expectRight+ now <- getCurrentTime+ identifier <- case journal.slices of+ [slice] -> pure slice.cellRun+ _ -> expectationFailure "expected one golden slice" >> error "unreachable"+ applyTransition now identifier (Session.ResultsFetched "tree") journal `shouldSatisfy` isLeft+ submitted <- expectRight (applyTransition now identifier Session.SubmissionPublished journal)+ fmap (.state) submitted.slices `shouldBe` [SliceSubmitted]+ applyTransition now identifier Session.SubmissionPublished submitted `shouldSatisfy` isLeft++ it "reconciles an exact marker after interruption before its publication checkpoint without writing another marker" $ withSystemTempDirectory "kenshou-marker-recovery" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ let path = root </> "out" </> "session.json"+ marker = ObjectName (submissionPrefix submission <> "submission.json")+ (_, _) <- journalCheckpoint path submission+ absent <- resumeObservedSlices store cellRef (Bucket "results") path (const (pure ())) >>= either (fail . show) pure+ fmap (.state) absent.slices `shouldBe` [SlicePlanned]+ store.getObject cellRef.controlBucket marker `shouldReturn` Nothing+ _ <- store.putObject cellRef.controlBucket marker "application/json" DoesNotExist (encode submission)+ retained <- store.getObject cellRef.controlBucket marker+ observed <- resumeObservedSlices store cellRef (Bucket "results") path (const (pure ())) >>= either (fail . show) pure+ fmap (.state) observed.slices `shouldBe` [SliceSubmitted]+ store.getObject cellRef.controlBucket marker `shouldReturn` retained+ releaseLease store cellRef handle `shouldReturn` True++ it "stops before publication when the lease was cancelled" $ withSystemTempDirectory "kenshou-session" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ requestCancel store cellRef (Just submission.leaseId) False `shouldReturn` True+ runSubmission store cellRef (Bucket "results") handle submission workBytes (root <> "/out") (const (pure ())) `shouldReturn` Left LeaseLost+ store.listObjects cellRef.controlBucket (submissionPrefix submission) `shouldReturn` []++ it "returns the cell's rejection reason after publishing the marker" $ withSystemTempDirectory "kenshou-session" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ let journalPath = root </> "out" </> "session.json"+ (_, checkpoint) <- journalCheckpoint journalPath submission+ _ <- forkIO do+ let marker = ObjectName (submissionPrefix submission <> "submission.json")+ let awaitMarker = do+ visible <- store.statObject cellRef.controlBucket marker+ case visible of+ Nothing -> threadDelay 10000 >> awaitMarker+ Just _ -> pure ()+ awaitMarker+ now <- getCurrentTime+ _ <- store.putObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "rejected.json")) "application/json" DoesNotExist (encode (Rejected submission.runId "unsafe-command" now))+ pure ()+ Session.runSubmissionWithTransitions store cellRef (Bucket "results") handle submission workBytes (root </> "out") (const (pure ())) checkpoint `shouldReturn` Left (SubmissionRejected "unsafe-command")+ journal <- readSessionJournal journalPath >>= expectRight+ fmap (.state) journal.slices `shouldBe` [SliceRejected]+ fmap (.rejectionReason) journal.slices `shouldBe` [Just "unsafe-command"]++ it "takes one submission through a sealed local protocol agent and writes the index" $ withSystemTempDirectory "kenshou-session" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ let journalPath = root </> "out" </> "session.json"+ (nested, checkpoint) <- journalCheckpoint journalPath submission+ workerResult <- newEmptyMVar+ _ <- forkIO do+ completed <- try (publishSealedFixture store root submission nested) :: IO (Either SomeException ())+ putMVar workerResult completed+ result <- timeout 10000000 (Session.runSubmissionWithTransitions store cellRef (Bucket "results") handle submission workBytes (root </> "out") (const (pure ())) checkpoint)+ case result of+ Nothing -> expectationFailure "session did not observe the sealed result within ten seconds"+ Just (Left failure) -> expectationFailure (show failure)+ Just (Right index) -> do+ index.cellRun `shouldBe` submission.runId+ indexed <- doesFileExist (root </> "out" </> Text.unpack (renderRunId submission.runId) </> "cell-run.json")+ indexed `shouldBe` True+ journal <- readSessionJournal journalPath >>= expectRight+ fmap (.state) journal.slices `shouldBe` [SliceVerified]+ fmap (.manifestSha256) journal.slices `shouldBe` [Just index.cellManifest.sha256]+ otherRun <- newRunId+ now <- getCurrentTime+ case journal.slices of+ [slice] -> do+ let mismatched = journal {slices = [slice {state = SliceFetched, entryExitCode = Nothing, runIds = [otherRun]}]}+ applyTransition now submission.runId (Session.ResultsVerified index) mismatched `shouldBe` Left "verified index nested runs differ from planned slice"+ let preEntryFailure = index {cellOutcome = InfrastructureFailure, runs = []}+ fetchedFailure = journal {slices = [slice {state = SliceFetched, entryExitCode = Nothing, cellOutcome = Just InfrastructureFailure}]}+ case applyTransition now submission.runId (Session.ResultsVerified preEntryFailure) fetchedFailure of+ Right completed -> fmap (.state) completed.slices `shouldBe` [SliceVerified]+ Left problem -> expectationFailure (Text.unpack problem)+ _ -> expectationFailure "expected one journal slice"+ worker <- takeMVar workerResult+ case worker of+ Left failure -> expectationFailure (show failure)+ Right () -> pure ()++ it "runs two prepared slices under one lease and checkpoints both results" $ withSystemTempDirectory "kenshou-session" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ first <- fixtureFor handle+ second <- fixtureFor handle+ firstNested <- newRunId+ secondNested <- newRunId+ sessionId <- newRunId+ now <- getCurrentTime+ let sessionDir = root </> "out" </> Text.unpack (renderRunId sessionId)+ journalPath = sessionDir </> "session.json"+ firstWork = "slice-0/work.json"+ secondWork = "slice-1/work.json"+ makeSlice index submission nested workPath = SliceJournal index submission.runId [index] [nested] submission.reset submission workPath SlicePlanned Nothing Nothing Nothing Nothing Nothing+ initial = SessionJournal sessionId "alpha" "file" Nothing "control" "results" first.leaseId Held Map.empty (Text.replicate 64 "a") [] [makeSlice 0 first firstNested firstWork, makeSlice 1 second secondNested secondWork] now now+ createDirectoryIfMissing True (sessionDir </> "slice-0")+ createDirectoryIfMissing True (sessionDir </> "slice-1")+ LazyByteString.writeFile (sessionDir </> firstWork) workBytes+ LazyByteString.writeFile (sessionDir </> secondWork) "wrong"+ runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())) `shouldReturn` Left (InvalidSession "prepared work digest or size differs: slice-1/work.json")+ doesFileExist journalPath `shouldReturn` False+ LazyByteString.writeFile (sessionDir </> secondWork) workBytes+ workerResult <- newEmptyMVar+ _ <- forkIO do+ completed <- try (forM_ [(first, firstNested), (second, secondNested)] \(submission, nested) -> publishSealedFixture store root submission nested) :: IO (Either SomeException ())+ putMVar workerResult completed+ result <- timeout 20000000 (runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())))+ case result of+ Nothing -> expectationFailure "two-slice session did not finish within twenty seconds"+ Just (Left failure) -> expectationFailure (show failure)+ Just (Right journal) -> do+ fmap (.state) journal.slices `shouldBe` [SliceVerified, SliceVerified]+ persisted <- readSessionJournal journalPath >>= expectRight+ persisted `shouldBe` journal+ forM_ [first, second] \submission -> do+ indexed <- doesFileExist (root </> "out" </> Text.unpack (renderRunId submission.runId) </> "cell-run.json")+ indexed `shouldBe` True+ worker <- takeMVar workerResult+ case worker of+ Left failure -> expectationFailure (show failure)+ Right () -> pure ()+ runPlannedSlices store cellRef (Bucket "results") handle journalPath initial (const (pure ())) `shouldReturn` Left (SessionFileExists journalPath)+ case journal.slices of+ [firstSlice, secondSlice] -> do+ let interrupted =+ journal+ { slices =+ [ firstSlice {state = SliceSealed, entryExitCode = Nothing, fetchedPath = Nothing},+ secondSlice {state = SliceSubmitted, cellOutcome = Nothing, entryExitCode = Nothing, manifestSha256 = Nothing, fetchedPath = Nothing}+ ]+ }+ writeSessionJournal journalPath interrupted+ runPendingSlices store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Left (UnresolvedSlice firstSlice.cellRun SliceSealed)+ resumed <- resumeObservedSlices store cellRef (Bucket "results") journalPath (const (pure ()))+ recovered <- case resumed of+ Left failure -> expectationFailure (show failure) >> error "unreachable"+ Right value -> pure value+ fmap (.state) recovered.slices `shouldBe` [SliceVerified, SliceVerified]+ readSessionJournal journalPath `shouldReturn` Right recovered+ resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Right recovered+ priorLeaseId <- newRunId+ let completedPriorLease = (recovered :: SessionJournal) {leaseId = priorLeaseId}+ writeSessionJournal journalPath completedPriorLease+ resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ())) `shouldReturn` Right completedPriorLease+ readSessionJournal journalPath `shouldReturn` Right completedPriorLease+ writeSessionJournal journalPath recovered+ case recovered.slices of+ [firstRecovered, secondRecovered] -> do+ writeSessionJournal journalPath (recovered {slices = [firstRecovered {state = SliceFetched, entryExitCode = Nothing}, secondRecovered]})+ fetchedAgain <- resumeObservedSlices store cellRef (Bucket "results") journalPath (const (pure ()))+ case fetchedAgain of+ Left failure -> expectationFailure (show failure)+ Right value -> fmap (.state) value.slices `shouldBe` [SliceVerified, SliceVerified]+ let oldMarker = secondRecovered {state = SlicePlanned, cellOutcome = Nothing, entryExitCode = Nothing, manifestSha256 = Nothing, fetchedPath = Nothing}+ writeSessionJournal journalPath (recovered {slices = [firstRecovered, oldMarker]})+ rebindPlannedSlices store cellRef handle journalPath `shouldReturn` Left (RemoteMarkerExists secondRecovered.cellRun)+ observedMarker <- resumeHeldSession store cellRef (Bucket "results") handle journalPath (const (pure ()))+ case observedMarker of+ Left failure -> expectationFailure (show failure)+ Right collected -> fmap (.state) collected.slices `shouldBe` [SliceVerified, SliceVerified]+ _ -> expectationFailure "expected two recovered slices"+ _ -> expectationFailure "expected two journal slices"+ fresh <- fixtureFor handle+ freshNested <- newRunId+ freshSessionId <- newRunId+ at <- getCurrentTime+ let freshDir = root </> "out" </> Text.unpack (renderRunId freshSessionId)+ freshPath = freshDir </> "session.json"+ freshSlice = SliceJournal 0 fresh.runId [0] [freshNested] fresh.reset fresh "slice-0/work.json" SlicePlanned Nothing Nothing Nothing Nothing Nothing+ freshJournal = SessionJournal freshSessionId "alpha" "file" Nothing "control" "results" fresh.leaseId Held Map.empty (Text.replicate 64 "a") [] [freshSlice] at at+ writeSessionJournal freshPath freshJournal+ rebound <- rebindPlannedSlices store cellRef handle freshPath+ case rebound of+ Left failure -> expectationFailure (show failure)+ Right updated -> do+ updated.leaseId `shouldBe` fresh.leaseId+ fmap (.state) updated.slices `shouldBe` [SlicePlanned]+ fmap (.cellRun) updated.slices `shouldNotBe` [fresh.runId]+ readSessionJournal freshPath `shouldReturn` Right updated+ case updated.slices of+ [pendingSlice] -> do+ createDirectoryIfMissing True (freshDir </> "slice-0")+ LazyByteString.writeFile (freshDir </> pendingSlice.workPath) workBytes+ pendingWorker <- newEmptyMVar+ _ <- forkIO do+ completed <- try (publishSealedFixture store root pendingSlice.submission freshNested) :: IO (Either SomeException ())+ putMVar pendingWorker completed+ continued <- timeout 10000000 (runPendingSlices store cellRef (Bucket "results") handle freshPath (const (pure ())))+ case continued of+ Nothing -> expectationFailure "rebound slice did not finish within ten seconds"+ Just (Left failure) -> expectationFailure (show failure)+ Just (Right finalJournal) -> do+ fmap (.state) finalJournal.slices `shouldBe` [SliceVerified]+ readSessionJournal freshPath `shouldReturn` Right finalJournal+ pendingOutcome <- takeMVar pendingWorker+ case pendingOutcome of+ Left failure -> expectationFailure (show failure)+ Right () -> pure ()+ _ -> expectationFailure "expected one rebound slice"++ it "retries an unaccepted submitted slice only after a fresh lease fences it" $ withSystemTempDirectory "kenshou-session" \root -> do+ store <- newFileStore root+ old <- acquireLease store cellRef request >>= expectAcquired+ submission <- fixtureFor old+ nested <- newRunId+ sessionId <- newRunId+ now <- getCurrentTime+ let sessionDir = root </> "out" </> Text.unpack (renderRunId sessionId)+ journalPath = sessionDir </> "session.json"+ slice = SliceJournal 0 submission.runId [0] [nested] submission.reset submission "slice-0/work.json" SliceSubmitted Nothing Nothing Nothing Nothing Nothing+ journal = SessionJournal sessionId "alpha" "file" Nothing "control" "results" submission.leaseId Held Map.empty (Text.replicate 64 "a") [] [slice] now now+ marker = ObjectName (submissionPrefix submission <> "submission.json")+ status = ObjectName (submissionPrefix submission <> "status.json")+ result = ObjectName ("runs/" <> renderRunId submission.runId <> "/cell/result.json")+ createDirectoryIfMissing True (sessionDir </> "slice-0")+ LazyByteString.writeFile (sessionDir </> slice.workPath) workBytes+ writeSessionJournal journalPath journal+ _ <- store.putObject cellRef.controlBucket marker "application/json" DoesNotExist (encode submission)+ rebindPlannedSlices store cellRef old journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)+ releaseLease store cellRef old `shouldReturn` True+ fresh <- acquireLease store cellRef request >>= expectAcquired+ _ <- store.putObject cellRef.controlBucket status "application/json" DoesNotExist "{}"+ rebindPlannedSlices store cellRef fresh journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)+ store.deleteObject cellRef.controlBucket status NoPrecondition `shouldReturn` True+ _ <- store.putObject (Bucket "results") result "application/json" DoesNotExist "{}"+ rebindPlannedSlices store cellRef fresh journalPath `shouldReturn` Left (Rebind.UnresolvedSlice submission.runId SliceSubmitted)+ store.deleteObject (Bucket "results") result NoPrecondition `shouldReturn` True+ rebound <- rebindPlannedSlices store cellRef fresh journalPath+ updated <- case rebound of+ Left failure -> expectationFailure (show failure) >> error "unreachable"+ Right value -> pure value+ fmap (.state) updated.slices `shouldBe` [SlicePlanned]+ updated.leaseId `shouldNotBe` submission.leaseId+ case updated.slices of+ [reboundSlice] -> do+ reboundSlice.cellRun `shouldNotBe` submission.runId+ reboundSlice.submission.leaseId `shouldBe` updated.leaseId+ store.statObject cellRef.controlBucket marker >>= (`shouldSatisfy` (maybe False (const True)))+ _ -> expectationFailure "expected one rebound slice"++publishSealedFixture :: ObjectStore -> FilePath -> Submission -> RunId -> IO ()+publishSealedFixture store root submission nested = do+ let marker = ObjectName (submissionPrefix submission <> "submission.json")+ results = Bucket "results"+ resultPrefix = "runs/" <> renderRunId submission.runId <> "/"+ awaitMarker = do+ visible <- store.statObject cellRef.controlBucket marker+ case visible of+ Nothing -> threadDelay 10000 >> awaitMarker+ Just _ -> pure ()+ awaitMarker+ (sourceTree, manifest) <- completeTreeWithNested (root </> "fixture" </> Text.unpack (renderRunId submission.runId)) submission.runId submission.leaseId nested False False False+ forM_ manifest.artifacts \artifact -> do+ bytes <- LazyByteString.readFile (sourceTree </> Text.unpack artifact.path)+ _ <- store.putObject results (ObjectName (resultPrefix <> artifact.path)) artifact.mediaType DoesNotExist bytes+ pure ()+ bytes <- LazyByteString.readFile (sourceTree </> "manifest.json")+ _ <- store.putObject results (ObjectName (resultPrefix <> "manifest.json")) "application/json" DoesNotExist bytes+ now <- getCurrentTime+ let digest = (workObjectFor "application/json" bytes).sha256+ status = CellStatus submission.runId Sealed (Just manifest.leaseSequence) now Nothing (LogChunks 0 0) (Just Completed) (Just digest) (Just [])+ _ <- store.putObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "status.json")) "application/json" DoesNotExist (encode status)+ pure ()++fixtureFor :: LeaseHandle -> IO Submission+fixtureFor handle = do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+ fixture <- either (ioError . userError) pure (eitherDecode bytes :: Either String Submission)+ identifier <- newRunId+ lease <- leaseSnapshot handle+ let bundle = fixture.payload.bundle+ payload = fixture.payload {bundle = bundle {uri = "gs://control/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"}}+ pure (Submission identifier lease.leaseId payload (workObjectFor "application/json" workBytes) fixture.env fixture.reset fixture.limits fixture.requires Nothing fixture.labels)++submissionPrefix :: Submission -> Text+submissionPrefix submission = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/"++workBytes :: LazyByteString.ByteString+workBytes = "{}"++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")++request :: LeaseRequest+request = LeaseRequest "tester@workstation" "verification" 120++expectAcquired :: AcquireOutcome -> IO LeaseHandle+expectAcquired (Acquired handle) = pure handle+expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"++expectRight :: Either Text value -> IO value+expectRight = either (\failure -> expectationFailure (Text.unpack failure) >> error "unreachable") pure++journalCheckpoint :: FilePath -> Submission -> IO (RunId, Session.SessionTransition -> IO ())+journalCheckpoint path submission = do+ now <- getCurrentTime+ identifier <- newRunId+ nested <- newRunId+ let slice = SliceJournal 0 submission.runId [0] [nested] submission.reset submission "slice-0/work.json" SlicePlanned Nothing Nothing Nothing Nothing Nothing+ initial = SessionJournal identifier "alpha" "file" Nothing "control" "results" submission.leaseId Held Map.empty (Text.replicate 64 "a") [] [slice] now now+ writeSessionJournal path initial+ journalRef <- newIORef initial+ pure+ ( nested,+ \transition -> do+ current <- readIORef journalRef+ at <- getCurrentTime+ changed <- either (ioError . userError . Text.unpack) pure (applyTransition at submission.runId transition current)+ writeSessionJournal path changed+ writeIORef journalRef changed+ )
+ test/SubmitSpec.hs view
@@ -0,0 +1,117 @@+module SubmitSpec (spec) where++import Data.Aeson (eitherDecode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.Text (Text)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (Submission (..), WorkObject (..))+import Kenshou.Remote.Cell.Lease (AcquireOutcome (..), CellRef (..), Lease (..), LeaseHandle, LeaseRequest (..), acquireLease, leaseSnapshot, requestCancel)+import Kenshou.Remote.Cell.Submit (PublishOutcome (..), publishSubmission, workObjectFor)+import Kenshou.Remote.Payload (Bundle (..), CellPayload (..))+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..), PutOutcome (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell submission publication" do+ it "publishes work before the create-only marker without changing the lease sequence" $ withSystemTempDirectory "kenshou-submit" \root -> do+ base <- newFileStore root+ handle <- acquireLease base cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ writes <- newIORef []+ let store =+ base+ { putObject = \bucket name media pre bytes -> do+ modifyIORef' writes (<> [(name, pre)])+ base.putObject bucket name media pre bytes+ }+ prefix = submissionPrefix submission+ workName = ObjectName (prefix <> "work")+ markerName = ObjectName (prefix <> "submission.json")+ publishSubmission store cellRef handle submission workBytes `shouldReturn` Submitted+ readIORef writes `shouldReturn` [(workName, DoesNotExist), (markerName, DoesNotExist)]+ fmap fst <$> base.getObject cellRef.controlBucket workName `shouldReturn` Just workBytes+ marker <- base.getObject cellRef.controlBucket markerName+ fmap (eitherDecode . fst) marker `shouldBe` Just (Right submission)+ lease <- leaseSnapshot handle+ lease.runsStarted `shouldBe` 0+ publishSubmission store cellRef handle submission workBytes `shouldReturn` WorkAlreadyExists+ readIORef writes `shouldReturn` [(workName, DoesNotExist), (markerName, DoesNotExist), (workName, DoesNotExist)]++ it "refuses incorrect work metadata before writing either object" $ withSystemTempDirectory "kenshou-submit" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ let badWork = WorkObject "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" submission.work.bytes submission.work.mediaType+ bad = submission {work = badWork}+ publishSubmission store cellRef handle bad workBytes `shouldThrow` anyIOException+ store.listObjects cellRef.controlBucket (submissionPrefix submission) `shouldReturn` []++ it "refuses a payload bundle from another control bucket" $ withSystemTempDirectory "kenshou-submit" \root -> do+ store <- newFileStore root+ handle <- acquireLease store cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ let bundle = submission.payload.bundle+ wrongBucket = submission {payload = submission.payload {bundle = bundle {uri = "gs://elsewhere/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"}}}+ publishSubmission store cellRef handle wrongBucket workBytes `shouldThrow` anyIOException+ store.listObjects cellRef.controlBucket (submissionPrefix submission) `shouldReturn` []++ it "leaves an orphan work object if the lease is cancelled during upload" $ withSystemTempDirectory "kenshou-submit" \root -> do+ base <- newFileStore root+ handle <- acquireLease base cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ let store =+ base+ { putObject = \bucket name media pre bytes -> do+ result <- base.putObject bucket name media pre bytes+ if name == ObjectName (submissionPrefix submission <> "work")+ then do+ requestCancel base cellRef (Just submission.leaseId) False `shouldReturn` True+ pure result+ else pure result+ }+ publishSubmission store cellRef handle submission workBytes `shouldReturn` LostLease+ work <- base.statObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "work"))+ work `shouldSatisfy` (/= Nothing)+ base.statObject cellRef.controlBucket (ObjectName (submissionPrefix submission <> "submission.json")) `shouldReturn` Nothing++ it "reports a marker collision without overwriting it" $ withSystemTempDirectory "kenshou-submit" \root -> do+ base <- newFileStore root+ handle <- acquireLease base cellRef request >>= expectAcquired+ submission <- fixtureFor handle+ let markerName = ObjectName (submissionPrefix submission <> "submission.json")+ store =+ base+ { putObject = \bucket name media pre bytes ->+ if name == markerName then pure PreconditionFailed else base.putObject bucket name media pre bytes+ }+ publishSubmission store cellRef handle submission workBytes `shouldReturn` SubmissionAlreadyExists+ base.statObject cellRef.controlBucket markerName `shouldReturn` Nothing++fixtureFor :: LeaseHandle -> IO Submission+fixtureFor handle = do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.submission.v1.json"+ fixture <- either (ioError . userError) pure (eitherDecode bytes)+ identifier <- newRunId+ lease <- leaseSnapshot handle+ let bundle = fixture.payload.bundle+ payload = fixture.payload {bundle = bundle {uri = "gs://control/payloads/sha256/" <> bundle.sha256 <> ".nar.zst"}}+ pure fixture {runId = identifier, leaseId = lease.leaseId, payload = payload, work = workObjectFor "application/json" workBytes}++submissionPrefix :: Submission -> Text+submissionPrefix submission = "cells/alpha/submissions/" <> renderRunId submission.runId <> "/"++workBytes :: LazyByteString.ByteString+workBytes = "{\"schema\":\"kenshou.run-plan/v1\",\"runs\":[]}"++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")++request :: LeaseRequest+request = LeaseRequest "tester@workstation" "verification" 120++expectAcquired :: AcquireOutcome -> IO LeaseHandle+expectAcquired (Acquired handle) = pure handle+expectAcquired _ = expectationFailure "expected acquired lease" >> error "unreachable"
+ test/WatchSpec.hs view
@@ -0,0 +1,83 @@+module WatchSpec (spec) where++import Data.Aeson (eitherDecode, encode)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Kenshou.Core.Id (RunId, newRunId, renderRunId)+import Kenshou.Remote.Cell.Docs (CellPhase (..), CellStatus (..), LogChunks (..), Rejected (..))+import Kenshou.Remote.Cell.Lease (CellRef (..))+import Kenshou.Remote.Cell.Watch (WatchEvent (..), WatchSnapshot (..), WatchTerminal (..), newWatchCursor, pollCellRun)+import Kenshou.Remote.Store (Bucket (..), ObjectName (..), ObjectStore (..), Precondition (..))+import Kenshou.Remote.Store.File (newFileStore)+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "cell status and log watcher" do+ it "waits for a status and then emits each phase and chunk exactly once" $ withSystemTempDirectory "kenshou-watch" \root -> do+ store <- newFileStore root+ identifier <- newRunId+ pollCellRun store cellRef identifier newWatchCursor `shouldReturn` WatchSnapshot newWatchCursor [] Nothing+ fixture <- statusFixture identifier+ let running = fixture {phase = Running, logChunks = LogChunks 1 1, outcome = Nothing, manifestSha256 = Nothing}+ put store identifier "log/stdout.0" "first out\n"+ put store identifier "log/stderr.0" "first err\n"+ put store identifier "status.json" (encode running)+ first <- pollCellRun store cellRef identifier newWatchCursor+ first.events `shouldBe` [PhaseChanged Running, StdoutChunk "first out\n", StderrChunk "first err\n"]+ first.terminal `shouldBe` Nothing+ unchanged <- pollCellRun store cellRef identifier first.cursor+ unchanged.events `shouldBe` []+ put store identifier "log/stdout.1" "second out\n"+ put store identifier "status.json" (encode fixture {logChunks = LogChunks 2 1})+ sealed <- pollCellRun store cellRef identifier unchanged.cursor+ sealed.events `shouldBe` [PhaseChanged Sealed, StdoutChunk "second out\n"]+ sealed.terminal `shouldBe` Just (RunSealed fixture {logChunks = LogChunks 2 1})++ it "gives rejection priority over status and checks the run identity" $ withSystemTempDirectory "kenshou-watch" \root -> do+ store <- newFileStore root+ identifier <- newRunId+ fixture <- statusFixture identifier+ rejected <- rejectedFixture identifier+ put store identifier "status.json" (encode fixture)+ put store identifier "rejected.json" (encode rejected)+ result <- pollCellRun store cellRef identifier newWatchCursor+ result.events `shouldBe` []+ result.terminal `shouldBe` Just (RunRejected rejected)+ other <- newRunId+ wrong <- rejectedFixture other+ put store identifier "rejected.json" (encode wrong)+ pollCellRun store cellRef identifier newWatchCursor `shouldThrow` anyIOException++ it "refuses missing chunks and a backwards chunk count" $ withSystemTempDirectory "kenshou-watch" \root -> do+ store <- newFileStore root+ identifier <- newRunId+ fixture <- statusFixture identifier+ let running = fixture {phase = Running, logChunks = LogChunks 1 0, outcome = Nothing, manifestSha256 = Nothing}+ put store identifier "status.json" (encode running)+ pollCellRun store cellRef identifier newWatchCursor `shouldThrow` anyIOException+ put store identifier "log/stdout.0" "first out\n"+ first <- pollCellRun store cellRef identifier newWatchCursor+ put store identifier "status.json" (encode running {logChunks = LogChunks 0 0})+ pollCellRun store cellRef identifier first.cursor `shouldThrow` anyIOException++statusFixture :: RunId -> IO CellStatus+statusFixture identifier = do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.status.v1.json"+ fixture <- either (ioError . userError) pure (eitherDecode bytes :: Either String CellStatus)+ pure (CellStatus identifier fixture.phase fixture.leaseSequence fixture.updatedAt fixture.phaseStartedAt fixture.logChunks fixture.outcome fixture.manifestSha256 fixture.reasons)++rejectedFixture :: RunId -> IO Rejected+rejectedFixture identifier = do+ bytes <- LazyByteString.readFile "test/golden/cell/cell.rejected.v1.json"+ fixture <- either (ioError . userError) pure (eitherDecode bytes :: Either String Rejected)+ pure (Rejected identifier fixture.reason fixture.at)++put :: ObjectStore -> RunId -> Text -> LazyByteString.ByteString -> IO ()+put store identifier suffix bytes = do+ let name = ObjectName ("cells/alpha/submissions/" <> renderRunId identifier <> "/" <> suffix)+ _ <- store.putObject cellRef.controlBucket name "application/json" NoPrecondition bytes+ pure ()++cellRef :: CellRef+cellRef = CellRef "alpha" (Bucket "control")
+ test/golden/cell-capabilities.json view
@@ -0,0 +1,13 @@+{+ "schema": "kenshou.cell-capabilities/v1",+ "cell": "alpha",+ "descriptorSha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "probedBy": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "probedAt": "2026-09-25T12:00:00Z",+ "capabilities": {+ "postgres.major": "18",+ "postgres.pg_partman": false,+ "broker": "redpanda v25.2.4",+ "otlp.null": true+ }+}
+ test/golden/cell-route.json view
@@ -0,0 +1,8 @@+{+ "schema": "kenshou.cell-route/v1",+ "planId": "0199a3f2-7c1e-7b8a-9d41-3e5f6a7b8c9d",+ "perCell": {"alpha": ["0199a3f2-7c20-7f00-8a11-0c0d0e0f1011"]},+ "localRunIds": [],+ "unroutable": [],+ "warnings": []+}
+ test/golden/cell-run.json view
@@ -0,0 +1,20 @@+{+ "schema": "kenshou.cell-run/v1",+ "cellRun": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "cell": "alpha",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "leaseSequence": 2,+ "resultsBaseUri": "gs://tan-nb-exp-cells-results/runs/0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "dataBaseUri": "gs://tan-nb-exp-cells-results/runs/0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9/output",+ "cellManifest": {"path": "manifest.json", "sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "bytes": 5210},+ "cellOutcome": "completed",+ "reasons": [],+ "entryExitCode": 0,+ "evidence": [+ {"kind": "cell-fingerprint", "path": "cell/fingerprint.json", "sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "bytes": 3904, "mediaType": "application/json"}+ ],+ "runs": [+ {"runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011", "path": "output/0199a3f2-7c20-7f00-8a11-0c0d0e0f1011", "scenario": "selftest/kernel/correctness/postgres-roundtrip", "manifestSha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", "recordedOutcome": "passed", "effectiveOutcome": "passed", "overrideReason": null}+ ],+ "verifiedAt": "2026-09-27T12:00:00Z"+}
+ test/golden/cell-session.json view
@@ -0,0 +1,118 @@+{+ "schema": "kenshou.cell-session/v1",+ "sessionId": "0199a3c3-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "cell": "alpha",+ "store": "gs://",+ "project": "tan-nb-exp",+ "controlBucket": "tan-nb-exp-cells-control",+ "resultsBucket": "tan-nb-exp-cells-results",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "leaseMode": "held",+ "payloads": {},+ "planSha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "rejectedRuns": [],+ "slices": [+ {+ "index": 0,+ "cellRun": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "ordinals": [+ 0+ ],+ "runIds": [+ "0199a3c4-7b1e-7c44-9d0a-3f5e2a61b7c9"+ ],+ "reset": {+ "postgres": {+ "major": 18,+ "databases": [+ {+ "name": "benchmark",+ "owner": "benchmark",+ "template": "template0"+ }+ ],+ "settings": {+ "checkpoint_timeout": "30min",+ "max_wal_size": "16GB"+ }+ },+ "cachePolicy": "cold",+ "broker": {+ "wipe": true+ }+ },+ "submission": {+ "schema": "cell.submission/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "payload": {+ "schema": "cell.payload/v1",+ "kind": "nix-nar-bundle",+ "bundle": {+ "uri": "gs://tan-nb-exp-cells-control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst",+ "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "bytes": 48211934+ },+ "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello",+ "narHash": "sha256-example",+ "closurePaths": [+ "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"+ ],+ "system": "x86_64-linux",+ "command": [+ "bin/cell-fixture-hello"+ ]+ },+ "work": {+ "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "bytes": 212,+ "mediaType": "application/json"+ },+ "env": {+ "EXAMPLE": "value"+ },+ "reset": {+ "postgres": {+ "major": 18,+ "databases": [+ {+ "name": "benchmark",+ "owner": "benchmark",+ "template": "template0"+ }+ ],+ "settings": {+ "checkpoint_timeout": "30min",+ "max_wal_size": "16GB"+ }+ },+ "cachePolicy": "cold",+ "broker": {+ "wipe": true+ }+ },+ "limits": {+ "wallClockSeconds": 3600,+ "memoryMaxBytes": 25769803776,+ "outputMaxBytes": 21474836480+ },+ "requires": {+ "protocol": "cell.protocol/v1",+ "minAgentVersion": "0.1.0"+ },+ "labels": {+ "arm": "candidate"+ }+ },+ "workPath": "slice-0/work.json",+ "state": "planned",+ "cellOutcome": null,+ "entryExitCode": null,+ "manifestSha256": null,+ "rejectionReason": null,+ "fetchedPath": null+ }+ ],+ "createdAt": "2026-09-27T12:00:00Z",+ "updatedAt": "2026-09-27T12:00:00Z"+}
+ test/golden/cell/SOURCE view
@@ -0,0 +1,19 @@+mori://shinzui/load-testing-infra+Commit: ef1596775d4df5a31b8de3a4b833adeeaf0f8021+Project-relative source directories (artifact-level URIs pending):+ schemas/cell/ (all cell.*.schema.json files)+ schemas/cell/examples/ (all cell.*.json files)++Execution-window schema and example extension:+ Commit: 915fb2f9ee8ca8eccafecd8341887638ac8b4218+ Project-relative paths (artifact-level URIs pending):+ schemas/cell/cell.execution-window.v1.schema.json+ schemas/cell/examples/cell.execution-window.v1.json++Kenshou extension (not in the upstream commit above):+ cell.environment.v1.schema.json and cell.submission.v1.schema.json carry an optional+ `resources` object (top level of the environment; inside `reset` of the submission),+ added by docs/plans/3-replace-kafka-specific-environment-fields-with-suite-provisioned-named-resources.md.+ It is pending adoption in mori://shinzui/load-testing-infra. Until then the client never+ sends `reset.resources` for the resource the legacy broker fields describe (see+ Kenshou.Remote.Cell.Legacy), and the example documents are unchanged.
+ test/golden/cell/cell.artifact-manifest.v1.json view
@@ -0,0 +1,21 @@+{+ "schema": "cell.artifact-manifest/v1",+ "layout": 1,+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "cell": "alpha",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "leaseSequence": 2,+ "sealedAt": "2026-09-25T12:21:07Z",+ "agentVersion": "0.1.0",+ "payload": {+ "kind": "nix-nar-bundle",+ "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"+ },+ "outcome": "completed",+ "retention": {"policy": "bucket-retention", "retentionSeconds": 34560000},+ "artifacts": [+ {"path": "cell/result.json", "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "bytes": 412, "mediaType": "application/json"},+ {"path": "output/result.json", "sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "bytes": 96, "mediaType": "application/json"}+ ]+}
+ test/golden/cell/cell.artifact-manifest.v1.schema.json view
@@ -0,0 +1,53 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.artifact-manifest.v1.schema.json",+ "title": "Sealed verification cell result manifest",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "layout", "runId", "cell", "leaseId", "leaseSequence", "sealedAt", "agentVersion", "payload", "outcome", "retention", "artifacts"],+ "properties": {+ "schema": {"const": "cell.artifact-manifest/v1"},+ "layout": {"const": 1},+ "runId": {"$ref": "#/$defs/identifier"},+ "cell": {"$ref": "#/$defs/identifier"},+ "leaseId": {"$ref": "#/$defs/identifier"},+ "leaseSequence": {"type": "integer", "minimum": 0},+ "sealedAt": {"type": "string", "format": "date-time"},+ "agentVersion": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"},+ "payload": {+ "type": "object", "additionalProperties": false,+ "required": ["kind", "sha256", "storePath"],+ "properties": {+ "kind": {"const": "nix-nar-bundle"},+ "sha256": {"$ref": "#/$defs/sha256"},+ "storePath": {"type": "string", "pattern": "^/nix/store/[a-z0-9]{32}-[^/]+$"}+ }+ },+ "outcome": {"enum": ["completed", "infrastructure-failure", "cancelled", "timed-out"]},+ "retention": {+ "type": "object", "additionalProperties": false,+ "required": ["policy", "retentionSeconds"],+ "properties": {+ "policy": {"const": "bucket-retention"},+ "retentionSeconds": {"type": "integer", "minimum": 0}+ }+ },+ "artifacts": {+ "type": "array", "minItems": 1,+ "items": {+ "type": "object", "additionalProperties": false,+ "required": ["path", "sha256", "bytes", "mediaType"],+ "properties": {+ "path": {"type": "string", "pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[^\\u0000]+$"},+ "sha256": {"$ref": "#/$defs/sha256"},+ "bytes": {"type": "integer", "minimum": 0},+ "mediaType": {"type": "string", "minLength": 1}+ }+ }+ }+ },+ "$defs": {+ "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}+ }+}
+ test/golden/cell/cell.descriptor.v1.json view
@@ -0,0 +1,31 @@+{+ "schema": "cell.descriptor/v1",+ "name": "alpha",+ "project": "tan-nb-exp",+ "zone": "us-west1-a",+ "postgresMajor": 18,+ "instances": {+ "postgres": "cell-alpha-postgres",+ "drivers": ["cell-alpha-driver-0"],+ "monitoring": "cell-alpha-monitoring"+ },+ "addresses": {+ "postgres": "10.0.0.2",+ "drivers": ["10.0.0.3"],+ "monitoring": "10.0.0.4"+ },+ "shape": {+ "postgres": {"machineType": "n2-standard-8", "diskType": "pd-ssd", "diskSizeGb": 200},+ "driver": {"machineType": "n2-standard-8", "count": 1, "bootDiskSizeGb": 100},+ "monitoring": {"machineType": "n2-standard-4"},+ "scheduling": {"onHostMaintenance": "MIGRATE", "compactPlacement": false}+ },+ "images": {+ "driver": "projects/tan-nb-exp/global/images/example-cell-driver",+ "postgres": "projects/tan-nb-exp/global/images/example-cell-postgres18",+ "monitoring": "projects/tan-nb-exp/global/images/example-cell-monitoring",+ "broker": ""+ },+ "buckets": {"control": "tan-nb-exp-cells-control", "results": "tan-nb-exp-cells-results"},+ "agentProtocol": "cell.protocol/v1"+}
+ test/golden/cell/cell.descriptor.v1.schema.json view
@@ -0,0 +1,72 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.descriptor.v1.schema.json",+ "title": "Verification cell descriptor",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "name", "project", "zone", "postgresMajor", "instances", "addresses", "shape", "images", "buckets", "agentProtocol"],+ "properties": {+ "schema": {"const": "cell.descriptor/v1"},+ "name": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,40}$"},+ "project": {"type": "string", "minLength": 1},+ "zone": {"type": "string", "minLength": 1},+ "postgresMajor": {"enum": [17, 18]},+ "instances": {+ "type": "object", "additionalProperties": false,+ "required": ["postgres", "drivers", "monitoring"],+ "properties": {+ "postgres": {"type": "string", "minLength": 1},+ "drivers": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}},+ "monitoring": {"type": "string", "minLength": 1}+ }+ },+ "addresses": {+ "type": "object", "additionalProperties": false,+ "required": ["postgres", "drivers", "monitoring"],+ "properties": {+ "postgres": {"type": "string", "format": "ipv4"},+ "drivers": {"type": "array", "minItems": 1, "items": {"type": "string", "format": "ipv4"}},+ "monitoring": {"type": "string", "format": "ipv4"}+ }+ },+ "shape": {+ "type": "object", "additionalProperties": false,+ "required": ["postgres", "driver", "monitoring", "scheduling"],+ "properties": {+ "postgres": {"type": "object", "required": ["machineType", "diskType", "diskSizeGb"], "properties": {+ "machineType": {"type": "string"}, "diskType": {"type": "string"},+ "diskSizeGb": {"type": "integer", "minimum": 1}, "provisionedIops": {"type": ["integer", "null"]}+ }},+ "driver": {"type": "object", "required": ["machineType", "count", "bootDiskSizeGb"], "properties": {+ "machineType": {"type": "string"}, "count": {"type": "integer", "minimum": 1},+ "bootDiskSizeGb": {"type": "integer", "minimum": 1}+ }},+ "monitoring": {"type": "object", "required": ["machineType"], "properties": {"machineType": {"type": "string"}}},+ "scheduling": {"type": "object", "required": ["onHostMaintenance", "compactPlacement"], "properties": {+ "onHostMaintenance": {"enum": ["MIGRATE", "TERMINATE"]},+ "minCpuPlatform": {"type": ["string", "null"]},+ "compactPlacement": {"type": "boolean"}+ }}+ }+ },+ "images": {+ "type": "object", "additionalProperties": false,+ "required": ["driver", "postgres", "monitoring", "broker"],+ "properties": {+ "driver": {"type": "string", "minLength": 1},+ "postgres": {"type": "string", "minLength": 1},+ "monitoring": {"type": "string", "minLength": 1},+ "broker": {"type": "string"}+ }+ },+ "buckets": {+ "type": "object", "additionalProperties": false,+ "required": ["control", "results"],+ "properties": {+ "control": {"type": "string", "minLength": 1},+ "results": {"type": "string", "minLength": 1}+ }+ },+ "agentProtocol": {"const": "cell.protocol/v1"}+ }+}
+ test/golden/cell/cell.environment.v1.json view
@@ -0,0 +1,18 @@+{+ "schema": "cell.environment/v1",+ "cell": "alpha",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "postgres": {+ "major": 18,+ "host": "10.0.0.2",+ "port": 5432,+ "database": "benchmark",+ "user": "benchmark",+ "connectionString": "host=10.0.0.2 port=5432 dbname=benchmark user=benchmark"+ },+ "broker": null,+ "otlp": null,+ "metrics": {"victoriaMetricsUrl": "http://10.0.0.4:8428"},+ "drivers": [{"index": 0, "host": "10.0.0.3"}]+}
+ test/golden/cell/cell.environment.v1.schema.json view
@@ -0,0 +1,76 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.environment.v1.schema.json",+ "title": "Environment made available to a verification cell payload",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "cell", "runId", "leaseId", "postgres", "broker", "otlp", "metrics", "drivers"],+ "properties": {+ "schema": {"const": "cell.environment/v1"},+ "cell": {"$ref": "#/$defs/identifier"},+ "runId": {"$ref": "#/$defs/identifier"},+ "leaseId": {"$ref": "#/$defs/identifier"},+ "postgres": {+ "type": "object", "additionalProperties": false,+ "required": ["major", "host", "port", "database", "user", "connectionString"],+ "properties": {+ "major": {"enum": [17, 18]},+ "host": {"type": "string", "format": "ipv4"},+ "port": {"type": "integer", "minimum": 1, "maximum": 65535},+ "database": {"type": "string", "minLength": 1},+ "user": {"type": "string", "minLength": 1},+ "connectionString": {"type": "string", "minLength": 1}+ }+ },+ "broker": {+ "oneOf": [+ {"type": "null"},+ {"type": "object", "additionalProperties": false, "required": ["bootstrapServers", "adminUrl", "implementation", "version"], "properties": {+ "bootstrapServers": {"type": "string", "minLength": 1},+ "adminUrl": {"type": "string", "format": "uri"},+ "implementation": {"type": "string", "minLength": 1},+ "version": {"type": "string", "minLength": 1}+ }}+ ]+ },+ "otlp": {+ "oneOf": [+ {"type": "null"},+ {"type": "object", "additionalProperties": false, "required": ["null", "file"], "properties": {+ "null": {"$ref": "#/$defs/otlpEndpoint"},+ "file": {"$ref": "#/$defs/otlpEndpoint"}+ }}+ ]+ },+ "metrics": {+ "type": "object", "additionalProperties": false,+ "required": ["victoriaMetricsUrl"],+ "properties": {"victoriaMetricsUrl": {"type": "string", "format": "uri"}}+ },+ "drivers": {+ "type": "array", "minItems": 1,+ "items": {"type": "object", "additionalProperties": false, "required": ["index", "host"], "properties": {+ "index": {"type": "integer", "minimum": 0},+ "host": {"type": "string", "format": "ipv4"}+ }}+ },+ "clock": {+ "type": "object", "additionalProperties": false,+ "required": ["skewBoundMicros"],+ "properties": {"skewBoundMicros": {"type": "integer", "minimum": 0}}+ },+ "faultHook": {"type": "string", "minLength": 1},+ "resources": {"type": "object", "description": "kenshou extension pending upstream adoption: bound values of named resources, keyed by resource name"}+ },+ "$defs": {+ "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "otlpEndpoint": {+ "type": "object", "additionalProperties": false,+ "required": ["grpc", "http"],+ "properties": {+ "grpc": {"type": "string", "format": "uri"},+ "http": {"type": "string", "format": "uri"}+ }+ }+ }+}
+ test/golden/cell/cell.execution-window.v1.json view
@@ -0,0 +1,7 @@+{+ "schema": "cell.execution-window/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "resetStarted": "2026-09-25T12:00:00Z",+ "runStarted": "2026-09-25T12:00:10Z",+ "runFinished": "2026-09-25T12:01:00Z"+}
+ test/golden/cell/cell.execution-window.v1.schema.json view
@@ -0,0 +1,33 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "type": "object",+ "additionalProperties": false,+ "required": [+ "schema",+ "runId",+ "resetStarted",+ "runStarted",+ "runFinished"+ ],+ "properties": {+ "schema": {+ "const": "cell.execution-window/v1"+ },+ "runId": {+ "type": "string",+ "format": "uuid"+ },+ "resetStarted": {+ "type": "string",+ "format": "date-time"+ },+ "runStarted": {+ "type": "string",+ "format": "date-time"+ },+ "runFinished": {+ "type": "string",+ "format": "date-time"+ }+ }+}
+ test/golden/cell/cell.health.v1.json view
@@ -0,0 +1,18 @@+{+ "schema": "cell.health/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "windowStart": "2026-09-25T12:00:00Z",+ "windowEnd": "2026-09-25T12:01:00Z",+ "passed": false,+ "gates": [+ {+ "name": "host-maintenance", "machine": "cell-alpha-postgres", "status": "tripped",+ "observed": "MIGRATE_ON_HOST_MAINTENANCE", "threshold": "NONE",+ "at": "2026-09-25T12:00:30Z", "detail": "maintenance event during run"+ },+ {+ "name": "background-load", "machine": "cell-alpha-driver-0", "status": "ok",+ "observed": 1.2, "threshold": 5, "at": "2026-09-25T12:00:10Z"+ }+ ]+}
+ test/golden/cell/cell.health.v1.schema.json view
@@ -0,0 +1,36 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.health.v1.schema.json",+ "title": "Verification cell health gate evidence",+ "type": "object", "additionalProperties": false,+ "required": ["schema", "runId", "windowStart", "windowEnd", "passed", "gates"],+ "properties": {+ "schema": {"const": "cell.health/v1"},+ "runId": {"$ref": "#/$defs/identifier"},+ "windowStart": {"type": "string", "format": "date-time"},+ "windowEnd": {"type": "string", "format": "date-time"},+ "passed": {"type": "boolean"},+ "gates": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/gate"}}+ },+ "allOf": [{+ "if": {"properties": {"passed": {"const": true}}, "required": ["passed"]},+ "then": {"properties": {"gates": {"items": {"properties": {"status": {"const": "ok"}}}}}},+ "else": {"properties": {"gates": {"contains": {"properties": {"status": {"const": "tripped"}}, "required": ["status"]}}}}+ }],+ "$defs": {+ "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "gate": {+ "type": "object", "additionalProperties": false,+ "required": ["name", "machine", "status", "observed", "threshold", "at"],+ "properties": {+ "name": {"enum": ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"]},+ "machine": {"$ref": "#/$defs/identifier"},+ "status": {"enum": ["ok", "tripped"]},+ "observed": {"type": ["number", "string", "boolean", "object"]},+ "threshold": {"type": ["number", "string", "boolean", "object", "null"]},+ "at": {"type": "string", "format": "date-time"},+ "detail": {"type": "string"}+ }+ }+ }+}
+ test/golden/cell/cell.lease.v1.json view
@@ -0,0 +1,12 @@+{+ "schema": "cell.lease/v1",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "cell": "alpha",+ "owner": "nadeem@workstation",+ "purpose": "fixture-abba",+ "ttlSeconds": 120,+ "acquiredAt": "2026-09-25T12:00:00Z",+ "heartbeatAt": "2026-09-25T12:00:40Z",+ "cancelRequested": false,+ "runsStarted": 2+}
+ test/golden/cell/cell.lease.v1.schema.json view
@@ -0,0 +1,20 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.lease.v1.schema.json",+ "title": "Exclusive verification cell lease",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "leaseId", "cell", "owner", "purpose", "ttlSeconds", "acquiredAt", "heartbeatAt", "cancelRequested", "runsStarted"],+ "properties": {+ "schema": {"const": "cell.lease/v1"},+ "leaseId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "cell": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "owner": {"type": "string", "minLength": 1},+ "purpose": {"type": "string", "minLength": 1},+ "ttlSeconds": {"type": "integer", "minimum": 1},+ "acquiredAt": {"type": "string", "format": "date-time"},+ "heartbeatAt": {"type": "string", "format": "date-time"},+ "cancelRequested": {"type": "boolean"},+ "runsStarted": {"type": "integer", "minimum": 0}+ }+}
+ test/golden/cell/cell.payload.v1.json view
@@ -0,0 +1,14 @@+{+ "schema": "cell.payload/v1",+ "kind": "nix-nar-bundle",+ "bundle": {+ "uri": "gs://tan-nb-exp-cells-control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst",+ "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "bytes": 48211934+ },+ "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello",+ "narHash": "sha256-example",+ "closurePaths": ["/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"],+ "system": "x86_64-linux",+ "command": ["bin/cell-fixture-hello"]+}
+ test/golden/cell/cell.payload.v1.schema.json view
@@ -0,0 +1,34 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.payload.v1.schema.json",+ "title": "Content-addressed verification cell payload",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "kind", "bundle", "storePath", "narHash", "closurePaths", "system", "command"],+ "properties": {+ "schema": {"const": "cell.payload/v1"},+ "kind": {"const": "nix-nar-bundle"},+ "bundle": {+ "type": "object",+ "additionalProperties": false,+ "required": ["uri", "sha256", "bytes"],+ "properties": {+ "uri": {"type": "string", "pattern": "^gs://[^/]+/payloads/sha256/[a-f0-9]{64}\\.nar\\.zst$"},+ "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},+ "bytes": {"type": "integer", "minimum": 1}+ }+ },+ "storePath": {"$ref": "#/$defs/storePath"},+ "narHash": {"type": "string", "minLength": 1},+ "closurePaths": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"$ref": "#/$defs/storePath"}},+ "system": {"const": "x86_64-linux"},+ "command": {+ "type": "array", "minItems": 1,+ "items": {"type": "string", "minLength": 1},+ "prefixItems": [{"type": "string", "pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))[^\\u0000]+$"}]+ }+ },+ "$defs": {+ "storePath": {"type": "string", "pattern": "^/nix/store/[a-z0-9]{32}-[^/]+$"}+ }+}
+ test/golden/cell/cell.policy.v1.json view
@@ -0,0 +1,12 @@+{+ "schema": "cell.policy/v1",+ "idleStopMinutes": 30,+ "keepWarmUntil": null,+ "allowFaultInjection": false,+ "health": {+ "settleCpuMaxPercent": 5,+ "runStealCpuMaxPercent": 2,+ "startFreeDiskMinPercent": 20,+ "runFreeDiskMinPercent": 5+ }+}
+ test/golden/cell/cell.policy.v1.schema.json view
@@ -0,0 +1,24 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.policy.v1.schema.json",+ "title": "Verification cell idle and fault policy",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "idleStopMinutes", "keepWarmUntil", "allowFaultInjection", "health"],+ "properties": {+ "schema": {"const": "cell.policy/v1"},+ "idleStopMinutes": {"type": "integer", "minimum": 1, "maximum": 1440},+ "keepWarmUntil": {"type": ["string", "null"], "format": "date-time"},+ "allowFaultInjection": {"type": "boolean"},+ "health": {+ "type": "object", "additionalProperties": false,+ "required": ["settleCpuMaxPercent", "runStealCpuMaxPercent", "startFreeDiskMinPercent", "runFreeDiskMinPercent"],+ "properties": {+ "settleCpuMaxPercent": {"type": "number", "minimum": 0, "maximum": 100},+ "runStealCpuMaxPercent": {"type": "number", "minimum": 0, "maximum": 100},+ "startFreeDiskMinPercent": {"type": "number", "minimum": 0, "maximum": 100},+ "runFreeDiskMinPercent": {"type": "number", "minimum": 0, "maximum": 100}+ }+ }+ }+}
+ test/golden/cell/cell.quarantine.v1.json view
@@ -0,0 +1,7 @@+{+ "schema": "cell.quarantine/v1",+ "cell": "alpha",+ "reason": "reset-failed",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "at": "2026-09-25T12:05:00Z"+}
+ test/golden/cell/cell.quarantine.v1.schema.json view
@@ -0,0 +1,15 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.quarantine.v1.schema.json",+ "title": "Verification cell quarantine record",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "cell", "reason", "runId", "at"],+ "properties": {+ "schema": {"const": "cell.quarantine/v1"},+ "cell": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "reason": {"type": "string", "minLength": 1},+ "runId": {"type": ["string", "null"], "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "at": {"type": "string", "format": "date-time"}+ }+}
+ test/golden/cell/cell.recovery.v1.json view
@@ -0,0 +1,6 @@+{+ "schema": "cell.recovery/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "reason": "agent-restarted",+ "at": "2026-09-25T17:30:00Z"+}
+ test/golden/cell/cell.recovery.v1.schema.json view
@@ -0,0 +1,14 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.recovery.v1.schema.json",+ "title": "Verification cell interrupted-run recovery",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "runId", "reason", "at"],+ "properties": {+ "schema": {"const": "cell.recovery/v1"},+ "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "reason": {"const": "agent-restarted"},+ "at": {"type": "string", "format": "date-time"}+ }+}
+ test/golden/cell/cell.rejected.v1.json view
@@ -0,0 +1,6 @@+{+ "schema": "cell.rejected/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "reason": "unsafe-command",+ "at": "2026-09-25T12:21:07Z"+}
+ test/golden/cell/cell.rejected.v1.schema.json view
@@ -0,0 +1,14 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.rejected.v1.schema.json",+ "title": "Rejected verification cell submission",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "runId", "reason", "at"],+ "properties": {+ "schema": {"const": "cell.rejected/v1"},+ "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "reason": {"type": "string", "minLength": 1},+ "at": {"type": "string", "format": "date-time"}+ }+}
+ test/golden/cell/cell.reset-evidence.v1.json view
@@ -0,0 +1,20 @@+{+ "schema": "cell.reset-evidence/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "cell": "alpha",+ "cachePolicy": "cold",+ "startedAt": "2026-09-25T12:00:00Z",+ "finishedAt": "2026-09-25T12:00:05Z",+ "verified": true,+ "steps": [+ {"name": "stop-driver-processes", "startedAt": "2026-09-25T12:00:00Z", "finishedAt": "2026-09-25T12:00:01Z", "status": "ok"},+ {"name": "reset-postgres", "startedAt": "2026-09-25T12:00:01Z", "finishedAt": "2026-09-25T12:00:05Z", "status": "ok"}+ ],+ "postgres": {+ "version": "PostgreSQL 18.1",+ "databases": ["benchmark", "postgres"],+ "settings": [{"name": "checkpoint_timeout", "setting": "1800", "source": "configuration file"}],+ "walLsn": "0/1734400",+ "checkpoint": {"numRequested": 1, "numDone": 1}+ }+}
+ test/golden/cell/cell.reset-evidence.v1.schema.json view
@@ -0,0 +1,47 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.reset-evidence.v1.schema.json",+ "title": "Verification cell reset evidence",+ "type": "object", "additionalProperties": false,+ "required": ["schema", "runId", "cell", "cachePolicy", "startedAt", "finishedAt", "verified", "steps", "postgres"],+ "properties": {+ "schema": {"const": "cell.reset-evidence/v1"},+ "runId": {"$ref": "#/$defs/identifier"},+ "cell": {"$ref": "#/$defs/identifier"},+ "cachePolicy": {"enum": ["warm", "cold"]},+ "startedAt": {"type": "string", "format": "date-time"},+ "finishedAt": {"type": "string", "format": "date-time"},+ "verified": {"type": "boolean"},+ "steps": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/step"}},+ "postgres": {"type": ["object", "null"], "additionalProperties": false,+ "required": ["version", "databases", "settings", "walLsn", "checkpoint"],+ "properties": {+ "version": {"type": "string", "minLength": 1},+ "databases": {"type": "array", "uniqueItems": true, "items": {"$ref": "#/$defs/identifier"}},+ "settings": {"type": "array", "items": {"type": "object", "additionalProperties": false,+ "required": ["name", "setting", "source"],+ "properties": {"name": {"type": "string"}, "setting": {"type": "string"}, "source": {"type": "string"}}}},+ "walLsn": {"type": "string"},+ "checkpoint": {"type": "object"}+ }+ }+ },+ "allOf": [{+ "if": {"properties": {"verified": {"const": true}}, "required": ["verified"]},+ "then": {"properties": {"steps": {"items": {"properties": {"status": {"const": "ok"}}}}}},+ "else": {"properties": {"steps": {"contains": {"properties": {"status": {"const": "failed"}}, "required": ["status"]}}}}+ }],+ "$defs": {+ "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "step": {"type": "object", "additionalProperties": false,+ "required": ["name", "startedAt", "finishedAt", "status"],+ "properties": {+ "name": {"type": "string", "minLength": 1},+ "startedAt": {"type": "string", "format": "date-time"},+ "finishedAt": {"type": "string", "format": "date-time"},+ "status": {"enum": ["ok", "failed"]},+ "detail": {"type": "string"}+ }+ }+ }+}
+ test/golden/cell/cell.reset-request.v1.json view
@@ -0,0 +1,6 @@+{+ "schema": "cell.reset-request/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "leaseId": "lease-001",+ "reset": {"cachePolicy": "warm"}+}
+ test/golden/cell/cell.reset-request.v1.schema.json view
@@ -0,0 +1,44 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.reset-request.v1.schema.json",+ "title": "Lease-authorized role reset request",+ "type": "object", "additionalProperties": false,+ "required": ["schema", "runId", "leaseId", "reset"],+ "properties": {+ "schema": {"const": "cell.reset-request/v1"},+ "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "leaseId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "reset": {+ "type": "object", "additionalProperties": false,+ "required": ["cachePolicy"],+ "properties": {+ "cachePolicy": {"enum": ["cold", "warm"]},+ "postgres": {+ "type": "object", "additionalProperties": false,+ "required": ["major", "databases", "settings"],+ "properties": {+ "major": {"enum": [17, 18]},+ "databases": {+ "type": "array", "uniqueItems": true,+ "items": {+ "type": "object", "additionalProperties": false,+ "required": ["name", "owner", "template"],+ "properties": {+ "name": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+ "owner": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+ "template": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"}+ }+ }+ },+ "settings": {"type": "object", "additionalProperties": {"type": "string"}, "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_.-]*$"}}+ }+ },+ "broker": {+ "type": "object", "additionalProperties": false,+ "required": ["wipe"],+ "properties": {"wipe": {"type": "boolean"}}+ }+ }+ }+ }+}
+ test/golden/cell/cell.run-result.v1.json view
@@ -0,0 +1,11 @@+{+ "schema": "cell.run-result/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "cell": "alpha",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "leaseSequence": 2,+ "outcome": "completed",+ "entryExitCode": 0,+ "entrySignal": null,+ "reasons": []+}
+ test/golden/cell/cell.run-result.v1.schema.json view
@@ -0,0 +1,22 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.run-result.v1.schema.json",+ "title": "Verification cell execution result",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "runId", "cell", "leaseId", "leaseSequence", "outcome", "entryExitCode", "entrySignal", "reasons"],+ "properties": {+ "schema": {"const": "cell.run-result/v1"},+ "runId": {"$ref": "#/$defs/identifier"},+ "cell": {"$ref": "#/$defs/identifier"},+ "leaseId": {"$ref": "#/$defs/identifier"},+ "leaseSequence": {"type": "integer", "minimum": 0},+ "outcome": {"enum": ["completed", "infrastructure-failure", "cancelled", "timed-out"]},+ "entryExitCode": {"type": ["integer", "null"]},+ "entrySignal": {"type": ["integer", "null"]},+ "reasons": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}+ },+ "$defs": {+ "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"}+ }+}
+ test/golden/cell/cell.status.v1.json view
@@ -0,0 +1,11 @@+{+ "schema": "cell.status/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "phase": "sealed",+ "updatedAt": "2026-09-25T12:00:00Z",+ "phaseStartedAt": "2026-09-25T11:59:59Z",+ "logChunks": {"stdout": 3, "stderr": 1},+ "outcome": "completed",+ "manifestSha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",+ "reasons": []+}
+ test/golden/cell/cell.status.v1.schema.json view
@@ -0,0 +1,34 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.status.v1.schema.json",+ "title": "Verification cell submission status",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "runId", "phase", "updatedAt", "logChunks"],+ "properties": {+ "schema": {"const": "cell.status/v1"},+ "runId": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "phase": {"enum": ["accepted", "resetting", "fetching", "running", "publishing", "sealed"]},+ "leaseSequence": {"type": "integer", "minimum": 0},+ "updatedAt": {"type": "string", "format": "date-time"},+ "phaseStartedAt": {"type": "string", "format": "date-time"},+ "logChunks": {+ "type": "object", "additionalProperties": false,+ "required": ["stdout", "stderr"],+ "properties": {+ "stdout": {"type": "integer", "minimum": 0},+ "stderr": {"type": "integer", "minimum": 0}+ }+ },+ "outcome": {"enum": ["completed", "infrastructure-failure", "cancelled", "timed-out"]},+ "manifestSha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},+ "reasons": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}+ },+ "allOf": [+ {+ "if": {"properties": {"phase": {"const": "sealed"}}, "required": ["phase"]},+ "then": {"required": ["outcome", "manifestSha256"]},+ "else": {"not": {"anyOf": [{"required": ["outcome"]}, {"required": ["manifestSha256"]}]}}+ }+ ]+}
+ test/golden/cell/cell.submission.v1.json view
@@ -0,0 +1,41 @@+{+ "schema": "cell.submission/v1",+ "runId": "0199a3c2-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "leaseId": "0199a3c1-0d52-7f10-8a77-6e0c4b2d9f01",+ "payload": {+ "schema": "cell.payload/v1",+ "kind": "nix-nar-bundle",+ "bundle": {+ "uri": "gs://tan-nb-exp-cells-control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst",+ "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "bytes": 48211934+ },+ "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello",+ "narHash": "sha256-example",+ "closurePaths": ["/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-cell-fixture-hello"],+ "system": "x86_64-linux",+ "command": ["bin/cell-fixture-hello"]+ },+ "work": {+ "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "bytes": 212,+ "mediaType": "application/json"+ },+ "env": {"EXAMPLE": "value"},+ "reset": {+ "postgres": {+ "major": 18,+ "databases": [{"name": "benchmark", "owner": "benchmark", "template": "template0"}],+ "settings": {"checkpoint_timeout": "30min", "max_wal_size": "16GB"}+ },+ "cachePolicy": "cold",+ "broker": {"wipe": true}+ },+ "limits": {+ "wallClockSeconds": 3600,+ "memoryMaxBytes": 25769803776,+ "outputMaxBytes": 21474836480+ },+ "requires": {"protocol": "cell.protocol/v1", "minAgentVersion": "0.1.0"},+ "labels": {"arm": "candidate"}+}
+ test/golden/cell/cell.submission.v1.schema.json view
@@ -0,0 +1,86 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "$id": "https://shinzui.dev/load-testing-infra/schemas/cell/cell.submission.v1.schema.json",+ "title": "Verification cell run submission",+ "type": "object",+ "additionalProperties": false,+ "required": ["schema", "runId", "leaseId", "payload", "work", "env", "reset", "limits", "requires", "labels"],+ "properties": {+ "schema": {"const": "cell.submission/v1"},+ "runId": {"$ref": "#/$defs/identifier"},+ "leaseId": {"$ref": "#/$defs/identifier"},+ "payload": {"$ref": "cell.payload.v1.schema.json"},+ "work": {+ "type": "object", "additionalProperties": false,+ "required": ["sha256", "bytes", "mediaType"],+ "properties": {+ "sha256": {"$ref": "#/$defs/sha256"},+ "bytes": {"type": "integer", "minimum": 0},+ "mediaType": {"type": "string", "minLength": 1}+ }+ },+ "env": {"$ref": "#/$defs/stringMap"},+ "reset": {+ "type": "object", "additionalProperties": false,+ "required": ["cachePolicy"],+ "properties": {+ "cachePolicy": {"enum": ["cold", "warm"]},+ "resources": {"type": "object", "description": "kenshou extension pending upstream adoption: per-resource reset requests, keyed by resource name"},+ "postgres": {+ "type": "object", "additionalProperties": false,+ "required": ["major", "databases", "settings"],+ "properties": {+ "major": {"enum": [17, 18]},+ "databases": {+ "type": "array", "uniqueItems": true,+ "items": {+ "type": "object", "additionalProperties": false,+ "required": ["name", "owner", "template"],+ "properties": {+ "name": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+ "owner": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"},+ "template": {"type": "string", "pattern": "^[a-z][a-z0-9_]{0,62}$"}+ }+ }+ },+ "settings": {"$ref": "#/$defs/stringMap"}+ }+ },+ "broker": {+ "type": "object", "additionalProperties": false,+ "required": ["wipe"],+ "properties": {"wipe": {"type": "boolean"}}+ }+ }+ },+ "limits": {+ "type": "object", "additionalProperties": false,+ "required": ["wallClockSeconds", "memoryMaxBytes", "outputMaxBytes"],+ "properties": {+ "wallClockSeconds": {"type": "integer", "minimum": 1},+ "memoryMaxBytes": {"type": "integer", "minimum": 1},+ "outputMaxBytes": {"type": "integer", "minimum": 1}+ }+ },+ "requires": {+ "type": "object", "additionalProperties": false,+ "required": ["protocol", "minAgentVersion"],+ "properties": {+ "protocol": {"const": "cell.protocol/v1"},+ "minAgentVersion": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"},+ "capabilities": {"type": "array", "uniqueItems": true, "items": {"type": "string", "minLength": 1}}+ }+ },+ "collect": {+ "type": "object", "additionalProperties": false,+ "required": ["traces"],+ "properties": {"traces": {"type": "boolean"}, "profiles": {"type": "boolean"}}+ },+ "labels": {"$ref": "#/$defs/stringMap"}+ },+ "$defs": {+ "identifier": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,62}$"},+ "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},+ "stringMap": {"type": "object", "additionalProperties": {"type": "string"}, "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_.-]*$"}}+ }+}
+ test/golden/parity-report.json view
@@ -0,0 +1,10 @@+{+ "schema": "kenshou.parity-report/v1",+ "localRunResultSha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000",+ "cellRunResultSha256": "sha256:1111111111111111111111111111111111111111111111111111111111111111",+ "equal": ["result.outcome", "spec.seed"],+ "intentional": [+ {"path": "result.fingerprint.placement", "local": "local", "cell": "cell"}+ ],+ "unexpected": []+}
+ test/golden/payload-identity.json view
@@ -0,0 +1,12 @@+{+ "schema": "kenshou.payload-identity/v1",+ "cohort": "released",+ "variant": "default",+ "system": "x86_64-linux",+ "compiler": "ghc-9.12.4",+ "nixpkgsRevision": "d5dfd8e6716dde34398bc14bc87c10dece9c8c68",+ "haskellNixRevision": "7b696dc80f8aaccaf1783fda0ab6a7f978a67134",+ "packages": [+ { "name": "kenshou-cli", "version": "0.1.0.0" }+ ]+}
+ test/golden/payload.json view
@@ -0,0 +1,79 @@+{+ "cell": {+ "bundle": {+ "bytes": 1,+ "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "uri": "gs://control/payloads/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.nar.zst"+ },+ "closurePaths": [+ "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released"+ ],+ "command": [+ "bin/kenshou",+ "cell",+ "exec"+ ],+ "kind": "nix-nar-bundle",+ "narHash": "sha256-example",+ "schema": "cell.payload/v1",+ "storePath": "/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-kenshou-released",+ "system": "x86_64-linux"+ },+ "cohort": "released",+ "cohortCheck": {+ "packagesChecked": 2,+ "status": "consistent"+ },+ "cohortIdentity": {+ "arch": "x86_64",+ "cabalVersion": "nix",+ "cohort": "released",+ "compiler": "ghc-9.12.4",+ "components": [+ {+ "id": "foo",+ "moriUri": "mori://example/foo",+ "packages": [+ {+ "name": "foo",+ "source": {+ "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "type": "hackage"+ },+ "version": "1.0.0"+ }+ ]+ },+ {+ "id": "bar",+ "moriUri": "mori://example/bar",+ "packages": [+ {+ "name": "bar",+ "source": {+ "location": "https://example.com/bar.git",+ "rev": "0123456789abcdef0123456789abcdef01234567",+ "subdir": "bar",+ "type": "git"+ },+ "version": "2.0.0"+ }+ ]+ }+ ],+ "descriptorSha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",+ "indexState": "2026-09-20T13:44:47Z",+ "os": "linux",+ "planHash": "sha256:857e6ff76844ccdf1903dc62d6518fe59b64e34d4bc626873efe2534934f1cc7",+ "resolver": "nix",+ "schema": "kenshou.cohort-identity/v1"+ },+ "createdAt": "2026-09-27T00:00:00Z",+ "flakeAttr": "packages.x86_64-linux.kenshou-released",+ "harness": {+ "dirty": false,+ "revision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ },+ "schema": "kenshou.payload/v1",+ "variant": "default"+}