packages feed

kenshou-core (empty) → 0.1.0.0

raw patch · 96 files changed

+10796/−0 lines, 96 filesdep +aesondep +basedep +base16-bytestring

Dependencies added: aeson, base, base16-bytestring, bytestring, containers, cryptohash-sha256, directory, ephemeral-pg, file-embed, filepath, hasql, hedgehog, hspec, hspec-hedgehog, kenshou-core, mmzk-typeid, optparse-applicative, pg-migrate, process, random, scientific, settei, settei-optparse-applicative, settei-yaml, splitmix, temporary, text, time, unix, uuid, vector

Files

+ CHANGELOG.md view
@@ -0,0 +1,17 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of the scenario kernel: run specifications, results, worker roles, knobs, dimensions, phases, cohort identity, change-aware planning, and the performance session protocol.+- Open layer identifiers (`LayerId`, `mkLayerId`) replace the closed `Layer` type, and a suite registers its layers as `LayerBundle` values built with `mkBundle`.+- Suite-supplied migration sets (`MigrationSet`) compose into one pg-migrate ledger per run database.+- Named resources (`ResourceProvider`, `requiresResource`) replace the broker-shaped environment fields; documents written in the legacy shape remain readable.+- Bundles contribute oracles, advisory-lock labels, pool methodology, routing rules, and their evidence subject.+- `Program`, `defaultProgram`, and `runCli` let an executable supply its bundles, commands, component graph, and performance contracts.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.+- The kernel no longer depends on any runtime migration library; its own self-test schema is the `kenshou-selftest` migration set.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+   list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+   this list of conditions and the following disclaimer in the documentation+   and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+   may be used to endorse or promote products derived from this software without+   specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ data/components.json view
@@ -0,0 +1,333 @@+{+  "schema": "kenshou.component-graph/v1",+  "ignoredPackages": [+    "pg-migrate-test-support"+  ],+  "components": [+    {+      "id": "postgresql",+      "kind": "service",+      "uri": null,+      "repository": "postgresql",+      "sourceRoots": [],+      "packages": [],+      "selectors": [],+      "verifiedAgainst": "PostgreSQL 17 and 18",+      "dependsOn": []+    },+    {+      "id": "pg-migrate",+      "kind": "library",+      "uri": "mori://shinzui/pg-migrate",+      "repository": "pg-migrate",+      "sourceRoots": [+        "src/",+        "pg-migrate.cabal"+      ],+      "packages": [+        "pg-migrate"+      ],+      "selectors": [],+      "verifiedAgainst": "pg-migrate 1.1.0.0",+      "dependsOn": [+        {+          "to": "postgresql",+          "kind": "runtime"+        }+      ]+    },+    {+      "id": "ephemeral-pg",+      "kind": "harness",+      "uri": "mori://shinzui/ephemeral-pg",+      "repository": "ephemeral-pg",+      "sourceRoots": [+        "src/",+        "ephemeral-pg.cabal"+      ],+      "packages": [+        "ephemeral-pg"+      ],+      "selectors": [+        "selftest/kernel/**"+      ],+      "verifiedAgainst": "ephemeral-pg 0.3.1.0",+      "dependsOn": [+        {+          "to": "postgresql",+          "kind": "runtime"+        }+      ]+    },+    {+      "id": "hs-opentelemetry",+      "kind": "library",+      "uri": "mori://iand675/hs-opentelemetry",+      "repository": "hs-opentelemetry",+      "sourceRoots": [],+      "packages": [+        "hs-opentelemetry-api",+        "hs-opentelemetry-sdk",+        "hs-opentelemetry-exporter-otlp",+        "hs-opentelemetry-propagator-w3c",+        "hs-opentelemetry-semantic-conventions"+      ],+      "selectors": [+        "selftest/telemetry/**"+      ],+      "changeImplies": {+        "minDimensionPolicy": "telemetry-corners"+      },+      "verifiedAgainst": "hs-opentelemetry 1.0 family",+      "dependsOn": []+    },+    {+      "id": "kenshou-harness",+      "kind": "harness",+      "uri": "mori://shinzui/kenshou/packages/kenshou-core",+      "repository": "kenshou",+      "sourceRoots": [+        "kenshou-core/"+      ],+      "packages": [+        "kenshou-core"+      ],+      "selectors": [+        "selftest/**"+      ],+      "verifiedAgainst": "kenshou-core 0.1.0.0",+      "subcomponents": [+        {+          "id": "kernel",+          "pathPrefixes": [],+          "selectors": [+            "selftest/kernel/**"+          ],+          "uses": []+        }+      ],+      "dependsOn": [+        {+          "to": "pg-migrate",+          "kind": "build"+        },+        {+          "to": "ephemeral-pg",+          "kind": "build"+        }+      ]+    }+  ],+  "repositoryPaths": [+    {+      "prefix": "kenshou-core/src/Kenshou/Plan/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "kenshou-core/data/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "kenshou-cli/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "kenshou-remote/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "kenshou-evidence/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "schemas/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "suites/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "policies/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "cohort/",+      "effect": {+        "kind": "cohort"+      }+    },+    {+      "prefix": "kenshou-core/",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "kenshou-measure/",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "kenshou-check/",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "kenshou-diagnose/",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "kenshou-telemetry/",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "cabal.project",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "flake.",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "nix/",+      "effect": {+        "kind": "all"+      }+    },+    {+      "prefix": "docs/",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": "agents/",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": ".seihou/",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": ".github/",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": ".claude/",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": ".agents/",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": "README.md",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": "mori.dhall",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": "mina.kdl",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": "justfile",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": "process-compose.yaml",+      "effect": {+        "kind": "nothing"+      }+    },+    {+      "prefix": "kenshou-synthetic/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    },+    {+      "prefix": "fixtures/",+      "effect": {+        "kind": "selectors",+        "selectors": [+          "selftest/**"+        ]+      }+    }+  ]+}
+ kenshou-core.cabal view
@@ -0,0 +1,169 @@+cabal-version: 3.0+name: kenshou-core+version: 0.1.0.0+synopsis: Scenario kernel and protocol for the Kenshou verification harness+description: Shared contracts and utilities for the Kenshou verification harness.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+  data/components.json+  test/fixtures/*.json+  test/fixtures/invalid-project/cohort/*.project+  test/fixtures/perf/*.json+  test/fixtures/plan/*.json+  test/fixtures/valid-project/cohort/*.project+  test/golden/*.json+  test/golden/*.jsonl+  test/golden/resources/*.json++common warnings+  ghc-options:+    -Wall+    -Wcompat+    -Widentities+    -Wincomplete-record-updates+    -Wincomplete-uni-patterns+    -Wpartial-fields+    -Wredundant-constraints++  default-language: GHC2024+  default-extensions:+    BlockArguments+    DuplicateRecordFields+    ImportQualifiedPost+    NoFieldSelectors+    OverloadedLabels+    OverloadedRecordDot+    OverloadedStrings++library+  import: warnings+  hs-source-dirs: src+  exposed-modules:+    Kenshou.Core.Bundle+    Kenshou.Core.Canonical+    Kenshou.Core.Cli+    Kenshou.Core.Cli.Config+    Kenshou.Core.Cohort+    Kenshou.Core.Compat+    Kenshou.Core.Context+    Kenshou.Core.Contribution+    Kenshou.Core.Dimension+    Kenshou.Core.Env+    Kenshou.Core.Env.Migration+    Kenshou.Core.Env.Postgres+    Kenshou.Core.Fingerprint+    Kenshou.Core.Id+    Kenshou.Core.Knob+    Kenshou.Core.Log+    Kenshou.Core.Manifest+    Kenshou.Core.Outcome+    Kenshou.Core.Phase+    Kenshou.Core.Resource+    Kenshou.Core.Resource.Types+    Kenshou.Core.Role+    Kenshou.Core.Role.Dispatch+    Kenshou.Core.Role.Spawn+    Kenshou.Core.Run+    Kenshou.Core.RunResult+    Kenshou.Core.RunSpec+    Kenshou.Core.RunSpec.Resolve+    Kenshou.Core.Scenario+    Kenshou.Core.Selector+    Kenshou.Core.Selftest+    Kenshou.Core.Selftest.Schema+    Kenshou.Core.Version+    Kenshou.Perf.Artifacts+    Kenshou.Perf.Calibration+    Kenshou.Perf.Contracts+    Kenshou.Perf.Plan+    Kenshou.Perf.Prepared+    Kenshou.Perf.Request+    Kenshou.Perf.Result+    Kenshou.Perf.Source+    Kenshou.Plan.Catalog+    Kenshou.Plan.Change+    Kenshou.Plan.Change.Cohort+    Kenshou.Plan.Change.Git+    Kenshou.Plan.Components+    Kenshou.Plan.Components.Check+    Kenshou.Plan.Execute+    Kenshou.Plan.Matrix+    Kenshou.Plan.Policy+    Kenshou.Plan.RunPlan+    Kenshou.Plan.Selector+    Kenshou.Plan.Suite+    Kenshou.Plan.Summary++  other-modules: Paths_kenshou_core+  autogen-modules: Paths_kenshou_core+  build-depends:+    aeson >=2.2 && <2.3,+    base >=4.21 && <4.22,+    base16-bytestring >=1.0 && <1.1,+    bytestring >=0.12 && <0.13,+    containers >=0.7 && <0.8,+    cryptohash-sha256 >=0.11 && <0.12,+    directory >=1.3 && <1.4,+    ephemeral-pg >=0.3.1 && <0.4,+    file-embed >=0.0.16 && <0.1,+    filepath >=1.5 && <1.6,+    hasql >=1.10 && <1.11,+    mmzk-typeid >=0.7 && <0.8,+    optparse-applicative >=0.19 && <0.20,+    pg-migrate ^>=1.1.0.0 || ^>=1.2.0.0,+    process >=1.6 && <1.7,+    random >=1.2.1 && <1.4,+    scientific >=0.3 && <0.4,+    settei ^>=0.2.0.0,+    settei-optparse-applicative ^>=0.2.0.0,+    settei-yaml ^>=0.2.0.0,+    splitmix >=0.1 && <0.2,+    text >=2.1 && <2.2,+    time >=1.14 && <1.15,+    unix >=2.8 && <2.9,+    uuid >=1.3 && <1.4,+    vector >=0.13 && <0.14,++test-suite kenshou-core-test+  import: warnings+  type: exitcode-stdio-1.0+  hs-source-dirs: test+  main-is: Main.hs+  other-modules:+    Kenshou.ConsumerBaseline+    Kenshou.PerfCalibrationSpec+    Kenshou.PerfPreparedSpec+    Kenshou.PerfResultSpec+    Kenshou.PerfSpec+    Kenshou.PlanSpec+    Kenshou.ResourceSpec++  build-depends:+    aeson >=2.2 && <2.3,+    base >=4.21 && <4.22,+    bytestring,+    containers,+    directory >=1.3 && <1.4,+    filepath,+    hedgehog >=1.4 && <1.8,+    hspec >=2.11 && <2.12,+    hspec-hedgehog >=0.0 && <0.4,+    kenshou-core,+    pg-migrate,+    process >=1.6 && <1.7,+    temporary,+    text >=2.1 && <2.2,+    time >=1.14 && <1.15,++source-repository head+  type: git+  location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Core/Bundle.hs view
@@ -0,0 +1,272 @@+module Kenshou.Core.Bundle+  ( LayerBundle (..),+    mkBundle,+    lookupBundle,+    lookupResource,+    registrySubjects,+    registryRoutingRules,+    ScenarioRule (..),+    requirePgVersionsFor,+    Registry,+    registryMigrationSets,+    RegistryError (..),+    ListFilter (..),+    emptyListFilter,+    mkRegistry,+    allScenarios,+    allRoles,+    lookupScenario,+    lookupRole,+    selectScenarios,+    scenarioListDocument,+    scenarioListDocumentFor,+  )+where++import Data.Aeson (Value, object, toJSON, (.=))+import Data.Foldable (toList)+import Data.List (group, sort, sortOn)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Contribution (AdvisoryLabel, OracleSpec (..), PoolMethodology, RoutingRule)+import Kenshou.Core.Dimension (DimensionSupport (..), PgDurability (..), PgVersion (..), Support (..), Supported (..))+import Kenshou.Core.Env (EnvRequirements (..), MigrationSet (..), MigrationSetName, PostgresRequirement (..), mkMigrationSetName, renderMigrationSetName)+import Kenshou.Core.Id (Kind (..), LayerId, ScenarioId (..), renderKind, renderLayer, renderScenarioId, unSegment)+import Kenshou.Core.Knob (KnobSpec (..), renderKnobName)+import Kenshou.Core.Resource (ResourceName, ResourceProvider (..), renderResourceName)+import Kenshou.Core.Role (RoleName, WorkerRole (..), renderRoleName)+import Kenshou.Core.Scenario (KnownDefect (..), Placement, Scenario (..), Tier, individualKnownDefects, renderPlacement, renderTier)+import Kenshou.Core.Selector (ScenarioSelector, matchesSelector)+import Kenshou.Core.Version (suiteVersion)++-- | What a suite registers with the kernel. Build one with 'mkBundle' and set the+-- optional contributions with record update, so new fields never break a suite.+data LayerBundle = LayerBundle+  { layer :: LayerId,+    scenarios :: [Scenario],+    roles :: [WorkerRole],+    -- | Migration sets this bundle defines. Any scenario in the registry may+    -- require them by name.+    migrations :: [MigrationSet],+    -- | Checks this bundle contributes. 'mkRegistry' applies every bundle's rules+    -- to every scenario in the registry.+    rules :: [ScenarioRule],+    -- | Providers for named resources. Any scenario in the registry may require a+    -- resource a bundle provides.+    resources :: [ResourceProvider],+    -- | SQL oracles the suite's scenarios reconcile, by name.+    oracles :: [OracleSpec],+    -- | Advisory-lock labels a stall capture of the suite's scenarios names.+    advisoryLabels :: [AdvisoryLabel],+    -- | The connection-pool size the suite recommends, when it has one.+    methodology :: Maybe PoolMethodology,+    -- | The canonical Mori URI that evidence records of this layer name as their+    -- subject.+    evidenceSubject :: Maybe Text,+    -- | Cell capabilities the suite's runs need.+    routingRules :: [RoutingRule]+  }++mkBundle :: LayerId -> [Scenario] -> [WorkerRole] -> LayerBundle+mkBundle layer scenarios roles = LayerBundle {layer, scenarios, roles, migrations = [], rules = [], resources = [], oracles = [], advisoryLabels = [], methodology = Nothing, evidenceSubject = Nothing, routingRules = []}++-- | A named registry check. Each message 'check' returns becomes a 'RegistryError'+-- prefixed with the scenario identifier.+data ScenarioRule = ScenarioRule+  { name :: Text,+    check :: Scenario -> [Text]+  }++-- | Require that a scenario using the migration set supports only the given+-- PostgreSQL versions, for a set whose migrations need a particular server.+requirePgVersionsFor :: MigrationSetName -> NonEmpty PgVersion -> ScenarioRule+requirePgVersionsFor set versions =+  ScenarioRule+    { name = renderMigrationSetName set <> "-postgres-versions",+      check = \scenario ->+        [ renderMigrationSetName set <> " scenarios require PostgreSQL " <> Text.intercalate " or " (fmap versionText (toList versions))+        | Just requirement <- [scenario.requires.postgres],+          set `elem` requirement.schemas,+          not (supportsOnly scenario.dimensions.pgVersion)+        ]+    }+  where+    supportsOnly (Supported support) = all (`elem` versions) support.values+    supportsOnly NotApplicable = False+    versionText Pg17 = "17"+    versionText Pg18 = "18"++newtype Registry = Registry [LayerBundle]++newtype RegistryError = RegistryError Text+  deriving stock (Eq, Show)++data ListFilter = ListFilter+  { selectors :: [ScenarioSelector],+    layers :: [LayerId],+    kinds :: [Kind],+    maxTier :: Maybe Tier,+    placements :: [Placement]+  }++emptyListFilter :: ListFilter+emptyListFilter = ListFilter [] [] [] Nothing []++mkRegistry :: [LayerBundle] -> Either (NonEmpty RegistryError) Registry+mkRegistry bundles = case errors of+  [] -> Right (Registry bundles)+  first : rest -> Left (first :| rest)+  where+    scenarios = concatMap (.scenarios) bundles+    roles = concatMap (.roles) bundles+    errors =+      [RegistryError ("scenario layer differs from bundle: " <> renderScenarioId scenario.id) | bundle <- bundles, scenario <- bundle.scenarios, scenario.id.layer /= bundle.layer]+        <> duplicateErrors "scenario" (fmap (renderScenarioId . (.id)) scenarios)+        <> duplicateErrors "role" (fmap (renderRoleName . (.name)) roles)+        <> [RegistryError ("role layer differs from bundle: " <> renderRoleName role.name) | bundle <- bundles, role <- bundle.roles, not ((renderLayer bundle.layer <> "/") `Text.isPrefixOf` renderRoleName role.name)]+        <> concatMap validateScenario scenarios+        <> migrationErrors bundles+        <> resourceErrors bundles+        <> [RegistryError ("evidence subject must be a canonical Mori URI: " <> subject) | bundle <- bundles, Just subject <- [bundle.evidenceSubject], not ("mori://" `Text.isPrefixOf` subject)]+        <> concat [duplicateErrors ("oracle in " <> renderLayer bundle.layer) (fmap (.name) bundle.oracles) | bundle <- bundles]+        <> [RegistryError (renderScenarioId scenario.id <> ": " <> message) | rule <- concatMap (.rules) bundles, scenario <- scenarios, message <- rule.check scenario]++-- Every set and required name is a valid pg-migrate component name, no two bundles+-- define the same set, and every set a scenario or another set requires is defined.+migrationErrors :: [LayerBundle] -> [RegistryError]+migrationErrors bundles =+  [RegistryError problem | name <- definedNames <> requiredNames, Left problem <- [mkMigrationSetName (renderMigrationSetName name)]]+    <> duplicateErrors "migration set" (fmap renderMigrationSetName definedNames)+    <> [ RegistryError (renderScenarioId scenario.id <> ": no registered bundle defines migration set " <> renderMigrationSetName name)+       | scenario <- concatMap (.scenarios) bundles,+         name <- scenarioSets scenario,+         name `notElem` definedNames+       ]+    <> [ RegistryError ("migration set " <> renderMigrationSetName set.name <> " requires undefined set " <> renderMigrationSetName name)+       | set <- sets,+         name <- set.requires,+         name `notElem` definedNames+       ]+  where+    sets = concatMap (.migrations) bundles+    definedNames = fmap (.name) sets+    requiredNames = concatMap scenarioSets (concatMap (.scenarios) bundles) <> concatMap (.requires) sets+    scenarioSets scenario = concatMap (.schemas) (maybe [] pure scenario.requires.postgres <> fmap snd scenario.requires.extraPostgres)++-- No two bundles provide the same resource, and every resource a scenario requires+-- has a provider.+resourceErrors :: [LayerBundle] -> [RegistryError]+resourceErrors bundles =+  duplicateErrors "resource provider" (fmap (renderResourceName . (.name)) providers)+    <> [ RegistryError ("scenario requires unprovided resource " <> renderResourceName required <> ": " <> renderScenarioId scenario.id)+       | scenario <- concatMap (.scenarios) bundles,+         required <- scenario.requires.resources,+         required `notElem` fmap (.name) providers+       ]+  where+    providers = concatMap (.resources) bundles++-- | The first registered bundle of a layer. Several bundles may share a layer;+-- suite contributions are read from the first.+lookupBundle :: Registry -> LayerId -> Maybe LayerBundle+lookupBundle (Registry bundles) wanted = case filter ((== wanted) . (.layer)) bundles of+  bundle : _ -> Just bundle+  [] -> Nothing++-- | The evidence subject each layer's first bundle that declares one names.+registrySubjects :: Registry -> Map.Map LayerId Text+registrySubjects (Registry bundles) = Map.fromListWith (\_ first -> first) [(bundle.layer, subject) | bundle <- bundles, Just subject <- [bundle.evidenceSubject]]++-- | Every bundle's routing rules, in registry order.+registryRoutingRules :: Registry -> [RoutingRule]+registryRoutingRules (Registry bundles) = concatMap (.routingRules) bundles++lookupResource :: Registry -> ResourceName -> Maybe ResourceProvider+lookupResource (Registry bundles) wanted = case [provider | bundle <- bundles, provider <- bundle.resources, provider.name == wanted] of+  provider : _ -> Just provider+  [] -> Nothing++registryMigrationSets :: Registry -> Map.Map MigrationSetName MigrationSet+registryMigrationSets (Registry bundles) = Map.fromList [(set.name, set) | bundle <- bundles, set <- bundle.migrations]++duplicateErrors :: Text -> [Text] -> [RegistryError]+duplicateErrors label values = [RegistryError ("duplicate " <> label <> ": " <> item) | items@(item : _) <- group (sort values), length items > 1]++validateScenario :: Scenario -> [RegistryError]+validateScenario scenario =+  [RegistryError (renderScenarioId scenario.id <> ": revision must be positive") | scenario.revision < 1]+    <> [RegistryError (renderScenarioId scenario.id <> ": duplicate knob " <> name) | name <- duplicateNames]+    <> postgresErrors+    <> benchmarkErrors+    <> knownDefectErrors+  where+    duplicateNames = [item | items@(item : _) <- group (sort (fmap (renderKnobName . (.name)) scenario.knobs)), length items > 1]+    hasPostgres = isJust scenario.requires.postgres+    pgApplicable = case (scenario.dimensions.pgDurability, scenario.dimensions.pgVersion) of+      (NotApplicable, NotApplicable) -> False+      _ -> True+    postgresErrors =+      [RegistryError (renderScenarioId scenario.id <> ": PostgreSQL dimensions disagree with environment requirement") | hasPostgres /= pgApplicable]+    benchmarkErrors = case (scenario.id.kind, scenario.requires.postgres, scenario.dimensions.pgDurability) of+      (Benchmark, Just _, Supported support) | PgFsyncOff `elem` support.values -> [RegistryError (renderScenarioId scenario.id <> ": benchmark supports fsync-off")]+      _ -> []+    knownDefectErrors =+      [ RegistryError (renderScenarioId scenario.id <> ": invalid known-defect reference")+      | defect <- maybe [] individualKnownDefects scenario.knownDefect,+        not ("mori://" `Text.isPrefixOf` defect.reference || "https://" `Text.isPrefixOf` defect.reference)+      ]++allScenarios :: Registry -> [Scenario]+allScenarios (Registry bundles) = sortOn (renderScenarioId . (.id)) (concatMap (.scenarios) bundles)++allRoles :: Registry -> [WorkerRole]+allRoles (Registry bundles) = sortOn (renderRoleName . (.name)) (concatMap (.roles) bundles)++lookupScenario :: Registry -> ScenarioId -> Maybe Scenario+lookupScenario registry wanted = Map.lookup wanted (Map.fromList [(scenario.id, scenario) | scenario <- allScenarios registry])++lookupRole :: Registry -> RoleName -> Maybe WorkerRole+lookupRole registry wanted = Map.lookup wanted (Map.fromList [(role.name, role) | role <- allRoles registry])++selectScenarios :: Registry -> ListFilter -> [Scenario]+selectScenarios registry filterValue = filter selected (allScenarios registry)+  where+    selected scenario =+      (null filterValue.selectors || any (`matchesSelector` scenario.id) filterValue.selectors)+        && (null filterValue.layers || scenario.id.layer `elem` filterValue.layers)+        && (null filterValue.kinds || scenario.id.kind `elem` filterValue.kinds)+        && maybe True (scenario.tier <=) filterValue.maxTier+        && (null filterValue.placements || scenario.placement `elem` filterValue.placements)++scenarioListDocument :: Registry -> Value+scenarioListDocument registry = scenarioListDocumentFor (allScenarios registry) (allRoles registry)++scenarioListDocumentFor :: [Scenario] -> [WorkerRole] -> Value+scenarioListDocumentFor scenarios roles =+  object+    [ "schema" .= ("kenshou.scenario-list/v1" :: Text),+      "suiteVersion" .= suiteVersion,+      "scenarios" .= fmap scenarioValue scenarios,+      "roles" .= fmap roleValue roles+    ]+  where+    scenarioValue scenario =+      object+        [ "id" .= renderScenarioId scenario.id,+          "layer" .= renderLayer scenario.id.layer,+          "component" .= unSegment scenario.id.component,+          "kind" .= renderKind scenario.id.kind,+          "name" .= unSegment scenario.id.name,+          "revision" .= scenario.revision,+          "summary" .= scenario.summary,+          "tier" .= renderTier scenario.tier,+          "placement" .= renderPlacement scenario.placement,+          "knownDefect" .= fmap defectReferences scenario.knownDefect+        ]+    roleValue role = object ["name" .= renderRoleName role.name, "summary" .= role.summary]+    defectReferences defect = case individualKnownDefects defect of+      [single] -> toJSON single.reference+      many -> toJSON (fmap (.reference) many)
+ src/Kenshou/Core/Canonical.hs view
@@ -0,0 +1,39 @@+module Kenshou.Core.Canonical (canonicalEncode, sha256Hex) where++import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (Value (..), encode)+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString (ByteString)+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Builder (Builder, char8, lazyByteString, toLazyByteString)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (sortOn)+import Data.Text (Text)+import Data.Text.Encoding qualified as Text+import Data.Vector qualified as Vector++canonicalEncode :: Value -> ByteString+canonicalEncode = LazyByteString.toStrict . toLazyByteString . render+  where+    render Null = "null"+    render (Bool True) = "true"+    render (Bool False) = "false"+    render (Number number) = lazyByteString (encode number)+    render (String value) = lazyByteString (encode value)+    render (Array values) = enclosed '[' ']' (fmap render (Vector.toList values))+    render (Object values) =+      enclosed+        '{'+        '}'+        [ lazyByteString (encode (Key.toText key)) <> char8 ':' <> render value+        | (key, value) <- sortOn (Text.encodeUtf8 . Key.toText . fst) (KeyMap.toList values)+        ]++    enclosed :: Char -> Char -> [Builder] -> Builder+    enclosed open close values = char8 open <> separated values <> char8 close+    separated [] = mempty+    separated (value : values) = value <> foldMap (char8 ',' <>) values++sha256Hex :: ByteString -> Text+sha256Hex = ("sha256:" <>) . Text.decodeUtf8 . Base16.encode . SHA256.hash
+ src/Kenshou/Core/Cli.hs view
@@ -0,0 +1,128 @@+module Kenshou.Core.Cli+  ( Program (..),+    defaultProgram,+    CliEnv (..),+    CliGroup (..),+    CliCommand (..),+    HelpTopic (..),+    InputSource (..),+    parseInputSource,+    readInputSource,+    runCli,+    exitWithOutcome,+  )+where++import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle, Registry, mkRegistry)+import Kenshou.Core.Outcome (Outcome, outcomeExitCode)+import Kenshou.Perf.Contracts (PerfContracts, noPerfContracts)+import Kenshou.Plan.Components (selftestGraphBytes)+import Options.Applicative+import System.Exit (ExitCode (..))+import System.IO (hPutStrLn, stderr, stdin, stdout)++-- | Everything an executable supplies: its name and help header, the bundles it+-- registers, its commands and help topics, the component graph that change-aware+-- planning reads when no graph file is given, and its performance contracts.+data Program = Program+  { name :: String,+    header :: String,+    bundles :: [LayerBundle],+    commands :: [CliCommand],+    topics :: [HelpTopic],+    componentGraph :: ByteString,+    perf :: PerfContracts+  }++-- | A program named @kenshou@ with no bundles or commands, the self-test component+-- graph, and no performance contracts. Executables extend it with record update.+defaultProgram :: Program+defaultProgram =+  Program+    { name = "kenshou",+      header = "kenshou - run verification scenarios",+      bundles = [],+      commands = [],+      topics = [],+      componentGraph = selftestGraphBytes,+      perf = noPerfContracts+    }++data CliEnv = CliEnv+  { registry :: Registry,+    programName :: Text,+    componentGraph :: ByteString,+    perf :: PerfContracts+  }++data CliGroup = Discovery | Execution | Analysis | Evidence | Maintenance | Internal deriving stock (Eq, Ord, Show, Enum, Bounded)++data CliCommand = CliCommand+  { name :: String,+    description :: String,+    group :: CliGroup,+    hidden :: Bool,+    parser :: Parser (CliEnv -> IO ExitCode)+  }++data HelpTopic = HelpTopic {name :: Text, description :: Text, content :: Text}++data InputSource = InputFile FilePath | InputStdin deriving stock (Eq, Show)++parseInputSource :: String -> InputSource+parseInputSource "-" = InputStdin+parseInputSource path = InputFile path++readInputSource :: InputSource -> IO ByteString+readInputSource InputStdin = ByteString.hGetContents stdin+readInputSource (InputFile path) = ByteString.readFile path++runCli :: Program -> [String] -> IO ExitCode+runCli program arguments = case mkRegistry program.bundles of+  Left problems -> do+    mapM_ (hPutStrLn stderr . show) (NonEmpty.toList problems)+    pure (ExitFailure 4)+  Right registry -> case execParserPure defaultPrefs (parserInfo program) arguments of+    Success action -> action (CliEnv registry (Text.pack program.name) program.componentGraph program.perf)+    Failure failure -> do+      let (message, code) = renderFailure failure program.name+      case code of+        ExitSuccess -> hPutStrLn stdout message >> pure ExitSuccess+        ExitFailure _ -> hPutStrLn stderr message >> pure (ExitFailure 2)+    CompletionInvoked completion -> execCompletion completion program.name >>= putStr >> pure ExitSuccess++parserInfo :: Program -> ParserInfo (CliEnv -> IO ExitCode)+parserInfo program = info (commandParser program.commands <**> helper) (fullDesc <> header program.header)++commandParser :: [CliCommand] -> Parser (CliEnv -> IO ExitCode)+commandParser commands = visibleParser <|> hiddenParser+  where+    visibleCommands = filter (not . (.hidden)) commands+    hiddenCommands = filter (.hidden) commands+    populatedGroups = filter (not . null . commandsIn) [Discovery .. Internal]+    commandsIn groupValue = filter ((== groupValue) . (.group)) visibleCommands+    visibleParser = foldr1 (<|>) (fmap groupParser populatedGroups)+    hiddenParser = subparser (foldMap hiddenCommand hiddenCommands)+    groupParser groupValue = subparser (groupModifier groupValue <> foldMap commandModifier (commandsIn groupValue))+    groupModifier Internal = mempty+    groupModifier groupValue = commandGroup (groupLabel groupValue)+    commandModifier commandValue = command commandValue.name (info (commandValue.parser <**> helper) (progDesc commandValue.description))+    hiddenCommand commandValue = command commandValue.name (info (commandValue.parser <**> helper) (progDesc commandValue.description)) <> internal++groupLabel :: CliGroup -> String+groupLabel Discovery = "Discovery"+groupLabel Execution = "Execution"+groupLabel Analysis = "Analysis"+groupLabel Evidence = "Evidence"+groupLabel Maintenance = "Maintenance"+groupLabel Internal = "Internal"++exitWithOutcome :: Outcome -> ExitCode+exitWithOutcome outcome = case outcomeExitCode outcome of+  0 -> ExitSuccess+  code -> ExitFailure code
+ src/Kenshou/Core/Cli/Config.hs view
@@ -0,0 +1,36 @@+module Kenshou.Core.Cli.Config+  ( ConfigInputs (..),+    configInputsParser,+    loadConfigSources,+  )+where++import Data.List.NonEmpty qualified as NonEmpty+import Data.Maybe (catMaybes)+import Data.Text (Text)+import Data.Text qualified as Text+import Options.Applicative (Parser)+import Options.Applicative qualified as Options+import Settei (Source)+import Settei.Optparse (DiagnosticMode, configPathOptions, diagnosticModeOptions)+import Settei.Yaml (readYamlSource, renderYamlErrorsText, yamlSourceOptions)++data ConfigInputs = ConfigInputs+  { paths :: [FilePath],+    namedSources :: [Source],+    diagnostic :: DiagnosticMode+  }++configInputsParser :: Parser [Maybe Source] -> Parser ConfigInputs+configInputsParser namedParser =+  ConfigInputs+    <$> Options.parserOptionGroup "Configuration" configPathOptions+    <*> (catMaybes <$> Options.parserOptionGroup "Configuration" namedParser)+    <*> Options.parserOptionGroup "Configuration diagnostics" diagnosticModeOptions++loadConfigSources :: ConfigInputs -> IO (Either Text [Source])+loadConfigSources inputs = do+  loaded <- sequence [readYamlSource (yamlSourceOptions ("configuration " <> Text.pack (show index))) path | (index, path) <- zip [1 :: Int ..] inputs.paths]+  pure $ case traverse (either (Left . renderYamlErrorsText) Right) loaded of+    Left err -> Left err+    Right sources -> Right (sources <> inputs.namedSources)
+ src/Kenshou/Core/Cohort.hs view
@@ -0,0 +1,636 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Core.Cohort+  ( CohortName (..),+    ComponentId (..),+    PlanHash (..),+    CohortDescriptor (..),+    ComponentSpec (..),+    PackagePin (..),+    SourceSpec (..),+    PinException (..),+    loadCohortDescriptor,+    CohortIdentity (..),+    ResolvedComponent (..),+    ResolvedPackage (..),+    PackageSource (..),+    CohortSource (..),+    CohortError (..),+    activeCohortName,+    resolveCohortIdentity,+    identityFromPlan,+    planHash,+    CohortMismatch (..),+    checkCohort,+    renderCohortIdentity,+  )+where++import Control.Exception (IOException, displayException, try)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson+  ( FromJSON (parseJSON),+    Object,+    ToJSON (toJSON),+    Value (..),+    eitherDecodeStrict',+    object,+    withObject,+    withText,+    (.!=),+    (.:),+    (.:?),+    (.=),+  )+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.Char (isAlphaNum)+import Data.List (sort, sortOn)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, fromMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Data.Word (Word8)+import Numeric (showHex)+import System.FilePath ((</>))++newtype CohortName = CohortName {unCohortName :: Text}+  deriving stock (Eq, Ord, Show)++newtype ComponentId = ComponentId {unComponentId :: Text}+  deriving stock (Eq, Ord, Show)++newtype PlanHash = PlanHash {unPlanHash :: Text}+  deriving stock (Eq, Ord, Show)++data SourceSpec+  = HackageSource+  | GitSource Text Text+  deriving stock (Eq, Show)++data PackagePin = PackagePin+  { pinName :: Text,+    pinVersion :: Text,+    pinSubdir :: Maybe Text+  }+  deriving stock (Eq, Show)++data ComponentSpec = ComponentSpec+  { componentId :: ComponentId,+    componentMoriUri :: Text,+    componentSource :: SourceSpec,+    componentPackages :: [PackagePin]+  }+  deriving stock (Eq, Show)++data PinException = PinException+  { exceptionPackage :: Text,+    exceptionSelected :: Text,+    exceptionLatestObserved :: Text,+    exceptionReason :: Text,+    exceptionRecheckAfter :: Text+  }+  deriving stock (Eq, Show)++data CohortDescriptor = CohortDescriptor+  { descriptorSchema :: Text,+    descriptorName :: CohortName,+    descriptorDescription :: Text,+    descriptorVerifiedAt :: Text,+    descriptorCompiler :: Text,+    descriptorIndexState :: Text,+    descriptorProjectFile :: FilePath,+    descriptorComponents :: [ComponentSpec],+    descriptorAllowNewer :: [Text],+    descriptorExceptions :: [PinException]+  }+  deriving stock (Eq, Show)++data PackageSource+  = FromHackage (Maybe Text)+  | FromGit Text Text (Maybe Text)+  | FromBoot+  | FromLocalPath FilePath+  deriving stock (Eq, Show)++data ResolvedPackage = ResolvedPackage+  { resolvedPackageName :: Text,+    resolvedPackageVersion :: Text,+    resolvedPackageSource :: PackageSource+  }+  deriving stock (Eq, Show)++data ResolvedComponent = ResolvedComponent+  { resolvedComponentId :: ComponentId,+    resolvedComponentMoriUri :: Text,+    resolvedComponentPackages :: [ResolvedPackage]+  }+  deriving stock (Eq, Show)++data CohortIdentity = CohortIdentity+  { identityCohort :: CohortName,+    identityCompiler :: Text,+    identityCabalVersion :: Text,+    identityOs :: Text,+    identityArch :: Text,+    identityIndexState :: Maybe Text,+    identityPlanHash :: PlanHash,+    identityDescriptorSha256 :: Text,+    identityComponents :: [ResolvedComponent],+    identityResolver :: Maybe Text+  }+  deriving stock (Eq, Show)++data CohortSource+  = FromProject FilePath (Maybe FilePath) (Maybe FilePath)+  | FromIdentityFile FilePath+  deriving stock (Eq, Show)++data CohortError+  = CohortIoError Text+  | CohortDecodeError Text+  | CohortPlanError Text+  | CohortInvalidActive Text+  deriving stock (Eq, Show)++data CohortMismatch+  = CohortNameMismatch CohortName CohortName+  | CompilerMismatch Text Text+  | MissingPackage Text+  | VersionMismatch Text Text Text+  | SourceMismatch Text SourceSpec PackageSource+  | LocalPathSource Text FilePath+  deriving stock (Eq, Show)++instance ToJSON CohortName where+  toJSON = toJSON . unCohortName++instance FromJSON CohortName where+  parseJSON = withText "CohortName" (pure . CohortName)++instance ToJSON ComponentId where+  toJSON = toJSON . unComponentId++instance FromJSON ComponentId where+  parseJSON = withText "ComponentId" (pure . ComponentId)++instance ToJSON PlanHash where+  toJSON = toJSON . unPlanHash++instance FromJSON PlanHash where+  parseJSON = withText "PlanHash" (pure . PlanHash)++instance ToJSON SourceSpec where+  toJSON HackageSource = object ["type" .= ("hackage" :: Text)]+  toJSON (GitSource gitLocation gitRevision) =+    object ["type" .= ("git" :: Text), "location" .= gitLocation, "rev" .= gitRevision]++instance FromJSON SourceSpec where+  parseJSON = withObject "SourceSpec" \value -> do+    sourceType <- value .: "type"+    case (sourceType :: Text) of+      "hackage" -> pure HackageSource+      "git" -> GitSource <$> value .: "location" <*> value .: "rev"+      other -> fail ("unknown cohort source type: " <> Text.unpack other)++instance ToJSON PackagePin where+  toJSON PackagePin {pinName, pinVersion, pinSubdir} =+    object (["name" .= pinName, "version" .= pinVersion] <> maybe [] (pure . ("subdir" .=)) pinSubdir)++instance FromJSON PackagePin where+  parseJSON = withObject "PackagePin" \value ->+    PackagePin <$> value .: "name" <*> value .: "version" <*> value .:? "subdir"++instance ToJSON ComponentSpec where+  toJSON ComponentSpec {componentId, componentMoriUri, componentSource, componentPackages} =+    object+      [ "id" .= componentId,+        "moriUri" .= componentMoriUri,+        "source" .= componentSource,+        "packages" .= componentPackages+      ]++instance FromJSON ComponentSpec where+  parseJSON = withObject "ComponentSpec" \value ->+    ComponentSpec+      <$> value .: "id"+      <*> value .: "moriUri"+      <*> value .: "source"+      <*> value .: "packages"++instance ToJSON PinException where+  toJSON PinException {exceptionPackage, exceptionSelected, exceptionLatestObserved, exceptionReason, exceptionRecheckAfter} =+    object+      [ "package" .= exceptionPackage,+        "selected" .= exceptionSelected,+        "latestObserved" .= exceptionLatestObserved,+        "reason" .= exceptionReason,+        "recheckAfter" .= exceptionRecheckAfter+      ]++instance FromJSON PinException where+  parseJSON = withObject "PinException" \value ->+    PinException+      <$> value .: "package"+      <*> value .: "selected"+      <*> value .: "latestObserved"+      <*> value .: "reason"+      <*> value .: "recheckAfter"++instance ToJSON CohortDescriptor where+  toJSON CohortDescriptor {descriptorSchema, descriptorName, descriptorDescription, descriptorVerifiedAt, descriptorCompiler, descriptorIndexState, descriptorProjectFile, descriptorComponents, descriptorAllowNewer, descriptorExceptions} =+    object+      [ "schema" .= descriptorSchema,+        "name" .= descriptorName,+        "description" .= descriptorDescription,+        "verifiedAt" .= descriptorVerifiedAt,+        "compiler" .= descriptorCompiler,+        "indexState" .= descriptorIndexState,+        "projectFile" .= descriptorProjectFile,+        "components" .= descriptorComponents,+        "allowNewer" .= descriptorAllowNewer,+        "exceptions" .= descriptorExceptions+      ]++instance FromJSON CohortDescriptor where+  parseJSON = withObject "CohortDescriptor" \value -> do+    descriptorSchema <- value .: "schema"+    if descriptorSchema /= ("kenshou.cohort/v1" :: Text)+      then fail ("unsupported cohort descriptor schema: " <> Text.unpack descriptorSchema)+      else+        CohortDescriptor descriptorSchema+          <$> value .: "name"+          <*> value .: "description"+          <*> value .: "verifiedAt"+          <*> value .: "compiler"+          <*> value .: "indexState"+          <*> value .: "projectFile"+          <*> value .: "components"+          <*> value .:? "allowNewer" .!= []+          <*> value .:? "exceptions" .!= []++instance ToJSON PackageSource where+  toJSON (FromHackage packageSha256) = object ["type" .= ("hackage" :: Text), "sha256" .= packageSha256]+  toJSON (FromGit packageLocation packageRevision packageSubdir) =+    object+      [ "type" .= ("git" :: Text),+        "location" .= packageLocation,+        "rev" .= packageRevision,+        "subdir" .= packageSubdir+      ]+  toJSON FromBoot = object ["type" .= ("boot" :: Text)]+  toJSON (FromLocalPath path) = object ["type" .= ("local" :: Text), "path" .= path]++instance FromJSON PackageSource where+  parseJSON = withObject "PackageSource" \value -> do+    sourceType <- value .: "type"+    case (sourceType :: Text) of+      "hackage" -> FromHackage <$> value .:? "sha256"+      "git" -> FromGit <$> value .: "location" <*> value .: "rev" <*> value .:? "subdir"+      "boot" -> pure FromBoot+      "local" -> FromLocalPath <$> value .: "path"+      other -> fail ("unknown resolved package source type: " <> Text.unpack other)++instance ToJSON ResolvedPackage where+  toJSON ResolvedPackage {resolvedPackageName, resolvedPackageVersion, resolvedPackageSource} =+    object ["name" .= resolvedPackageName, "version" .= resolvedPackageVersion, "source" .= resolvedPackageSource]++instance FromJSON ResolvedPackage where+  parseJSON = withObject "ResolvedPackage" \value ->+    ResolvedPackage <$> value .: "name" <*> value .: "version" <*> value .: "source"++instance ToJSON ResolvedComponent where+  toJSON ResolvedComponent {resolvedComponentId, resolvedComponentMoriUri, resolvedComponentPackages} =+    object ["id" .= resolvedComponentId, "moriUri" .= resolvedComponentMoriUri, "packages" .= resolvedComponentPackages]++instance FromJSON ResolvedComponent where+  parseJSON = withObject "ResolvedComponent" \value ->+    ResolvedComponent <$> value .: "id" <*> value .: "moriUri" <*> value .: "packages"++instance ToJSON CohortIdentity where+  toJSON CohortIdentity {identityCohort, identityCompiler, identityCabalVersion, identityOs, identityArch, identityIndexState, identityPlanHash, identityDescriptorSha256, identityComponents, identityResolver} =+    object+      ( [ "schema" .= ("kenshou.cohort-identity/v1" :: Text),+          "cohort" .= identityCohort,+          "compiler" .= identityCompiler,+          "cabalVersion" .= identityCabalVersion,+          "os" .= identityOs,+          "arch" .= identityArch,+          "indexState" .= identityIndexState,+          "planHash" .= identityPlanHash,+          "descriptorSha256" .= identityDescriptorSha256,+          "components" .= identityComponents+        ]+          <> maybe [] (\resolver -> ["resolver" .= resolver]) identityResolver+      )++instance FromJSON CohortIdentity where+  parseJSON = withObject "CohortIdentity" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.cohort-identity/v1" :: Text)+      then fail ("unsupported cohort identity schema: " <> Text.unpack schema)+      else do+        resolver <- value .:? "resolver"+        case (resolver :: Maybe Text) of+          Just "cabal" -> pure ()+          Just "nix" -> pure ()+          Nothing -> pure ()+          Just other -> fail ("unsupported cohort resolver: " <> Text.unpack other)+        CohortIdentity+          <$> value .: "cohort"+          <*> value .: "compiler"+          <*> value .: "cabalVersion"+          <*> value .: "os"+          <*> value .: "arch"+          <*> value .:? "indexState"+          <*> value .: "planHash"+          <*> value .: "descriptorSha256"+          <*> value .: "components"+          <*> pure resolver++loadCohortDescriptor :: FilePath -> IO (Either CohortError CohortDescriptor)+loadCohortDescriptor path = do+  bytesResult <- readBytes path+  pure (bytesResult >>= decodeBytes path)++activeCohortName :: FilePath -> IO (Either CohortError CohortName)+activeCohortName projectDir = do+  bytesResult <- readBytes (projectDir </> "cohort" </> "active.project")+  pure do+    bytes <- bytesResult+    text <- either (Left . CohortDecodeError . Text.pack . displayException) Right (Text.Encoding.decodeUtf8' bytes)+    case Text.lines text of+      [line]+        | Just project <- Text.stripPrefix "import: " line,+          Just name <- Text.stripSuffix ".project" project,+          not (Text.null name),+          Text.all validNameCharacter name ->+            Right (CohortName name)+      _ -> Left (CohortInvalidActive "cohort/active.project must contain exactly one line: import: <name>.project")+  where+    validNameCharacter character = isAlphaNum character || character == '-'++resolveCohortIdentity :: CohortSource -> IO (Either CohortError CohortIdentity)+resolveCohortIdentity (FromIdentityFile path) = do+  bytesResult <- readBytes path+  pure (bytesResult >>= decodeBytes path)+resolveCohortIdentity (FromProject sourceProjectDir sourcePlanJson sourceDescriptor) = do+  nameResult <- activeCohortName sourceProjectDir+  case nameResult of+    Left err -> pure (Left err)+    Right cohortName@(CohortName name) -> do+      let descriptorPath = fromMaybe (sourceProjectDir </> "cohort" </> Text.unpack name <> ".json") sourceDescriptor+          planPath = fromMaybe (sourceProjectDir </> "dist-newstyle" </> "cache" </> "plan.json") sourcePlanJson+      descriptorBytesResult <- readBytes descriptorPath+      planBytesResult <- readBytes planPath+      pure do+        descriptorBytes <- descriptorBytesResult+        planBytes <- planBytesResult+        descriptor <- decodeBytes descriptorPath descriptorBytes+        if descriptorName descriptor /= cohortName+          then Left (CohortDecodeError "active cohort name does not match descriptor name")+          else do+            planValue <- decodeBytes planPath planBytes+            identityFromPlan descriptor (sha256Hex descriptorBytes) planValue++identityFromPlan :: CohortDescriptor -> Text -> Value -> Either CohortError CohortIdentity+identityFromPlan descriptor descriptorSha value = do+  plan <- parsePlan value+  identityPlanHash <- planHash value+  let unitsByName = Map.fromListWith preferRuntimeUnit [(planUnitName unit, unit) | unit <- planUnits plan]+  identityComponents <- traverse (resolveComponent unitsByName) (descriptorComponents descriptor)+  pure+    CohortIdentity+      { identityCohort = descriptorName descriptor,+        identityCompiler = planCompiler plan,+        identityCabalVersion = planCabalVersion plan,+        identityOs = planOs plan,+        identityArch = planArch plan,+        identityIndexState = Just (descriptorIndexState descriptor),+        identityPlanHash,+        identityDescriptorSha256 = descriptorSha,+        identityComponents,+        identityResolver = Nothing+      }+  where+    preferRuntimeUnit left right+      | planUnitStyle left == Just "local" = right+      | otherwise = left++planHash :: Value -> Either CohortError PlanHash+planHash value = do+  plan <- parsePlan value+  renderedUnits <- traverse renderPlanUnit (filter ((/= Just "local") . planUnitStyle) (planUnits plan))+  let material = Text.intercalate "\n" ("compiler " <> planCompiler plan : sort (Set.toList (Set.fromList renderedUnits)))+  pure (PlanHash ("sha256:" <> sha256Hex (Text.Encoding.encodeUtf8 material)))++checkCohort :: CohortDescriptor -> CohortIdentity -> [CohortMismatch]+checkCohort descriptor identity = identityMismatches <> concatMap checkComponent (descriptorComponents descriptor)+  where+    identityMismatches =+      [CohortNameMismatch (descriptorName descriptor) (identityCohort identity) | descriptorName descriptor /= identityCohort identity]+        <> [CompilerMismatch (descriptorCompiler descriptor) (identityCompiler identity) | descriptorCompiler descriptor /= identityCompiler identity]++    resolvedByName =+      Map.fromList+        [ (resolvedPackageName package, package)+        | component <- identityComponents identity,+          package <- resolvedComponentPackages component+        ]++    checkComponent component = concatMap (checkPackage (componentSource component)) (componentPackages component)++    checkPackage expectedSource pin =+      case Map.lookup (pinName pin) resolvedByName of+        Nothing -> [MissingPackage (pinName pin)]+        Just actual -> versionMismatch pin actual <> sourceMismatch expectedSource pin actual++    versionMismatch pin actual+      | pinVersion pin == resolvedPackageVersion actual = []+      | otherwise = [VersionMismatch (pinName pin) (pinVersion pin) (resolvedPackageVersion actual)]++    sourceMismatch _ pin ResolvedPackage {resolvedPackageSource = FromLocalPath path} = [LocalPathSource (pinName pin) path]+    sourceMismatch HackageSource _ ResolvedPackage {resolvedPackageSource = FromHackage _} = []+    sourceMismatch expected@HackageSource pin ResolvedPackage {resolvedPackageSource = actual} = [SourceMismatch (pinName pin) expected actual]+    sourceMismatch expected@(GitSource gitLocation gitRevision) pin ResolvedPackage {resolvedPackageSource = actual@(FromGit packageLocation packageRevision _)}+      | normaliseGitLocation gitLocation == normaliseGitLocation packageLocation,+        gitRevision == packageRevision =+          []+      | otherwise = [SourceMismatch (pinName pin) expected actual]+    sourceMismatch expected@(GitSource _ _) pin ResolvedPackage {resolvedPackageSource = actual} = [SourceMismatch (pinName pin) expected actual]++renderCohortIdentity :: CohortIdentity -> Text+renderCohortIdentity identity =+  Text.unlines+    ( [ "cohort     " <> unCohortName (identityCohort identity),+        "compiler   " <> identityCompiler identity <> "   " <> fromMaybe "cabal" (identityResolver identity) <> " " <> identityCabalVersion identity <> "   " <> identityOs identity <> "/" <> identityArch identity,+        "index      " <> fromMaybe "-" (identityIndexState identity),+        "plan       " <> unPlanHash (identityPlanHash identity)+      ]+        <> map renderComponent (identityComponents identity)+    )+  where+    renderComponent component =+      Text.justifyLeft 22 ' ' (unComponentId (resolvedComponentId component))+        <> Text.intercalate ", " (map renderPackage (resolvedComponentPackages component))+    renderPackage package =+      resolvedPackageName package+        <> " "+        <> resolvedPackageVersion package+        <> " ["+        <> renderSource (resolvedPackageSource package)+        <> "]"+    renderSource (FromHackage _) = "hackage"+    renderSource (FromGit _ packageRevision _) = "git " <> Text.take 12 packageRevision+    renderSource FromBoot = "boot"+    renderSource FromLocalPath {} = "local"++data Plan = Plan+  { planCompiler :: Text,+    planCabalVersion :: Text,+    planOs :: Text,+    planArch :: Text,+    planUnits :: [PlanUnit]+  }++data PlanUnit = PlanUnit+  { planUnitType :: Text,+    planUnitStyle :: Maybe Text,+    planUnitName :: Text,+    planUnitVersion :: Text,+    planUnitSource :: Maybe Object,+    planUnitSha256 :: Maybe Text,+    planUnitFlags :: Map Text Bool+  }++parsePlan :: Value -> Either CohortError Plan+parsePlan (Object root) = do+  planCompiler <- requiredText "compiler-id" root+  planCabalVersion <- requiredText "cabal-version" root+  planOs <- requiredText "os" root+  planArch <- requiredText "arch" root+  unitValues <- requiredArray "install-plan" root+  planUnits <- traverse parseUnit unitValues+  pure Plan {planCompiler, planCabalVersion, planOs, planArch, planUnits}+parsePlan _ = Left (CohortPlanError "plan.json root must be an object")++parseUnit :: Value -> Either CohortError PlanUnit+parseUnit (Object unit) = do+  planUnitType <- requiredText "type" unit+  planUnitName <- requiredText "pkg-name" unit+  planUnitVersion <- requiredText "pkg-version" unit+  let planUnitStyle = optionalText "style" unit+      planUnitSource = optionalObject "pkg-src" unit+      planUnitSha256 = optionalText "pkg-src-sha256" unit+      planUnitFlags = fromMaybe Map.empty (optionalFlags "flags" unit)+  pure PlanUnit {planUnitType, planUnitStyle, planUnitName, planUnitVersion, planUnitSource, planUnitSha256, planUnitFlags}+parseUnit _ = Left (CohortPlanError "install-plan entries must be objects")++resolveComponent :: Map Text PlanUnit -> ComponentSpec -> Either CohortError ResolvedComponent+resolveComponent units component = do+  resolvedComponentPackages <- catMaybes <$> traverse resolvePackage (componentPackages component)+  pure+    ResolvedComponent+      { resolvedComponentId = componentId component,+        resolvedComponentMoriUri = componentMoriUri component,+        resolvedComponentPackages+      }+  where+    resolvePackage pin = case Map.lookup (pinName pin) units of+      Nothing -> pure Nothing+      Just unit -> Just . ResolvedPackage (planUnitName unit) (planUnitVersion unit) <$> packageSource unit++renderPlanUnit :: PlanUnit -> Either CohortError Text+renderPlanUnit unit = do+  source <- packageSource unit+  let flags =+        if Map.null (planUnitFlags unit)+          then "-"+          else+            Text.intercalate+              ","+              [ (if enabled then "+" else "-") <> name+              | (name, enabled) <- sortOn fst (Map.toList (planUnitFlags unit))+              ]+  pure (planUnitName unit <> " " <> planUnitVersion unit <> " " <> renderPlanSource source <> " " <> flags)++packageSource :: PlanUnit -> Either CohortError PackageSource+packageSource PlanUnit {planUnitType = "pre-existing"} = Right FromBoot+packageSource PlanUnit {planUnitSource = Nothing} = Right FromBoot+packageSource PlanUnit {planUnitSource = Just source, planUnitSha256} = do+  sourceType <- requiredText "type" source+  case sourceType of+    "repo-tar" -> Right (FromHackage planUnitSha256)+    "local" -> FromLocalPath . Text.unpack <$> requiredText "path" source+    "source-repo" -> parseSourceRepo source+    other -> Left (CohortPlanError ("unsupported plan package source: " <> other))++parseSourceRepo :: Object -> Either CohortError PackageSource+parseSourceRepo source = do+  let repo = fromMaybe source (optionalObject "source-repo" source)+  packageLocation <- requiredText "location" repo+  packageRevision <- requiredText "tag" repo+  let packageSubdir = optionalText "subdir" repo+  pure (FromGit packageLocation packageRevision packageSubdir)++renderPlanSource :: PackageSource -> Text+renderPlanSource (FromHackage packageSha256) = "hackage:" <> fromMaybe "-" packageSha256+renderPlanSource (FromGit packageLocation packageRevision packageSubdir) =+  "git:" <> packageLocation <> "@" <> packageRevision <> "#" <> fromMaybe "-" packageSubdir+renderPlanSource FromBoot = "boot"+renderPlanSource (FromLocalPath path) = "local:" <> Text.pack path++requiredText :: Text -> Object -> Either CohortError Text+requiredText key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+  Just (String value) -> Right value+  _ -> Left (CohortPlanError ("plan field " <> key <> " must be text"))++optionalText :: Text -> Object -> Maybe Text+optionalText key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+  Just (String value) -> Just value+  _ -> Nothing++optionalObject :: Text -> Object -> Maybe Object+optionalObject key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+  Just (Object value) -> Just value+  _ -> Nothing++requiredArray :: Text -> Object -> Either CohortError [Value]+requiredArray key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+  Just (Array values) -> Right (foldr (:) [] values)+  _ -> Left (CohortPlanError ("plan field " <> key <> " must be an array"))++optionalFlags :: Text -> Object -> Maybe (Map Text Bool)+optionalFlags key objectValue = do+  flags <- optionalObject key objectValue+  traverse valueToBool (Map.fromList [(Key.toText name, value) | (name, value) <- KeyMap.toList flags])+  where+    valueToBool (Bool value) = Just value+    valueToBool _ = Nothing++readBytes :: FilePath -> IO (Either CohortError ByteString)+readBytes path = do+  result <- try @IOException (ByteString.readFile path)+  pure (either (\err -> Left (CohortIoError (Text.pack (path <> ": " <> displayException err)))) Right result)++decodeBytes :: (FromJSON value) => FilePath -> ByteString -> Either CohortError value+decodeBytes path bytes =+  either (\err -> Left (CohortDecodeError (Text.pack (path <> ": " <> err)))) Right (eitherDecodeStrict' bytes)++sha256Hex :: ByteString -> Text+sha256Hex = Text.pack . concatMap renderByte . ByteString.unpack . SHA256.hash+  where+    renderByte :: Word8 -> String+    renderByte byte = case showHex byte "" of+      [digit] -> ['0', digit]+      digits -> digits++normaliseGitLocation :: Text -> Text+normaliseGitLocation location =+  let withoutSlash = Text.dropWhileEnd (== '/') location+   in fromMaybe withoutSlash (Text.stripSuffix ".git" withoutSlash)
+ src/Kenshou/Core/Compat.hs view
@@ -0,0 +1,118 @@+module Kenshou.Core.Compat+  ( CompatField (..),+    CompatInputs (..),+    compatInputs,+    comparisonKey,+    seriesKey,+    compatibleExcept,+    renderResourceInputs,+  )+where++import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.Types (Pair)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))+import Kenshou.Core.Dimension (DimensionName (..), Dimensions (..))+import Kenshou.Core.Env (MigrationSetName, renderMigrationSetName)+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot)+import Kenshou.Core.Knob (KnobName)+import Kenshou.Core.Knob qualified as Knob+import Kenshou.Core.Resource.Types (ResourceName, legacyKafkaName, renderResourceName)+import Kenshou.Core.RunSpec (EffectiveRunSpec (..), EnvironmentSpec (..))+import Kenshou.Core.Version (suiteVersion)++data CompatField+  = CfSuiteVersion+  | CfScenario+  | CfKnob KnobName+  | CfDimension DimensionName+  | CfPhases+  | CfMachineProfile+  | CfPostgresProfile+  | CfSchemas+  | CfCohort+  | CfResource ResourceName+  deriving stock (Eq, Show)++data CompatInputs = CompatInputs+  { suiteVersion :: Text,+    spec :: EffectiveRunSpec,+    postgres :: Maybe PgSettingsSnapshot,+    schemas :: [MigrationSetName],+    cohortPlanHash :: Text+  }+  deriving stock (Eq, Show)++compatInputs :: EffectiveRunSpec -> Maybe PgSettingsSnapshot -> [MigrationSetName] -> CohortIdentity -> CompatInputs+compatInputs spec postgres schemas cohort = CompatInputs suiteVersion spec postgres schemas (unPlanHash cohort.identityPlanHash)++comparisonKey :: CompatInputs -> Text+comparisonKey = sha256Hex . canonicalEncode . comparisonValue++seriesKey :: CompatInputs -> Text+seriesKey = sha256Hex . canonicalEncode . toJSON++compatibleExcept :: [CompatField] -> CompatInputs -> CompatInputs -> Either (NonEmpty CompatField) ()+compatibleExcept allowed left right = case filter (`notElem` allowed) (differences left right) of+  [] -> Right ()+  first : rest -> Left (first :| rest)++instance ToJSON CompatInputs where+  toJSON = compatibilityValue True++comparisonValue :: CompatInputs -> Value+comparisonValue = compatibilityValue False++compatibilityValue :: Bool -> CompatInputs -> Value+compatibilityValue includeCohort inputs =+  object $+    [ "suiteVersion" .= inputs.suiteVersion,+      "scenario" .= inputs.spec.scenario,+      "scenarioRevision" .= inputs.spec.scenarioRevision,+      "knobs" .= inputs.spec.knobs,+      "dimensions" .= inputs.spec.dimensions,+      "phases" .= inputs.spec.phases,+      "machineProfile" .= inputs.spec.environment.machineProfile,+      "postgresProfile" .= inputs.postgres,+      "schemas" .= fmap renderMigrationSetName inputs.schemas+    ]+      <> renderResourceInputs inputs.spec.environment.resources+      <> ["cohortPlanHash" .= inputs.cohortPlanHash | includeCohort]++-- | compat-key/v1 shim: keeps Kafka comparison keys stable for recorded baselines.+-- The resource that documents once carried as a top-level field renders there; every+-- other resource renders under @resources@, which is omitted when empty.+renderResourceInputs :: Map ResourceName Value -> [Pair]+renderResourceInputs resources =+  maybe [] (pure . (Key.fromText (renderResourceName legacyKafkaName) .=)) (Map.lookup legacyKafkaName resources)+    <> ["resources" .= others | not (Map.null others)]+  where+    others = Map.delete legacyKafkaName resources++differences :: CompatInputs -> CompatInputs -> [CompatField]+differences left right =+  [CfSuiteVersion | left.suiteVersion /= right.suiteVersion]+    <> [CfScenario | left.spec.scenario /= right.spec.scenario || left.spec.scenarioRevision /= right.spec.scenarioRevision]+    <> [CfPhases | left.spec.phases /= right.spec.phases]+    <> [CfMachineProfile | left.spec.environment.machineProfile /= right.spec.environment.machineProfile]+    <> [CfPostgresProfile | left.postgres /= right.postgres]+    <> [CfCohort | left.cohortPlanHash /= right.cohortPlanHash]+    <> [CfSchemas | left.schemas /= right.schemas]+    <> [CfDimension DimTracing | left.spec.dimensions.tracing /= right.spec.dimensions.tracing]+    <> [CfDimension DimMetrics | left.spec.dimensions.metrics /= right.spec.dimensions.metrics]+    <> [CfDimension DimPgDurability | left.spec.dimensions.pgDurability /= right.spec.dimensions.pgDurability]+    <> [CfDimension DimPgVersion | left.spec.dimensions.pgVersion /= right.spec.dimensions.pgVersion]+    <> [CfKnob name | name <- Set.toAscList knobNames, Map.lookup name leftKnobs /= Map.lookup name rightKnobs]+    <> [CfResource name | name <- Set.toAscList resourceNames, Map.lookup name left.spec.environment.resources /= Map.lookup name right.spec.environment.resources]+  where+    resourceNames = Map.keysSet left.spec.environment.resources <> Map.keysSet right.spec.environment.resources+    leftKnobs = Knob.resolvedKnobsMap left.spec.knobs+    rightKnobs = Knob.resolvedKnobsMap right.spec.knobs+    knobNames = Map.keysSet leftKnobs <> Map.keysSet rightKnobs
+ src/Kenshou/Core/Context.hs view
@@ -0,0 +1,135 @@+module Kenshou.Core.Context+  ( SummarySection (..),+    ArtifactDir (..),+    Observation (..),+    Environment (..),+    RunContext (..),+    RunState,+    PhaseTiming (..),+    InfrastructureError (..),+    newRunState,+    readRunState,+    requirePostgres,+    genFor,+    withPhase,+    putSummary,+    observe,+    artifactPath,+    declareMediaType,+  )+where++import Control.Exception (Exception, bracket_)+import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.IORef+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime, getCurrentTime)+import Data.Word (Word64)+import GHC.Clock (getMonotonicTimeNSec)+import GHC.Stack (HasCallStack)+import Kenshou.Core.Dimension (Dimensions)+import Kenshou.Core.Env.Postgres (PostgresEnv)+import Kenshou.Core.Id (RunId, ScenarioId, Seed, deriveGen)+import Kenshou.Core.Knob (ResolvedKnobs)+import Kenshou.Core.Log (Logger, Severity)+import Kenshou.Core.Phase (PhaseName, PhasePlan)+import Kenshou.Core.Resource.Types (ProvisionedResource, ResourceName)+import Kenshou.Core.RunSpec (ComparisonMembership, EnvironmentSpec)+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.Random.SplitMix (SMGen)++data SummarySection = Measurements | Verdicts | Diagnosis | Telemetry deriving stock (Eq, Ord, Show)++data ArtifactDir = SamplesDir | SeriesDir | VerdictsDir | DiagnosisDir | LogsDir deriving stock (Eq, Ord, Show)++data Observation = Observation {source :: Text, severity :: Severity, message :: Text, at :: UTCTime} deriving stock (Eq, Show)++data Environment = Environment {postgres :: Maybe PostgresEnv, extraPostgres :: Map Text PostgresEnv, resources :: Map ResourceName ProvisionedResource}++data PhaseTiming = PhaseTiming+  { name :: PhaseName,+    startedAt :: UTCTime,+    endedAt :: UTCTime,+    startedMonotonicNs :: Word64,+    endedMonotonicNs :: Word64+  }+  deriving stock (Eq, Show)++instance ToJSON PhaseTiming where+  toJSON timing = object ["name" .= timing.name, "startedAt" .= timing.startedAt, "endedAt" .= timing.endedAt, "startedMonotonicNs" .= timing.startedMonotonicNs, "endedMonotonicNs" .= timing.endedMonotonicNs]++data RunState = RunState+  { summaries :: IORef (Map SummarySection (Map Text Value)),+    observations :: IORef [Observation],+    phaseTimings :: IORef [PhaseTiming],+    mediaTypes :: IORef (Map FilePath Text)+  }++data RunContext = RunContext+  { runId :: RunId,+    scenario :: ScenarioId,+    knobs :: ResolvedKnobs,+    dimensions :: Dimensions,+    seed :: Seed,+    phases :: PhasePlan,+    env :: Environment,+    environmentSpec :: EnvironmentSpec,+    comparison :: Maybe ComparisonMembership,+    outDir :: FilePath,+    logger :: Logger,+    state :: RunState+  }++newtype InfrastructureError = InfrastructureError Text deriving stock (Eq, Show)++instance Exception InfrastructureError++newRunState :: IO RunState+newRunState = RunState <$> newIORef Map.empty <*> newIORef [] <*> newIORef [] <*> newIORef Map.empty++readRunState :: RunState -> IO (Map SummarySection (Map Text Value), [Observation], [PhaseTiming], Map FilePath Text)+readRunState state = (,,,) <$> readIORef state.summaries <*> readIORef state.observations <*> readIORef state.phaseTimings <*> readIORef state.mediaTypes++requirePostgres :: (HasCallStack) => RunContext -> PostgresEnv+requirePostgres context = maybe (error "requirePostgres: scenario did not declare PostgreSQL") id context.env.postgres++genFor :: RunContext -> Text -> SMGen+genFor context = deriveGen context.seed++withPhase :: RunContext -> PhaseName -> IO value -> IO value+withPhase context phase action = do+  startedAt <- getCurrentTime+  startedMono <- getMonotonicTimeNSec+  action `finallyRecord` do+    endedAt <- getCurrentTime+    endedMono <- getMonotonicTimeNSec+    modifyIORef' context.state.phaseTimings (<> [PhaseTiming phase startedAt endedAt startedMono endedMono])+  where+    finallyRecord body cleanup = bracket_ (pure ()) cleanup body++putSummary :: RunContext -> SummarySection -> Text -> Value -> IO ()+putSummary context section key value = modifyIORef' context.state.summaries (Map.alter (Just . Map.insert key value . maybe Map.empty id) section)++observe :: RunContext -> Text -> Severity -> Text -> IO ()+observe context source severity message = do+  at <- getCurrentTime+  modifyIORef' context.state.observations (<> [Observation source severity message at])++artifactPath :: RunContext -> ArtifactDir -> FilePath -> IO FilePath+artifactPath context directory name = do+  let root = context.outDir </> artifactDir directory+  createDirectoryIfMissing True root+  pure (root </> name)++declareMediaType :: RunContext -> FilePath -> Text -> IO ()+declareMediaType context path mediaType = modifyIORef' context.state.mediaTypes (Map.insert path mediaType)++artifactDir :: ArtifactDir -> FilePath+artifactDir SamplesDir = "samples"+artifactDir SeriesDir = "series"+artifactDir VerdictsDir = "verdicts"+artifactDir DiagnosisDir = "diagnosis"+artifactDir LogsDir = "logs"
+ src/Kenshou/Core/Contribution.hs view
@@ -0,0 +1,97 @@+-- | Data a suite contributes through its layer bundle for the toolkits to consume:+-- SQL oracles the correctness toolkit runs, advisory-lock labels stall diagnoses+-- name, and a connection-pool methodology. The types live in the kernel because+-- 'Kenshou.Core.Bundle.LayerBundle' carries them; the toolkits re-export them from+-- the modules that use them.+module Kenshou.Core.Contribution+  ( OracleSpec (..),+    AdvisoryKeySpec (..),+    AdvisoryLabel (..),+    PoolMethodology (..),+    RuleCondition (..),+    RoutingRule (..),+  )+where++import Control.Applicative ((<|>))+import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), object, withObject, (.:), (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Plan.Selector (Selector, parseSelector)++-- | A named SQL query whose rows a correctness scenario reconciles.+data OracleSpec = OracleSpec {name :: !Text, sql :: !Text}+  deriving stock (Eq, Show)++-- | How PostgreSQL derives an advisory-lock key from a label's text.+data AdvisoryKeySpec = HashTextExtended !Text | HashText !Text | LiteralKey !Int64+  deriving stock (Eq, Show)++-- | A human label for an advisory lock, so a stall capture names who holds it.+data AdvisoryLabel = AdvisoryLabel {label :: !Text, key :: !AdvisoryKeySpec}+  deriving stock (Eq, Show)++-- | The pool size a suite recommends for its scenarios and the range its+-- measurements justify.+data PoolMethodology = PoolMethodology {recommendedPoolSize :: !Int, poolRange :: !(Int, Int)}+  deriving stock (Eq, Show)++-- | Which runs a routing rule applies to: a knob with a given text value, or the+-- scenarios a selector matches.+data RuleCondition = KnobEquals Text Text | ScenarioMatches Selector+  deriving stock (Eq, Show)++-- | A cell capability a run needs, with the reason; cell routing and preparation+-- refuse a cell that reports the capability absent.+data RoutingRule = RoutingRule+  { condition :: !RuleCondition,+    requires :: !Text,+    why :: !Text+  }+  deriving stock (Eq, Show)++instance FromJSON RoutingRule where+  parseJSON = withObject "cell routing rule" \fields -> do+    condition <- fields .: "when" >>= parseCondition+    capability <- fields .: "requires"+    explanation <- fields .: "why"+    if Text.null capability || Text.null explanation then fail "cell routing rule requires a capability and explanation" else pure ()+    pure (RoutingRule condition capability explanation)+    where+      parseCondition = withObject "cell routing condition" \fields ->+        case (KeyMap.lookup "knob" fields, KeyMap.lookup "scenario" fields) of+          (Just (String knob), Nothing) -> do+            _ <- either (fail . Text.unpack) pure (mkKnobName knob)+            KnobEquals knob <$> fields .: "equals"+          (Nothing, Just (String patternText)) -> ScenarioMatches <$> either (fail . Text.unpack) pure (parseSelector patternText)+          _ -> fail "cell routing condition must select one knob or scenario"++instance ToJSON OracleSpec where+  toJSON spec = object ["name" .= spec.name, "sql" .= spec.sql]++instance FromJSON OracleSpec where+  parseJSON = withObject "OracleSpec" \value -> OracleSpec <$> value .: "name" <*> value .: "sql"++instance ToJSON AdvisoryKeySpec where+  toJSON (HashTextExtended text) = object ["hashTextExtended" .= text]+  toJSON (HashText text) = object ["hashText" .= text]+  toJSON (LiteralKey key) = object ["literal" .= key]++instance FromJSON AdvisoryKeySpec where+  parseJSON = withObject "AdvisoryKeySpec" \value ->+    (HashTextExtended <$> value .: "hashTextExtended") <|> (HashText <$> value .: "hashText") <|> (LiteralKey <$> value .: "literal")++instance ToJSON AdvisoryLabel where+  toJSON advisory = object ["label" .= advisory.label, "key" .= advisory.key]++instance FromJSON AdvisoryLabel where+  parseJSON = withObject "AdvisoryLabel" \value -> AdvisoryLabel <$> value .: "label" <*> value .: "key"++instance ToJSON PoolMethodology where+  toJSON methodology = object ["recommendedPoolSize" .= methodology.recommendedPoolSize, "poolRange" .= methodology.poolRange]++instance FromJSON PoolMethodology where+  parseJSON = withObject "PoolMethodology" \value -> PoolMethodology <$> value .: "recommendedPoolSize" <*> value .: "poolRange"
+ src/Kenshou/Core/Dimension.hs view
@@ -0,0 +1,163 @@+module Kenshou.Core.Dimension+  ( TracingArm (..),+    MetricsArm (..),+    PgDurability (..),+    PgVersion (..),+    DimensionName (..),+    Support (..),+    Supported (..),+    DimensionSupport (..),+    Dimensions (..),+    noDimensions,+    emptyDimensions,+    allTelemetryArms,+    postgresDimensions,+    resolveDimensions,+    renderDimensions,+    renderDimensionName,+    renderTracing,+    parseTracing,+    renderMetrics,+    parseMetrics,+    renderDurability,+    parseDurability,+    renderVersion,+    parseVersion,+  )+where++import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:?), (.=))+import Data.Aeson.Key qualified as Key+import Data.List (group, sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text++data TracingArm = TracingOff | TracingNoop | TracingSdkInMemory | TracingSdkOtlp deriving stock (Eq, Ord, Show)++data MetricsArm = MetricsOff | MetricsCollect | MetricsServe | MetricsServeScraped deriving stock (Eq, Ord, Show)++data PgDurability = PgFsyncOff | PgDurable deriving stock (Eq, Ord, Show)++data PgVersion = Pg17 | Pg18 deriving stock (Eq, Ord, Show)++data DimensionName = DimTracing | DimMetrics | DimPgDurability | DimPgVersion deriving stock (Eq, Ord, Show)++data Support a = Support {values :: NonEmpty a, def :: a} deriving stock (Eq, Show)++data Supported a = NotApplicable | Supported (Support a) deriving stock (Eq, Show)++data DimensionSupport = DimensionSupport+  { tracing :: Supported TracingArm,+    metrics :: Supported MetricsArm,+    pgDurability :: Supported PgDurability,+    pgVersion :: Supported PgVersion+  }+  deriving stock (Eq, Show)++data Dimensions = Dimensions+  { tracing :: Maybe TracingArm,+    metrics :: Maybe MetricsArm,+    pgDurability :: Maybe PgDurability,+    pgVersion :: Maybe PgVersion+  }+  deriving stock (Eq, Show)++noDimensions :: DimensionSupport+noDimensions = DimensionSupport NotApplicable NotApplicable NotApplicable NotApplicable++emptyDimensions :: Dimensions+emptyDimensions = Dimensions Nothing Nothing Nothing Nothing++allTelemetryArms :: DimensionSupport -> DimensionSupport+allTelemetryArms support = support {tracing = Supported (Support (TracingOff :| [TracingNoop, TracingSdkInMemory, TracingSdkOtlp]) TracingOff), metrics = Supported (Support (MetricsOff :| [MetricsCollect, MetricsServe, MetricsServeScraped]) MetricsOff)}++postgresDimensions :: NonEmpty PgDurability -> NonEmpty PgVersion -> DimensionSupport -> DimensionSupport+postgresDimensions durabilities versions support = support {pgDurability = Supported (Support durabilities (NonEmpty.head durabilities)), pgVersion = Supported (Support versions (NonEmpty.head versions))}++resolveDimensions :: DimensionSupport -> [(Text, Text)] -> Either (NonEmpty Text) Dimensions+resolveDimensions support assignments = case errors of+  [] -> Right (Dimensions tracing metrics durability version)+  first : rest -> Left (first :| rest)+  where+    names = fmap fst assignments+    duplicateErrors = [name <> ": assigned more than once" | values@(name : _) <- group (sort names), length values > 1]+    unknownErrors = [name <> ": unknown dimension" | name <- names, name `notElem` fmap renderDimensionName [DimTracing, DimMetrics, DimPgDurability, DimPgVersion]]+    lookupValue name = lookup (renderDimensionName name) assignments+    (tracing, tracingErrors) = resolveOne DimTracing renderTracing parseTracing support.tracing (lookupValue DimTracing)+    (metrics, metricsErrors) = resolveOne DimMetrics renderMetrics parseMetrics support.metrics (lookupValue DimMetrics)+    (durability, durabilityErrors) = resolveOne DimPgDurability renderDurability parseDurability support.pgDurability (lookupValue DimPgDurability)+    (version, versionErrors) = resolveOne DimPgVersion renderVersion parseVersion support.pgVersion (lookupValue DimPgVersion)+    errors = duplicateErrors <> unknownErrors <> tracingErrors <> metricsErrors <> durabilityErrors <> versionErrors++resolveOne :: (Eq a) => DimensionName -> (a -> Text) -> (Text -> Maybe a) -> Supported a -> Maybe Text -> (Maybe a, [Text])+resolveOne _ _ _ NotApplicable Nothing = (Nothing, [])+resolveOne name _ _ NotApplicable (Just _) = (Nothing, [renderDimensionName name <> ": not applicable to this scenario"])+resolveOne _ _ _ (Supported support) Nothing = (Just support.def, [])+resolveOne name _render parse (Supported support) (Just raw) = case parse raw of+  Nothing -> (Nothing, [renderDimensionName name <> ": unknown value \"" <> raw <> "\""])+  Just value | value `elem` support.values -> (Just value, [])+  Just _ -> (Nothing, [renderDimensionName name <> ": value \"" <> raw <> "\" is not supported"])++renderDimensions :: Dimensions -> [(Text, Text)]+renderDimensions dimensions =+  concat+    [ maybe [] (pure . (renderDimensionName DimTracing,) . renderTracing) dimensions.tracing,+      maybe [] (pure . (renderDimensionName DimMetrics,) . renderMetrics) dimensions.metrics,+      maybe [] (pure . (renderDimensionName DimPgDurability,) . renderDurability) dimensions.pgDurability,+      maybe [] (pure . (renderDimensionName DimPgVersion,) . renderVersion) dimensions.pgVersion+    ]++renderDimensionName :: DimensionName -> Text+renderDimensionName DimTracing = "telemetry.tracing"+renderDimensionName DimMetrics = "telemetry.metrics"+renderDimensionName DimPgDurability = "pg.durability"+renderDimensionName DimPgVersion = "pg.version"++renderTracing :: TracingArm -> Text+renderTracing TracingOff = "off"+renderTracing TracingNoop = "noop"+renderTracing TracingSdkInMemory = "sdk-inmemory"+renderTracing TracingSdkOtlp = "sdk-otlp"++parseTracing :: Text -> Maybe TracingArm+parseTracing value = lookup value [(renderTracing item, item) | item <- [TracingOff, TracingNoop, TracingSdkInMemory, TracingSdkOtlp]]++renderMetrics :: MetricsArm -> Text+renderMetrics MetricsOff = "off"+renderMetrics MetricsCollect = "collect"+renderMetrics MetricsServe = "serve"+renderMetrics MetricsServeScraped = "serve-scraped"++parseMetrics :: Text -> Maybe MetricsArm+parseMetrics value = lookup value [(renderMetrics item, item) | item <- [MetricsOff, MetricsCollect, MetricsServe, MetricsServeScraped]]++renderDurability :: PgDurability -> Text+renderDurability PgFsyncOff = "fsync-off"+renderDurability PgDurable = "durable"++parseDurability :: Text -> Maybe PgDurability+parseDurability value = lookup value [(renderDurability item, item) | item <- [PgFsyncOff, PgDurable]]++renderVersion :: PgVersion -> Text+renderVersion Pg17 = "17"+renderVersion Pg18 = "18"++parseVersion :: Text -> Maybe PgVersion+parseVersion value = lookup value [(renderVersion item, item) | item <- [Pg17, Pg18]]++instance ToJSON Dimensions where+  toJSON dimensions = object [Key.fromText name .= value | (name, value) <- renderDimensions dimensions]++instance FromJSON Dimensions where+  parseJSON = withObject "Dimensions" \value ->+    Dimensions+      <$> optionalArm value "telemetry.tracing" parseTracing+      <*> optionalArm value "telemetry.metrics" parseMetrics+      <*> optionalArm value "pg.durability" parseDurability+      <*> optionalArm value "pg.version" parseVersion+    where+      optionalArm objectValue name parse = do+        raw <- objectValue .:? Key.fromText name+        traverse (maybe (fail ("unknown dimension value for " <> Text.unpack name)) pure . parse) raw
+ src/Kenshou/Core/Env.hs view
@@ -0,0 +1,61 @@+module Kenshou.Core.Env+  ( MigrationSetName (..),+    mkMigrationSetName,+    renderMigrationSetName,+    MigrationSet (..),+    PostgresRequirement (..),+    EnvRequirements (..),+    noEnvironment,+    requiresResource,+  )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Database.PostgreSQL.Migrate (MigrationComponent, componentName)+import Kenshou.Core.Resource.Types (ResourceName)++-- | The name of a migration set, which is also its pg-migrate component name and the+-- text a scenario's comparison key records. 'Kenshou.Core.Bundle.mkRegistry' rejects+-- names that 'mkMigrationSetName' would reject.+newtype MigrationSetName = MigrationSetName Text+  deriving stock (Eq, Ord, Show)++mkMigrationSetName :: Text -> Either Text MigrationSetName+mkMigrationSetName value = either (\problem -> Left ("invalid migration set name \"" <> value <> "\": " <> Text.pack (show problem))) (const (Right (MigrationSetName value))) (componentName value)++renderMigrationSetName :: MigrationSetName -> Text+renderMigrationSetName (MigrationSetName value) = value++-- | A named group of database migrations a registered bundle contributes. Sets are+-- resolved by name across the whole registry, so a scenario may require a set that+-- another bundle defines. 'requires' names sets that must be applied first. A+-- 'Left' component names a set this build cannot provide; it fails only when a+-- scenario requires it.+data MigrationSet = MigrationSet+  { name :: MigrationSetName,+    requires :: [MigrationSetName],+    component :: Either Text MigrationComponent+  }++data PostgresRequirement = PostgresRequirement+  { schemas :: [MigrationSetName],+    settings :: [(Text, Text)],+    needsServerControl :: Bool+  }+  deriving stock (Eq, Show)++data EnvRequirements = EnvRequirements+  { postgres :: Maybe PostgresRequirement,+    extraPostgres :: [(Text, PostgresRequirement)],+    -- | Named resources the scenario needs, provisioned by registered providers.+    resources :: [ResourceName]+  }+  deriving stock (Eq, Show)++noEnvironment :: EnvRequirements+noEnvironment = EnvRequirements Nothing [] []++-- | An environment that needs one named resource and nothing else.+requiresResource :: ResourceName -> EnvRequirements+requiresResource name = noEnvironment {resources = [name]}
+ src/Kenshou/Core/Env/Migration.hs view
@@ -0,0 +1,54 @@+module Kenshou.Core.Env.Migration+  ( MigrationSetupError (..),+    composePlan,+    orderMigrationSets,+    migrateDatabase,+  )+where++import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Database.PostgreSQL.Migrate (MigrationPlan, defaultRunOptions, migrationPlan, runMigrationPlan)+import Hasql.Connection.Settings qualified as Settings+import Kenshou.Core.Env (MigrationSet (..), MigrationSetName, renderMigrationSetName)++newtype MigrationSetupError = MigrationSetupError Text deriving stock (Eq, Show)++-- | Compose the requested migration sets into one ordered pg-migrate plan, applied+-- as one ledger, in the order 'orderMigrationSets' gives.+composePlan :: Map MigrationSetName MigrationSet -> [MigrationSetName] -> Either MigrationSetupError MigrationPlan+composePlan sets requested = do+  ordered <- orderMigrationSets sets requested+  components <- traverse component ordered+  case components of+    [] -> Left (MigrationSetupError "cannot compose an empty migration plan")+    first : rest -> mapError (migrationPlan (first :| rest))+  where+    component set = either (\problem -> Left (MigrationSetupError (renderMigrationSetName set.name <> ": " <> problem))) Right set.component+    mapError :: (Show problem) => Either problem value -> Either MigrationSetupError value+    mapError = either (Left . MigrationSetupError . Text.pack . show) Right++-- | The sets to apply, in application order. Sets keep the order the scenario+-- declared them; each set's 'requires' are inserted, transitively, before the set+-- that needs them, and a set appears once, at its first position.+orderMigrationSets :: Map MigrationSetName MigrationSet -> [MigrationSetName] -> Either MigrationSetupError [MigrationSet]+orderMigrationSets sets requested = fmap (fmap snd) (foldlM (visit []) [] requested)+  where+    visit path placed wanted+      | wanted `elem` fmap fst placed = Right placed+      | wanted `elem` path = Left (MigrationSetupError ("migration sets require one another in a cycle: " <> render (reverse (wanted : path))))+      | otherwise = case Map.lookup wanted sets of+          Nothing -> Left (MigrationSetupError ("no registered bundle defines migration set " <> renderMigrationSetName wanted))+          Just set -> do+            withRequired <- foldlM (visit (wanted : path)) placed set.requires+            Right (withRequired <> [(wanted, set)])+    render = Text.intercalate " -> " . fmap renderMigrationSetName+    foldlM step initial = foldl (\accumulated item -> accumulated >>= (`step` item)) (Right initial)++migrateDatabase :: Text -> MigrationPlan -> IO (Either MigrationSetupError ())+migrateDatabase connectionString plan = do+  result <- runMigrationPlan defaultRunOptions (Settings.connectionString connectionString) plan+  pure (either (Left . MigrationSetupError . Text.pack . show) (const (Right ())) result)
+ src/Kenshou/Core/Env/Postgres.hs view
@@ -0,0 +1,318 @@+module Kenshou.Core.Env.Postgres+  ( PgSettingsSnapshot (..),+    PostgresMode (..),+    StopMode (..),+    ServerControl (..),+    PostgresEnv (..),+    EnvError (..),+    withPostgresEnv,+    withPostgresEnvKeeping,+    withPostgresEnvNamed,+  )+where++import Control.Exception (finally)+import Control.Monad (unless)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.Char (isAlphaNum, toLower)+import Data.IORef+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Monoid (Last (..))+import Data.Text (Text)+import Data.Text qualified as Text+import EphemeralPg qualified as Pg+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..))+import Kenshou.Core.Env (MigrationSet, MigrationSetName, PostgresRequirement (..))+import Kenshou.Core.Env.Migration (composePlan, migrateDatabase)+import Kenshou.Core.Id (RunId, renderRunId)+import Kenshou.Core.Log (Logger, Severity (Warning), logAt)+import Kenshou.Core.RunSpec (ConnectionSource (..), PostgresSpec (..))+import System.Directory (copyFile, createDirectoryIfMissing, doesFileExist)+import System.Environment (getEnv, lookupEnv, setEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Posix.User (getRealUserID)+import System.Process (readProcessWithExitCode)++data PgSettingsSnapshot = PgSettingsSnapshot+  { serverVersion :: Text,+    serverVersionNum :: Int,+    settings :: Map Text Text,+    superuser :: Bool,+    collation :: Text,+    encoding :: Text+  }+  deriving stock (Eq, Show)++data PostgresMode = PgEphemeral | PgExternal deriving stock (Eq, Show)++data StopMode = StopFast | StopImmediate deriving stock (Eq, Show)++data ServerControl = ServerControl {currentServer :: IO Pg.Database, restartServer :: IO (), stopServer :: StopMode -> IO (), startServer :: IO ()}++data PostgresEnv = PostgresEnv+  { mode :: PostgresMode,+    connectionString :: Text,+    tcpEndpoint :: Maybe (Text, Int),+    adminConnectionString :: Text,+    databaseName :: Text,+    newDatabase :: Text -> IO Text,+    snapshot :: PgSettingsSnapshot,+    control :: Maybe ServerControl+  }++newtype EnvError = EnvError Text deriving stock (Eq, Show)++instance ToJSON PgSettingsSnapshot where+  toJSON snapshot = object ["serverVersion" .= snapshot.serverVersion, "serverVersionNum" .= snapshot.serverVersionNum, "settings" .= snapshot.settings, "superuser" .= snapshot.superuser, "collation" .= snapshot.collation, "encoding" .= snapshot.encoding]++withPostgresEnv :: Map MigrationSetName MigrationSet -> Logger -> FilePath -> RunId -> PostgresRequirement -> PostgresSpec -> Dimensions -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withPostgresEnv sets = withPostgresEnvKeeping sets False++withPostgresEnvKeeping :: Map MigrationSetName MigrationSet -> Bool -> Logger -> FilePath -> RunId -> PostgresRequirement -> PostgresSpec -> Dimensions -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withPostgresEnvKeeping sets keepEnvironment = withPostgresEnvNamed sets keepEnvironment Nothing++-- | Like 'withPostgresEnvKeeping', for a named context of a scenario. On an+-- external server several contexts share one server, so the context name+-- becomes part of every database name the run creates there.+withPostgresEnvNamed :: Map MigrationSetName MigrationSet -> Bool -> Maybe Text -> Logger -> FilePath -> RunId -> PostgresRequirement -> PostgresSpec -> Dimensions -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withPostgresEnvNamed sets keepEnvironment contextName logger runDirectory runId requirement postgresSpec dimensions action = case postgresSpec of+  PostgresEphemeral extraSettings -> withSelectedBinaries dimensions.pgVersion $ do+    createDirectoryIfMissing True (runDirectory </> "logs")+    uid <- getRealUserID+    let temporaryRoot = "/tmp/ephpg-kenshou-" <> show uid+    createDirectoryIfMissing True temporaryRoot+    let durabilitySettings = case dimensions.pgDurability of+          Just PgDurable -> [("fsync", "on"), ("synchronous_commit", "on"), ("full_page_writes", "on")]+          _ -> [("fsync", "off"), ("synchronous_commit", "off"), ("full_page_writes", "off")]+        settings = [("listen_addresses", "'127.0.0.1'"), ("log_min_messages", "'WARNING'"), ("log_line_prefix", "'%m [%p] %a '"), ("logging_collector", "on"), ("log_directory", "'log'"), ("log_filename", "'postgres.log'")] <> durabilitySettings <> requirement.settings <> extraSettings+        config = Pg.defaultConfig {Pg.databaseName = "kenshou_template", Pg.temporaryRoot = Last (Just temporaryRoot), Pg.postgresSettings = settings}+    started <- Pg.startCached config Pg.defaultCacheConfig+    case started of+      Left err -> pure (Left (EnvError (Text.pack (show err))))+      Right database -> do+        databaseRef <- newIORef database+        let cleanup = do+              current <- readIORef databaseRef+              let postgresLog = current.dataDirectory </> "log" </> "postgres.log"+              exists <- doesFileExist postgresLog+              if exists then copyFile postgresLog (runDirectory </> "logs" </> "postgres.log") else pure ()+              Pg.stop current+        if keepEnvironment+          then setup databaseRef database >>= either (pure . Left) (\environment -> logAt logger Warning "leaving ephemeral PostgreSQL running" [] >> fmap Right (action environment))+          else (setup databaseRef database >>= either (pure . Left) (fmap Right . action)) `finally` cleanup+  PostgresExternal source -> do+    connection <- resolveConnection source+    case connection of+      Left err -> pure (Left err)+      Right maintenanceConnection -> do+        initialSnapshot <- snapshotDatabase maintenanceConnection+        case initialSnapshot >>= \snapshot -> validateExternal dimensions snapshot *> validatePreloadedLibraries requirement snapshot of+          Left err -> pure (Left err)+          Right () -> do+            unless (null (filter ((/= "shared_preload_libraries") . fst) requirement.settings)) $ logAt logger Warning "external PostgreSQL settings cannot be applied" []+            withExternalDatabases sets keepEnvironment maintenanceConnection runId contextName requirement action+  where+    setup databaseRef database = do+      migration <- migrateIfNeeded sets (Pg.connectionString database) requirement+      case migration of+        Left err -> pure (Left err)+        Right () -> do+          cloned <- runCommand "createdb" ["-h", database.socketDirectory, "-p", show database.port, "-U", Text.unpack database.user, "-T", "kenshou_template", "kenshou_run"]+          case cloned of+            Left err -> pure (Left err)+            Right () -> do+              let connection = connectionFor database "kenshou_run"+              snapshot <- snapshotDatabase connection+              pure (makeEnvironment databaseRef database connection <$> snapshot)++    makeEnvironment databaseRef database connection snapshot =+      PostgresEnv PgEphemeral connection (Just ("127.0.0.1", fromIntegral database.port)) (Pg.connectionString database) "kenshou_run" (cloneDatabase database) snapshot (Just (serverControl databaseRef))++    cloneDatabase database label = do+      let name = "kenshou_" <> label+      result <- runCommand "createdb" ["-h", database.socketDirectory, "-p", show database.port, "-U", Text.unpack database.user, "-T", "kenshou_template", Text.unpack name]+      either (ioError . userError . show) (const (pure (connectionFor database name))) result++    serverControl databaseRef = ServerControl (readIORef databaseRef) restart stopNow start+      where+        restart = do+          current <- readIORef databaseRef+          Pg.restart current >>= either (ioError . userError . show) (writeIORef databaseRef)+        stopNow mode = do+          current <- readIORef databaseRef+          let modeText = case mode of StopFast -> "fast"; StopImmediate -> "immediate"+          runCommand "pg_ctl" ["-D", current.dataDirectory, "stop", "-m", modeText, "-w"] >>= either (ioError . userError . show) pure+        start = do+          current <- readIORef databaseRef+          Pg.restart current >>= either (ioError . userError . show) (writeIORef databaseRef)++withExternalDatabases :: Map MigrationSetName MigrationSet -> Bool -> Text -> RunId -> Maybe Text -> PostgresRequirement -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withExternalDatabases sets keepEnvironment maintenanceConnection runId contextName requirement action = do+  counter <- newIORef (0 :: Int)+  created <- newIORef []+  let context = maybe "" (\name -> "_" <> Text.take 16 (Text.map (\character -> if isAlphaNum character then toLower character else '_') name)) contextName+      prefix = Text.take 13 (Text.filter (/= '-') (renderRunId runId)) <> context+      templateName = "kenshou_" <> prefix <> "_template"+      runName = "kenshou_" <> prefix+      remember name = modifyIORef' created (name :)+      create name template = do+        result <- psqlUnit maintenanceConnection ("create database " <> name <> maybe "" (" template " <>) template)+        case result of Left err -> pure (Left err); Right () -> remember name >> pure (Right ())+      cleanup = readIORef created >>= mapM_ (\name -> psqlUnit maintenanceConnection ("drop database if exists " <> name <> " with (force)") >> pure ())+      setup = do+        createdTemplate <- create templateName Nothing+        case createdTemplate of+          Left err -> pure (Left err)+          Right () -> do+            let templateConnection = connectionForExternal maintenanceConnection templateName+            migrated <- migrateIfNeeded sets templateConnection requirement+            case migrated of+              Left err -> pure (Left err)+              Right () -> do+                createdRun <- create runName (Just templateName)+                case createdRun of+                  Left err -> pure (Left err)+                  Right () -> do+                    let runConnection = connectionForExternal maintenanceConnection runName+                    snapshot <- snapshotDatabase runConnection+                    pure $ fmap (externalEnvironment counter created maintenanceConnection templateName runName runConnection) snapshot+  if keepEnvironment+    then setup >>= either (pure . Left) (fmap Right . action)+    else (setup >>= either (pure . Left) (fmap Right . action)) `finally` cleanup++externalEnvironment :: IORef Int -> IORef [Text] -> Text -> Text -> Text -> Text -> PgSettingsSnapshot -> PostgresEnv+externalEnvironment counter created maintenanceConnection templateName runName connection snapshot =+  PostgresEnv PgExternal connection (externalTcpEndpoint maintenanceConnection) maintenanceConnection runName clone snapshot Nothing+  where+    clone label = do+      number <- atomicModifyIORef' counter (\value -> let next = value + 1 in (next, next))+      let safeLabel = Text.map (\character -> if isAlphaNum character then character else '_') label+          name = "kenshou_" <> safeLabel <> "_" <> Text.pack (show number)+      result <- psqlUnit maintenanceConnection ("create database " <> name <> " template " <> templateName)+      case result of+        Left err -> ioError (userError (show err))+        Right () -> modifyIORef' created (name :) >> pure (connectionForExternal maintenanceConnection name)++resolveConnection :: ConnectionSource -> IO (Either EnvError Text)+resolveConnection (ConnLiteral value) = pure (Right value)+resolveConnection (ConnFromEnv name) = maybe (Left (EnvError ("environment variable is not set: " <> name))) (Right . Text.pack) <$> lookupEnv (Text.unpack name)++migrateIfNeeded :: Map MigrationSetName MigrationSet -> Text -> PostgresRequirement -> IO (Either EnvError ())+migrateIfNeeded _ _ requirement | null requirement.schemas = pure (Right ())+migrateIfNeeded sets connection requirement = case composePlan sets requirement.schemas of+  Left err -> pure (Left (EnvError (Text.pack (show err))))+  Right plan -> fmap (either (Left . EnvError . Text.pack . show) Right) (migrateDatabase connection plan)++connectionFor :: Pg.Database -> Text -> Text+connectionFor database name = Text.unwords ["host=" <> Text.pack database.socketDirectory, "port=" <> Text.pack (show database.port), "dbname=" <> name, "user=" <> database.user]++snapshotDatabase :: Text -> IO (Either EnvError PgSettingsSnapshot)+snapshotDatabase connection = do+  let setting name = psql connection ("show " <> name)+  version <- setting "server_version"+  versionNumber <- setting "server_version_num"+  settingPairs <- traverse (\name -> fmap ((name,) <$>) (setting name)) fingerprintSettings+  collation <- psql connection "select datcollate from pg_database where datname=current_database()"+  encoding <- psql connection "select pg_encoding_to_char(encoding) from pg_database where datname=current_database()"+  superuser <- psql connection "select rolsuper::text from pg_roles where rolname=current_user"+  pure do+    serverVersion <- version+    numberText <- versionNumber+    serverVersionNum <- maybe (Left (EnvError "invalid server_version_num")) Right (readMaybeText numberText)+    resolvedSettings <- sequence settingPairs+    databaseCollation <- collation+    databaseEncoding <- encoding+    isSuperuser <- (== "true") . Text.toCaseFold <$> superuser+    pure (PgSettingsSnapshot serverVersion serverVersionNum (Map.fromList resolvedSettings) isSuperuser databaseCollation databaseEncoding)++fingerprintSettings :: [Text]+fingerprintSettings =+  [ "fsync",+    "synchronous_commit",+    "full_page_writes",+    "wal_level",+    "shared_buffers",+    "effective_cache_size",+    "work_mem",+    "max_connections",+    "checkpoint_timeout",+    "checkpoint_completion_target",+    "max_wal_size",+    "wal_buffers",+    "random_page_cost",+    "autovacuum",+    "shared_preload_libraries"+  ]++validateExternal :: Dimensions -> PgSettingsSnapshot -> Either EnvError ()+validateExternal dimensions snapshot = do+  case dimensions.pgVersion of+    Just Pg17 | snapshot.serverVersionNum < 170000 || snapshot.serverVersionNum >= 180000 -> Left (EnvError "external PostgreSQL does not match pg.version=17")+    Just Pg18 | snapshot.serverVersionNum < 180000 || snapshot.serverVersionNum >= 190000 -> Left (EnvError "external PostgreSQL does not match pg.version=18")+    _ -> Right ()++  let actual name = Text.toCaseFold <$> Map.lookup name snapshot.settings+  case dimensions.pgDurability of+    Just PgFsyncOff | actual "fsync" /= Just "off" || actual "synchronous_commit" /= Just "off" -> Left (EnvError "external PostgreSQL does not match pg.durability=fsync-off")+    Just PgDurable | actual "fsync" /= Just "on" || actual "synchronous_commit" == Just "off" -> Left (EnvError "external PostgreSQL does not match pg.durability=durable")+    _ -> Right ()++validatePreloadedLibraries :: PostgresRequirement -> PgSettingsSnapshot -> Either EnvError ()+validatePreloadedLibraries requirement snapshot =+  case filter (`notElem` available) requested of+    [] -> Right ()+    missing -> Left (EnvError ("external PostgreSQL is missing required preloaded libraries: " <> Text.intercalate ", " missing))+  where+    requested = concatMap (libraries . snd) (filter ((== "shared_preload_libraries") . fst) requirement.settings)+    available = libraries (Map.findWithDefault "" "shared_preload_libraries" snapshot.settings)+    libraries = filter (not . Text.null) . fmap (Text.toCaseFold . Text.strip) . Text.splitOn "," . Text.dropAround (== '\'') . Text.strip++psql :: Text -> Text -> IO (Either EnvError Text)+psql connection query = do+  (code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+  pure case code of ExitSuccess -> Right (Text.strip (Text.pack output)); _ -> Left (EnvError (Text.strip (Text.pack err)))++psqlUnit :: Text -> Text -> IO (Either EnvError ())+psqlUnit connection query = fmap (const ()) <$> psql connection query++connectionForExternal :: Text -> Text -> Text+connectionForExternal maintenanceConnection name = maintenanceConnection <> " dbname=" <> name++externalTcpEndpoint :: Text -> Maybe (Text, Int)+externalTcpEndpoint connection = do+  host <- lookupWord "host"+  portText <- lookupWord "port"+  port <- readMaybeText portText+  pure (host, port)+  where+    pairs = [(key, Text.drop 1 rest) | word <- Text.words connection, let (key, rest) = Text.breakOn "=" word, not (Text.null rest)]+    lookupWord key = lookup key pairs++runCommand :: FilePath -> [String] -> IO (Either EnvError ())+runCommand command arguments = do+  (code, _, err) <- readProcessWithExitCode command arguments ""+  pure case code of ExitSuccess -> Right (); _ -> Left (EnvError (Text.strip (Text.pack err)))++withSelectedBinaries :: Maybe PgVersion -> IO (Either EnvError value) -> IO (Either EnvError value)+withSelectedBinaries version action = do+  (original, validation) <- prepare+  case validation of+    Left err -> restore original >> pure (Left err)+    Right () -> action `finally` restore original+  where+    prepare = do+      original <- getEnv "PATH"+      let (variable, expectedMajor) = case version of Just Pg17 -> ("KENSHOU_PG17_BIN", 17 :: Int); _ -> ("KENSHOU_PG18_BIN", 18)+      binaryDirectory <- lookupEnv variable+      maybe (pure ()) (\directory -> setEnv "PATH" (directory <> ":" <> original)) binaryDirectory+      (code, output, err) <- readProcessWithExitCode "postgres" ["--version"] ""+      let versionText = Text.pack output+          matches = (" " <> Text.pack (show expectedMajor) <> ".") `Text.isInfixOf` versionText+          validation = if code == ExitSuccess && matches then Right () else Left (EnvError (Text.pack (variable <> " is unset or does not select PostgreSQL " <> show expectedMajor <> ": " <> err <> output)))+      pure (original, validation)+    restore = setEnv "PATH"++readMaybeText :: (Read value) => Text -> Maybe value+readMaybeText value = case reads (Text.unpack value) of [(parsed, "")] -> Just parsed; _ -> Nothing
+ src/Kenshou/Core/Fingerprint.hs view
@@ -0,0 +1,118 @@+module Kenshou.Core.Fingerprint+  ( HostFingerprint (..),+    collectHostFingerprint,+    hostValue,+    runtimeValue,+    kenshouValue,+  )+where++import Control.Exception (IOException, try)+import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.ByteString qualified as ByteString+import Data.Text (Text)+import Data.Text qualified as Text+import GHC.Conc (getNumCapabilities)+import GHC.Stats (getRTSStatsEnabled)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Version (suiteVersion)+import System.Environment (getExecutablePath, lookupEnv)+import System.Info qualified as System+import System.Process (readProcess)++data HostFingerprint = HostFingerprint+  { os :: Text,+    arch :: Text,+    kernel :: Text,+    cpuModel :: Maybe Text,+    logicalCores :: Int,+    memoryBytes :: Maybe Integer,+    hostname :: Text,+    ghc :: Text,+    capabilities :: Int,+    threaded :: Bool,+    rtsStats :: Bool,+    kenshouVersion :: Text,+    executableSha256 :: Text,+    revision :: Maybe Text,+    dirty :: Maybe Bool+  }+  deriving stock (Eq, Show)++instance ToJSON HostFingerprint where+  toJSON = hostValue++hostValue :: HostFingerprint -> Value+hostValue fingerprint =+  object+    [ "os" .= fingerprint.os,+      "arch" .= fingerprint.arch,+      "kernel" .= fingerprint.kernel,+      "cpuModel" .= fingerprint.cpuModel,+      "logicalCores" .= fingerprint.logicalCores,+      "memoryBytes" .= fingerprint.memoryBytes,+      "hostname" .= fingerprint.hostname+    ]++runtimeValue :: HostFingerprint -> Value+runtimeValue fingerprint = object ["ghc" .= fingerprint.ghc, "capabilities" .= fingerprint.capabilities, "threaded" .= fingerprint.threaded, "rtsStats" .= fingerprint.rtsStats]++kenshouValue :: HostFingerprint -> Value+kenshouValue fingerprint = object ["version" .= fingerprint.kenshouVersion, "executableSha256" .= fingerprint.executableSha256, "revision" .= fingerprint.revision, "dirty" .= fingerprint.dirty]++collectHostFingerprint :: IO HostFingerprint+collectHostFingerprint = do+  kernel <- commandText "uname" ["-r"]+  hostname <- commandText "hostname" []+  cpuModel <- optionalCommandText (if System.os == "darwin" then "sysctl" else "sh") (if System.os == "darwin" then ["-n", "machdep.cpu.brand_string"] else ["-c", "sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo | head -1"])+  memoryBytes <- collectMemoryBytes+  capabilities <- getNumCapabilities+  rtsStats <- getRTSStatsEnabled+  executable <- getExecutablePath+  bytes <- ByteString.readFile executable+  revision <- firstPresent "KENSHOU_HARNESS_REVISION" (optionalCommandText "git" ["rev-parse", "HEAD"])+  dirtyOverride <- lookupEnv "KENSHOU_HARNESS_DIRTY"+  dirty <- case fmap Text.toCaseFold (Text.pack <$> dirtyOverride) of+    Just "true" -> pure (Just True)+    Just "false" -> pure (Just False)+    Just "1" -> pure (Just True)+    Just "0" -> pure (Just False)+    Just _ -> pure Nothing+    Nothing -> fmap (fmap (not . Text.null)) (optionalCommandText "git" ["status", "--porcelain"])+  pure+    HostFingerprint+      { os = Text.pack System.os,+        arch = Text.pack System.arch,+        kernel,+        cpuModel,+        logicalCores = capabilities,+        memoryBytes,+        hostname,+        ghc = Text.pack (show System.compilerVersion),+        capabilities,+        threaded = True,+        rtsStats,+        kenshouVersion = suiteVersion,+        executableSha256 = sha256Hex bytes,+        revision,+        dirty+      }++collectMemoryBytes :: IO (Maybe Integer)+collectMemoryBytes+  | System.os == "darwin" = optionalCommandText "sysctl" ["-n", "hw.memsize"] >>= pure . (>>= readInteger)+  | otherwise = optionalCommandText "sh" ["-c", "awk '/MemTotal:/ {print $2 * 1024}' /proc/meminfo"] >>= pure . (>>= readInteger)++firstPresent :: String -> IO (Maybe Text) -> IO (Maybe Text)+firstPresent variable fallback = lookupEnv variable >>= maybe fallback (pure . Just . Text.pack)++readInteger :: Text -> Maybe Integer+readInteger input = case reads (Text.unpack input) of [(value, "")] -> Just value; _ -> Nothing++commandText :: FilePath -> [String] -> IO Text+commandText command arguments = Text.strip . Text.pack <$> readProcess command arguments ""++optionalCommandText :: FilePath -> [String] -> IO (Maybe Text)+optionalCommandText command arguments = do+  result <- try (commandText command arguments) :: IO (Either IOException Text)+  pure (either (const Nothing) Just result)
+ src/Kenshou/Core/Id.hs view
@@ -0,0 +1,174 @@+module Kenshou.Core.Id+  ( LayerId,+    Kind (..),+    Segment,+    ScenarioId (..),+    RunId,+    Seed,+    mkLayerId,+    selftestLayer,+    renderLayer,+    parseLayer,+    renderKind,+    parseKind,+    mkSegment,+    unSegment,+    parseScenarioId,+    renderScenarioId,+    newRunId,+    parseRunId,+    renderRunId,+    mkSeed,+    unSeed,+    deriveGen,+  )+where++import Data.Aeson (FromJSON (..), ToJSON (..), withText)+import Data.Bits (xor)+import Data.ByteString qualified as ByteString+import Data.Char (isAsciiLower, isDigit)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Data.UUID (UUID)+import Data.UUID qualified as UUID+import Data.UUID.V7 qualified as UUID.V7+import Data.Word (Word64)+import System.Random.SplitMix (SMGen, mkSMGen)++-- | The first segment of a scenario identifier: the name of what a registered layer+-- bundle verifies. Any valid segment is a layer; suites choose their own.+newtype LayerId = LayerId Segment+  deriving stock (Eq, Ord, Show)++data Kind = Correctness | Concurrency | Soak | Benchmark+  deriving stock (Eq, Ord, Show, Enum, Bounded)++newtype Segment = Segment Text+  deriving stock (Eq, Ord, Show)++data ScenarioId = ScenarioId+  { layer :: LayerId,+    component :: Segment,+    kind :: Kind,+    name :: Segment+  }+  deriving stock (Eq, Ord, Show)++newtype RunId = RunId UUID+  deriving stock (Eq, Ord, Show)++newtype Seed = Seed Word64+  deriving stock (Eq, Ord, Show)++mkLayerId :: Text -> Either Text LayerId+mkLayerId value = either (\problem -> Left ("invalid layer \"" <> value <> "\": " <> problem)) (Right . LayerId) (mkSegment value)++-- | The layer of the kernel's and toolkits' own self-test scenarios.+selftestLayer :: LayerId+selftestLayer = LayerId (Segment "selftest")++renderLayer :: LayerId -> Text+renderLayer (LayerId segment) = unSegment segment++parseLayer :: Text -> Either Text LayerId+parseLayer = mkLayerId++renderKind :: Kind -> Text+renderKind Correctness = "correctness"+renderKind Concurrency = "concurrency"+renderKind Soak = "soak"+renderKind Benchmark = "benchmark"++parseKind :: Text -> Either Text Kind+parseKind value = maybe (Left ("unknown kind \"" <> value <> "\"")) Right (lookup value pairs)+  where+    pairs = [(renderKind item, item) | item <- [minBound .. maxBound]]++mkSegment :: Text -> Either Text Segment+mkSegment value+  | Text.length value > 48 = Left "segment is longer than 48 characters"+  | Text.null value = Left "segment is empty"+  | valid = Right (Segment value)+  | otherwise = Left ("invalid segment \"" <> value <> "\"")+  where+    chunks = Text.splitOn "-" value+    validChunk chunk =+      not (Text.null chunk)+        && isAsciiLower (Text.head chunk)+        && Text.all (\c -> isAsciiLower c || isDigit c) chunk+    valid = all validChunk chunks++unSegment :: Segment -> Text+unSegment (Segment value) = value++parseScenarioId :: Text -> Either Text ScenarioId+parseScenarioId value = case Text.splitOn "/" value of+  [layerText, componentText, kindText, nameText] ->+    ScenarioId+      <$> parseLayer layerText+      <*> mkSegment componentText+      <*> parseKind kindText+      <*> mkSegment nameText+  _ -> Left ("scenario identifier must have four segments: \"" <> value <> "\"")++renderScenarioId :: ScenarioId -> Text+renderScenarioId value =+  Text.intercalate+    "/"+    [renderLayer value.layer, unSegment value.component, renderKind value.kind, unSegment value.name]++newRunId :: IO RunId+newRunId = RunId <$> UUID.V7.genUUID++parseRunId :: Text -> Either Text RunId+parseRunId value = case UUID.fromText value of+  Nothing -> Left ("invalid UUIDv7 \"" <> value <> "\"")+  Just uuid+    | UUID.toText uuid /= value -> Left "run id must use canonical lowercase UUID text"+    | Text.index value 14 /= '7' -> Left "run id must be UUIDv7"+    | otherwise -> Right (RunId uuid)++renderRunId :: RunId -> Text+renderRunId (RunId uuid) = UUID.toText uuid++mkSeed :: Word64 -> Either Text Seed+mkSeed value+  | value <= 9007199254740991 = Right (Seed value)+  | otherwise = Left "seed must be between 0 and 9007199254740991"++unSeed :: Seed -> Word64+unSeed (Seed value) = value++deriveGen :: Seed -> Text -> SMGen+deriveGen (Seed seed) label = mkSMGen (seed `xor` fnv1a64 (Text.encodeUtf8 label))++fnv1a64 :: ByteString.ByteString -> Word64+fnv1a64 = ByteString.foldl' step 14695981039346656037+  where+    step hash byte = (hash `xor` fromIntegral byte) * 1099511628211++instance ToJSON LayerId where toJSON = toJSON . renderLayer++instance FromJSON LayerId where parseJSON = withText "LayerId" (either (fail . Text.unpack) pure . parseLayer)++instance ToJSON Kind where toJSON = toJSON . renderKind++instance FromJSON Kind where parseJSON = withText "Kind" (either (fail . Text.unpack) pure . parseKind)++instance ToJSON Segment where toJSON = toJSON . unSegment++instance FromJSON Segment where parseJSON = withText "Segment" (either (fail . Text.unpack) pure . mkSegment)++instance ToJSON ScenarioId where toJSON = toJSON . renderScenarioId++instance FromJSON ScenarioId where parseJSON = withText "ScenarioId" (either (fail . Text.unpack) pure . parseScenarioId)++instance ToJSON RunId where toJSON = toJSON . renderRunId++instance FromJSON RunId where parseJSON = withText "RunId" (either (fail . Text.unpack) pure . parseRunId)++instance ToJSON Seed where toJSON = toJSON . unSeed++instance FromJSON Seed where parseJSON value = parseJSON value >>= either (fail . Text.unpack) pure . mkSeed
+ src/Kenshou/Core/Knob.hs view
@@ -0,0 +1,177 @@+module Kenshou.Core.Knob+  ( KnobName,+    KnobType (..),+    KnobValue (..),+    Allowed (..),+    KnobSpec (..),+    RawKnob (..),+    ResolvedKnobs,+    KnobError (..),+    mkKnobName,+    renderKnobName,+    emptyKnobs,+    parseAssignment,+    resolveKnobs,+    resolvedKnobsMap,+    knobBool,+    knobInt,+    knobDouble,+    knobText,+  )+where++import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), withObject, withText)+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Char (isAsciiLower, isDigit)+import Data.Int (Int64)+import Data.List (group, sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Scientific qualified as Scientific+import Data.Text (Text)+import Data.Text qualified as Text++newtype KnobName = KnobName Text+  deriving stock (Eq, Ord, Show)++data KnobType = KnobBool | KnobInt | KnobDouble | KnobText+  deriving stock (Eq, Ord, Show)++data KnobValue = VBool Bool | VInt Int64 | VDouble Double | VText Text+  deriving stock (Eq, Show)++data Allowed = AnyValue | OneOf (NonEmpty KnobValue) | IntRange Int64 Int64 | DoubleRange Double Double+  deriving stock (Eq, Show)++data KnobSpec = KnobSpec+  { name :: KnobName,+    summary :: Text,+    knobType :: KnobType,+    def :: KnobValue,+    allowed :: Allowed,+    variants :: [KnobValue]+  }+  deriving stock (Eq, Show)++newtype ResolvedKnobs = ResolvedKnobs (Map KnobName KnobValue)+  deriving stock (Eq, Show)++data RawKnob = RawText Text | RawJson Value deriving stock (Eq, Show)++newtype KnobError = KnobError Text deriving stock (Eq, Show)++mkKnobName :: Text -> Either Text KnobName+mkKnobName value+  | length pieces < 2 = Left "knob name must contain a dot"+  | all validSegment pieces = Right (KnobName value)+  | otherwise = Left ("invalid knob name \"" <> value <> "\"")+  where+    pieces = Text.splitOn "." value+    validSegment piece =+      not (Text.null piece)+        && isAsciiLower (Text.head piece)+        && Text.all (\c -> isAsciiLower c || isDigit c || c == '-') piece++renderKnobName :: KnobName -> Text+renderKnobName (KnobName value) = value++emptyKnobs :: ResolvedKnobs+emptyKnobs = ResolvedKnobs Map.empty++parseAssignment :: Text -> Either KnobError (KnobName, RawKnob)+parseAssignment input = case Text.breakOn "=" input of+  (name, assignment)+    | Text.null assignment -> Left (KnobError "expected NAME=VALUE")+    | otherwise -> case mkKnobName name of+        Left err -> Left (KnobError err)+        Right parsed -> Right (parsed, RawText (Text.drop 1 assignment))++resolveKnobs :: [KnobSpec] -> [(KnobName, RawKnob)] -> Either (NonEmpty KnobError) ResolvedKnobs+resolveKnobs specs supplied = case errors of+  [] -> Right (ResolvedKnobs values)+  first : rest -> Left (first :| rest)+  where+    declarations = Map.fromList [(spec.name, spec) | spec <- specs]+    duplicateNames = [head names | names <- group (sort (fmap fst supplied)), length names > 1]+    unknown = [name | (name, _) <- supplied, Map.notMember name declarations]+    parsed = [(name, resolveOne spec raw) | (name, raw) <- supplied, Just spec <- [Map.lookup name declarations]]+    parseErrors = [KnobError (renderKnobName name <> ": " <> message) | (name, Left message) <- parsed]+    overrides = Map.fromList [(name, value) | (name, Right value) <- parsed]+    values = Map.fromList [(spec.name, Map.findWithDefault spec.def spec.name overrides) | spec <- specs]+    errors =+      [KnobError (renderKnobName name <> ": assigned more than once") | name <- duplicateNames]+        <> [KnobError (renderKnobName name <> ": unknown knob") | name <- unknown]+        <> parseErrors++resolveOne :: KnobSpec -> RawKnob -> Either Text KnobValue+resolveOne spec raw = do+  parsed <- parseByType spec.knobType raw+  if allowedValue spec.allowed parsed then Right parsed else Left "value is outside the allowed set"++parseByType :: KnobType -> RawKnob -> Either Text KnobValue+parseByType KnobBool (RawText "true") = Right (VBool True)+parseByType KnobBool (RawText "false") = Right (VBool False)+parseByType KnobBool (RawJson (Bool value)) = Right (VBool value)+parseByType KnobInt (RawText value) = maybe (Left "expected an integer") (Right . VInt) (readMaybeText value)+parseByType KnobInt (RawJson (Number value)) = maybe (Left "expected an integer") (Right . VInt) (Scientific.toBoundedInteger value)+parseByType KnobDouble (RawText value) = maybe (Left "expected a number") (Right . VDouble) (readMaybeText value)+parseByType KnobDouble (RawJson (Number value)) = Right (VDouble (Scientific.toRealFloat value))+parseByType KnobText (RawText value) = Right (VText value)+parseByType KnobText (RawJson (String value)) = Right (VText value)+parseByType _ _ = Left "value has the wrong JSON type"++readMaybeText :: (Read value) => Text -> Maybe value+readMaybeText value = case reads (Text.unpack value) of [(parsed, "")] -> Just parsed; _ -> Nothing++allowedValue :: Allowed -> KnobValue -> Bool+allowedValue AnyValue _ = True+allowedValue (OneOf values) value = value `elem` values+allowedValue (IntRange low high) (VInt value) = value >= low && value <= high+allowedValue (DoubleRange low high) (VDouble value) = value >= low && value <= high+allowedValue _ _ = False++resolvedKnobsMap :: ResolvedKnobs -> Map KnobName KnobValue+resolvedKnobsMap (ResolvedKnobs values) = values++knobBool :: ResolvedKnobs -> KnobName -> Bool+knobBool knobs name = case Map.lookup name (resolvedKnobsMap knobs) of Just (VBool value) -> value; _ -> error "knobBool: missing or wrong type"++knobInt :: ResolvedKnobs -> KnobName -> Int64+knobInt knobs name = case Map.lookup name (resolvedKnobsMap knobs) of Just (VInt value) -> value; _ -> error "knobInt: missing or wrong type"++knobDouble :: ResolvedKnobs -> KnobName -> Double+knobDouble knobs name = case Map.lookup name (resolvedKnobsMap knobs) of+  Just (VDouble value) -> value+  -- JSON number decoding cannot recover whether an integral value was+  -- originally a decimal knob. Worker init messages use this round trip.+  Just (VInt value) -> fromIntegral value+  _ -> error "knobDouble: missing or wrong type"++knobText :: ResolvedKnobs -> KnobName -> Text+knobText knobs name = case Map.lookup name (resolvedKnobsMap knobs) of Just (VText value) -> value; _ -> error "knobText: missing or wrong type"++instance ToJSON KnobName where toJSON = toJSON . renderKnobName++instance FromJSON KnobName where parseJSON = withText "KnobName" (either (fail . Text.unpack) pure . mkKnobName)++instance ToJSON KnobValue where+  toJSON (VBool value) = Bool value+  toJSON (VInt value) = toJSON value+  toJSON (VDouble value) = toJSON value+  toJSON (VText value) = String value++instance ToJSON ResolvedKnobs where+  toJSON (ResolvedKnobs values) = Object (KeyMap.fromList [(Key.fromText (renderKnobName name), toJSON value) | (name, value) <- Map.toAscList values])++instance FromJSON ResolvedKnobs where+  parseJSON = withObject "ResolvedKnobs" \values -> ResolvedKnobs . Map.fromList <$> traverse parseEntry (KeyMap.toList values)+    where+      parseEntry (key, value) = (,) <$> either (fail . Text.unpack) pure (mkKnobName (Key.toText key)) <*> parseJSON value++instance FromJSON KnobValue where+  parseJSON (Bool value) = pure (VBool value)+  parseJSON (String value) = pure (VText value)+  parseJSON (Number value) = pure (maybe (VDouble (Scientific.toRealFloat value)) VInt (Scientific.toBoundedInteger value))+  parseJSON _ = fail "knob values must be JSON scalars"
+ src/Kenshou/Core/Log.hs view
@@ -0,0 +1,57 @@+module Kenshou.Core.Log+  ( Severity (..),+    Logger,+    withLogger,+    logAt,+    nullLogger,+  )+where++import Data.Aeson (Value, encode, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text.IO qualified as Text.IO+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, renderRunId)+import System.IO (BufferMode (LineBuffering), Handle, IOMode (AppendMode), hSetBuffering, stderr, withFile)+import System.Posix.Process (getProcessID)++data Severity = Debug | Info | Warning | Error deriving stock (Eq, Ord, Show)++newtype Logger = Logger (Severity -> Text -> [(Text, Value)] -> IO ())++withLogger :: FilePath -> RunId -> (Logger -> IO value) -> IO value+withLogger path runId action = withFile path AppendMode \handle -> do+  hSetBuffering handle LineBuffering+  action (fileLogger handle runId)++fileLogger :: Handle -> RunId -> Logger+fileLogger handle runId = Logger \severity message fields -> do+  now <- getCurrentTime+  pid <- getProcessID+  LazyByteString.hPutStr handle $+    encode+      ( object+          [ "ts" .= now,+            "level" .= severityText severity,+            "msg" .= message,+            "fields" .= object [Key.fromText name .= value | (name, value) <- fields],+            "runId" .= renderRunId runId,+            "process" .= show pid+          ]+      )+      <> "\n"+  Text.IO.hPutStrLn stderr ("[" <> severityText severity <> "] " <> message)++logAt :: Logger -> Severity -> Text -> [(Text, Value)] -> IO ()+logAt (Logger write) = write++nullLogger :: Logger+nullLogger = Logger \_ _ _ -> pure ()++severityText :: Severity -> Text+severityText Debug = "debug"+severityText Info = "info"+severityText Warning = "warning"+severityText Error = "error"
+ src/Kenshou/Core/Manifest.hs view
@@ -0,0 +1,94 @@+module Kenshou.Core.Manifest+  ( ManifestFile (..),+    Manifest (..),+    ManifestProblem (..),+    writeManifest,+    verifyManifest,+  )+where++import Data.Aeson+import Data.ByteString qualified as ByteString+import Data.List (sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (RunId)+import System.Directory (doesDirectoryExist, doesFileExist, listDirectory)+import System.FilePath (makeRelative, takeExtension, (</>))++data ManifestFile = ManifestFile {path :: FilePath, sha256 :: Text, bytes :: Integer, mediaType :: Text} deriving stock (Eq, Show)++data Manifest = Manifest {runId :: RunId, createdAt :: UTCTime, files :: [ManifestFile]} deriving stock (Eq, Show)++newtype ManifestProblem = ManifestProblem Text deriving stock (Eq, Show)++instance ToJSON ManifestFile where+  toJSON file = object ["path" .= file.path, "sha256" .= file.sha256, "bytes" .= file.bytes, "mediaType" .= file.mediaType]++instance FromJSON ManifestFile where+  parseJSON = withObject "ManifestFile" \value -> ManifestFile <$> value .: "path" <*> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"++instance ToJSON Manifest where+  toJSON manifest = object ["schema" .= ("kenshou.artifact-manifest/v1" :: Text), "runId" .= manifest.runId, "createdAt" .= manifest.createdAt, "algorithm" .= ("sha256" :: Text), "files" .= manifest.files]++instance FromJSON Manifest where+  parseJSON = withObject "Manifest" \value -> do+    schema <- value .: "schema"+    algorithm <- value .: "algorithm"+    if schema /= ("kenshou.artifact-manifest/v1" :: Text) || algorithm /= ("sha256" :: Text) then fail "unsupported artifact manifest" else pure ()+    Manifest <$> value .: "runId" <*> value .: "createdAt" <*> value .: "files"++writeManifest :: FilePath -> RunId -> Map FilePath Text -> IO Manifest+writeManifest root runId mediaTypes = do+  paths <- filter (/= "manifest.json") . sort <$> regularFiles root+  files <- traverse fileRecord paths+  createdAt <- getCurrentTime+  pure Manifest {runId, createdAt, files}+  where+    fileRecord relativePath = do+      content <- ByteString.readFile (root </> relativePath)+      pure ManifestFile {path = relativePath, sha256 = sha256Hex content, bytes = fromIntegral (ByteString.length content), mediaType = Map.findWithDefault (mediaTypeFor relativePath) relativePath mediaTypes}++verifyManifest :: FilePath -> IO (Either (NonEmpty ManifestProblem) ())+verifyManifest root = do+  decoded <- eitherDecodeFileStrict' (root </> "manifest.json") :: IO (Either String Manifest)+  case decoded of+    Left err -> pure (Left (ManifestProblem ("manifest.json: " <> Text.pack err) :| []))+    Right manifest -> do+      problems <- concat <$> traverse checkFile manifest.files+      pure case problems of [] -> Right (); first : rest -> Left (first :| rest)+  where+    checkFile :: ManifestFile -> IO [ManifestProblem]+    checkFile file = do+      let fullPath = root </> file.path+      exists <- doesFileExist fullPath+      if not exists+        then pure [ManifestProblem (Text.pack file.path <> ": missing")]+        else do+          content <- ByteString.readFile fullPath+          pure [ManifestProblem (Text.pack file.path <> ": digest mismatch") | sha256Hex content /= file.sha256]++regularFiles :: FilePath -> IO [FilePath]+regularFiles root = go root+  where+    go directory = do+      names <- listDirectory directory+      concat <$> traverse (visit directory) names+    visit directory name = do+      let path = directory </> name+      isDirectory <- doesDirectoryExist path+      if isDirectory then go path else pure [makeRelative root path]++mediaTypeFor :: FilePath -> Text+mediaTypeFor path = case takeExtension path of+  ".json" -> "application/json"+  ".jsonl" -> "application/x-ndjson"+  ".csv" -> "text/csv"+  ".log" -> "text/plain"+  ".txt" -> "text/plain"+  _ -> "application/octet-stream"
+ src/Kenshou/Core/Outcome.hs view
@@ -0,0 +1,55 @@+module Kenshou.Core.Outcome+  ( Outcome (..),+    renderOutcome,+    parseOutcome,+    outcomeExitCode,+    usageExitCode,+    worstOutcome,+  )+where++import Data.Aeson (FromJSON (..), ToJSON (..), withText)+import Data.List (maximumBy)+import Data.List.NonEmpty (NonEmpty)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text++data Outcome = Passed | Failed | Errored | Inconclusive | InfrastructureFailure+  deriving stock (Eq, Show, Enum, Bounded)++renderOutcome :: Outcome -> Text+renderOutcome Passed = "passed"+renderOutcome Failed = "failed"+renderOutcome Errored = "errored"+renderOutcome Inconclusive = "inconclusive"+renderOutcome InfrastructureFailure = "infrastructure-failure"++parseOutcome :: Text -> Either Text Outcome+parseOutcome value = maybe (Left ("unknown outcome \"" <> value <> "\"")) Right (lookup value pairs)+  where+    pairs = [(renderOutcome item, item) | item <- [minBound .. maxBound]]++outcomeExitCode :: Outcome -> Int+outcomeExitCode Passed = 0+outcomeExitCode Failed = 1+outcomeExitCode Inconclusive = 3+outcomeExitCode Errored = 4+outcomeExitCode InfrastructureFailure = 4++usageExitCode :: Int+usageExitCode = 2++worstOutcome :: NonEmpty Outcome -> Outcome+worstOutcome = maximumBy (compare `onRank` rank) . NonEmpty.toList+  where+    onRank comparator project left right = comparator (project left) (project right)+    rank Passed = 0 :: Int+    rank Inconclusive = 1+    rank InfrastructureFailure = 2+    rank Errored = 3+    rank Failed = 4++instance ToJSON Outcome where toJSON = toJSON . renderOutcome++instance FromJSON Outcome where parseJSON = withText "Outcome" (either (fail . Text.unpack) pure . parseOutcome)
+ src/Kenshou/Core/Phase.hs view
@@ -0,0 +1,44 @@+module Kenshou.Core.Phase+  ( PhaseName (..),+    PhasePlan (..),+    zeroPhases,+    renderPhaseName,+  )+where++import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, withText, (.:), (.=))+import Data.Text (Text)+import Data.Text qualified as Text++data PhaseName = WarmUp | Steady | Drain+  deriving stock (Eq, Ord, Show)++data PhasePlan = PhasePlan+  { warmUpSeconds :: Double,+    steadySeconds :: Double,+    drainSeconds :: Double+  }+  deriving stock (Eq, Show)++zeroPhases :: PhasePlan+zeroPhases = PhasePlan 0 0 0++renderPhaseName :: PhaseName -> Text+renderPhaseName WarmUp = "warm-up"+renderPhaseName Steady = "steady"+renderPhaseName Drain = "drain"++instance ToJSON PhaseName where toJSON = toJSON . renderPhaseName++instance FromJSON PhaseName where+  parseJSON = withText "PhaseName" \case+    "warm-up" -> pure WarmUp+    "steady" -> pure Steady+    "drain" -> pure Drain+    other -> fail ("unknown phase " <> Text.unpack other)++instance ToJSON PhasePlan where+  toJSON value = object ["warmUpSeconds" .= value.warmUpSeconds, "steadySeconds" .= value.steadySeconds, "drainSeconds" .= value.drainSeconds]++instance FromJSON PhasePlan where+  parseJSON = withObject "PhasePlan" \value -> PhasePlan <$> value .: "warmUpSeconds" <*> value .: "steadySeconds" <*> value .: "drainSeconds"
+ src/Kenshou/Core/Resource.hs view
@@ -0,0 +1,37 @@+-- | Named resources a scenario requires besides PostgreSQL, and the providers that+-- suites register for them. A scenario names its resources in+-- 'Kenshou.Core.Env.EnvRequirements'; a bundle contributes one 'ResourceProvider'+-- per name; the kernel normalizes the specification value, provisions the resource+-- around a local run, hands each worker the provisioned value, and records a+-- fingerprint in the run result.+module Kenshou.Core.Resource+  ( ResourceName,+    mkResourceName,+    renderResourceName,+    ProvisionedResource (..),+    CellBinding (..),+    ResourceProvider (..),+  )+where++import Data.Aeson (Value)+import Data.Text (Text)+import Kenshou.Core.Context (RunContext)+import Kenshou.Core.Resource.Types++data ResourceProvider = ResourceProvider+  { name :: ResourceName,+    summary :: Text,+    -- | Turn the run specification's value, absent when the specification omits+    -- the resource, into the complete validated value the provider acts on. This+    -- is where defaults apply and bad input is rejected; the effective run+    -- specification records the result, so the comparison key hashes it.+    normalize :: Maybe Value -> Either Text Value,+    -- | Acquire the resource for one local run and release it when the action+    -- returns or throws. The run directory in the context is where a provider keeps+    -- its logs.+    withLocal :: forall a. RunContext -> Value -> (ProvisionedResource -> IO a) -> IO a,+    -- | How the resource is bound on a leased cell; 'Nothing' rejects cell placement+    -- for scenarios that require it.+    cell :: Maybe CellBinding+  }
+ src/Kenshou/Core/Resource/Types.hs view
@@ -0,0 +1,71 @@+-- | Names and values of the resources a scenario may require besides PostgreSQL:+-- anything a suite provisions for a run, such as a message broker or an object+-- store. 'Kenshou.Core.Resource' adds the provider contract; these types live apart+-- so the environment and context modules can name them without an import cycle.+module Kenshou.Core.Resource.Types+  ( ResourceName,+    mkResourceName,+    renderResourceName,+    legacyKafkaName,+    ProvisionedResource (..),+    CellBinding (..),+  )+where++import Data.Aeson (FromJSON (..), FromJSONKey (..), FromJSONKeyFunction (..), ToJSON (..), ToJSONKey (..), Value, withText)+import Data.Aeson.Types (toJSONKeyText)+import Data.Char (isAsciiLower, isDigit)+import Data.Text (Text)+import Data.Text qualified as Text++-- | Lowercase ASCII letters, digits, and hyphens, starting with a letter, at most+-- 63 characters, so a name is usable as a JSON object key without escaping.+newtype ResourceName = ResourceName Text+  deriving stock (Eq, Ord, Show)++mkResourceName :: Text -> Either Text ResourceName+mkResourceName value+  | valid = Right (ResourceName value)+  | otherwise = Left ("invalid resource name \"" <> value <> "\"")+  where+    valid =+      Text.length value >= 1+        && Text.length value <= 63+        && isAsciiLower (Text.head value)+        && Text.all (\character -> isAsciiLower character || isDigit character || character == '-') value++renderResourceName :: ResourceName -> Text+renderResourceName (ResourceName value) = value++-- | The resource name that documents written before named resources carried as a+-- top-level field. It exists only for the legacy readers and the compat-key/v1 shim;+-- the coupling allow-list names it.+legacyKafkaName :: ResourceName+legacyKafkaName = ResourceName "kafka"++-- | What a provider hands a run for one resource.+data ProvisionedResource = ProvisionedResource+  { -- | Sent to every worker in @WorkerInit.resources@.+    workerValue :: Value,+    -- | Recorded in the run result fingerprint under @resources.<name>@.+    fingerprint :: Value+  }+  deriving stock (Eq, Show)++-- | How a resource is bound on a leased verification cell.+data CellBinding = CellBinding+  { -- | The capability key a cell must report.+    capability :: Text,+    -- | The value for @cell.submission@ @reset.resources.<name>@, if the cell must+    -- reset the resource before the run.+    reset :: Maybe Value+  }+  deriving stock (Eq, Show)++instance ToJSON ResourceName where toJSON = toJSON . renderResourceName++instance FromJSON ResourceName where parseJSON = withText "ResourceName" (either (fail . Text.unpack) pure . mkResourceName)++instance ToJSONKey ResourceName where toJSONKey = toJSONKeyText renderResourceName++instance FromJSONKey ResourceName where fromJSONKey = FromJSONKeyTextParser (either (fail . Text.unpack) pure . mkResourceName)
+ src/Kenshou/Core/Role.hs view
@@ -0,0 +1,142 @@+module Kenshou.Core.Role+  ( RoleName,+    WorkerRole (..),+    PostgresConnInfo (..),+    WorkerInit (..),+    ControlMessage (..),+    WorkerMessage (..),+    RoleContext (..),+    mkRoleName,+    renderRoleName,+  )+where++import Data.Aeson+import Data.Aeson.Types (Pair, Parser)+import Data.Int (Int64)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Dimension (Dimensions)+import Kenshou.Core.Id (RunId, ScenarioId, Seed)+import Kenshou.Core.Knob (ResolvedKnobs)+import Kenshou.Core.Log (Logger)+import Kenshou.Core.Phase (PhaseName (..), renderPhaseName)+import Kenshou.Core.Resource.Types (ResourceName)+import Kenshou.Core.RunSpec (withLegacyResource)++newtype RoleName = RoleName Text deriving stock (Eq, Ord, Show)++data WorkerRole = WorkerRole+  { name :: RoleName,+    summary :: Text,+    run :: RoleContext -> IO ()+  }++data PostgresConnInfo = PostgresConnInfo {connectionString :: Text, adminConnectionString :: Text} deriving stock (Eq, Show)++data WorkerInit = WorkerInit+  { runId :: RunId,+    scenario :: ScenarioId,+    role :: RoleName,+    instanceName :: Text,+    seed :: Seed,+    knobs :: ResolvedKnobs,+    dimensions :: Dimensions,+    postgres :: Maybe PostgresConnInfo,+    resources :: Map ResourceName Value,+    telemetry :: Maybe Value,+    outDir :: FilePath,+    args :: Value+  }+  deriving stock (Eq, Show)++data ControlMessage = CtlInit WorkerInit | CtlStart | CtlPhase PhaseName | CtlStop Int | CtlCustom Text Value deriving stock (Eq, Show)++data WorkerMessage = WrkReady | WrkProgress Int64 UTCTime | WrkFacts [Value] | WrkCustom Text Value | WrkDone (Maybe Text) | WrkError Text deriving stock (Eq, Show)++data RoleContext = RoleContext+  { init :: WorkerInit,+    receive :: IO (Maybe ControlMessage),+    send :: WorkerMessage -> IO (),+    logger :: Logger+  }++mkRoleName :: Text -> Either Text RoleName+mkRoleName value = case Text.splitOn "/" value of+  [layer, role] | not (Text.null layer) && not (Text.null role) -> Right (RoleName value)+  _ -> Left "role name must be <layer>/<name>"++renderRoleName :: RoleName -> Text+renderRoleName (RoleName value) = value++instance ToJSON RoleName where toJSON = toJSON . renderRoleName++instance FromJSON RoleName where parseJSON = withText "RoleName" (either (fail . Text.unpack) pure . mkRoleName)++instance ToJSON PostgresConnInfo where toJSON value = object ["connectionString" .= value.connectionString, "adminConnectionString" .= value.adminConnectionString]++instance FromJSON PostgresConnInfo where parseJSON = withObject "PostgresConnInfo" \value -> PostgresConnInfo <$> value .: "connectionString" <*> value .: "adminConnectionString"++instance ToJSON WorkerInit where+  toJSON value = object ["schema" .= ("kenshou.worker-init/v1" :: Text), "runId" .= value.runId, "scenario" .= value.scenario, "role" .= value.role, "instance" .= value.instanceName, "seed" .= value.seed, "knobs" .= value.knobs, "dimensions" .= value.dimensions, "postgres" .= value.postgres, "resources" .= value.resources, "telemetry" .= value.telemetry, "outDir" .= value.outDir, "args" .= value.args]++instance FromJSON WorkerInit where+  parseJSON = withObject "WorkerInit" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.worker-init/v1" :: Text) then fail "unsupported worker-init schema" else pure ()+    resources <- (value .:? "resources" .!= Map.empty) >>= withLegacyResource "worker-init" value+    WorkerInit <$> value .: "runId" <*> value .: "scenario" <*> value .: "role" <*> value .: "instance" <*> value .: "seed" <*> value .: "knobs" <*> value .: "dimensions" <*> value .:? "postgres" <*> pure resources <*> value .:? "telemetry" <*> value .: "outDir" <*> value .: "args"++instance ToJSON ControlMessage where+  toJSON (CtlInit value) = tagged "init" ["init" .= value]+  toJSON CtlStart = tagged "start" []+  toJSON (CtlPhase name) = tagged "phase" ["name" .= renderPhaseName name]+  toJSON (CtlStop grace) = tagged "stop" ["graceMillis" .= grace]+  toJSON (CtlCustom name payload) = tagged "custom" ["name" .= name, "payload" .= payload]++instance FromJSON ControlMessage where+  parseJSON = withObject "ControlMessage" \value -> do+    version <- value .: "v"+    if version /= (1 :: Int) then fail "unsupported worker protocol version" else pure ()+    messageType <- value .: "type" :: Parser Text+    case messageType of+      "init" -> CtlInit <$> value .: "init"+      "start" -> pure CtlStart+      "phase" -> CtlPhase <$> (value .: "name" >>= parsePhase)+      "stop" -> CtlStop <$> value .: "graceMillis"+      "custom" -> CtlCustom <$> value .: "name" <*> value .: "payload"+      _ -> fail "unknown control message"++instance ToJSON WorkerMessage where+  toJSON WrkReady = tagged "ready" []+  toJSON (WrkProgress count at) = tagged "progress" ["count" .= count, "at" .= at]+  toJSON (WrkFacts batch) = tagged "facts" ["batch" .= batch]+  toJSON (WrkCustom name payload) = tagged "custom" ["name" .= name, "payload" .= payload]+  toJSON (WrkDone detail) = tagged "done" ["detail" .= detail]+  toJSON (WrkError message) = tagged "error" ["message" .= message]++instance FromJSON WorkerMessage where+  parseJSON = withObject "WorkerMessage" \value -> do+    version <- value .: "v"+    if version /= (1 :: Int) then fail "unsupported worker protocol version" else pure ()+    messageType <- value .: "type" :: Parser Text+    case messageType of+      "ready" -> pure WrkReady+      "progress" -> WrkProgress <$> value .: "count" <*> value .: "at"+      "facts" -> WrkFacts <$> value .: "batch"+      "custom" -> WrkCustom <$> value .: "name" <*> value .: "payload"+      "done" -> WrkDone <$> value .:? "detail"+      "error" -> WrkError <$> value .: "message"+      _ -> fail "unknown worker message"++tagged :: Text -> [Pair] -> Value+tagged messageType fields = object (["v" .= (1 :: Int), "type" .= messageType] <> fields)++parsePhase :: Text -> Parser PhaseName+parsePhase "warm-up" = pure WarmUp+parsePhase "steady" = pure Steady+parsePhase "drain" = pure Drain+parsePhase _ = fail "unknown phase"
+ src/Kenshou/Core/Role/Dispatch.hs view
@@ -0,0 +1,45 @@+module Kenshou.Core.Role.Dispatch (runWorker) where++import Control.Exception (SomeException, displayException, try)+import Data.Aeson qualified as Aeson+import Data.ByteString.Char8 qualified as ByteString+import Data.ByteString.Lazy.Char8 qualified as LazyByteString+import Data.IORef+import Data.Text qualified as Text+import GHC.IO.Handle (hDuplicate, hDuplicateTo)+import Kenshou.Core.Bundle (Registry, lookupRole)+import Kenshou.Core.Log (nullLogger)+import Kenshou.Core.Role+import System.Exit (ExitCode (..))+import System.IO (Handle, hFlush, hIsEOF, stderr, stdin, stdout)++runWorker :: Registry -> RoleName -> IO ExitCode+runWorker registry roleName = case lookupRole registry roleName of+  Nothing -> pure (ExitFailure 2)+  Just role -> do+    channel <- hDuplicate stdout+    hDuplicateTo stderr stdout+    initial <- receiveControl stdin+    case initial of+      Just (CtlInit workerInit) | workerInit.role == roleName -> do+        parentGone <- newIORef False+        let receive = receiveControl stdin >>= \case Nothing -> writeIORef parentGone True >> pure Nothing; message -> pure message+        outcome <- try (role.run (RoleContext workerInit receive (sendWorker channel) nullLogger)) :: IO (Either SomeException ())+        case outcome of+          Left exception -> sendWorker channel (WrkError (Text.pack (displayException exception))) >> pure (ExitFailure 70)+          Right () -> do+            orphaned <- readIORef parentGone+            if orphaned then pure (ExitFailure 75) else sendWorker channel (WrkDone Nothing) >> pure ExitSuccess+      _ -> pure (ExitFailure 2)++receiveControl :: Handle -> IO (Maybe ControlMessage)+receiveControl handle = do+  atEnd <- hIsEOF handle+  if atEnd+    then pure Nothing+    else do+      line <- ByteString.hGetLine handle+      pure (either (const Nothing) Just (Aeson.eitherDecodeStrict' line))++sendWorker :: Handle -> WorkerMessage -> IO ()+sendWorker handle message = LazyByteString.hPutStrLn handle (Aeson.encode message) >> hFlush handle
+ src/Kenshou/Core/Role/Spawn.hs view
@@ -0,0 +1,66 @@+module Kenshou.Core.Role.Spawn (WorkerHandle (..), withWorker) where++import Control.Exception (bracket)+import Data.Aeson (Value, eitherDecodeStrict', encode)+import Data.ByteString.Char8 qualified as ByteString+import Data.ByteString.Lazy.Char8 qualified as LazyByteString+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (ArtifactDir (..), Environment (..), RunContext (..), artifactPath)+import Kenshou.Core.Env.Postgres (PostgresEnv (..))+import Kenshou.Core.Resource.Types (ProvisionedResource (..))+import Kenshou.Core.Role+import Kenshou.Core.RunSpec (EnvironmentSpec (..))+import System.Environment (getExecutablePath)+import System.Exit (ExitCode)+import System.IO (BufferMode (LineBuffering), Handle, IOMode (AppendMode), hClose, hFlush, hIsEOF, hSetBuffering, openFile)+import System.Posix.Types (CPid)+import System.Process+import System.Timeout (timeout)++data WorkerHandle = WorkerHandle+  { instanceName :: Text,+    pid :: CPid,+    send :: ControlMessage -> IO (),+    receive :: Int -> IO (Maybe WorkerMessage),+    waitExit :: IO ExitCode+  }++withWorker :: RunContext -> RoleName -> Text -> Value -> (WorkerHandle -> IO value) -> IO value+withWorker context roleName instanceName arguments action = do+  logPath <- artifactPath context LogsDir ("worker-" <> Text.unpack instanceName <> ".stderr.log")+  bracket (start logPath) stop (action . fst)+  where+    start logPath = do+      executable <- getExecutablePath+      errorHandle <- openFile logPath AppendMode+      (Just input, Just output, _, processHandle) <- createProcess (proc executable ["worker", "--role", Text.unpack (renderRoleName roleName), "--diagnosis-root", context.outDir]) {std_in = CreatePipe, std_out = CreatePipe, std_err = UseHandle errorHandle}+      hSetBuffering input LineBuffering+      processId <- getPid processHandle >>= maybe (ioError (userError "worker has no process id")) pure+      let sendMessage message = LazyByteString.hPutStrLn input (encode message) >> hFlush input+          receiveMessage timeoutMillis = timeout (timeoutMillis * 1000) (readWorker output) >>= pure . joinMaybe+          workerHandle = WorkerHandle instanceName processId sendMessage receiveMessage (waitForProcess processHandle)+          postgres = fmap (\environment -> PostgresConnInfo environment.connectionString environment.adminConnectionString) context.env.postgres+          workerInit = WorkerInit context.runId context.scenario roleName instanceName context.seed context.knobs context.dimensions postgres (fmap (.workerValue) context.env.resources) context.environmentSpec.telemetry context.outDir arguments+      sendMessage (CtlInit workerInit)+      pure (workerHandle, (input, output, errorHandle, processHandle))+    stop (_, (input, output, errorHandle, processHandle)) = do+      sendLine input (CtlStop 2000)+      hClose input+      finished <- timeout 2000000 (waitForProcess processHandle)+      case finished of+        Just _ -> pure ()+        Nothing -> terminateProcess processHandle >> waitForProcess processHandle >> pure ()+      hClose output+      hClose errorHandle++readWorker :: Handle -> IO (Maybe WorkerMessage)+readWorker handle = do+  atEnd <- hIsEOF handle+  if atEnd then pure Nothing else either (const Nothing) Just . eitherDecodeStrict' <$> ByteString.hGetLine handle++sendLine :: Handle -> ControlMessage -> IO ()+sendLine handle message = LazyByteString.hPutStrLn handle (encode message) >> hFlush handle++joinMaybe :: Maybe (Maybe value) -> Maybe value+joinMaybe = maybe Nothing id
+ src/Kenshou/Core/Run.hs view
@@ -0,0 +1,169 @@+module Kenshou.Core.Run+  ( RunnerConfig (..),+    RunOutput (..),+    executeRun,+    defectDisposition,+  )+where++import Control.Exception (SomeException, displayException, fromException, try)+import Control.Monad (when)+import Data.Aeson (ToJSON, Value, encode, object, (.=))+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (diffUTCTime, getCurrentTime)+import Kenshou.Core.Bundle (Registry, lookupResource, registryMigrationSets)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))+import Kenshou.Core.Compat (comparisonKey, compatInputs, seriesKey)+import Kenshou.Core.Context+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Env.Postgres (EnvError (..), PgSettingsSnapshot, PostgresEnv (..), withPostgresEnvKeeping, withPostgresEnvNamed)+import Kenshou.Core.Fingerprint (HostFingerprint (..), collectHostFingerprint, hostValue, kenshouValue, runtimeValue)+import Kenshou.Core.Id (renderRunId, unSeed)+import Kenshou.Core.Log (Severity (Warning), withLogger)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Core.Outcome (Outcome (..), outcomeExitCode)+import Kenshou.Core.Resource (ProvisionedResource (..), ResourceName, ResourceProvider (..), renderResourceName)+import Kenshou.Core.RunResult+import Kenshou.Core.RunSpec (CohortExpectation (..), EffectiveRunSpec (..), EnvironmentSpec (..), RunSpec)+import Kenshou.Core.RunSpec.Resolve (SpecError (..), resolveRunSpec)+import Kenshou.Core.Scenario (KnownDefect (..), Scenario (..), ScenarioReport (..), cohortScopeApplies, individualKnownDefects, infrastructureFailureBecause)+import System.Directory (createDirectory, createDirectoryIfMissing, doesDirectoryExist, getCurrentDirectory, renameFile)+import System.FilePath ((</>))+import System.Timeout (timeout)++data RunnerConfig = RunnerConfig+  { registry :: Registry,+    outRoot :: FilePath,+    cohort :: CohortIdentity,+    cellFingerprint :: Maybe Value,+    keepEnvironment :: Bool,+    strictKnownDefects :: Bool,+    argv :: [String]+  }++data RunOutput = RunOutput {directory :: FilePath, result :: RunResult, manifestSha256 :: Text}++executeRun :: RunnerConfig -> RunSpec -> IO (Either (NonEmpty SpecError) RunOutput)+executeRun config input = do+  resolution <- resolveRunSpec config.registry input+  case resolution of+    Left problems -> pure (Left problems)+    Right (scenario, spec) -> do+      let directory = config.outRoot </> Text.unpack (renderRunId spec.runId)+      exists <- doesDirectoryExist directory+      if exists+        then pure (Left (SpecError ("run directory already exists: " <> Text.pack directory) :| []))+        else do+          createDirectoryIfMissing True config.outRoot+          createDirectory directory+          createDirectory (directory </> "logs")+          atomicEncode (directory </> "run-spec.json") spec+          specBytes <- ByteString.readFile (directory </> "run-spec.json")+          (result, mediaTypes) <- withLogger (directory </> "logs" </> "harness.jsonl") spec.runId \logger -> do+            startedAt <- getCurrentTime+            state <- newRunState+            let context = RunContext spec.runId spec.scenario spec.knobs spec.dimensions spec.seed spec.phases (Environment Nothing Map.empty Map.empty) spec.environment spec.comparison directory logger state+            (report, postgresSnapshot, extraPostgresSnapshots, resourceFingerprints) <- runScenario config scenario spec context+            endedAt <- getCurrentTime+            host <- collectHostFingerprint+            when (host.revision == Nothing) (observe context "harness" Warning "harness revision is unavailable")+            (summaries, observations, phaseTimings, mediaTypes) <- readRunState state+            workingDirectory <- getCurrentDirectory+            let (defect, blocking, exitCode) = defectDisposition config.strictKnownDefects config.cohort scenario report+                schemas = maybe [] (.schemas) scenario.requires.postgres+                inputs = compatInputs spec postgresSnapshot schemas config.cohort+                compatibility = object ["algorithm" .= ("kenshou.compat-key/v1" :: Text), "comparisonKey" .= comparisonKey inputs, "seriesKey" .= seriesKey inputs, "inputs" .= inputs]+                fingerprint =+                  object $+                    ["placement" .= spec.environment.placement, "machineProfile" .= spec.environment.machineProfile, "host" .= hostValue host, "runtime" .= runtimeValue host, "kenshou" .= kenshouValue host, "postgres" .= postgresSnapshot, "extraPostgres" .= extraPostgresSnapshots, "cell" .= config.cellFingerprint]+                      <> ["resources" .= resourceFingerprints | not (Map.null resourceFingerprints)]+                invocation = object ["argv" .= config.argv, "workingDirectory" .= workingDirectory]+                result = RunResult spec.runId spec.scenario spec.scenarioRevision scenario.tier report.outcome blocking exitCode report.reason report.failures defect (unSeed spec.seed) (sha256Hex specBytes) spec.comparison startedAt endedAt (realToFrac (diffUTCTime endedAt startedAt)) phaseTimings config.cohort fingerprint compatibility summaries observations invocation+            atomicEncode (directory </> "run-result.json") result+            pure (result, mediaTypes)+          manifest <- writeManifest directory spec.runId mediaTypes+          atomicEncode (directory </> "manifest.json") manifest+          manifestBytes <- ByteString.readFile (directory </> "manifest.json")+          pure (Right (RunOutput directory result (sha256Hex manifestBytes)))++runScenario :: RunnerConfig -> Scenario -> EffectiveRunSpec -> RunContext -> IO (ScenarioReport, Maybe PgSettingsSnapshot, Map.Map Text PgSettingsSnapshot, Map.Map ResourceName Value)+runScenario config scenario spec context+  | cohortMismatch config.cohort spec.cohortExpectation = pure (infrastructureFailureBecause "cohort-mismatch", Nothing, Map.empty, Map.empty)+  | otherwise = do+      nested <- withPrimary scenario.requires.postgres spec.environment.postgres \primary ->+        withExtras scenario.requires.extraPostgres spec.environment.extraPostgres Map.empty \extras ->+          withResources scenario.requires.resources Map.empty \resources -> do+            let environment = Environment primary extras resources+            report <- timed context {env = environment}+            pure (report, (.snapshot) <$> primary, fmap (.snapshot) extras, fmap (.fingerprint) resources)+      let provisioned = nested >>= id >>= id+      pure (either (\err -> (infrastructureFailureBecause (Text.pack (show err)), Nothing, Map.empty, Map.empty)) id provisioned)+  where+    -- Providers are bracketed in declaration order inside the PostgreSQL contexts; a+    -- provider failure is an infrastructure failure like a PostgreSQL one.+    withResources [] accumulated action = Right <$> action accumulated+    withResources (name : rest) accumulated action = case (lookupResource config.registry name, Map.lookup name spec.environment.resources) of+      (Nothing, _) -> pure (Left (EnvError ("no registered provider for resource " <> renderResourceName name)))+      (_, Nothing) -> pure (Left (EnvError ("resource specification is missing: " <> renderResourceName name)))+      (Just ResourceProvider {withLocal = acquire}, Just value) -> do+        attempted <- try (acquire context value \provisioned -> withResources rest (Map.insert name provisioned accumulated) action)+        pure case attempted of+          Left (exception :: SomeException) -> Left (EnvError ("resource " <> renderResourceName name <> " failed: " <> Text.pack (displayException exception)))+          Right inner -> inner++    withPrimary Nothing _ action = Right <$> action Nothing+    withPrimary (Just requirement) (Just postgresSpec) action = withPostgresEnvKeeping (registryMigrationSets config.registry) config.keepEnvironment context.logger context.outDir spec.runId requirement postgresSpec spec.dimensions (action . Just)+    withPrimary (Just _) Nothing _ = pure (Left (EnvError "PostgreSQL specification is missing"))++    withExtras [] _ accumulated action = Right <$> action accumulated+    withExtras ((name, requirement) : rest) postgresSpecs accumulated action = case Map.lookup name postgresSpecs of+      Nothing -> pure (Left (EnvError ("extra PostgreSQL specification is missing: " <> name)))+      Just postgresSpec -> do+        provisioned <- withPostgresEnvNamed (registryMigrationSets config.registry) config.keepEnvironment (Just name) context.logger context.outDir spec.runId requirement postgresSpec spec.dimensions \postgres ->+          withExtras rest postgresSpecs (Map.insert name postgres accumulated) action+        pure (provisioned >>= id)++    timed runContext = do+      execution <- try (timeout (spec.timeoutSeconds * 1000000) (scenario.run runContext)) :: IO (Either SomeException (Maybe ScenarioReport))+      pure case execution of+        Left exception -> case fromException exception of+          Just (InfrastructureError message) -> ScenarioReport InfrastructureFailure (Just message) []+          Nothing -> ScenarioReport Errored (Just (Text.pack (displayException exception))) []+        Right Nothing -> ScenarioReport Errored (Just ("timeout after " <> Text.pack (show spec.timeoutSeconds) <> " s")) []+        Right (Just report) -> report++cohortMismatch :: CohortIdentity -> Maybe CohortExpectation -> Bool+cohortMismatch _ Nothing = False+cohortMismatch cohort (Just expectation) = unPlanHash cohort.identityPlanHash /= expectation.planHash++defectDisposition :: Bool -> CohortIdentity -> Scenario -> ScenarioReport -> (Maybe (KnownDefect, KnownDefectStatus), Bool, Int)+defectDisposition strict cohort scenario report = case NonEmpty.nonEmpty applicable of+  Nothing -> ordinary+  Just defects ->+    let status+          | report.outcome == Failed && not (null report.failures) && all (`elem` coveredLabels) report.failures = DefectReproduced+          | report.outcome == Failed = DefectDifferentFailure+          | otherwise = DefectNotReproduced+        blocking = report.outcome /= Passed && status /= DefectReproduced+        exitCode = if status == DefectReproduced && not strict then 0 else outcomeExitCode report.outcome+        identified = case NonEmpty.toList defects of+          [single] -> single+          _ -> KnownDefectGroup defects+     in (Just (identified, status), blocking, exitCode)+  where+    applicable = filter (cohortScopeApplies cohort . (.appliesTo)) (maybe [] individualKnownDefects scenario.knownDefect)+    coveredLabels = concatMap (.expectedFailures) applicable+    ordinary = (Nothing, report.outcome /= Passed, outcomeExitCode report.outcome)++atomicEncode :: (ToJSON value) => FilePath -> value -> IO ()+atomicEncode path value = do+  let temporary = path <> ".tmp"+  LazyByteString.writeFile temporary (encode value <> "\n")+  renameFile temporary path
+ src/Kenshou/Core/RunResult.hs view
@@ -0,0 +1,114 @@+module Kenshou.Core.RunResult+  ( RunResult (..),+    KnownDefectStatus (..),+  )+where++import Data.Aeson hiding (Error)+import Data.Aeson.Key qualified as Key+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime)+import Data.Word (Word64)+import Kenshou.Core.Cohort (CohortIdentity)+import Kenshou.Core.Context (Observation (..), PhaseTiming, SummarySection (..))+import Kenshou.Core.Id (RunId, ScenarioId (..), renderKind, renderLayer, renderScenarioId, unSegment)+import Kenshou.Core.Log (Severity (..))+import Kenshou.Core.Outcome (Outcome)+import Kenshou.Core.RunSpec (ComparisonMembership)+import Kenshou.Core.Scenario (KnownDefect (..), Tier, individualKnownDefects, renderTier)++data KnownDefectStatus = DefectReproduced | DefectDifferentFailure | DefectNotReproduced deriving stock (Eq, Show)++data RunResult = RunResult+  { runId :: RunId,+    scenario :: ScenarioId,+    scenarioRevision :: Int,+    tier :: Tier,+    outcome :: Outcome,+    blocking :: Bool,+    exitCode :: Int,+    reason :: Maybe Text,+    failures :: [Text],+    knownDefect :: Maybe (KnownDefect, KnownDefectStatus),+    seed :: Word64,+    specSha256 :: Text,+    comparison :: Maybe ComparisonMembership,+    startedAt :: UTCTime,+    endedAt :: UTCTime,+    durationSeconds :: Double,+    phases :: [PhaseTiming],+    cohort :: CohortIdentity,+    fingerprint :: Value,+    compatibility :: Value,+    summaries :: Map SummarySection (Map Text Value),+    observations :: [Observation],+    invocation :: Value+  }+  deriving stock (Eq, Show)++instance ToJSON RunResult where+  toJSON result =+    object+      [ "schema" .= ("kenshou.run-result/v1" :: Text),+        "runId" .= result.runId,+        "scenario" .= renderScenarioId result.scenario,+        "scenarioRevision" .= result.scenarioRevision,+        "layer" .= renderLayer result.scenario.layer,+        "component" .= unSegment result.scenario.component,+        "kind" .= renderKind result.scenario.kind,+        "tier" .= renderTier result.tier,+        "outcome" .= result.outcome,+        "blocking" .= result.blocking,+        "exitCode" .= result.exitCode,+        "reason" .= result.reason,+        "failures" .= result.failures,+        "knownDefect" .= fmap (knownDefectValue result.failures) result.knownDefect,+        "seed" .= result.seed,+        "spec" .= object ["path" .= ("run-spec.json" :: Text), "sha256" .= result.specSha256],+        "comparison" .= result.comparison,+        "timings" .= object ["clock" .= ("GHC.Clock.getMonotonicTimeNSec" :: Text), "startedAt" .= result.startedAt, "endedAt" .= result.endedAt, "durationSeconds" .= result.durationSeconds, "phases" .= result.phases],+        "cohort" .= result.cohort,+        "fingerprint" .= result.fingerprint,+        "compatibility" .= result.compatibility,+        "summaries" .= summaryValue result.summaries,+        "observations" .= fmap observationValue result.observations,+        "invocation" .= result.invocation+      ]++knownDefectValue :: [Text] -> (KnownDefect, KnownDefectStatus) -> Value+knownDefectValue failures (defect, status) = case individualKnownDefects defect of+  [single] -> singleValue single status+  many ->+    object+      [ "status" .= statusText status,+        "defects" .= fmap (\item -> singleValue item (if any (`elem` item.expectedFailures) failures then DefectReproduced else DefectNotReproduced)) many+      ]+  where+    singleValue item itemStatus = object ["reference" .= item.reference, "summary" .= item.summary, "expectedFailures" .= item.expectedFailures, "status" .= statusText itemStatus]++statusText :: KnownDefectStatus -> Text+statusText DefectReproduced = "reproduced"+statusText DefectDifferentFailure = "different-failure"+statusText DefectNotReproduced = "not-reproduced"++summaryValue :: Map SummarySection (Map Text Value) -> Value+summaryValue summaries =+  object+    [ "measurements" .= section Measurements,+      "verdicts" .= section Verdicts,+      "diagnosis" .= section Diagnosis,+      "telemetry" .= section Telemetry+    ]+  where+    section name = object [Key.fromText key .= value | (key, value) <- Map.toAscList (Map.findWithDefault Map.empty name summaries)]++observationValue :: Observation -> Value+observationValue observation = object ["source" .= observation.source, "severity" .= severityText observation.severity, "message" .= observation.message, "at" .= observation.at]++severityText :: Severity -> Text+severityText Debug = "debug"+severityText Info = "info"+severityText Warning = "warning"+severityText Error = "error"
+ src/Kenshou/Core/RunSpec.hs view
@@ -0,0 +1,223 @@+module Kenshou.Core.RunSpec+  ( SpecPlacement (..),+    ConnectionSource (..),+    PostgresSpec (..),+    EnvironmentSpec (..),+    CohortExpectation (..),+    ComparisonMembership (..),+    RunSpec (..),+    EffectiveRunSpec (..),+    minimalRunSpec,+    redactConnectionString,+    withLegacyResource,+  )+where++import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.Types (Parser)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Dimension (Dimensions, renderDimensions)+import Kenshou.Core.Id (RunId, ScenarioId, Seed)+import Kenshou.Core.Knob (KnobName, RawKnob (..), ResolvedKnobs)+import Kenshou.Core.Knob qualified+import Kenshou.Core.Phase (PhasePlan)+import Kenshou.Core.Resource.Types (ResourceName, legacyKafkaName)++data SpecPlacement = RunLocal | RunOnCell deriving stock (Eq, Show)++data ConnectionSource = ConnLiteral Text | ConnFromEnv Text deriving stock (Eq, Show)++data PostgresSpec = PostgresEphemeral [(Text, Text)] | PostgresExternal ConnectionSource deriving stock (Eq, Show)++data EnvironmentSpec = EnvironmentSpec+  { placement :: SpecPlacement,+    machineProfile :: Maybe Text,+    postgres :: Maybe PostgresSpec,+    extraPostgres :: Map Text PostgresSpec,+    -- | Per-resource specification values, keyed by the resource name.+    resources :: Map ResourceName Value,+    telemetry :: Maybe Value+  }+  deriving stock (Eq, Show)++data CohortExpectation = CohortExpectation {name :: Maybe Text, planHash :: Text} deriving stock (Eq, Show)++data ComparisonMembership = ComparisonMembership {group :: Text, arm :: Text, trial :: Int, position :: Int} deriving stock (Eq, Show)++data RunSpec = RunSpec+  { runId :: Maybe RunId,+    scenario :: ScenarioId,+    scenarioRevision :: Maybe Int,+    knobs :: [(KnobName, RawKnob)],+    dimensions :: [(Text, Text)],+    seed :: Maybe Seed,+    phases :: Maybe PhasePlan,+    timeoutSeconds :: Maybe Int,+    environment :: EnvironmentSpec,+    cohortExpectation :: Maybe CohortExpectation,+    comparison :: Maybe ComparisonMembership,+    labels :: Map Text Text+  }+  deriving stock (Eq, Show)++data EffectiveRunSpec = EffectiveRunSpec+  { runId :: RunId,+    scenario :: ScenarioId,+    scenarioRevision :: Int,+    knobs :: ResolvedKnobs,+    dimensions :: Dimensions,+    seed :: Seed,+    phases :: PhasePlan,+    timeoutSeconds :: Int,+    environment :: EnvironmentSpec,+    cohortExpectation :: Maybe CohortExpectation,+    comparison :: Maybe ComparisonMembership,+    labels :: Map Text Text+  }+  deriving stock (Eq, Show)++minimalRunSpec :: ScenarioId -> RunSpec+minimalRunSpec scenario = RunSpec Nothing scenario Nothing [] [] Nothing Nothing Nothing (EnvironmentSpec RunLocal Nothing Nothing Map.empty Map.empty Nothing) Nothing Nothing Map.empty++redactConnectionString :: Text -> Text+redactConnectionString value = redactUri (Text.unwords (fmap redactWord (Text.words value)))+  where+    redactWord word+      | "password=" `Text.isPrefixOf` Text.toCaseFold word = "password=<redacted>"+      | otherwise = word+    redactUri input = case Text.breakOn "://" input of+      (scheme, rest)+        | not (Text.null rest) ->+            let afterScheme = Text.drop 3 rest+                (authority, suffix) = Text.breakOn "/" afterScheme+             in case Text.breakOnEnd "@" authority of+                  (credentials, host)+                    | not (Text.null credentials) ->+                        let (user, password) = Text.breakOn ":" (Text.dropEnd 1 credentials)+                         in if Text.null password then input else scheme <> "://" <> user <> ":<redacted>@" <> host <> suffix+                  _ -> input+      _ -> input++instance FromJSON RunSpec where+  parseJSON = withObject "RunSpec" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.run-spec/v1" :: Text) then fail "unsupported run-spec schema" else pure ()+    runId <- value .:? "runId"+    scenario <- value .: "scenario"+    scenarioRevision <- value .:? "scenarioRevision"+    rawKnobs <- value .:? "knobs" .!= Map.empty+    knobs <- traverse parseKnob (Map.toList rawKnobs)+    dimensions <- Map.toList <$> value .:? "dimensions" .!= Map.empty+    seed <- value .:? "seed"+    phases <- value .:? "phases"+    timeoutSeconds <- value .:? "timeoutSeconds"+    environment <- value .:? "environment" .!= EnvironmentSpec RunLocal Nothing Nothing Map.empty Map.empty Nothing+    cohortExpectation <- value .:? "cohortExpectation"+    comparison <- value .:? "comparison"+    labels <- value .:? "labels" .!= Map.empty+    pure RunSpec {runId, scenario, scenarioRevision, knobs, dimensions, seed, phases, timeoutSeconds, environment, cohortExpectation, comparison, labels}+    where+      parseKnob (name, raw) = case Kenshou.Core.Knob.mkKnobName name of+        Left err -> fail (Text.unpack err)+        Right parsed -> pure (parsed, RawJson raw)++instance ToJSON RunSpec where+  toJSON spec =+    object $+      [ "schema" .= ("kenshou.run-spec/v1" :: Text),+        "scenario" .= spec.scenario,+        "knobs" .= Map.fromList [(Kenshou.Core.Knob.renderKnobName name, rawValue raw) | (name, raw) <- spec.knobs],+        "dimensions" .= Map.fromList spec.dimensions,+        "environment" .= spec.environment,+        "cohortExpectation" .= spec.cohortExpectation,+        "comparison" .= spec.comparison,+        "labels" .= spec.labels+      ]+        <> maybe [] (pure . ("runId" .=)) spec.runId+        <> maybe [] (pure . ("scenarioRevision" .=)) spec.scenarioRevision+        <> maybe [] (pure . ("seed" .=)) spec.seed+        <> maybe [] (pure . ("phases" .=)) spec.phases+        <> maybe [] (pure . ("timeoutSeconds" .=)) spec.timeoutSeconds+    where+      rawValue (RawText value) = String value+      rawValue (RawJson value) = value++instance ToJSON EffectiveRunSpec where+  toJSON spec =+    object+      [ "schema" .= ("kenshou.run-spec/v1" :: Text),+        "runId" .= spec.runId,+        "scenario" .= spec.scenario,+        "scenarioRevision" .= spec.scenarioRevision,+        "knobs" .= spec.knobs,+        "dimensions" .= object [Key.fromText name .= value | (name, value) <- renderDimensions spec.dimensions],+        "seed" .= spec.seed,+        "phases" .= spec.phases,+        "timeoutSeconds" .= spec.timeoutSeconds,+        "environment" .= spec.environment,+        "cohortExpectation" .= spec.cohortExpectation,+        "comparison" .= spec.comparison,+        "labels" .= spec.labels+      ]++instance ToJSON EnvironmentSpec where+  toJSON environment =+    object $+      ["placement" .= placementText environment.placement, "machineProfile" .= environment.machineProfile, "postgres" .= environment.postgres]+        <> ["extraPostgres" .= environment.extraPostgres | not (Map.null environment.extraPostgres)]+        <> ["resources" .= environment.resources | not (Map.null environment.resources)]+        <> ["telemetry" .= environment.telemetry]++instance FromJSON EnvironmentSpec where+  parseJSON = withObject "EnvironmentSpec" \value -> do+    resources <- (value .:? "resources" .!= Map.empty) >>= withLegacyResource "environment" value+    EnvironmentSpec <$> (value .:? "placement" .!= RunLocal) <*> value .:? "machineProfile" <*> value .:? "postgres" <*> (value .:? "extraPostgres" .!= Map.empty) <*> pure resources <*> value .:? "telemetry"++-- | Documents written before named resources carried the one resource they knew as a+-- top-level field; read a non-null one as that named resource.+withLegacyResource :: Text -> Object -> Map ResourceName Value -> Parser (Map ResourceName Value)+withLegacyResource label value resources = do+  legacy <- value .:? "kafka"+  case (legacy, Map.lookup legacyKafkaName resources) of+    (Nothing, _) -> pure resources+    (Just Null, _) -> pure resources+    (Just found, Nothing) -> pure (Map.insert legacyKafkaName found resources)+    (Just found, Just current)+      | found == current -> pure resources+      | otherwise -> fail (Text.unpack label <> ".kafka and " <> Text.unpack label <> ".resources.kafka disagree")++instance ToJSON SpecPlacement where toJSON = String . placementText++instance FromJSON SpecPlacement where parseJSON = withText "SpecPlacement" \case "local" -> pure RunLocal; "cell" -> pure RunOnCell; other -> fail ("unknown placement " <> Text.unpack other)++placementText :: SpecPlacement -> Text+placementText RunLocal = "local"+placementText RunOnCell = "cell"++instance ToJSON PostgresSpec where+  toJSON (PostgresEphemeral settings) = object ["mode" .= ("ephemeral" :: Text), "settings" .= Map.fromList settings]+  toJSON (PostgresExternal (ConnFromEnv name)) = object ["mode" .= ("external" :: Text), "connectionStringEnv" .= name]+  toJSON (PostgresExternal (ConnLiteral value)) = object ["mode" .= ("external" :: Text), "connectionString" .= redactConnectionString value]++instance FromJSON PostgresSpec where+  parseJSON = withObject "PostgresSpec" \value -> do+    mode <- value .: "mode" :: Parser Text+    case mode of+      "ephemeral" -> PostgresEphemeral . Map.toList <$> value .:? "settings" .!= Map.empty+      "external" -> do+        env <- value .:? "connectionStringEnv"+        literal <- value .:? "connectionString"+        case (env, literal) of (Just name, Nothing) -> pure (PostgresExternal (ConnFromEnv name)); (Nothing, Just connection) -> pure (PostgresExternal (ConnLiteral connection)); _ -> fail "external PostgreSQL requires exactly one connection source"+      _ -> fail "unknown PostgreSQL mode"++instance ToJSON CohortExpectation where toJSON value = object ["name" .= value.name, "planHash" .= value.planHash]++instance FromJSON CohortExpectation where parseJSON = withObject "CohortExpectation" \value -> CohortExpectation <$> value .:? "name" <*> value .: "planHash"++instance ToJSON ComparisonMembership where toJSON value = object ["group" .= value.group, "arm" .= value.arm, "trial" .= value.trial, "position" .= value.position]++instance FromJSON ComparisonMembership where parseJSON = withObject "ComparisonMembership" \value -> ComparisonMembership <$> value .: "group" <*> value .: "arm" <*> value .: "trial" <*> value .: "position"
+ src/Kenshou/Core/RunSpec/Resolve.hs view
@@ -0,0 +1,106 @@+module Kenshou.Core.RunSpec.Resolve (SpecError (..), resolveRunSpec) where++import Data.Aeson (Value)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (Registry, lookupResource, lookupScenario)+import Kenshou.Core.Dimension (resolveDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (mkSeed, newRunId)+import Kenshou.Core.Knob (resolveKnobs)+import Kenshou.Core.Phase (PhasePlan (..))+import Kenshou.Core.Resource (ResourceName, ResourceProvider (..), renderResourceName)+import Kenshou.Core.RunSpec+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Core.Scenario (Placement (..), Scenario (..), Tier (..))+import System.Info qualified as System+import System.Random (randomIO)++newtype SpecError = SpecError Text deriving stock (Eq, Show)++resolveRunSpec :: Registry -> RunSpec -> IO (Either (NonEmpty SpecError) (Scenario, EffectiveRunSpec))+resolveRunSpec registry spec = case lookupScenario registry spec.scenario of+  Nothing -> pure (Left (SpecError ("unknown scenario " <> Text.pack (show spec.scenario)) :| []))+  Just scenario -> do+    generatedRunId <- maybe newRunId pure spec.runId+    randomSeed <- randomIO+    let seedResult = maybe (mkSeed (randomSeed `mod` 9007199254740992)) Right spec.seed+        knobsResult = resolveKnobs scenario.knobs spec.knobs+        dimensionsResult = resolveDimensions scenario.dimensions spec.dimensions+    pure do+      checkedSeed <- firstOne seedResult+      checkedKnobs <- mapErrors knobsResult+      checkedDimensions <- mapErrors dimensionsResult+      if maybe True (== scenario.revision) spec.scenarioRevision then pure () else Left (SpecError "scenario revision does not match registry" :| [])+      validatePlacement scenario.placement spec.environment.placement+      validatePostgres scenario spec.environment.postgres+      validateExtraPostgres scenario spec.environment.extraPostgres+      resources <- resolveResources registry scenario spec.environment.resources+      let environment = (applyEnvironmentDefaults scenario spec.environment) {RunSpec.resources = resources}+          phases = maybe scenario.phases id spec.phases+          effective = EffectiveRunSpec generatedRunId scenario.id scenario.revision checkedKnobs checkedDimensions checkedSeed phases (maybe (tierTimeout scenario.tier phases) id spec.timeoutSeconds) environment spec.cohortExpectation spec.comparison spec.labels+      pure (scenario, effective)+  where+    firstOne :: Either Text value -> Either (NonEmpty SpecError) value+    firstOne = either (Left . (:| []) . SpecError) Right+    mapErrors :: (Show problem) => Either (NonEmpty problem) value -> Either (NonEmpty SpecError) value+    mapErrors = either (Left . fmap (SpecError . Text.pack . show)) Right++-- | Every resource the specification names must be required by the scenario, and+-- every required resource's value is normalized by its registered provider, so the+-- effective specification records the complete value.+resolveResources :: Registry -> Scenario -> Map ResourceName Value -> Either (NonEmpty SpecError) (Map ResourceName Value)+resolveResources registry scenario supplied = do+  case [name | name <- Map.keys supplied, name `notElem` scenario.requires.resources] of+    [] -> pure ()+    extra : _ -> Left (SpecError ("environment names a resource the scenario does not require: " <> renderResourceName extra) :| [])+  normalized <- traverse normalizeOne scenario.requires.resources+  pure (Map.fromList normalized)+  where+    normalizeOne name = case lookupResource registry name of+      Nothing -> Left (SpecError ("no registered provider for resource " <> renderResourceName name) :| [])+      Just provider -> either (\problem -> Left (SpecError ("resource " <> renderResourceName name <> ": " <> problem) :| [])) (Right . (name,)) (provider.normalize (Map.lookup name supplied))++validatePlacement :: Placement -> SpecPlacement -> Either (NonEmpty SpecError) ()+validatePlacement PlaceCell RunLocal = Left (SpecError "scenario requires cell placement" :| [])+validatePlacement PlaceLocal RunOnCell = Left (SpecError "scenario requires local placement" :| [])+validatePlacement _ _ = Right ()++validatePostgres :: Scenario -> Maybe PostgresSpec -> Either (NonEmpty SpecError) ()+validatePostgres scenario postgresSpec = case (scenario.requires.postgres, postgresSpec) of+  (Nothing, Just _) -> Left (SpecError "PostgreSQL is not applicable to this scenario" :| [])+  (Just requirement, Just (PostgresExternal _)) | requirement.needsServerControl -> Left (SpecError "scenario requires control of an ephemeral PostgreSQL server" :| [])+  _ -> Right ()++validateExtraPostgres :: Scenario -> Map.Map Text PostgresSpec -> Either (NonEmpty SpecError) ()+validateExtraPostgres scenario supplied =+  case errors of+    [] -> Right ()+    first : rest -> Left (first :| rest)+  where+    required = Map.fromList scenario.requires.extraPostgres+    unknown = Map.keys (supplied `Map.difference` required)+    controlErrors =+      [ SpecError ("extra PostgreSQL " <> name <> " requires an ephemeral server")+      | (name, requirement) <- Map.toList required,+        requirement.needsServerControl,+        Just (PostgresExternal _) <- [Map.lookup name supplied]+      ]+    errors = fmap (SpecError . ("unknown extra PostgreSQL environment " <>)) unknown <> controlErrors++applyEnvironmentDefaults :: Scenario -> EnvironmentSpec -> EnvironmentSpec+applyEnvironmentDefaults scenario environment =+  environment+    { machineProfile = Just (maybe ("local/" <> Text.pack System.os <> "-" <> Text.pack System.arch) id environment.machineProfile),+      postgres = case (scenario.requires.postgres, environment.postgres) of (Just _, Nothing) -> Just (PostgresEphemeral []); (_, existing) -> existing,+      extraPostgres = Map.union environment.extraPostgres (Map.fromList [(name, PostgresEphemeral []) | (name, _) <- scenario.requires.extraPostgres])+    }++tierTimeout :: Tier -> PhasePlan -> Int+tierTimeout TierSmoke _ = 120+tierTimeout TierStandard _ = 1200+tierTimeout TierExtended _ = 7200+tierTimeout TierSoak phases = ceiling (phases.warmUpSeconds + phases.steadySeconds + phases.drainSeconds) + 1800
+ src/Kenshou/Core/Scenario.hs view
@@ -0,0 +1,119 @@+module Kenshou.Core.Scenario+  ( Tier (..),+    Placement (..),+    KnownDefect (..),+    individualKnownDefects,+    CohortScope (..),+    PackageCondition (..),+    cohortScopeApplies,+    ScenarioReport (..),+    Scenario (..),+    renderTier,+    renderPlacement,+    passed,+    failedWith,+    inconclusiveBecause,+    infrastructureFailureBecause,+  )+where++import Data.List.NonEmpty (NonEmpty)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Version (parseVersion)+import Kenshou.Core.Cohort (CohortIdentity (..), PackageSource (..), ResolvedComponent (..), ResolvedPackage (..))+import Kenshou.Core.Context (RunContext)+import Kenshou.Core.Dimension (DimensionSupport)+import Kenshou.Core.Env (EnvRequirements)+import Kenshou.Core.Id (ScenarioId)+import Kenshou.Core.Knob (KnobSpec)+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.Phase (PhasePlan)+import Text.ParserCombinators.ReadP (readP_to_S)++data Tier = TierSmoke | TierStandard | TierExtended | TierSoak deriving stock (Eq, Ord, Show)++data Placement = PlaceLocal | PlaceCell | PlaceEither deriving stock (Eq, Ord, Show)++data KnownDefect+  = KnownDefect+      { reference :: Text,+        summary :: Text,+        expectedFailures :: [Text],+        appliesTo :: CohortScope+      }+  | KnownDefectGroup (NonEmpty KnownDefect)+  deriving stock (Eq, Show)++individualKnownDefects :: KnownDefect -> [KnownDefect]+individualKnownDefects (KnownDefectGroup defects) = concatMap individualKnownDefects defects+individualKnownDefects defect = [defect]++data CohortScope = AllCohorts | OnlyWhen (NonEmpty PackageCondition) deriving stock (Eq, Show)++data PackageCondition+  = ResolvedFromHackage Text+  | ResolvedFromGit Text+  | VersionBelow Text Text+  | RevisionIs Text Text+  deriving stock (Eq, Show)++cohortScopeApplies :: CohortIdentity -> CohortScope -> Bool+cohortScopeApplies _ AllCohorts = True+cohortScopeApplies cohort (OnlyWhen conditions) = all (conditionApplies packages) conditions+  where+    packages = concatMap (.resolvedComponentPackages) cohort.identityComponents++conditionApplies :: [ResolvedPackage] -> PackageCondition -> Bool+conditionApplies packages condition = any matches packages+  where+    matches package = case condition of+      ResolvedFromHackage name -> package.resolvedPackageName == name && case package.resolvedPackageSource of FromHackage _ -> True; _ -> False+      ResolvedFromGit name -> package.resolvedPackageName == name && case package.resolvedPackageSource of FromGit _ _ _ -> True; _ -> False+      VersionBelow name boundary -> package.resolvedPackageName == name && maybe False (uncurry (<)) ((,) <$> parse package.resolvedPackageVersion <*> parse boundary)+      RevisionIs name revision -> package.resolvedPackageName == name && case package.resolvedPackageSource of FromGit _ actual _ -> actual == revision; _ -> False+    parse value = case [version | (version, "") <- readP_to_S parseVersion (Text.unpack value)] of [] -> Nothing; versions -> Just (last versions)++data ScenarioReport = ScenarioReport+  { outcome :: Outcome,+    reason :: Maybe Text,+    failures :: [Text]+  }+  deriving stock (Eq, Show)++data Scenario = Scenario+  { id :: ScenarioId,+    revision :: Int,+    summary :: Text,+    tier :: Tier,+    placement :: Placement,+    knobs :: [KnobSpec],+    dimensions :: DimensionSupport,+    phases :: PhasePlan,+    requires :: EnvRequirements,+    knownDefect :: Maybe KnownDefect,+    run :: RunContext -> IO ScenarioReport+  }++renderTier :: Tier -> Text+renderTier TierSmoke = "smoke"+renderTier TierStandard = "standard"+renderTier TierExtended = "extended"+renderTier TierSoak = "soak"++renderPlacement :: Placement -> Text+renderPlacement PlaceLocal = "local"+renderPlacement PlaceCell = "cell"+renderPlacement PlaceEither = "either"++passed :: ScenarioReport+passed = ScenarioReport Passed Nothing []++failedWith :: [Text] -> Text -> ScenarioReport+failedWith labels message = ScenarioReport Failed (Just message) labels++inconclusiveBecause :: Text -> ScenarioReport+inconclusiveBecause message = ScenarioReport Inconclusive (Just message) []++infrastructureFailureBecause :: Text -> ScenarioReport+infrastructureFailureBecause message = ScenarioReport InfrastructureFailure (Just message) []
+ src/Kenshou/Core/Selector.hs view
@@ -0,0 +1,40 @@+module Kenshou.Core.Selector+  ( ScenarioSelector,+    parseSelector,+    renderSelector,+    matchesSelector,+  )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (ScenarioId, renderScenarioId)++newtype ScenarioSelector = ScenarioSelector [Text]+  deriving stock (Eq, Show)++parseSelector :: Text -> Either Text ScenarioSelector+parseSelector value+  | null pieces = Left "selector is empty"+  | "**" `elem` initSafe pieces = Left "** is permitted only as the final segment"+  | length pieces < 4 && last pieces /= "**" = Left "short selectors must end in **"+  | length pieces > 4 = Left "selector has more than four segments"+  | any invalid pieces = Left ("invalid selector \"" <> value <> "\"")+  | otherwise = Right (ScenarioSelector pieces)+  where+    pieces = Text.splitOn "/" value+    initSafe [] = []+    initSafe xs = init xs+    invalid piece = Text.null piece || (piece /= "*" && piece /= "**" && Text.any (== '*') piece)++renderSelector :: ScenarioSelector -> Text+renderSelector (ScenarioSelector pieces) = Text.intercalate "/" pieces++matchesSelector :: ScenarioSelector -> ScenarioId -> Bool+matchesSelector (ScenarioSelector patterns) scenario = go patterns (Text.splitOn "/" (renderScenarioId scenario))+  where+    go ["**"] _ = True+    go [] [] = True+    go (patternPart : restPatterns) (actual : restActual) =+      (patternPart == "*" || patternPart == actual) && go restPatterns restActual+    go _ _ = False
+ src/Kenshou/Core/Selftest.hs view
@@ -0,0 +1,151 @@+module Kenshou.Core.Selftest (bundle) where++import Control.Concurrent (threadDelay)+import Data.Aeson (object, (.=))+import Data.Either (isLeft)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Context (RunContext (..), SummarySection (..), putSummary, requirePostgres)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), noDimensions, postgresDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..), noEnvironment)+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot (..), PostgresEnv (..))+import Kenshou.Core.Id (parseScenarioId, selftestLayer)+import Kenshou.Core.Knob+import Kenshou.Core.Outcome (parseOutcome)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.Role+import Kenshou.Core.Role.Spawn (WorkerHandle (..), withWorker)+import Kenshou.Core.Scenario+import Kenshou.Core.Selftest.Schema (selftestMigrationSet, selftestSchema)+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)++bundle :: LayerBundle+bundle = (mkBundle selftestLayer [alwaysFail, alwaysPass, errors, outcomeScenario, knownDefectScenario, postgresRoundtripScenario, workerEchoScenario] [echoRole]) {migrations = [selftestMigrationSet]}++alwaysPass, alwaysFail, errors :: Scenario+alwaysPass = scenario "selftest/kernel/correctness/always-pass" "Always reports passed." (const (pure passed))+alwaysFail = scenario "selftest/kernel/correctness/always-fail" "Always reports failed." (const (pure (failedWith ["seeded-failure"] "this scenario always fails")))+errors = scenario "selftest/kernel/correctness/errors" "Always throws." (const (ioError (userError "seeded self-test error")))++outcomeScenario :: Scenario+outcomeScenario =+  (scenario "selftest/kernel/correctness/outcome" "Returns an operator-selected outcome." runOutcome)+    { knobs =+        [ KnobSpec outcomeKnob "Outcome to report" KnobText (VText "passed") (OneOf (VText "passed" :| fmap VText ["failed", "errored", "inconclusive", "infrastructure-failure"])) [],+          KnobSpec sleepKnob "Delay before reporting" KnobDouble (VDouble 0) (DoubleRange 0 3600) []+        ]+    }++knownDefectScenario :: Scenario+knownDefectScenario =+  (scenario "selftest/kernel/correctness/known-defect" "Reproduces a declared known defect." (const (pure (failedWith ["seeded-defect"] "seeded known defect"))))+    { knownDefect = Just (KnownDefect "mori://shinzui/kenshou/plans/2-open-the-kernel-layer-identity-and-schema-composition-to-registered-suites" "Seeded harness defect" ["seeded-defect"] AllCohorts)+    }++workerEchoScenario :: Scenario+workerEchoScenario =+  (scenario "selftest/kernel/concurrency/worker-echo" "Exchanges messages with a child worker process." runWorkerEcho)+    { knobs = [KnobSpec messagesKnob "Messages to echo" KnobInt (VInt 1) (IntRange 1 1000) []]+    }++postgresRoundtripScenario :: Scenario+postgresRoundtripScenario =+  (scenario "selftest/kernel/correctness/postgres-roundtrip" "Verifies a composed migration ledger and PostgreSQL durability." runPostgresRoundtrip)+    { revision = 2,+      knobs = [KnobSpec eventsKnob "Rows to round-trip" KnobInt (VInt 3) (IntRange 1 1000) [VInt 1, VInt 100]],+      dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| []) noDimensions,+      requires = noEnvironment {postgres = Just (PostgresRequirement [selftestSchema] [] False)}+    }++echoRole :: WorkerRole+echoRole = WorkerRole (roleName "selftest/echo") "Echoes custom worker messages." \context -> do+  context.send WrkReady+  let loop =+        context.receive >>= \case+          Just (CtlCustom "echo" payload) -> context.send (WrkCustom "echo" payload) >> loop+          Just (CtlStop _) -> pure ()+          Just _ -> loop+          Nothing -> pure ()+  loop++runWorkerEcho :: RunContext -> IO ScenarioReport+runWorkerEcho context = withWorker context (roleName "selftest/echo") "echo-1" (object []) \worker -> do+  ready <- worker.receive 10000+  case ready of+    Just WrkReady -> echoMessages worker [1 .. fromIntegral (knobInt context.knobs messagesKnob)]+    _ -> pure (failedWith ["echo-timeout"] "worker did not become ready")+  where+    echoMessages worker [] = do+      putSummary context Verdicts "worker-echo" (object ["childPid" .= show worker.pid])+      pure passed+    echoMessages worker (number : rest) = do+      let payload = object ["n" .= (number :: Int)]+      worker.send (CtlCustom "echo" payload)+      reply <- worker.receive 10000+      case reply of+        Just (WrkCustom "echo" echoed) | echoed == payload -> echoMessages worker rest+        Nothing -> pure (failedWith ["echo-timeout"] "worker reply timed out")+        _ -> pure (failedWith ["echo-mismatch"] "worker reply differed")++runPostgresRoundtrip :: RunContext -> IO ScenarioReport+runPostgresRoundtrip context = do+  let environment = requirePostgres context+      count = knobInt context.knobs eventsKnob+  schemas <- psql environment.connectionString "select string_agg(schema_name, ',' order by schema_name) from information_schema.schemata where schema_name not like 'pg\\_%' and schema_name not in ('information_schema','public')"+  ledger <- psql environment.connectionString "select string_agg(distinct component, ',' order by component) from pgmigrate.migrations"+  inserted <- psql environment.connectionString ("insert into kenshou_selftest.roundtrip (id, payload) select n, 'row-' || n from generate_series(1," <> Text.pack (show count) <> ") as n")+  roundtrip <- psql environment.connectionString "select count(*) from kenshou_selftest.roundtrip where payload = 'row-' || id"+  let actualFsync = Map.lookup "fsync" environment.snapshot.settings+      expectedFsync = case context.dimensions.pgDurability of Just PgDurable -> "on"; _ -> "off"+      failures =+        ["schemas" | schemas /= Right "kenshou_selftest,pgmigrate"]+          <> ["one-ledger" | ledger /= Right "kenshou-selftest"]+          <> ["roundtrip" | isLeft inserted || roundtrip /= Right (Text.pack (show count))]+          <> ["durability-honoured" | actualFsync /= Just expectedFsync]+  putSummary context Verdicts "postgres-roundtrip" (object ["rows" .= count, "ledgerComponents" .= (["kenshou-selftest"] :: [Text])])+  pure (if null failures then passed else failedWith failures "PostgreSQL round-trip checks failed")++psql :: Text -> Text -> IO (Either Text Text)+psql connection query = do+  (code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+  pure case code of ExitSuccess -> Right (Text.strip (Text.pack output)); _ -> Left (Text.strip (Text.pack err))++runOutcome :: RunContext -> IO ScenarioReport+runOutcome context = do+  let delay = knobDouble context.knobs sleepKnob+  threadDelay (round (delay * 1000000))+  pure case parseOutcome (knobText context.knobs outcomeKnob) of+    Right result -> ScenarioReport result Nothing []+    Left message -> failedWith ["invalid-outcome"] message++outcomeKnob, sleepKnob, messagesKnob, eventsKnob :: KnobName+outcomeKnob = knobName "selftest.outcome"+sleepKnob = knobName "selftest.sleep-seconds"+messagesKnob = knobName "selftest.messages"+eventsKnob = knobName "selftest.events"++knobName :: Text -> KnobName+knobName = either (error . show) id . mkKnobName++roleName :: Text -> RoleName+roleName = either (error . show) id . mkRoleName++scenario :: Text -> Text -> (RunContext -> IO ScenarioReport) -> Scenario+scenario identifier summary action =+  Scenario+    { id = either (error . show) id (parseScenarioId identifier),+      revision = 1,+      summary,+      tier = TierSmoke,+      placement = PlaceEither,+      knobs = [],+      dimensions = noDimensions,+      phases = zeroPhases,+      requires = noEnvironment,+      knownDefect = Nothing,+      run = action+    }
+ src/Kenshou/Core/Selftest/Schema.hs view
@@ -0,0 +1,26 @@+-- | The kernel's own migration set: one schema and one table that the self-test+-- scenarios of every toolkit may use without depending on a suite's schema.+module Kenshou.Core.Selftest.Schema+  ( selftestSchema,+    selftestMigrationSet,+  )+where++import Data.List.NonEmpty (NonEmpty (..))+import Data.Set qualified as Set+import Data.Text qualified as Text+import Database.PostgreSQL.Migrate (migrationComponent, sqlMigration)+import Kenshou.Core.Env (MigrationSet (..), MigrationSetName (..))++selftestSchema :: MigrationSetName+selftestSchema = MigrationSetName "kenshou-selftest"++selftestMigrationSet :: MigrationSet+selftestMigrationSet =+  MigrationSet+    { name = selftestSchema,+      requires = [],+      component = either (Left . Text.pack . show) Right do+        roundtrip <- sqlMigration "create-roundtrip" "create schema kenshou_selftest;\ncreate table kenshou_selftest.roundtrip (id bigint primary key, payload text not null);\n"+        migrationComponent "kenshou-selftest" Set.empty (roundtrip :| [])+    }
+ src/Kenshou/Core/Version.hs view
@@ -0,0 +1,9 @@+module Kenshou.Core.Version (suiteVersion) where++import Data.Text (Text)+import Data.Text qualified as Text+import Data.Version (showVersion)+import Paths_kenshou_core (version)++suiteVersion :: Text+suiteVersion = Text.pack (showVersion version)
+ src/Kenshou/Perf/Artifacts.hs view
@@ -0,0 +1,64 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Artifacts (verifyPerfManifest, sealPerfDirectory, safePath) where++import Control.Exception (IOException, displayException, try)+import Control.Monad (unless)+import Data.Aeson (eitherDecodeFileStrict', encodeFile)+import Data.ByteString qualified as ByteString+import Data.List (nub, sort)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (newRunId)+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..), writeManifest)+import System.Directory (doesDirectoryExist, listDirectory, pathIsSymbolicLink)+import System.FilePath (isAbsolute, makeRelative, splitDirectories, (</>))++-- Performance documents are replayed after retrieval. Reject links, escaping+-- paths, omitted files and duplicate entries before opening any manifest path.+verifyPerfManifest :: FilePath -> IO (Either Text ())+verifyPerfManifest root = ioResult do+  decoded <- eitherDecodeFileStrict' (root </> "manifest.json")+  case decoded of+    Left problem -> pure (Left (Text.pack problem))+    Right manifest -> do+      paths <- regularFiles root+      let listed = map (.path) (manifest :: Manifest).files+      if any (not . safePath) listed || nub listed /= listed || sort listed /= filter (/= "manifest.json") paths+        then pure (Left "performance manifest file inventory is unsafe or incomplete")+        else do+          problems <- traverse verifyFile manifest.files+          pure (sequence_ problems)+  where+    verifyFile file = do+      bytes <- ByteString.readFile (root </> file.path)+      pure do+        unless (sha256Hex bytes == file.sha256) (Left (Text.pack file.path <> ": digest mismatch"))+        unless (fromIntegral (ByteString.length bytes) == file.bytes) (Left (Text.pack file.path <> ": byte count mismatch"))++sealPerfDirectory :: FilePath -> IO (Either Text ())+sealPerfDirectory root = ioResult do+  _ <- regularFiles root+  identifier <- newRunId+  manifest <- writeManifest root identifier Map.empty+  encodeFile (root </> "manifest.json") manifest+  pure (Right ())++regularFiles :: FilePath -> IO [FilePath]+regularFiles root = sort <$> visit root+  where+    visit path = do+      link <- pathIsSymbolicLink path+      if link then ioError (userError "performance evidence may not contain symbolic links") else pure ()+      directory <- doesDirectoryExist path+      if directory then listDirectory path >>= fmap concat . traverse (visit . (path </>)) else pure [makeRelative root path]++safePath :: FilePath -> Bool+safePath path = not (null path) && not (isAbsolute path) && all (`notElem` ["", ".", ".."]) (splitDirectories path) && '\\' `notElem` path++ioResult :: IO (Either Text a) -> IO (Either Text a)+ioResult action = do+  result <- try @IOException action+  pure (either (Left . Text.pack . displayException) id result)
+ src/Kenshou/Perf/Calibration.hs view
@@ -0,0 +1,185 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Calibration+  ( CalibrationReference (..),+    CalibrationDefinition (..),+    CalibrationSlot (..),+    CalibrationAcceptance (..),+    CalibrationReceipt (..),+    VerifiedCalibrationTrial (..),+    validateCalibration,+    validateCalibrationDefinition,+    toolchainDigest,+  )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List (nub, sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Id (RunId)+import Kenshou.Perf.Prepared (BuildInventory (..))+import Kenshou.Perf.Request (canonicalProject, immutableRevision, strictObject)++data CalibrationReference = CalibrationReference {definitionUri :: Text, definitionSha256 :: Text, acceptanceUri :: Text, acceptanceSha256 :: Text}+  deriving stock (Eq, Show)++data CalibrationDefinition = CalibrationDefinition+  { batchId :: RunId,+    project :: Text,+    profile :: Text,+    machineProfile :: Text,+    healthPolicySha256 :: Text,+    workloadSha256 :: Text,+    toolchainSha256 :: Text,+    harnessSemanticsSha256 :: Text,+    baselineRevision :: Text,+    slowdownRevision :: Text,+    injectionMechanism :: Text,+    targetMetric :: Text,+    targetRelativeEffect :: Double,+    targetAbsoluteEffect :: Double,+    seeds :: [Integer],+    infrastructureRetryCap :: Int,+    expiresAt :: UTCTime+  }+  deriving stock (Eq, Show)++data CalibrationReceipt = CalibrationReceipt {uri :: Text, sha256 :: Text}+  deriving stock (Eq, Show)++data CalibrationSlot = CalibrationSlot {seed :: Integer, receipts :: [CalibrationReceipt]}+  deriving stock (Eq, Show)++data CalibrationAcceptance = CalibrationAcceptance {batchId :: RunId, aa :: [CalibrationSlot], slowdown :: [CalibrationSlot]}+  deriving stock (Eq, Show)++-- These observations are constructed only after the ordinary full report replay.+-- They are not accepted as a calibration input document or catalog assertion.+data VerifiedCalibrationTrial = VerifiedCalibrationTrial+  { sessionId :: RunId,+    seed :: Integer,+    control :: Text,+    project :: Text,+    baselineRevision :: Text,+    candidateRevision :: Text,+    profile :: Text,+    trialMachineProfile :: Text,+    trialHealthPolicySha256 :: Text,+    workloadSha256 :: Text,+    trialToolchainSha256 :: Text,+    trialHarnessSemanticsSha256 :: Text,+    verdict :: Text,+    infrastructureState :: Text,+    runtimeBlockers :: [Text],+    createdAt :: UTCTime,+    elapsedSeconds :: Double+  }+  deriving stock (Eq, Show)++-- Commit identity belongs to preparation, not this toolchain identity: attaching+-- a calibration link in a new catalog commit must not invalidate itself.+toolchainDigest :: BuildInventory -> Text+toolchainDigest inventory = sha256Hex (canonicalEncode (object ["compiler" .= inventory.compiler, "platform" .= inventory.platform, "toolchain" .= stable]))+  where+    stable = case inventory.toolchain of Object value -> Object (KeyMap.delete "harnessRevision" value); value -> value++validateCalibration :: UTCTime -> UTCTime -> CalibrationDefinition -> CalibrationAcceptance -> [[VerifiedCalibrationTrial]] -> [[VerifiedCalibrationTrial]] -> Either Text Double+validateCalibration now declaredAt definition acceptance aaTrials slowdownTrials = do+  validateCalibrationDefinition now declaredAt definition+  require (acceptance.batchId == definition.batchId) "calibration batch identity differs"+  mapM_ (\slots -> require (map (.seed) slots == definition.seeds && all (not . null . (.receipts)) slots) "acceptance omitted or reordered a declared slot") [acceptance.aa, acceptance.slowdown]+  let references = concatMap (.receipts) (acceptance.aa <> acceptance.slowdown)+      trials = concat (aaTrials <> slowdownTrials)+  require (nub (map (.uri) references) == map (.uri) references && nub (map (.sessionId) trials) == map (.sessionId) trials) "duplicate calibration evidence or session"+  aaAccepted <- check "aa" acceptance.aa aaTrials+  slowdownAccepted <- check "none" acceptance.slowdown slowdownTrials+  require (length (filter ((== "failed") . (.infrastructureState)) trials) <= definition.infrastructureRetryCap) "calibration infrastructure retry cap exceeded"+  require (counts "pass" aaAccepted >= 19 && counts "regression" aaAccepted == 0 && counts "inconclusive" aaAccepted <= 1) "A/A statistical acceptance failed"+  require (counts "regression" slowdownAccepted >= 19 && counts "pass" slowdownAccepted == 0 && counts "inconclusive" slowdownAccepted <= 1) "slowdown sensitivity acceptance failed"+  let elapsed = sort (map (sum . map (.elapsedSeconds)) aaTrials)+      p95 = elapsed !! 18 -- nearest rank ceil(0.95 * 20), including every retained retry+  require (p95 <= 900) "A/A warm turnaround p95 exceeds fifteen minutes"+  pure p95+  where+    require condition message = unless condition (Left message)+    counts outcome = length . filter ((== outcome) . (.verdict))+    check control slots groups = do+      require (length groups == 20 && length groups == length slots) "missing calibration trial group"+      traverse (checkSlot control) (zip slots groups)+    checkSlot control (slot, trials) = do+      require (not (null trials) && length trials == length slot.receipts) "calibration receipt/trial count differs"+      mapM_ (checkTrial slot.seed control) trials+      require (all ((== "failed") . (.infrastructureState)) (init trials) && (.infrastructureState) (last trials) == "healthy") "statistical outcomes were discarded or slot never completed"+      pure (last trials)+    checkTrial seed control trial = do+      require (trial.createdAt >= declaredAt && trial.createdAt <= definition.expiresAt) "trial is outside the frozen batch lifetime"+      require ((trial.seed, trial.control, trial.project, trial.profile, trial.trialMachineProfile, trial.trialHealthPolicySha256, trial.workloadSha256, trial.trialToolchainSha256, trial.trialHarnessSemanticsSha256) == (seed, control, definition.project, definition.profile, definition.machineProfile, definition.healthPolicySha256, definition.workloadSha256, definition.toolchainSha256, definition.harnessSemanticsSha256)) "calibration workload, seed, toolchain or harness drift"+      require (trial.baselineRevision == definition.baselineRevision && trial.candidateRevision == if control == "aa" then definition.baselineRevision else definition.slowdownRevision) "calibration source revision differs"+      require (null trial.runtimeBlockers) "runtime blocker invalidates calibration"+      require (trial.infrastructureState `elem` ["healthy", "failed"] && trial.verdict `elem` ["pass", "regression", "inconclusive", "not-run"]) "unsupported calibration outcome"+      require (trial.infrastructureState /= "healthy" || trial.verdict /= "not-run") "unexecuted slot is not a statistical outcome"+      require (not (isNaN trial.elapsedSeconds || isInfinite trial.elapsedSeconds) && trial.elapsedSeconds >= 0) "invalid retained calibration timing"++finitePositive :: Double -> Bool+finitePositive value = not (isNaN value || isInfinite value) && value > 0++validateCalibrationDefinition :: UTCTime -> UTCTime -> CalibrationDefinition -> Either Text ()+validateCalibrationDefinition now declaredAt definition = do+  require (canonicalProject definition.project && all immutableRevision [definition.baselineRevision, definition.slowdownRevision] && definition.baselineRevision /= definition.slowdownRevision) "invalid calibration source identity"+  require (length definition.seeds == 20 && nub definition.seeds == definition.seeds && all (\seed -> seed >= 0 && seed <= 9007199254740991) definition.seeds) "calibration needs twenty unique declared seeds"+  require (definition.infrastructureRetryCap >= 0 && definition.infrastructureRetryCap <= 20) "invalid calibration infrastructure retry cap"+  require (declaredAt <= now && declaredAt < definition.expiresAt && now <= definition.expiresAt) "calibration expired or predates its declaration"+  require (not (Text.null definition.injectionMechanism) && not (Text.null definition.targetMetric) && finitePositive definition.targetRelativeEffect && finitePositive definition.targetAbsoluteEffect) "missing or invalid injected effect"+  require (not (Text.null definition.profile) && not (Text.null definition.machineProfile) && not ("pending" `Text.isInfixOf` definition.machineProfile) && all digest [definition.healthPolicySha256, definition.workloadSha256, definition.toolchainSha256, definition.harnessSemanticsSha256]) "calibration needs immutable workload, toolchain and harness digests"+  where+    require condition message = unless condition (Left message)+    digest value = Text.length value == 71 && "sha256:" `Text.isPrefixOf` value && Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) (Text.drop 7 value)++instance ToJSON CalibrationReference where+  toJSON value = object ["definitionUri" .= value.definitionUri, "definitionSha256" .= value.definitionSha256, "acceptanceUri" .= value.acceptanceUri, "acceptanceSha256" .= value.acceptanceSha256]++instance FromJSON CalibrationReference where+  parseJSON = withObject "CalibrationReference" \v -> do+    strictObject ["definitionUri", "definitionSha256", "acceptanceUri", "acceptanceSha256"] v+    CalibrationReference <$> v .: "definitionUri" <*> v .: "definitionSha256" <*> v .: "acceptanceUri" <*> v .: "acceptanceSha256"++instance ToJSON CalibrationDefinition where+  toJSON value = object ["schema" .= ("kenshou.perf-calibration-definition/v1" :: Text), "batchId" .= value.batchId, "project" .= value.project, "profile" .= value.profile, "machineProfile" .= value.machineProfile, "healthPolicySha256" .= value.healthPolicySha256, "workloadSha256" .= value.workloadSha256, "toolchainSha256" .= value.toolchainSha256, "harnessSemanticsSha256" .= value.harnessSemanticsSha256, "baselineRevision" .= value.baselineRevision, "slowdownRevision" .= value.slowdownRevision, "injectionMechanism" .= value.injectionMechanism, "targetMetric" .= value.targetMetric, "targetRelativeEffect" .= value.targetRelativeEffect, "targetAbsoluteEffect" .= value.targetAbsoluteEffect, "seeds" .= value.seeds, "infrastructureRetryCap" .= value.infrastructureRetryCap, "expiresAt" .= value.expiresAt]++instance FromJSON CalibrationDefinition where+  parseJSON = withObject "CalibrationDefinition" \v -> do+    strictObject ["schema", "batchId", "project", "profile", "machineProfile", "healthPolicySha256", "workloadSha256", "toolchainSha256", "harnessSemanticsSha256", "baselineRevision", "slowdownRevision", "injectionMechanism", "targetMetric", "targetRelativeEffect", "targetAbsoluteEffect", "seeds", "infrastructureRetryCap", "expiresAt"] v+    schema <- v .: "schema"+    unless (schema == ("kenshou.perf-calibration-definition/v1" :: Text)) (fail "unsupported calibration definition")+    CalibrationDefinition <$> v .: "batchId" <*> v .: "project" <*> v .: "profile" <*> v .: "machineProfile" <*> v .: "healthPolicySha256" <*> v .: "workloadSha256" <*> v .: "toolchainSha256" <*> v .: "harnessSemanticsSha256" <*> v .: "baselineRevision" <*> v .: "slowdownRevision" <*> v .: "injectionMechanism" <*> v .: "targetMetric" <*> v .: "targetRelativeEffect" <*> v .: "targetAbsoluteEffect" <*> v .: "seeds" <*> v .: "infrastructureRetryCap" <*> v .: "expiresAt"++instance ToJSON CalibrationReceipt where+  toJSON value = object ["uri" .= value.uri, "sha256" .= value.sha256]++instance FromJSON CalibrationReceipt where+  parseJSON = withObject "CalibrationReceipt" \v -> do+    strictObject ["uri", "sha256"] v+    CalibrationReceipt <$> v .: "uri" <*> v .: "sha256"++instance ToJSON CalibrationSlot where+  toJSON value = object ["seed" .= value.seed, "receipts" .= value.receipts]++instance FromJSON CalibrationSlot where+  parseJSON = withObject "CalibrationSlot" \v -> do+    strictObject ["seed", "receipts"] v+    CalibrationSlot <$> v .: "seed" <*> v .: "receipts"++instance ToJSON CalibrationAcceptance where+  toJSON value = object ["schema" .= ("kenshou.perf-calibration-acceptance/v1" :: Text), "batchId" .= value.batchId, "aa" .= value.aa, "slowdown" .= value.slowdown]++instance FromJSON CalibrationAcceptance where+  parseJSON = withObject "CalibrationAcceptance" \v -> do+    strictObject ["schema", "batchId", "aa", "slowdown"] v+    schema <- v .: "schema"+    unless (schema == ("kenshou.perf-calibration-acceptance/v1" :: Text)) (fail "unsupported calibration acceptance")+    CalibrationAcceptance <$> v .: "batchId" <*> v .: "aa" <*> v .: "slowdown"
+ src/Kenshou/Perf/Contracts.hs view
@@ -0,0 +1,40 @@+-- | What a suite supplies to performance planning and preparation. The kernel's+-- validators are generic; which scenario revisions honour count-bound steady work,+-- which knob carries that count, and how an isolated cohort lane is built are+-- facts about the suite's scenarios and packages, so the executable passes them in+-- through 'Kenshou.Core.Cli.Program'.+module Kenshou.Perf.Contracts+  ( PerfContracts (..),+    noPerfContracts,+  )+where++import Data.Text (Text)+import Kenshou.Core.Cohort (CohortDescriptor)+import Kenshou.Core.Knob (KnobName)++data PerfContracts = PerfContracts+  { -- | Benchmark scenario identifiers and revisions whose phase contract has been+    -- reviewed: they implement count-bound steady work rather than timed phases.+    -- A profile naming any other scenario revision is rejected.+    reviewedPhaseContracts :: [(Text, Int)],+    -- | The knob whose effective value must equal a profile's steady count, when+    -- the suite has one.+    steadyCountKnob :: Maybe KnobName,+    -- | Extra Cabal project stanzas for a cohort lane, such as package flags that+    -- leave out parts of the suite the lane cannot build.+    laneStanzas :: CohortDescriptor -> [Text],+    -- | The harness packages, by directory name, that a cohort lane builds.+    lanePackages :: CohortDescriptor -> [String] -> [String]+  }++-- | No reviewed contracts: every performance profile is rejected, and cohort+-- lanes build every harness package without extra stanzas.+noPerfContracts :: PerfContracts+noPerfContracts =+  PerfContracts+    { reviewedPhaseContracts = [],+      steadyCountKnob = Nothing,+      laneStanzas = const [],+      lanePackages = const id+    }
+ src/Kenshou/Perf/Plan.hs view
@@ -0,0 +1,345 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Plan+  ( RequiredMetric (..),+    PerfProfile (..),+    ProfileCatalog (..),+    PlannedProfile (..),+    PerfPlan (..),+    validateProfile,+    workloadDigest,+    planPerformance,+    conservativeChanges,+  )+where++import Control.Monad (unless)+import Data.Aeson+import Data.List (nub, sort, sortOn)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), renderDimensions, resolveDimensions)+import Kenshou.Core.Id (Kind (..), ScenarioId (..), renderScenarioId)+import Kenshou.Core.Knob+import Kenshou.Perf.Contracts (PerfContracts (..))+import Kenshou.Perf.Request+import Kenshou.Plan.Catalog qualified as Catalog+import Kenshou.Plan.Change+import Kenshou.Plan.Components++data RequiredMetric = RequiredMetric {name :: Text, unit :: Text, minimumSteadySamples :: Int}+  deriving stock (Eq, Show)++data PerfProfile = PerfProfile+  { name :: Text,+    revision :: Int,+    scenario :: Text,+    scenarioRevision :: Int,+    knobs :: Map Text Value,+    dimensions :: Map Text Text,+    workloadPurpose :: Text,+    warmupSeconds :: Int,+    steadyCount :: Int,+    cooldownSeconds :: Int,+    timeoutSeconds :: Int,+    cachePolicy :: Text,+    policy :: FilePath,+    requiredMetrics :: [RequiredMetric],+    capabilities :: [Text],+    machineProfile :: Text,+    trialSeconds :: Int,+    resetSeconds :: Int,+    sealSeconds :: Int,+    minimumPairs :: Int,+    replacementPairs :: Int,+    calibration :: Maybe Value+  }+  deriving stock (Eq, Show)++newtype ProfileCatalog = ProfileCatalog {profiles :: [PerfProfile]}+  deriving stock (Eq, Show)++data PlannedProfile = PlannedProfile+  { profile :: PerfProfile,+    workloadSha256 :: Text,+    effectiveKnobs :: ResolvedKnobs,+    reasons :: [Value],+    estimateSeconds :: Integer+  }+  deriving stock (Eq, Show)++data PerfPlan = PerfPlan+  { request :: PerfRequest,+    requestSha256 :: Text,+    graphSha256 :: Text,+    catalogSha256 :: Text,+    profileCatalogSha256 :: Text,+    selected :: [PlannedProfile],+    excluded :: [(Text, Text)],+    affectedScenarios :: [Text],+    uncoveredScenarios :: [Text],+    unmappedPaths :: [FilePath],+    documentationProof :: [FilePath],+    coverage :: Text,+    changeCoverage :: Text,+    estimateSeconds :: Integer,+    gatingEligible :: Bool+  }+  deriving stock (Eq, Show)++-- Calibration references are deliberately excluded from the workload identity.+-- Attaching evidence to an unchanged workload must not create a digest cycle.+workloadDigest :: PerfProfile -> Text -> Text+workloadDigest profile policyDigest = sha256Hex (canonicalEncode (object ["workload" .= profile {calibration = Nothing}, "policySha256" .= policyDigest]))++validateProfile :: PerfContracts -> [Catalog.ScenarioInfo] -> PerfProfile -> Either Text ResolvedKnobs+validateProfile contracts catalog profile = do+  scenario <- case filter ((== profile.scenario) . renderScenarioId . (.id)) catalog of+    [value] -> Right value+    _ -> Left ("profile names an unknown or duplicate scenario: " <> profile.scenario)+  require (scenario.id.kind == Benchmark && scenario.revision == profile.scenarioRevision) "profile scenario revision/kind mismatch"+  require (profile.revision > 0 && not (Text.null profile.name) && Text.all (`elem` (['a' .. 'z'] <> ['0' .. '9'] <> ['-'])) profile.name) "invalid profile identity"+  require (profile.workloadPurpose == "fixed-load-latency") "unsupported workload purpose; paced throughput is not capacity"+  -- Only reviewed revisions implement count-bound steady work, not timed phases.+  require ((profile.scenario, profile.scenarioRevision) `elem` contracts.reviewedPhaseContracts) "scenario phase contract has not been reviewed"+  require (profile.warmupSeconds == 0 && profile.cooldownSeconds == 0 && profile.timeoutSeconds == 180) "scenario revision does not honor requested phases/timeout"+  require (profile.cachePolicy == "cold-reset") "unsupported cache policy"+  require (profile.minimumPairs >= 5 && profile.replacementPairs >= 0 && profile.replacementPairs <= 2) "invalid fixed pair/replacement allowance"+  require (profile.trialSeconds > 0 && profile.resetSeconds >= 0 && profile.sealSeconds >= 0) "cost estimate needs positive trial time and nonnegative overheads"+  require (not (null profile.requiredMetrics) && nub (map (.name) profile.requiredMetrics) == map (.name) profile.requiredMetrics) "required metrics must be nonempty and unique"+  require (all (\metric -> not (Text.null metric.name) && not (Text.null metric.unit) && metric.minimumSteadySamples > 0) profile.requiredMetrics) "invalid required metric"+  require (all ((<= profile.steadyCount) . (.minimumSteadySamples)) profile.requiredMetrics) "required sample count exceeds planned work"+  require (not (Text.null profile.machineProfile) && not (null profile.capabilities)) "profile needs a machine profile and cell capabilities"+  assignments <- traverse (\(key, value) -> (,RawJson value) <$> mkKnobName key) (Map.toAscList profile.knobs)+  resolved <- either (Left . Text.pack . show) Right (resolveKnobs scenario.knobs assignments)+  dimensionValues <- either (Left . Text.pack . show) Right (resolveDimensions scenario.dimensions (Map.toAscList profile.dimensions))+  require (dimensionValues.pgDurability /= Just PgFsyncOff) "performance profiles require durable PostgreSQL"+  rawKnob <- mkKnobName "measure.raw-samples"+  case contracts.steadyCountKnob of+    Just countKnob -> require (Map.lookup countKnob (resolvedKnobsMap resolved) == Just (VInt (fromIntegral profile.steadyCount))) "profile steady count disagrees with effective workload"+    Nothing -> pure ()+  require (Map.lookup rawKnob (resolvedKnobsMap resolved) == Just (VText "full")) "performance profiles require full raw samples"+  rateKnob <- mkKnobName "bench.rate"+  let offeredRate = case Map.lookup rateKnob (resolvedKnobsMap resolved) of Just (VInt rate) -> rate; _ -> 0+  require (offeredRate > 0 && fromIntegral profile.trialSeconds * offeredRate >= fromIntegral profile.steadyCount) "trial estimate is shorter than the declared offered workload"+  pure resolved+  where+    require condition message = unless condition (Left (profile.name <> ": " <> message))++-- Paths absent from the graph are not a documentation proof. Only explicit+-- documentation names outside all source roots can be excluded safely.+conservativeChanges :: ComponentGraph -> Text -> [FilePath] -> [Change] -> ([Change], [FilePath], [FilePath])+conservativeChanges graph repository paths mapped = (changes, unknown, documentation)+  where+    components = filter ((== repository) . (.repository)) graph.components+    inRoot path = any (\component -> any (\root -> Text.pack root `Text.isPrefixOf` Text.pack path) component.sourceRoots) components+    isDocumentation path = not (inRoot path) && ("docs/" `Text.isPrefixOf` Text.pack path || path `elem` ["README.md", "CHANGELOG.md", "LICENSE", "LICENSE.md"])+    documentation = filter isDocumentation paths+    unknown = filter (\path -> not (inRoot path) && not (isDocumentation path)) paths+    changes =+      if null components || not (null unknown)+        then mapped <> [Change (ComponentRef (ComponentId "all") Nothing) Everything "unmapped upstream input; conservative selection"]+        else mapped++-- The admission map is populated only by an evidence verifier, never by a+-- catalog's self-asserted calibrated flag. It maps workload digest to verified+-- calibration evidence digest. A catalog link cannot populate this map without+-- full receipt retrieval, raw report replay and acceptance-batch verification.+planPerformance :: PerfContracts -> PerfRequest -> ComponentGraph -> [Catalog.ScenarioInfo] -> ProfileCatalog -> [Change] -> [FilePath] -> [FilePath] -> Map Text Text -> Map Text Text -> Either Text PerfPlan+planPerformance contracts request graph catalog profiles changes unknown documentation policyDigests admitted = do+  validateRequest request+  unless (nub profileNames == profileNames) (Left "duplicate performance profile")+  unless (all (`elem` profileNames) request.profiles) (Left "unknown requested performance profile")+  checked <- traverse check (sortOn (.name) profiles.profiles)+  let affected = selectScenarios graph catalog changes+      affectedNames = sort (map (renderScenarioId . (.id) . (.scenario)) affected)+      candidates = filter (wanted affectedNames) checked+      (selected, excluded, _) = foldl admit ([], [], 0) candidates+      covered = map (.scenario) (map (.profile) selected)+      telemetryMissing = [renderScenarioId item.scenario.id | item <- affected, item.minPolicy == Just "telemetry-corners", not (any (coversTelemetry item) selected)]+      uncovered = nub (filter (`notElem` covered) affectedNames <> telemetryMissing)+      notApplicable = request.control == "none" && null changes && null unknown && not (null documentation) && null request.profiles+      scopeComplete = if request.coverageMode == "selected-profiles" then sort (map (.name) (map (.profile) selected)) == sort request.profiles else null uncovered && not (null selected)+      scope = if notApplicable then "not-applicable" else if scopeComplete && null excluded then "complete" else "incomplete"+  pure+    PerfPlan+      { request,+        requestSha256 = requestDigest request,+        graphSha256 = graphDigest graph,+        catalogSha256 = sha256Hex (canonicalEncode (toJSON catalog)),+        profileCatalogSha256 = sha256Hex (canonicalEncode (toJSON profiles)),+        selected,+        excluded,+        affectedScenarios = affectedNames,+        uncoveredScenarios = uncovered,+        unmappedPaths = unknown,+        documentationProof = documentation,+        coverage = scope,+        changeCoverage = if notApplicable then "not-applicable" else if null uncovered && not (null affectedNames) then "complete" else "incomplete",+        estimateSeconds = sum (map (.estimateSeconds) selected),+        gatingEligible = request.purpose == "gating" && scope == "complete" && all (\item -> Map.member item.workloadSha256 admitted) selected+      }+  where+    profileNames = map (.name) profiles.profiles+    check profile = do+      effective <- validateProfile contracts catalog profile+      scenario <- case filter ((== profile.scenario) . renderScenarioId . (.id)) catalog of+        [value] -> Right value+        _ -> Left "profile scenario is missing"+      effectiveDimensions <- either (Left . Text.pack . show) Right (resolveDimensions scenario.dimensions (Map.toAscList profile.dimensions))+      let normalized =+            (profile :: PerfProfile)+              { knobs = Map.fromList [(renderKnobName name, toJSON value) | (name, value) <- Map.toAscList (resolvedKnobsMap effective)],+                dimensions = Map.fromList (renderDimensions effectiveDimensions)+              }+      policyDigest <- maybe (Left ("missing policy digest: " <> Text.pack profile.policy)) Right (Map.lookup (Text.pack profile.policy) policyDigests)+      pure (normalized, (effective, workloadDigest normalized policyDigest))+    wanted affected (profile, _) = (null request.profiles || profile.name `elem` request.profiles) && (request.control == "aa" || request.coverageMode == "selected-profiles" || profile.scenario `elem` affected)+    admit (selected, excluded, spent) (profile, (effective, digest))+      | request.purpose /= "calibration" && Map.notMember digest admitted = (selected, excluded <> [(profile.name, "uncalibrated")], spent)+      | spent + cost > fromIntegral request.budgetMinutes * 60 = (selected, excluded <> [(profile.name, "over-budget")], spent)+      | otherwise = (selected <> [PlannedProfile profile digest effective reasons cost], excluded, spent + cost)+      where+        cost = 2 * (fromIntegral profile.minimumPairs + fromIntegral profile.replacementPairs) * (fromIntegral profile.trialSeconds + fromIntegral profile.resetSeconds + fromIntegral profile.sealSeconds)+        reasons =+          [toJSON reason | item <- selectScenarios graph catalog changes, renderScenarioId item.scenario.id == profile.scenario, reason <- NonEmpty.toList item.reasons]+            <> [ object+                   [ "component" .= ("request/profile/" <> profile.name),+                     "source" .= ("named" :: Text),+                     "via" .= [if request.control == "aa" then "aa-control" else "selected-profiles" :: Text],+                     "selector" .= profile.scenario,+                     "distance" .= (0 :: Int)+                   ]+               | profile.name `elem` request.profiles+               ]+    coversTelemetry item selected =+      selected.profile.scenario == renderScenarioId item.scenario.id+        && Map.lookup "telemetry.tracing" selected.profile.dimensions `elem` [Just "sdk-inmemory", Just "sdk-otlp"]+        && Map.lookup "telemetry.metrics" selected.profile.dimensions `elem` [Just "collect", Just "serve", Just "serve-scraped"]++instance ToJSON RequiredMetric where+  toJSON metric = object ["name" .= metric.name, "unit" .= metric.unit, "minimumSteadySamples" .= metric.minimumSteadySamples]++instance FromJSON RequiredMetric where+  parseJSON = withObject "RequiredMetric" \value -> do+    strictObject ["name", "unit", "minimumSteadySamples"] value+    RequiredMetric <$> value .: "name" <*> value .: "unit" <*> value .: "minimumSteadySamples"++instance ToJSON PerfProfile where+  toJSON profile =+    object+      [ "name" .= profile.name,+        "revision" .= profile.revision,+        "scenario" .= profile.scenario,+        "scenarioRevision" .= profile.scenarioRevision,+        "knobs" .= profile.knobs,+        "dimensions" .= profile.dimensions,+        "workloadPurpose" .= profile.workloadPurpose,+        "warmupSeconds" .= profile.warmupSeconds,+        "steadyCount" .= profile.steadyCount,+        "cooldownSeconds" .= profile.cooldownSeconds,+        "timeoutSeconds" .= profile.timeoutSeconds,+        "cachePolicy" .= profile.cachePolicy,+        "policy" .= profile.policy,+        "requiredMetrics" .= profile.requiredMetrics,+        "capabilities" .= profile.capabilities,+        "machineProfile" .= profile.machineProfile,+        "trialSeconds" .= profile.trialSeconds,+        "resetSeconds" .= profile.resetSeconds,+        "sealSeconds" .= profile.sealSeconds,+        "minimumPairs" .= profile.minimumPairs,+        "replacementPairs" .= profile.replacementPairs,+        "calibration" .= profile.calibration+      ]++instance FromJSON PerfProfile where+  parseJSON = withObject "PerfProfile" \value -> do+    strictObject ["name", "revision", "scenario", "scenarioRevision", "knobs", "dimensions", "workloadPurpose", "warmupSeconds", "steadyCount", "cooldownSeconds", "timeoutSeconds", "cachePolicy", "policy", "requiredMetrics", "capabilities", "machineProfile", "trialSeconds", "resetSeconds", "sealSeconds", "minimumPairs", "replacementPairs", "calibration"] value+    PerfProfile+      <$> value .: "name"+      <*> value .: "revision"+      <*> value .: "scenario"+      <*> value .: "scenarioRevision"+      <*> value .: "knobs"+      <*> value .: "dimensions"+      <*> value .: "workloadPurpose"+      <*> value .: "warmupSeconds"+      <*> value .: "steadyCount"+      <*> value .: "cooldownSeconds"+      <*> value .: "timeoutSeconds"+      <*> value .: "cachePolicy"+      <*> value .: "policy"+      <*> value .: "requiredMetrics"+      <*> value .: "capabilities"+      <*> value .: "machineProfile"+      <*> value .: "trialSeconds"+      <*> value .: "resetSeconds"+      <*> value .: "sealSeconds"+      <*> value .: "minimumPairs"+      <*> value .: "replacementPairs"+      <*> value .:? "calibration"++instance ToJSON ProfileCatalog where+  toJSON catalog = object ["schema" .= ("kenshou.perf-catalog/v1" :: Text), "profiles" .= catalog.profiles]++instance FromJSON ProfileCatalog where+  parseJSON = withObject "ProfileCatalog" \value -> do+    strictObject ["schema", "profiles"] value+    schema <- value .: "schema"+    unless (schema == ("kenshou.perf-catalog/v1" :: Text)) (fail "unsupported performance catalog schema")+    ProfileCatalog <$> value .: "profiles"++instance ToJSON PlannedProfile where+  toJSON planned = object ["profile" .= planned.profile, "workloadSha256" .= planned.workloadSha256, "effectiveKnobs" .= planned.effectiveKnobs, "reasons" .= planned.reasons, "estimateSeconds" .= planned.estimateSeconds]++instance ToJSON PerfPlan where+  toJSON plan =+    object+      [ "schema" .= ("kenshou.perf-plan/v1" :: Text),+        "request" .= plan.request,+        "requestSha256" .= plan.requestSha256,+        "graphSha256" .= plan.graphSha256,+        "catalogSha256" .= plan.catalogSha256,+        "profileCatalogSha256" .= plan.profileCatalogSha256,+        "selected" .= plan.selected,+        "excluded" .= [object ["profile" .= name, "reason" .= reason] | (name, reason) <- plan.excluded],+        "affectedScenarios" .= plan.affectedScenarios,+        "uncoveredScenarios" .= plan.uncoveredScenarios,+        "unmappedPaths" .= plan.unmappedPaths,+        "documentationProof" .= plan.documentationProof,+        "coverage" .= plan.coverage,+        "changeCoverage" .= plan.changeCoverage,+        "estimateSeconds" .= plan.estimateSeconds,+        "gatingEligible" .= plan.gatingEligible+      ]++instance FromJSON PlannedProfile where+  parseJSON = withObject "PlannedProfile" \value -> do+    strictObject ["profile", "workloadSha256", "effectiveKnobs", "reasons", "estimateSeconds"] value+    planned <- PlannedProfile <$> value .: "profile" <*> value .: "workloadSha256" <*> value .: "effectiveKnobs" <*> value .: "reasons" <*> value .: "estimateSeconds"+    unless (planned.estimateSeconds > 0) (fail "invalid planned configuration estimate")+    pure planned++instance FromJSON PerfPlan where+  parseJSON = withObject "PerfPlan" \value -> do+    strictObject ["schema", "request", "requestSha256", "graphSha256", "catalogSha256", "profileCatalogSha256", "selected", "excluded", "affectedScenarios", "uncoveredScenarios", "unmappedPaths", "documentationProof", "coverage", "changeCoverage", "estimateSeconds", "gatingEligible"] value+    schema <- value .: "schema"+    unless (schema == ("kenshou.perf-plan/v1" :: Text)) (fail "unsupported performance plan schema")+    omissions <-+      value .: "excluded"+        >>= traverse+          ( withObject "ExcludedProfile" \item -> do+              strictObject ["profile", "reason"] item+              (,) <$> item .: "profile" <*> item .: "reason"+          )+    plan <- PerfPlan <$> value .: "request" <*> value .: "requestSha256" <*> value .: "graphSha256" <*> value .: "catalogSha256" <*> value .: "profileCatalogSha256" <*> value .: "selected" <*> pure omissions <*> value .: "affectedScenarios" <*> value .: "uncoveredScenarios" <*> value .: "unmappedPaths" <*> value .: "documentationProof" <*> value .: "coverage" <*> value .: "changeCoverage" <*> value .: "estimateSeconds" <*> value .: "gatingEligible"+    unless (plan.requestSha256 == requestDigest plan.request) (fail "performance plan request digest mismatch")+    unless (plan.estimateSeconds == sum (map (.estimateSeconds) plan.selected) && plan.estimateSeconds <= fromIntegral plan.request.budgetMinutes * 60) (fail "performance plan estimate mismatch or budget exceeded")+    unless (all (`elem` ["complete", "incomplete", "not-applicable"]) [plan.coverage, plan.changeCoverage]) (fail "unknown performance coverage")+    unless (nub (map (.name) (map (.profile) plan.selected)) == map (.name) (map (.profile) plan.selected)) (fail "duplicate selected performance profile")+    unless (not plan.gatingEligible || plan.request.purpose == "gating" && plan.coverage == "complete" && not (null plan.selected)) (fail "invalid gating eligibility")+    pure plan
+ src/Kenshou/Perf/Prepared.hs view
@@ -0,0 +1,352 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Prepared+  ( InventoryPackage (..),+    BuildInventory (..),+    PerfPrepared (..),+    PreparedArm (..),+    cabalInventory,+    compareInventories,+    inventoryDigest,+    preparedDigest,+    buildKeyFor,+    validatePrepared,+    validateArmInventories,+    renderBoundsProject,+  )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser, parseEither)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (nub, sortOn)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Cohort+import Kenshou.Perf.Contracts (PerfContracts (..))+import Kenshou.Perf.Plan (PerfPlan (..), PerfProfile (..), PlannedProfile (..), validateProfile, workloadDigest)+import Kenshou.Perf.Request+import Kenshou.Perf.Source (SourceSnapshot, descriptorForSnapshot, legacyDescriptorForSnapshot, targetPackages)+import Kenshou.Plan.Catalog (ScenarioInfo)+import System.FilePath (makeRelative, normalise, splitDirectories)++data InventoryPackage = InventoryPackage+  {name :: Text, version :: Text, source :: Value, flags :: Map Text Bool, settings :: Value}+  deriving stock (Eq, Show)++data BuildInventory = BuildInventory+  { resolver :: Text,+    compiler :: Text,+    platform :: Text,+    toolchain :: Value,+    packages :: [InventoryPackage]+  }+  deriving stock (Eq, Show)++data PreparedArm = PreparedArm+  { descriptor :: CohortDescriptor,+    sourceLock :: Value,+    boundsInventory :: BuildInventory,+    buildInventory :: BuildInventory,+    payload :: Value,+    buildKey :: Text+  }+  deriving stock (Eq, Show)++data PerfPrepared = PerfPrepared+  { plan :: PerfPlan,+    inputs :: Value,+    harnessRevision :: Text,+    baseline :: PreparedArm,+    candidate :: PreparedArm+  }+  deriving stock (Eq, Show)++inventoryDigest :: BuildInventory -> Text+inventoryDigest = sha256Hex . canonicalEncode . toJSON++preparedDigest :: PerfPrepared -> Text+preparedDigest = sha256Hex . canonicalEncode . toJSON++buildKeyFor :: Text -> CohortDescriptor -> Value -> BuildInventory -> BuildInventory -> Text+buildKeyFor revision descriptor lock inventory bounds =+  sha256Hex+    ( canonicalEncode+        ( object+            ["harness" .= revision, "descriptorSha256" .= sha256Hex (LazyByteString.toStrict (encode descriptor)), "sourceLock" .= lock, "inventory" .= inventory, "boundsInventory" .= bounds]+        )+    )++-- Recheck the relationships inside a sealed preparation before any cell action.+-- The original files and their manifest are verified separately by the loader.+validatePrepared :: PerfContracts -> PerfPrepared -> Either Text ()+validatePrepared contracts prepared = do+  (request, template, before, after, catalog, policies) <-+    parse+      "pinned inputs"+      ( withObject "PinnedInputs" \v -> do+          request <- v .: "request"+          harness <- v .: "harness"+          withObject+            "PinnedHarness"+            ( \h -> do+                revision <- h .: "revision"+                dirty <- h .: "dirty"+                unless (revision == prepared.harnessRevision && not dirty) (fail "pinned harness must be the clean prepared commit")+            )+            harness+          graph <- v .: "graph"+          scenarios <- v .: "scenarioCatalog"+          profiles <- v .: "profileCatalog"+          unless (sha256Hex (canonicalEncode graph) == prepared.plan.graphSha256 && sha256Hex (canonicalEncode scenarios) == prepared.plan.catalogSha256 && sha256Hex (canonicalEncode profiles) == prepared.plan.profileCatalogSha256) (fail "pinned graph/catalog digest mismatch")+          (,,,,,) request <$> v .: "template" <*> v .: "baselineSource" <*> v .: "candidateSource" <*> parseJSON scenarios <*> v .: "policies"+      )+      prepared.inputs+  unless (request == prepared.plan.request) (Left "prepared request differs from the plan")+  targets <- targetPackages request template+  baselineDescriptor <- descriptorForSnapshot request template "baseline" (before :: SourceSnapshot)+  candidateDescriptor <- descriptorForSnapshot request template "candidate" after+  legacyBaseline <- legacyDescriptorForSnapshot request template "baseline" before+  legacyCandidate <- legacyDescriptorForSnapshot request template "candidate" after+  unless ((baselineDescriptor == prepared.baseline.descriptor && candidateDescriptor == prepared.candidate.descriptor) || (legacyBaseline == prepared.baseline.descriptor && legacyCandidate == prepared.candidate.descriptor)) (Left "prepared arm descriptor differs from pinned source manifests")+  compareInventories targets prepared.baseline.boundsInventory prepared.candidate.boundsInventory+  compareInventories targets prepared.baseline.buildInventory prepared.candidate.buildInventory+  mapM_ (validateSelected catalog policies) prepared.plan.selected+  mapM_ validateArm [prepared.baseline, prepared.candidate]+  where+    parse label parser value = either (Left . ((label <> ": ") <>) . Text.pack) Right (parseEither parser value)+    validateSelected :: [ScenarioInfo] -> Map FilePath Value -> PlannedProfile -> Either Text ()+    validateSelected catalog policies selected = do+      effective <- validateProfile contracts catalog selected.profile+      unless (toJSON effective == toJSON selected.effectiveKnobs && toJSON effective == toJSON selected.profile.knobs) (Left "prepared effective workload differs from its profile")+      policy <- maybe (Left "pinned profile policy missing") Right (Map.lookup selected.profile.policy policies)+      unless (workloadDigest selected.profile (sha256Hex (canonicalEncode policy)) == selected.workloadSha256) (Left "prepared workload/policy digest mismatch")+    validateArm arm = do+      unless (arm.buildInventory.resolver == "nix" && arm.buildInventory.platform == "x86_64-linux" && arm.buildInventory.compiler == arm.descriptor.descriptorCompiler && arm.boundsInventory.resolver == "cabal-bounds" && arm.boundsInventory.compiler == arm.descriptor.descriptorCompiler) (Left "prepared inventory compiler/platform/resolver mismatch")+      _ <-+        parse+          "prepared toolchain"+          ( withObject "Toolchain" \t -> do+              revision <- t .: "harnessRevision"+              strict <- t .: "strictBounds"+              nixpkgs <- t .: "nixpkgsRevision"+              haskellNix <- t .: "haskellNixRevision"+              unless (revision == prepared.harnessRevision && strict && immutableRevision nixpkgs && immutableRevision haskellNix) (fail "unverified or mutable prepared toolchain")+          )+          arm.buildInventory.toolchain+      expected <- parse "pinned build toolchain" (withObject "PinnedInputs" (.:? "buildToolchain")) prepared.inputs+      case expected of+        Nothing -> unless (not prepared.plan.gatingEligible) (Left "gating preparation requires a pinned build toolchain")+        Just value -> case arm.buildInventory.toolchain of+          Object actual -> unless (Object (KeyMap.delete "harnessRevision" actual) == value) (Left "actual build toolchain differs from frozen planning inputs")+          _ -> Left "actual build toolchain is not an object"+      let descriptorDigest = sha256Hex (LazyByteString.toStrict (encode arm.descriptor))+      _ <-+        parse+          "source lock"+          ( withObject "SourceLock" \v -> do+              schema <- v .: "schema"+              name <- v .: "cohort"+              digest <- v .: "descriptorSha256"+              unless (schema == ("kenshou.cohort-nix-lock/v1" :: Text) && name == arm.descriptor.descriptorName && digest == Text.drop 7 descriptorDigest) (fail "source lock does not bind its descriptor")+          )+          arm.sourceLock+      unless (arm.buildKey == buildKeyFor prepared.harnessRevision arm.descriptor arm.sourceLock arm.buildInventory arm.boundsInventory) (Left "prepared build key mismatch")+      validateArmInventories arm.descriptor arm.buildInventory arm.boundsInventory+      _ <-+        parse+          "payload identity"+          ( withObject "Payload" \v -> do+              identity <- v .: "cohortIdentity"+              harness <- v .: "harness"+              withObject+                "PayloadHarness"+                ( \h -> do+                    revision <- h .: "revision"+                    dirty <- h .: "dirty"+                    unless (revision == prepared.harnessRevision && not dirty) (fail "payload harness mismatch")+                )+                harness+              unless (null (checkCohort arm.descriptor identity) && identity.identityDescriptorSha256 == Text.drop 7 descriptorDigest && identity.identityResolver == Just "nix") (fail "payload cohort identity mismatch")+          )+          arm.payload+      pure ()++-- Run this before publication, and repeat it during offline preparation loading.+validateArmInventories :: CohortDescriptor -> BuildInventory -> BuildInventory -> Either Text ()+validateArmInventories descriptor nix bounds = do+  compareInventories [] nix nix+  compareInventories [] bounds bounds+  mapM_ strictPackage nix.packages+  let nixPackages = Map.fromList [(entry.name, entry) | entry <- nix.packages]+      cabalPackages = Map.fromList [(entry.name, entry) | entry <- bounds.packages]+  mapM_+    ( \pin -> do+        actual <- maybe (Left "cohort pin missing from Nix inventory") Right (Map.lookup pin.pinName nixPackages)+        bound <- maybe (Left "cohort pin missing from bounds inventory") Right (Map.lookup pin.pinName cabalPackages)+        unless (actual.version == pin.pinVersion && bound.version == pin.pinVersion) (Left "resolved inventory version differs from cohort pin")+    )+    (concatMap (.componentPackages) descriptor.descriptorComponents)+  mapM_+    ( \entry -> case Map.lookup entry.name cabalPackages of+        Nothing -> Right ()+        Just bound -> unless (entry.version == bound.version && entry.flags == bound.flags) (Left ("Nix package version or flags differs from bounds plan: " <> entry.name))+    )+    nix.packages+  where+    strictPackage entry =+      either+        (Left . Text.pack)+        Right+        ( parseEither+            ( withObject "PackageSettings" \s -> do+                jailbreak <- s .: "jailbreak"+                flags <- s .: "configureFlags" :: Parser [Text]+                let allowed = map ("--allow-newer=" <>) descriptor.descriptorAllowNewer+                unless (not jailbreak && all (\flag -> not ("--allow-newer" `Text.isPrefixOf` flag) || flag `elem` allowed) flags) (fail "package has unreviewed bound lifting")+            )+            entry.settings+        )++-- Cabal's plan source and revision fields are documented by the upstream+-- ProjectPlanOutput implementation. Include description revisions and all+-- resolved flags, and identify local harness packages by the clean commit.+-- Build-product IDs are excluded: relinking a dependent is expected.+cabalInventory :: FilePath -> Text -> Value -> Either Text BuildInventory+cabalInventory harnessRoot harnessRevision document = do+  unless (immutableRevision harnessRevision) (Left "bounds inventory requires a clean harness commit")+  either (Left . Text.pack) Right (parseEither parsePlan document)+  where+    parsePlan = withObject "Cabal plan" \root -> do+      compiler <- root .: "compiler-id"+      cabal <- root .: "cabal-version" :: Parser Text+      os <- root .: "os" :: Parser Text+      arch <- root .: "arch" :: Parser Text+      units <- root .: "install-plan" >>= traverse parseUnit+      let grouped = Map.fromListWith (<>) [(unit.name, [unit]) | unit <- units]+      unless (all (\values -> nub values == take 1 values) (Map.elems grouped)) (fail "ambiguous package source/version/flags in Cabal inventory")+      pure (BuildInventory "cabal-bounds" compiler (arch <> "-" <> os) (object ["cabal" .= cabal, "harnessRevision" .= harnessRevision]) (sortOn (.name) (nub units)))+    parseUnit = withObject "Cabal unit" \unit -> do+      name <- unit .: "pkg-name"+      version <- unit .: "pkg-version"+      kind <- unit .: "type" :: Parser Text+      flags <- unit .:? "flags" .!= Map.empty+      source <-+        if kind == "pre-existing"+          then pure (object ["type" .= ("boot" :: Text)])+          else do+            descriptionHash <- unit .:? "pkg-cabal-sha256" :: Parser (Maybe Text)+            sourceHash <- unit .:? "pkg-src-sha256" :: Parser (Maybe Text)+            revision <- unit .:? "pkg-revision" :: Parser (Maybe Int)+            location <- unit .: "pkg-src" >>= normalizeSource name+            pure (object ["location" .= location, "sourceSha256" .= sourceHash, "descriptionSha256" .= descriptionHash, "descriptionRevision" .= revision])+      pure (InventoryPackage name version source flags (object []))+    normalizeSource name = withObject "Cabal source" \source -> do+      kind <- source .: "type" :: Parser Text+      case kind of+        "local" -> do+          path <- source .: "path"+          let relative = makeRelative (normalise harnessRoot) (normalise path)+          unless ("kenshou-" `Text.isPrefixOf` name && not (null relative) && all (/= "..") (splitDirectories relative) && relative /= normalise path) (fail "Cabal inventory contains a local source outside the pinned harness")+          pure (object ["type" .= ("harness" :: Text), "revision" .= harnessRevision, "path" .= relative])+        "source-repo" -> do+          repo <- source .: "source-repo"+          withObject+            "Cabal Git source"+            ( \fields -> do+                location <- fields .: "location" :: Parser Text+                revision <- fields .: "tag"+                branch <- fields .:? "branch" :: Parser (Maybe Text)+                unless ("https://" `Text.isPrefixOf` location && immutableRevision revision && branch == Nothing) (fail "Cabal inventory contains a mutable Git source")+                pure (Object source)+            )+            repo+        "repo-tar" -> pure (Object source)+        _ -> fail "Cabal inventory contains an unsupported source"++compareInventories :: [Text] -> BuildInventory -> BuildInventory -> Either Text ()+compareInventories targets baseline candidate = do+  unless (not (null baseline.packages) && not (null candidate.packages)) (Left "empty resolved inventory")+  unless ((baseline.resolver, baseline.compiler, baseline.platform, baseline.toolchain) == (candidate.resolver, candidate.compiler, candidate.platform, candidate.toolchain)) (Left "compiler, platform or toolchain drift")+  let before = Map.fromList [(entry.name, entry) | entry <- baseline.packages]+      after = Map.fromList [(entry.name, entry) | entry <- candidate.packages]+  unless (Map.size before == length baseline.packages && Map.size after == length candidate.packages && Map.keys before == Map.keys after) (Left "resolved package inventory changed or contains duplicate names")+  unless (all (`Map.member` before) targets) (Left "target package missing from resolved inventory")+  mapM_ (check after) (Map.toAscList before)+  where+    check after (name, previous) = case Map.lookup name after of+      Nothing -> Left ("missing resolved package: " <> name)+      Just current+        | previous.flags /= current.flags || previous.settings /= current.settings -> Left ("resolved flags/settings changed: " <> name)+        | name `elem` targets -> Right ()+        | previous /= current -> Left ("unrelated resolved dependency changed: " <> name)+        | otherwise -> Right ()++-- Standalone projects never import active.project or project.local. They pin+-- every cohort package and use only already reviewed, package-scoped exceptions.+-- The suite's lane stanzas follow the pins.+renderBoundsProject :: PerfContracts -> [FilePath] -> CohortDescriptor -> Either Text Text+renderBoundsProject contracts harnessPackages descriptor = do+  unless (not (null harnessPackages) && all safeAtom harnessPackages) (Left "unsafe or empty harness package paths")+  unless (all scopedException descriptor.descriptorAllowNewer) (Left "blanket bound lifting is forbidden in performance preparation")+  unless (all (safeAtom . Text.unpack) (descriptor.descriptorCompiler : descriptor.descriptorIndexState : map (.pinName) pins <> map (.pinVersion) pins)) (Left "unsafe cohort project field")+  gitStanzas <- traverse gitComponent descriptor.descriptorComponents+  pure+    ( Text.unlines+        (["packages: " <> Text.intercalate ", " (map Text.pack harnessPackages), "with-compiler: " <> descriptor.descriptorCompiler, "index-state: " <> descriptor.descriptorIndexState, "tests: False", "benchmarks: False", "extra-packages: " <> Text.intercalate ", " (map (.pinName) pins), "constraints: " <> Text.intercalate ", " [pin.pinName <> " ==" <> pin.pinVersion | pin <- pins], "allow-newer: " <> (if null descriptor.descriptorAllowNewer then "none" else Text.intercalate ", " descriptor.descriptorAllowNewer)] <> laneFlags <> concat gitStanzas)+    )+  where+    laneFlags = contracts.laneStanzas descriptor+    pins = concatMap (.componentPackages) descriptor.descriptorComponents+    safeAtom value = not (null value) && all (`notElem` ['\n', '\r', '\t', ' ', ',', '"']) value+    scopedException value = case Text.splitOn ":" value of+      [owner, dependency] -> all (\part -> not (Text.null part) && Text.all (\c -> c `elem` ['a' .. 'z'] || c `elem` ['0' .. '9'] || c == '-') part) [owner, dependency]+      _ -> False+    gitComponent component = case component.componentSource of+      HackageSource -> Right []+      GitSource location revision -> do+        unless ("https://" `Text.isPrefixOf` location && immutableRevision revision && safeAtom (Text.unpack location)) (Left "source project must use immutable HTTPS Git")+        let dirs = nub [maybe "." id pin.pinSubdir | pin <- component.componentPackages]+        unless (all (\path -> safeAtom (Text.unpack path) && not ("/" `Text.isPrefixOf` path) && all (/= "..") (Text.splitOn "/" path)) dirs) (Left "unsafe Git package subdirectory")+        pure ["", "source-repository-package", "  type: git", "  location: " <> location, "  tag: " <> revision, "  subdir: " <> Text.intercalate " " dirs]++instance ToJSON InventoryPackage where+  toJSON entry = object ["name" .= entry.name, "version" .= entry.version, "source" .= entry.source, "flags" .= entry.flags, "settings" .= entry.settings]++instance FromJSON InventoryPackage where+  parseJSON = withObject "InventoryPackage" \v -> do+    strictObject ["name", "version", "source", "flags", "settings"] v+    InventoryPackage <$> v .: "name" <*> v .: "version" <*> v .: "source" <*> v .: "flags" <*> v .: "settings"++instance ToJSON BuildInventory where+  toJSON inventory = object ["resolver" .= inventory.resolver, "compiler" .= inventory.compiler, "platform" .= inventory.platform, "toolchain" .= inventory.toolchain, "packages" .= sortOn (.name) inventory.packages]++instance FromJSON BuildInventory where+  parseJSON = withObject "BuildInventory" \v -> do+    strictObject ["resolver", "compiler", "platform", "toolchain", "packages"] v+    BuildInventory <$> v .: "resolver" <*> v .: "compiler" <*> v .: "platform" <*> v .: "toolchain" <*> v .: "packages"++instance ToJSON PreparedArm where+  toJSON arm = object ["descriptor" .= arm.descriptor, "sourceLock" .= arm.sourceLock, "boundsInventory" .= arm.boundsInventory, "buildInventory" .= arm.buildInventory, "payload" .= arm.payload, "buildKey" .= arm.buildKey]++instance FromJSON PreparedArm where+  parseJSON = withObject "PreparedArm" \v -> do+    strictObject ["descriptor", "sourceLock", "boundsInventory", "buildInventory", "payload", "buildKey"] v+    PreparedArm <$> v .: "descriptor" <*> v .: "sourceLock" <*> v .: "boundsInventory" <*> v .: "buildInventory" <*> v .: "payload" <*> v .: "buildKey"++instance ToJSON PerfPrepared where+  toJSON prepared = object ["schema" .= ("kenshou.perf-prepared/v1" :: Text), "plan" .= prepared.plan, "inputs" .= prepared.inputs, "harnessRevision" .= prepared.harnessRevision, "baseline" .= prepared.baseline, "candidate" .= prepared.candidate]++instance FromJSON PerfPrepared where+  parseJSON = withObject "PerfPrepared" \v -> do+    strictObject ["schema", "plan", "inputs", "harnessRevision", "baseline", "candidate"] v+    schema <- v .: "schema"+    unless (schema == ("kenshou.perf-prepared/v1" :: Text)) (fail "unsupported performance preparation schema")+    prepared <- PerfPrepared <$> v .: "plan" <*> v .: "inputs" <*> v .: "harnessRevision" <*> v .: "baseline" <*> v .: "candidate"+    unless (immutableRevision prepared.harnessRevision) (fail "invalid prepared harness revision")+    pure prepared
+ src/Kenshou/Perf/Request.hs view
@@ -0,0 +1,126 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Request+  ( PerfRequest (..),+    validateRequest,+    requestDigest,+    immutableRevision,+    canonicalProject,+    strictObject,+  )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser)+import Data.Char (isAsciiLower, isDigit)+import Data.List (nub, sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)++-- Lists are sets on the wire; encoding sorts them to give equivalent requests+-- the same identity. Duplicates are rejected rather than silently discarded.+data PerfRequest = PerfRequest+  { project :: Text,+    baselineRevision :: Text,+    candidateRevision :: Text,+    cohortTemplate :: FilePath,+    targetComponents :: [Text],+    suite :: Text,+    coverageMode :: Text,+    purpose :: Text,+    control :: Text,+    profiles :: [Text],+    seed :: Integer,+    cells :: [Text],+    waitSeconds :: Int,+    budgetMinutes :: Int+  }+  deriving stock (Eq, Show)++immutableRevision :: Text -> Bool+immutableRevision revision = Text.length revision == 40 && Text.all (\c -> c `elem` ['0' .. '9'] || c `elem` ['a' .. 'f']) revision++canonicalProject :: Text -> Bool+canonicalProject uri = case Text.splitOn "/" <$> Text.stripPrefix "mori://" uri of+  Just [namespace, projectName] -> valid namespace && valid projectName+  _ -> False+  where+    valid part = not (Text.null part) && Text.all (\c -> isAsciiLower c || isDigit c || c `elem` ("-_" :: String)) part++validateRequest :: PerfRequest -> Either Text ()+validateRequest request = do+  require (canonicalProject request.project) "project must be a canonical mori://namespace/project URI"+  require (all immutableRevision [request.baselineRevision, request.candidateRevision]) "revisions must be full lowercase 40-character Git commits"+  require (not (null request.cohortTemplate)) "cohortTemplate must be nonempty"+  require (request.suite == "change-performance") "unsupported performance suite"+  require (request.coverageMode `elem` ["affected", "selected-profiles"]) "unknown coverageMode"+  require (request.purpose `elem` ["gating", "calibration"]) "unknown purpose"+  require (request.control `elem` ["none", "aa"]) "unknown control"+  require (request.control /= "aa" || request.baselineRevision == request.candidateRevision) "A/A control requires identical revisions"+  require (request.control /= "aa" || not (null request.profiles)) "A/A control requires explicit profiles"+  require (request.coverageMode /= "selected-profiles" || not (null request.profiles)) "selected-profiles requires explicit profiles"+  require (request.seed >= 0 && request.seed <= 9007199254740991) "seed must be an exactly representable nonnegative JSON integer"+  require (request.waitSeconds >= 0 && request.budgetMinutes > 0) "waitSeconds must be nonnegative and budgetMinutes positive"+  require (not (null request.targetComponents) && not (null request.cells)) "targetComponents and cells must be nonempty"+  mapM_ validateSet [request.targetComponents, request.profiles, request.cells]+  where+    require condition message = unless condition (Left message)+    validateSet values = do+      require (all (\v -> not (Text.null v) && Text.strip v == v) values) "set entries must be nonempty and have no surrounding whitespace"+      require (nub values == values) "duplicate set entry"++requestDigest :: PerfRequest -> Text+requestDigest = sha256Hex . canonicalEncode . toJSON++strictObject :: [Text] -> Object -> Parser ()+strictObject allowed value = unless (null unknown) (fail ("unknown fields: " <> show unknown))+  where+    unknown = filter (`notElem` allowed) (map Key.toText (KeyMap.keys value))++instance ToJSON PerfRequest where+  toJSON request =+    object+      [ "schema" .= ("kenshou.perf-request/v1" :: Text),+        "project" .= request.project,+        "baselineRevision" .= request.baselineRevision,+        "candidateRevision" .= request.candidateRevision,+        "cohortTemplate" .= request.cohortTemplate,+        "targetComponents" .= sort request.targetComponents,+        "suite" .= request.suite,+        "coverageMode" .= request.coverageMode,+        "purpose" .= request.purpose,+        "control" .= request.control,+        "profiles" .= sort request.profiles,+        "seed" .= request.seed,+        "cells" .= sort request.cells,+        "waitSeconds" .= request.waitSeconds,+        "budgetMinutes" .= request.budgetMinutes+      ]++instance FromJSON PerfRequest where+  parseJSON = withObject "PerfRequest" \value -> do+    strictObject ["schema", "project", "baselineRevision", "candidateRevision", "cohortTemplate", "targetComponents", "suite", "coverageMode", "purpose", "control", "profiles", "seed", "cells", "waitSeconds", "budgetMinutes"] value+    schema <- value .: "schema"+    unless (schema == ("kenshou.perf-request/v1" :: Text)) (fail "unsupported performance request schema")+    request <-+      PerfRequest+        <$> value .: "project"+        <*> value .: "baselineRevision"+        <*> value .: "candidateRevision"+        <*> value .: "cohortTemplate"+        <*> value .: "targetComponents"+        <*> value .:? "suite" .!= "change-performance"+        <*> value .:? "coverageMode" .!= "affected"+        <*> value .:? "purpose" .!= "gating"+        <*> value .:? "control" .!= "none"+        <*> value .:? "profiles" .!= []+        <*> value .: "seed"+        <*> value .: "cells"+        <*> value .:? "waitSeconds" .!= 90+        <*> value .:? "budgetMinutes" .!= 30+    either (fail . Text.unpack) pure (validateRequest request)+    pure request {targetComponents = sort request.targetComponents, profiles = sort request.profiles, cells = sort request.cells}
+ src/Kenshou/Perf/Result.hs view
@@ -0,0 +1,192 @@+module Kenshou.Perf.Result+  ( ConfigurationResult (..),+    PerfResult (..),+    ArtifactReference (..),+    aggregateResult,+    validateResult,+    semanticComparison,+  )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.Types (Parser, parseEither)+import Data.List (nub, sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Perf.Plan+import Kenshou.Perf.Request++-- Attempts include unsuccessful infrastructure attempts. Only the complete,+-- healthy comparison represented by comparison supplies validPairs.+data ConfigurationResult = ConfigurationResult+  { profile :: Text,+    workloadSha256 :: Text,+    performanceVerdict :: Text,+    infrastructureState :: Text,+    runtimeFailures :: [Text],+    validPairs :: Int,+    requiredPairs :: Int,+    attempts :: [Value],+    comparison :: Maybe Value+  }+  deriving stock (Eq, Show)++data PerfResult = PerfResult+  { requestSha256 :: Text,+    preparedSha256 :: Text,+    purpose :: Text,+    coverage :: Text,+    changeCoverage :: Text,+    performanceVerdict :: Text,+    infrastructureState :: Text,+    blockingRuntimeFailures :: [Text],+    gatingEligible :: Bool,+    configurations :: [ConfigurationResult],+    exclusions :: [(Text, Text)],+    uncoveredScenarios :: [Text],+    evidenceVerified :: Bool,+    exitCode :: Int,+    stageSeconds :: Map Text Double,+    artifactReferences :: [ArtifactReference]+  }+  deriving stock (Eq, Show)++-- The root manifest and retrieval receipt anchor their own digests externally,+-- avoiding a self-referential report. Other links carry an artifact digest.+data ArtifactReference = ArtifactReference {kind :: Text, uri :: Text, sha256 :: Maybe Text}+  deriving stock (Eq, Show)++instance ToJSON ArtifactReference where+  toJSON reference = object ["kind" .= reference.kind, "uri" .= reference.uri, "sha256" .= reference.sha256]++instance FromJSON ArtifactReference where+  parseJSON = withObject "PerformanceArtifactReference" \v -> do+    strictObject ["kind", "uri", "sha256"] v+    reference <- ArtifactReference <$> v .: "kind" <*> v .: "uri" <*> v .:? "sha256"+    unless (not (Text.null reference.kind) && "gs://" `Text.isPrefixOf` reference.uri && not (Text.any (`elem` ['\n', '\r', ' ', '?', '#']) reference.uri) && maybe True digest reference.sha256) (fail "invalid performance artifact reference")+    pure reference++-- An aggregate's ordinary exit mapping also applies to calibration exercises;+-- their purpose and gatingEligible fields keep them out of production gates.+aggregateResult :: PerfPlan -> Text -> Bool -> [ConfigurationResult] -> Either Text PerfResult+aggregateResult plan preparedDigest verified configurations = do+  mapM_ validateConfiguration configurations+  let expected = sort [entry.profile.name | entry <- plan.selected]+      supplied = map (.profile) configurations+  unless (nub supplied == supplied && all (`elem` expected) supplied) (Left "unexpected or duplicate configuration result")+  mapM_ checkBinding configurations+  let complete = sort supplied == expected && all ((/= "pending") . (.infrastructureState)) configurations+      scope = if plan.coverage == "not-applicable" then "not-applicable" else if plan.coverage == "complete" && complete then "complete" else "incomplete"+      blockers = concatMap (.runtimeFailures) configurations+      infrastructure = if not verified || any ((== "failed") . (.infrastructureState)) configurations then "failed" else if any ((== "pending") . (.infrastructureState)) configurations || not complete then "pending" else "healthy"+      regression = any ((== "regression") . (.performanceVerdict)) configurations+      accepted = all ((== "pass") . (.performanceVerdict)) configurations && (not (null configurations) || scope == "not-applicable")+      verdict = if scope == "not-applicable" then "not-applicable" else if regression then "regression" else if accepted && scope == "complete" && infrastructure == "healthy" then "pass" else "inconclusive"+      code = outcomeExit infrastructure blockers scope verdict+      eligible = plan.gatingEligible && code == 0 && verdict == "pass"+      result = PerfResult plan.requestSha256 preparedDigest plan.request.purpose scope plan.changeCoverage verdict infrastructure blockers eligible configurations plan.excluded plan.uncoveredScenarios verified code Map.empty []+  validateResult result+  pure result+  where+    checkBinding result = case filter ((== result.profile) . (.name) . (.profile)) plan.selected of+      [entry] -> unless (result.workloadSha256 == entry.workloadSha256 && result.requiredPairs == entry.profile.minimumPairs) (Left "configuration workload or fixed pair count differs from pinned plan")+      _ -> Left "configuration has no unique selected profile"++outcomeExit :: Text -> [Text] -> Text -> Text -> Int+outcomeExit infrastructure blockers scope verdict+  | infrastructure == "failed" = 4+  | not (null blockers) || verdict == "regression" = 1+  | infrastructure /= "healthy" || scope == "incomplete" || verdict == "inconclusive" = 3+  | otherwise = 0++validateResult :: PerfResult -> Either Text ()+validateResult result = do+  unless (digest result.requestSha256 && digest result.preparedSha256) (Left "invalid aggregate input digest")+  unless (result.purpose `elem` ["gating", "calibration"] && result.coverage `elem` ["complete", "incomplete", "not-applicable"] && result.changeCoverage `elem` ["complete", "incomplete", "not-applicable"] && result.performanceVerdict `elem` ["pass", "regression", "inconclusive", "not-applicable"]) (Left "invalid aggregate purpose, scope or verdict")+  unless (result.infrastructureState `elem` ["healthy", "pending", "failed"]) (Left "invalid aggregate infrastructure state")+  mapM_ validateConfiguration result.configurations+  unless (all (\seconds -> not (isNaN seconds || isInfinite seconds) && seconds >= 0) (Map.elems result.stageSeconds)) (Left "invalid performance stage timing")+  mapM_ (\reference -> case fromJSON (toJSON reference) :: Result ArtifactReference of Error problem -> Left (Text.pack problem); Success _ -> Right ()) result.artifactReferences+  let names = map (.profile) result.configurations+      blockers = concatMap (.runtimeFailures) result.configurations+      regression = any ((== "regression") . (.performanceVerdict)) result.configurations+  unless (nub names == names && result.blockingRuntimeFailures == blockers) (Left "aggregate duplicates configurations or omits runtime blockers")+  unless (result.exitCode == outcomeExit result.infrastructureState blockers result.coverage result.performanceVerdict) (Left "aggregate exit mapping mismatch")+  unless (result.evidenceVerified || result.infrastructureState == "failed") (Left "unverified publication must be an infrastructure failure")+  unless (not (any ((== "failed") . (.infrastructureState)) result.configurations) || result.infrastructureState == "failed") (Left "aggregate hides unresolved infrastructure failure")+  unless (not regression || result.performanceVerdict == "regression") (Left "aggregate hides a regression")+  unless (not result.gatingEligible || (result.purpose == "gating" && result.exitCode == 0 && result.performanceVerdict == "pass")) (Left "ineligible aggregate claims a production gate")+  unless (result.performanceVerdict /= "pass" || (result.coverage == "complete" && result.infrastructureState == "healthy" && not (null result.configurations) && all ((== "pass") . (.performanceVerdict)) result.configurations)) (Left "aggregate performance pass lacks complete healthy evidence")+  unless (result.performanceVerdict /= "not-applicable" || (result.coverage == "not-applicable" && null result.configurations)) (Left "not-applicable aggregate contains executed configurations")++validateConfiguration :: ConfigurationResult -> Either Text ()+validateConfiguration result = do+  unless (not (Text.null result.profile) && digest result.workloadSha256 && result.requiredPairs >= 5 && result.validPairs >= 0 && result.validPairs <= result.requiredPairs) (Left "invalid configuration identity or pair count")+  unless (result.performanceVerdict `elem` ["pass", "regression", "inconclusive", "not-run"] && result.infrastructureState `elem` ["healthy", "pending", "failed"]) (Left "invalid configuration outcome")+  unless (result.performanceVerdict `notElem` ["pass", "regression"] || (result.validPairs == result.requiredPairs && result.infrastructureState == "healthy")) (Left "statistical verdict lacks complete healthy pairs")+  case result.comparison of+    Nothing -> unless (result.performanceVerdict `elem` ["inconclusive", "not-run"]) (Left "statistical verdict lacks its retained comparison")+    Just document ->+      either+        (Left . Text.pack)+        Right+        ( parseEither+            ( withObject "RetainedComparison" \v -> do+                schema <- v .: "schema"+                verdict <- v .: "verdict"+                count <- v .: "pairCount"+                let sameVerdict = verdict == result.performanceVerdict || (verdict == "infrastructure-failure" && result.infrastructureState == "failed" && result.performanceVerdict == "inconclusive")+                unless (schema == ("kenshou.comparison/v1" :: Text) && sameVerdict && count == result.validPairs) (fail "retained comparison differs from configuration result")+            )+            document+        )++digest :: Text -> Bool+digest value = "sha256:" `Text.isPrefixOf` value && Text.length value == 71 && Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) (Text.drop 7 value)++-- Reanalysis creates a new analysis ID, clock and host fingerprint. Compare the+-- evidence identities and calculation instead; preserve original sealed bytes.+semanticComparison :: Value -> Either Text Value+semanticComparison document = either (Left . Text.pack) Right (parseEither parser document)+  where+    parser = withObject "Comparison" \v -> do+      schema <- v .: "schema"+      unless (schema == ("kenshou.comparison/v1" :: Text)) (fail "unsupported retained comparison")+      fields <-+        traverse+          (\name -> (Key.fromText name,) <$> (v .: Key.fromText name :: Parser Value))+          ["schema", "design", "baselineRuns", "candidateRuns", "algorithm", "policy", "variedFactors", "pairCount", "metrics", "reasons", "verdict", "exitCode"]+      pure (object fields)++instance ToJSON ConfigurationResult where+  toJSON result = object ["profile" .= result.profile, "workloadSha256" .= result.workloadSha256, "performanceVerdict" .= result.performanceVerdict, "infrastructureState" .= result.infrastructureState, "runtimeFailures" .= result.runtimeFailures, "validPairs" .= result.validPairs, "requiredPairs" .= result.requiredPairs, "attempts" .= result.attempts, "comparison" .= result.comparison]++instance FromJSON ConfigurationResult where+  parseJSON = withObject "ConfigurationResult" \v -> do+    strictObject ["profile", "workloadSha256", "performanceVerdict", "infrastructureState", "runtimeFailures", "validPairs", "requiredPairs", "attempts", "comparison"] v+    result <- ConfigurationResult <$> v .: "profile" <*> v .: "workloadSha256" <*> v .: "performanceVerdict" <*> v .: "infrastructureState" <*> v .: "runtimeFailures" <*> v .: "validPairs" <*> v .: "requiredPairs" <*> v .: "attempts" <*> v .:? "comparison"+    either (fail . Text.unpack) pure (validateConfiguration result)+    pure result++instance ToJSON PerfResult where+  toJSON result = object ["schema" .= ("kenshou.perf-result/v1" :: Text), "requestSha256" .= result.requestSha256, "preparedSha256" .= result.preparedSha256, "purpose" .= result.purpose, "coverage" .= result.coverage, "changeCoverage" .= result.changeCoverage, "performanceVerdict" .= result.performanceVerdict, "infrastructureState" .= result.infrastructureState, "blockingRuntimeFailures" .= result.blockingRuntimeFailures, "gatingEligible" .= result.gatingEligible, "configurations" .= result.configurations, "exclusions" .= [object ["profile" .= name, "reason" .= reason] | (name, reason) <- result.exclusions], "uncoveredScenarios" .= result.uncoveredScenarios, "evidenceVerified" .= result.evidenceVerified, "exitCode" .= result.exitCode, "stageSeconds" .= result.stageSeconds, "artifactReferences" .= result.artifactReferences]++instance FromJSON PerfResult where+  parseJSON = withObject "PerfResult" \v -> do+    strictObject ["schema", "requestSha256", "preparedSha256", "purpose", "coverage", "changeCoverage", "performanceVerdict", "infrastructureState", "blockingRuntimeFailures", "gatingEligible", "configurations", "exclusions", "uncoveredScenarios", "evidenceVerified", "exitCode", "stageSeconds", "artifactReferences"] v+    schema <- v .: "schema"+    unless (schema == ("kenshou.perf-result/v1" :: Text)) (fail "unsupported performance result schema")+    exclusions <-+      v .: "exclusions"+        >>= traverse+          ( withObject "Exclusion" \e -> do+              strictObject ["profile", "reason"] e+              (,) <$> e .: "profile" <*> e .: "reason"+          )+    result <- PerfResult <$> v .: "requestSha256" <*> v .: "preparedSha256" <*> v .: "purpose" <*> v .: "coverage" <*> v .: "changeCoverage" <*> v .: "performanceVerdict" <*> v .: "infrastructureState" <*> v .: "blockingRuntimeFailures" <*> v .: "gatingEligible" <*> v .: "configurations" <*> pure exclusions <*> v .: "uncoveredScenarios" <*> v .: "evidenceVerified" <*> v .: "exitCode" <*> (v .:? "stageSeconds" .!= Map.empty) <*> (v .:? "artifactReferences" .!= [])+    either (fail . Text.unpack) pure (validateResult result)+    pure result
+ src/Kenshou/Perf/Source.hs view
@@ -0,0 +1,173 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Source+  ( PackageManifest (..),+    SourceSnapshot (..),+    parsePackageManifest,+    targetPackages,+    sourceSnapshot,+    descriptorForSnapshot,+    legacyDescriptorForSnapshot,+  )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Char (isAsciiLower, isDigit)+import Data.List (nub, sort)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Cohort+import Kenshou.Perf.Request+import System.FilePath (takeDirectory)++data PackageManifest = PackageManifest+  { name :: Text,+    version :: Text,+    path :: FilePath,+    sha256 :: Text+  }+  deriving stock (Eq, Show)++data SourceSnapshot = SourceSnapshot+  { project :: Text,+    location :: Text,+    revision :: Text,+    packages :: [PackageManifest]+  }+  deriving stock (Eq, Show)++-- Source manifests are inspected without executing candidate code. Cabal will+-- perform the full syntax/bounds validation during preparation. Reject an+-- ambiguous identity here rather than guessing which declaration Cabal means.+parsePackageManifest :: FilePath -> Text -> Either Text PackageManifest+parsePackageManifest path contents = do+  name <- field "name"+  version <- field "version"+  unless (validName name && validVersion version) (Left ("invalid manifest identity: " <> Text.pack path))+  unless (safePath path) (Left "unsafe manifest path")+  pure (PackageManifest name version path (sha256Hex (Text.encodeUtf8 contents)))+  where+    field key = case [Text.strip (fst (Text.breakOn "--" rest)) | line <- Text.lines contents, let (label, value) = Text.breakOn ":" line, Text.toLower (Text.strip label) == key, let rest = Text.drop 1 value] of+      [value] | not (Text.null value) -> Right value+      _ -> Left ("missing or ambiguous " <> key <> " in " <> Text.pack path)+    validName value = not (Text.null value) && Text.all (\c -> isAsciiLower c || isDigit c || c == '-') value+    validVersion value = all (\part -> not (Text.null part) && Text.all isDigit part) (Text.splitOn "." value)++targetPackages :: PerfRequest -> CohortDescriptor -> Either Text [Text]+targetPackages request descriptor = do+  validateRequest request+  unless (nub ids == ids && nub names == names) (Left "cohort component/package IDs are not unique")+  unless (all (`elem` ids) request.targetComponents) (Left "unknown target component in cohort template")+  unless (all owns targets) (Left "all target components must belong to the request's source project")+  unless (not (null targets) && all (not . null . (.componentPackages)) targets) (Left "target components must declare packages")+  mapM_ checkSource descriptor.descriptorComponents+  pure (sort (concatMap (map (.pinName) . (.componentPackages)) targets))+  where+    ids = map (unComponentId . (.componentId)) descriptor.descriptorComponents+    names = concatMap (map (.pinName) . (.componentPackages)) descriptor.descriptorComponents+    targets = filter ((`elem` request.targetComponents) . unComponentId . (.componentId)) descriptor.descriptorComponents+    owns component = component.componentMoriUri == request.project || (request.project <> "/packages/") `Text.isPrefixOf` component.componentMoriUri+    checkSource component = case component.componentSource of+      HackageSource -> Right ()+      GitSource location revision -> unless (validLocation location && immutableRevision revision) (Left "template Git sources must be immutable HTTPS sources")++sourceSnapshot :: PerfRequest -> CohortDescriptor -> Text -> Text -> [PackageManifest] -> Either Text SourceSnapshot+sourceSnapshot request descriptor location revision manifests = do+  expected <- targetPackages request descriptor+  unless (validLocation location && immutableRevision revision) (Left "source snapshot requires immutable HTTPS revision")+  unless (revision `elem` [request.baselineRevision, request.candidateRevision]) (Left "source snapshot is not one of the requested revisions")+  unless (all sameRepository targets) (Left "target template Git source belongs to another repository")+  let selected = filter ((`elem` expected) . (.name)) manifests+  unless (sort (map (.name) selected) == expected) (Left "source revision has a missing or duplicate target package")+  mapM_ validateManifest selected+  pure (SourceSnapshot request.project location revision (sortOnName selected))+  where+    targets = filter ((`elem` request.targetComponents) . unComponentId . (.componentId)) descriptor.descriptorComponents+    sameRepository component = case component.componentSource of+      HackageSource -> True+      GitSource upstream _ -> Text.dropWhileEnd (== '/') (maybe upstream id (Text.stripSuffix ".git" upstream)) == Text.dropWhileEnd (== '/') (maybe location id (Text.stripSuffix ".git" location))+    sortOnName values = map snd (Map.toAscList (Map.fromList [(value.name, value) | value <- values]))++descriptorForSnapshot :: PerfRequest -> CohortDescriptor -> Text -> SourceSnapshot -> Either Text CohortDescriptor+descriptorForSnapshot = descriptorWithIdentity False++-- Read compatibility for previously sealed v1 preparations. New preparation+-- always uses the resolved-cohort identity; legacy names remain bound to the+-- exact retained request and independent manifests, never accepted arbitrarily.+legacyDescriptorForSnapshot :: PerfRequest -> CohortDescriptor -> Text -> SourceSnapshot -> Either Text CohortDescriptor+legacyDescriptorForSnapshot = descriptorWithIdentity True++descriptorWithIdentity :: Bool -> PerfRequest -> CohortDescriptor -> Text -> SourceSnapshot -> Either Text CohortDescriptor+descriptorWithIdentity legacy request descriptor arm snapshot = do+  expected <- targetPackages request descriptor+  unless (snapshot.project == request.project && sort (map (.name) snapshot.packages) == expected) (Left "snapshot project/package inventory differs from the request")+  unless (validLocation snapshot.location && immutableRevision snapshot.revision) (Left "snapshot source must be immutable HTTPS Git")+  unless (arm `elem` ["baseline", "candidate"]) (Left "unknown comparison arm")+  unless (snapshot.revision == if arm == "baseline" then request.baselineRevision else request.candidateRevision) (Left "snapshot revision differs from its arm")+  mapM_ validateManifest snapshot.packages+  components <- traverse replace descriptor.descriptorComponents+  let effectiveArm = if request.baselineRevision == request.candidateRevision then "aa" else arm+      resolved =+        descriptor+          { descriptorName = CohortName "perf",+            descriptorDescription = "Exact-revision performance cohort for " <> request.project <> "; template scope: " <> descriptor.descriptorDescription,+            descriptorProjectFile = "perf.project",+            descriptorComponents = components+          }+      generatedName =+        if legacy+          then "perf-" <> Text.take 16 (Text.drop 7 (requestDigest request)) <> "-" <> effectiveArm+          else "perf-" <> Text.drop 7 (sha256Hex (canonicalEncode (toJSON resolved)))+  pure+    resolved+      { descriptorName = CohortName generatedName,+        descriptorDescription = if legacy then "Exact-revision " <> effectiveArm <> " for " <> request.project <> "; template scope: " <> descriptor.descriptorDescription else resolved.descriptorDescription,+        descriptorProjectFile = Text.unpack generatedName <> ".project"+      }+  where+    replace component+      | unComponentId component.componentId `notElem` request.targetComponents = Right component+      | otherwise = do+          pins <- traverse pin component.componentPackages+          pure component {componentSource = GitSource snapshot.location snapshot.revision, componentPackages = pins}+    pin previous = case filter ((== previous.pinName) . (.name)) snapshot.packages of+      [manifest] -> pure previous {pinVersion = manifest.version, pinSubdir = let directory = takeDirectory manifest.path in if directory == "." then Nothing else Just (Text.pack directory)}+      _ -> Left "snapshot package identity mismatch"++instance ToJSON PackageManifest where+  toJSON manifest = object ["name" .= manifest.name, "version" .= manifest.version, "path" .= manifest.path, "sha256" .= manifest.sha256]++instance FromJSON PackageManifest where+  parseJSON = withObject "PackageManifest" \value -> do+    strictObject ["name", "version", "path", "sha256"] value+    manifest <- PackageManifest <$> value .: "name" <*> value .: "version" <*> value .: "path" <*> value .: "sha256"+    either (fail . Text.unpack) pure (validateManifest manifest)+    pure manifest++instance ToJSON SourceSnapshot where+  toJSON snapshot = object ["project" .= snapshot.project, "location" .= snapshot.location, "revision" .= snapshot.revision, "packages" .= snapshot.packages]++instance FromJSON SourceSnapshot where+  parseJSON = withObject "SourceSnapshot" \value -> do+    strictObject ["project", "location", "revision", "packages"] value+    snapshot <- SourceSnapshot <$> value .: "project" <*> value .: "location" <*> value .: "revision" <*> value .: "packages"+    unless (canonicalProject snapshot.project && validLocation snapshot.location && immutableRevision snapshot.revision) (fail "invalid immutable source identity")+    unless (not (null snapshot.packages) && nub (map (.name) snapshot.packages) == map (.name) snapshot.packages) (fail "invalid source package inventory")+    pure snapshot++validLocation :: Text -> Bool+validLocation location = "https://" `Text.isPrefixOf` location && Text.length location > 8 && not (Text.any (`elem` ['@', '#', '?', '\n', '\r', ' ']) location)++safePath :: FilePath -> Bool+safePath value = not (null value) && not ("/" `Text.isPrefixOf` Text.pack value) && all (`notElem` ["", ".", ".."]) (Text.splitOn "/" (Text.pack value)) && not (Text.any (`elem` ['\n', '\r', '\t', ' ', '\\']) (Text.pack value))++validateManifest :: PackageManifest -> Either Text ()+validateManifest manifest = do+  unless (not (Text.null manifest.name) && Text.all (\c -> isAsciiLower c || isDigit c || c == '-') manifest.name) (Left "invalid manifest package name")+  unless (all (\part -> not (Text.null part) && Text.all isDigit part) (Text.splitOn "." manifest.version)) (Left "invalid manifest package version")+  unless (safePath manifest.path && ".cabal" `Text.isSuffixOf` Text.pack manifest.path) (Left "unsafe manifest path")+  unless (maybe False (\digest -> Text.length digest == 64 && Text.all (\c -> c `elem` ['0' .. '9'] || c `elem` ['a' .. 'f']) digest) (Text.stripPrefix "sha256:" manifest.sha256)) (Left "invalid manifest digest")
+ src/Kenshou/Plan/Catalog.hs view
@@ -0,0 +1,208 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Catalog+  ( ScenarioInfo (..),+    CatalogDocument (..),+    fromScenario,+    fromBundles,+    readCatalogFile,+    decodeCatalog,+  )+where++import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser)+import Data.Bifunctor (first)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle, allScenarios, mkRegistry)+import Kenshou.Core.Dimension+import Kenshou.Core.Id (ScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Scenario++data ScenarioInfo = ScenarioInfo+  { id :: ScenarioId,+    revision :: Int,+    summary :: Text,+    tier :: Tier,+    placement :: Placement,+    knobs :: [KnobSpec],+    dimensions :: DimensionSupport,+    knownDefect :: Maybe KnownDefect+  }+  deriving stock (Eq, Show)++newtype CatalogDocument = CatalogDocument {scenarios :: [ScenarioInfo]}+  deriving stock (Eq, Show)++fromScenario :: Scenario -> ScenarioInfo+fromScenario scenario =+  ScenarioInfo+    { id = scenario.id,+      revision = scenario.revision,+      summary = scenario.summary,+      tier = scenario.tier,+      placement = scenario.placement,+      knobs = scenario.knobs,+      dimensions = scenario.dimensions,+      knownDefect = scenario.knownDefect+    }++fromBundles :: [LayerBundle] -> [ScenarioInfo]+fromBundles bundles = case mkRegistry bundles of+  Left _ -> []+  Right registry -> fmap fromScenario (allScenarios registry)++readCatalogFile :: FilePath -> IO (Either Text [ScenarioInfo])+readCatalogFile path = decodeCatalog <$> ByteString.readFile path++decodeCatalog :: ByteString -> Either Text [ScenarioInfo]+decodeCatalog bytes = first Text.pack do+  document <- eitherDecodeStrict' bytes :: Either String CatalogDocument+  pure document.scenarios++instance FromJSON CatalogDocument where+  parseJSON = withObject "CatalogDocument" \value -> do+    schema <- value .:? "schema" .!= ("kenshou.scenario-list/v1" :: Text)+    if schema /= "kenshou.scenario-list/v1" then fail "unsupported catalog schema" else pure ()+    CatalogDocument <$> value .: "scenarios"++instance ToJSON CatalogDocument where+  toJSON document = object ["schema" .= ("kenshou.scenario-list/v1" :: Text), "scenarios" .= document.scenarios, "roles" .= ([] :: [Value])]++instance FromJSON ScenarioInfo where+  parseJSON = withObject "ScenarioInfo" \value ->+    ScenarioInfo+      <$> value .: "id"+      <*> value .:? "revision" .!= 1+      <*> value .:? "summary" .!= ""+      <*> (value .:? "tier" .!= "smoke" >>= parseTier)+      <*> (value .:? "placement" .!= "either" >>= parsePlacement)+      <*> value .:? "knobs" .!= []+      <*> value .:? "dimensions" .!= noDimensions+      <*> pure Nothing++instance ToJSON ScenarioInfo where+  toJSON scenario =+    object+      [ "id" .= scenario.id,+        "revision" .= scenario.revision,+        "summary" .= scenario.summary,+        "tier" .= renderTier scenario.tier,+        "placement" .= renderPlacement scenario.placement,+        "knobs" .= scenario.knobs,+        "dimensions" .= scenario.dimensions,+        "knownDefect" .= fmap defectReferences scenario.knownDefect+      ]+    where+      defectReferences defect = case individualKnownDefects defect of+        [single] -> toJSON single.reference+        many -> toJSON (fmap (.reference) many)++parseTier :: Text -> Parser Tier+parseTier = \case+  "smoke" -> pure TierSmoke+  "standard" -> pure TierStandard+  "extended" -> pure TierExtended+  "soak" -> pure TierSoak+  value -> fail ("unknown tier " <> Text.unpack value)++parsePlacement :: Text -> Parser Placement+parsePlacement = \case+  "local" -> pure PlaceLocal+  "cell" -> pure PlaceCell+  "either" -> pure PlaceEither+  value -> fail ("unknown placement " <> Text.unpack value)++instance ToJSON KnobSpec where+  toJSON spec =+    object+      [ "name" .= spec.name,+        "summary" .= spec.summary,+        "type" .= knobTypeText spec.knobType,+        "default" .= spec.def,+        "allowed" .= allowedValue spec.allowed,+        "variants" .= spec.variants+      ]++instance FromJSON KnobSpec where+  parseJSON = withObject "KnobSpec" \value -> do+    name <- value .: "name"+    summary <- value .:? "summary" .!= ""+    knobType <- value .: "type" >>= parseKnobType+    def <- value .: "default"+    allowed <- value .:? "allowed" >>= maybe (pure AnyValue) parseAllowed+    variants <- value .:? "variants" .!= []+    pure KnobSpec {name, summary, knobType, def, allowed, variants}++knobTypeText :: KnobType -> Text+knobTypeText KnobBool = "bool"+knobTypeText KnobInt = "int"+knobTypeText KnobDouble = "double"+knobTypeText KnobText = "text"++parseKnobType :: Text -> Parser KnobType+parseKnobType = \case+  "bool" -> pure KnobBool+  "int" -> pure KnobInt+  "double" -> pure KnobDouble+  "text" -> pure KnobText+  value -> fail ("unknown knob type " <> Text.unpack value)++allowedValue :: Allowed -> Value+allowedValue AnyValue = object ["kind" .= ("any" :: Text)]+allowedValue (OneOf values) = object ["kind" .= ("one-of" :: Text), "values" .= values]+allowedValue (IntRange low high) = object ["kind" .= ("int-range" :: Text), "min" .= low, "max" .= high]+allowedValue (DoubleRange low high) = object ["kind" .= ("double-range" :: Text), "min" .= low, "max" .= high]++parseAllowed :: Value -> Parser Allowed+parseAllowed = withObject "Allowed" \value ->+  value .: "kind" >>= \case+    ("any" :: Text) -> pure AnyValue+    "one-of" -> do+      values <- value .: "values"+      case values of [] -> fail "one-of must not be empty"; first : rest -> pure (OneOf (first :| rest))+    "int-range" -> IntRange <$> value .: "min" <*> value .: "max"+    "double-range" -> DoubleRange <$> value .: "min" <*> value .: "max"+    other -> fail ("unknown allowed kind " <> Text.unpack other)++instance ToJSON DimensionSupport where+  toJSON support =+    object+      [ "telemetry.tracing" .= supported renderTracing support.tracing,+        "telemetry.metrics" .= supported renderMetrics support.metrics,+        "pg.durability" .= supported renderDurability support.pgDurability,+        "pg.version" .= supported renderVersion support.pgVersion+      ]+    where+      supported _ NotApplicable = Null+      supported render (Supported value) = object ["values" .= fmap render value.values, "default" .= render value.def]++instance FromJSON DimensionSupport where+  parseJSON = withObject "DimensionSupport" \value ->+    DimensionSupport+      <$> parseSupported value "telemetry.tracing" parseTracing+      <*> parseSupported value "telemetry.metrics" parseMetrics+      <*> parseSupported value "pg.durability" parseDurability+      <*> parseSupported value "pg.version" parseVersion+    where+      parseSupported objectValue name parse = case KeyMap.lookup (Key.fromText name) objectValue of+        Nothing -> pure NotApplicable+        Just Null -> pure NotApplicable+        Just raw ->+          withObject+            "Support"+            ( \supportValue -> do+                rawValues <- supportValue .: "values"+                rawDefault <- supportValue .: "default"+                values <- traverse (maybe (fail ("unknown value in " <> Text.unpack name)) pure . parse) rawValues+                def <- maybe (fail ("unknown default in " <> Text.unpack name)) pure (parse rawDefault)+                case values of [] -> fail "dimension support must not be empty"; first : rest -> pure (Supported (Support (first :| rest) def))+            )+            raw
+ src/Kenshou/Plan/Change.hs view
@@ -0,0 +1,131 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Change+  ( ChangeSource (..),+    Change (..),+    Warning (..),+    Reason (..),+    Selected (..),+    selectScenarios,+    selectAll,+    selectBySelectors,+    applySelectors,+  )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.List (sortOn)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (mapMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Components+import Kenshou.Plan.Selector++data ChangeSource = Named | CohortDiff | Since | UpstreamDiff | Everything+  deriving stock (Eq, Ord, Show)++data Change = Change+  { ref :: ComponentRef,+    source :: ChangeSource,+    detail :: Text+  }+  deriving stock (Eq, Show)++data Warning = Warning {code :: Text, message :: Text}+  deriving stock (Eq, Show)++data Reason = Reason+  { change :: Change,+    via :: [ComponentRef],+    selector :: Selector,+    distance :: Int+  }+  deriving stock (Eq, Show)++data Selected = Selected+  { scenario :: ScenarioInfo,+    reasons :: NonEmpty Reason,+    minPolicy :: Maybe Text,+    minKnobPolicy :: Maybe Text+  }+  deriving stock (Eq, Show)++selectScenarios :: ComponentGraph -> [ScenarioInfo] -> [Change] -> [Selected]+selectScenarios graph catalog changes+  | any ((== Everything) . (.source)) changes = selectAll catalog+  | otherwise = sortOn (scenarioKey . (.scenario)) (mapMaybe selected catalog)+  where+    affectedByChange changeValue = dependents graph (Set.singleton changeValue.ref)+    closure = [(changeValue, affectedByChange changeValue) | changeValue <- changes]+    selected scenario = case sortOn reasonKey (concatMap (reasonsFor scenario) closure) of+      [] -> Nothing+      first : rest ->+        Just+          Selected+            { scenario,+              reasons = first :| rest,+              minPolicy = maximumPolicy [minPolicyForRef graph changeValue.ref | changeValue <- changes, any ((== changeValue) . (.change)) (first : rest)],+              minKnobPolicy = Nothing+            }+    reasonsFor scenario (changeValue, affected) = do+      affectedValue <- Map.elems affected+      selector <- selectorsForRef graph affectedValue.ref+      if matches selector scenario.id+        then pure (Reason changeValue affectedValue.path selector affectedValue.distance)+        else []+    reasonKey reason = (reason.distance, renderRef reason.change.ref, renderSelector reason.selector)+    scenarioKey = show . (.id)++selectAll :: [ScenarioInfo] -> [Selected]+selectAll catalog = case parseSelector "**" of+  Left _ -> []+  Right selector ->+    [ Selected scenario (Reason change [change.ref] selector 0 :| []) Nothing Nothing+    | scenario <- catalog+    ]+  where+    change = Change (ComponentRef (ComponentId "all") Nothing) Everything "all scenarios"++selectBySelectors :: ChangeSource -> Text -> [Selector] -> [ScenarioInfo] -> [Selected]+selectBySelectors source detail selectors catalog =+  [ Selected scenario (Reason change [change.ref] selector 0 :| []) Nothing Nothing+  | scenario <- catalog,+    selector <- take 1 (filter (`matches` scenario.id) selectors)+  ]+  where+    change = Change (ComponentRef (ComponentId "repository-path") Nothing) source detail++applySelectors :: [Selector] -> [Selector] -> [Selected] -> [Selected]+applySelectors includes excludes = filter keep+  where+    keep selectedValue =+      (null includes || any (`matches` selectedValue.scenario.id) includes)+        && not (any (`matches` selectedValue.scenario.id) excludes)++maximumPolicy :: [Maybe Text] -> Maybe Text+maximumPolicy policies = case mapMaybe (\value -> value) policies of+  [] -> Nothing+  values -> Just (maximumByRank values)+  where+    maximumByRank = foldl1 (\left right -> if rank left >= rank right then left else right)+    rank "default-only" = (0 :: Int)+    rank "telemetry-corners" = 1+    rank "pairwise" = 2+    rank "full" = 3+    rank _ = 0++instance ToJSON Change where+  toJSON change = object ["component" .= renderRef change.ref, "source" .= renderChangeSource change.source, "detail" .= change.detail]++instance ToJSON Reason where+  toJSON reason = object ["component" .= renderRef reason.change.ref, "source" .= renderChangeSource reason.change.source, "via" .= fmap renderRef reason.via, "selector" .= renderSelector reason.selector, "distance" .= reason.distance]++renderChangeSource :: ChangeSource -> Text+renderChangeSource Named = "named"+renderChangeSource CohortDiff = "cohort-diff"+renderChangeSource Since = "since"+renderChangeSource UpstreamDiff = "upstream-diff"+renderChangeSource Everything = "all"
+ src/Kenshou/Plan/Change/Cohort.hs view
@@ -0,0 +1,87 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Change.Cohort+  ( CohortInput (..),+    PackageSource (..),+    parseCohortInput,+    readCohortPackages,+    packagesFromDescriptor,+    diffCohorts,+  )+where++import Data.Aeson qualified as Aeson+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Kenshou.Core.Cohort qualified as Cohort+import Kenshou.Plan.Change+import Kenshou.Plan.Components+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)++data CohortInput = CohortName Text | CohortFile FilePath | CohortGitObject Text+  deriving stock (Eq, Show)++data PackageSource = FromHackage Text | FromGit Text Text+  deriving stock (Eq, Ord, Show)++parseCohortInput :: Text -> CohortInput+parseCohortInput value+  | ":" `Text.isInfixOf` value = CohortGitObject value+  | "/" `Text.isInfixOf` value || ".json" `Text.isSuffixOf` value = CohortFile (Text.unpack value)+  | otherwise = CohortName value++readCohortPackages :: FilePath -> CohortInput -> IO (Either Text (Map Text PackageSource))+readCohortPackages root = \case+  CohortName name -> loadFile (root <> "/cohort/" <> Text.unpack name <> ".json")+  CohortFile path -> loadFile path+  CohortGitObject objectName -> do+    (code, output, err) <- readProcessWithExitCode "git" ["-C", root, "show", Text.unpack objectName] ""+    pure case code of+      ExitSuccess -> decodeBytes (Text.Encoding.encodeUtf8 (Text.pack output))+      _ -> Left (Text.strip (Text.pack err))+  where+    loadFile path = do+      descriptor <- Cohort.loadCohortDescriptor path+      pure (either (Left . Text.pack . show) (Right . packagesFromDescriptor) descriptor)+    decodeBytes bytes = case Aeson.eitherDecodeStrict' bytes of+      Left err -> Left (Text.pack err)+      Right descriptor -> Right (packagesFromDescriptor descriptor)++packagesFromDescriptor :: Cohort.CohortDescriptor -> Map Text PackageSource+packagesFromDescriptor descriptor = Map.fromList do+  component <- descriptor.descriptorComponents+  package <- component.componentPackages+  pure+    ( package.pinName,+      case component.componentSource of+        Cohort.HackageSource -> FromHackage package.pinVersion+        Cohort.GitSource gitLocation gitRevision -> FromGit gitLocation gitRevision+    )++diffCohorts :: ComponentGraph -> Map Text PackageSource -> Map Text PackageSource -> ([Change], [Warning])+diffCohorts graph old new = (changes, warnings)+  where+    packageOwners = Map.fromList [(package, componentValue.id) | componentValue <- graph.components, package <- componentValue.packages]+    names = Map.keysSet old <> Map.keysSet new+    changed = [name | name <- Set.toAscList names, Map.lookup name old /= Map.lookup name new]+    mapped = [(name, owner) | name <- changed, Just owner <- [Map.lookup name packageOwners]]+    changes =+      [ Change (ComponentRef owner Nothing) CohortDiff (name <> " " <> renderSource (Map.lookup name old) <> " -> " <> renderSource (Map.lookup name new))+      | (name, owner) <- mapped+      ]+        <> [Change (ComponentRef (ComponentId "all") Nothing) Everything ("unmapped package " <> name) | name <- changed, Map.notMember name packageOwners, name `notElem` graph.ignoredPackages]+    warnings =+      [ Warning "unmapped-package" ("unmapped package " <> name <> "; selecting everything")+      | name <- changed,+        Map.notMember name packageOwners,+        name `notElem` graph.ignoredPackages+      ]+        <> [Warning "ignored-package" ("ignored package changed: " <> name) | name <- changed, name `elem` graph.ignoredPackages]+    renderSource Nothing = "absent"+    renderSource (Just (FromHackage version)) = version+    renderSource (Just (FromGit _ revision)) = "git " <> revision
+ src/Kenshou/Plan/Change/Git.hs view
@@ -0,0 +1,128 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Change.Git+  ( UpstreamDiff (..),+    parseUpstreamDiff,+    changesSince,+    changesFromUpstream,+  )+where++import Data.Aeson qualified as Aeson+import Data.ByteString qualified as ByteString+import Data.List (isPrefixOf, maximumBy, nub)+import Data.Ord (comparing)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Kenshou.Core.Cohort (CohortDescriptor)+import Kenshou.Plan.Change hiding (UpstreamDiff)+import Kenshou.Plan.Change qualified as Change+import Kenshou.Plan.Change.Cohort (diffCohorts, packagesFromDescriptor)+import Kenshou.Plan.Components+import Kenshou.Plan.Selector (Selector)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Process (readProcessWithExitCode)++data UpstreamDiff = UpstreamDiff+  { repository :: Text,+    path :: FilePath,+    revisionA :: Text,+    revisionB :: Text+  }+  deriving stock (Eq, Show)++parseUpstreamDiff :: Text -> Either Text UpstreamDiff+parseUpstreamDiff raw = do+  let (repository, afterEquals) = Text.breakOn "=" raw+      withoutEquals = Text.drop 1 afterEquals+      (pathAndAt, revisions) = Text.breakOnEnd "@" withoutEquals+      path = Text.dropEnd 1 pathAndAt+      (revisionA, dotsAndB) = Text.breakOn ".." revisions+      revisionB = Text.drop 2 dotsAndB+  if any Text.null [repository, afterEquals, path, revisionA, dotsAndB, revisionB]+    then Left "expected REPO=PATH@REVA..REVB"+    else Right (UpstreamDiff repository (Text.unpack path) revisionA revisionB)++changesSince :: ComponentGraph -> FilePath -> Text -> IO (Either Text ([Change], [Selector], [Warning]))+changesSince graph root revision = do+  tracked <- git root ["diff", "--name-only", Text.unpack revision]+  untracked <- git root ["ls-files", "--others", "--exclude-standard"]+  case (tracked, untracked) of+    (Right trackedPaths, Right untrackedPaths) -> classifyPaths graph root revision (nub (trackedPaths <> untrackedPaths))+    (Left err, _) -> pure (Left err)+    (_, Left err) -> pure (Left err)++classifyPaths :: ComponentGraph -> FilePath -> Text -> [FilePath] -> IO (Either Text ([Change], [Selector], [Warning]))+classifyPaths graph root revision paths = do+  classified <- traverse classify paths+  pure do+    values <- sequence classified+    let changes = concatMap (\(items, _, _) -> items) values+        selectors = concatMap (\(_, items, _) -> items) values+        warnings = concatMap (\(_, _, items) -> items) values+    Right (changes, selectors, warnings)+  where+    classify path = case longestRule graph path of+      Nothing -> pure (Right ([everything ("unmapped repository path " <> Text.pack path)], [], [Warning "unmapped-path" ("unmapped repository path " <> Text.pack path <> "; selecting everything")]))+      Just rule -> case rule.effect of+        PathNothing -> pure (Right ([], [], []))+        PathAll -> pure (Right ([everything (Text.pack path)], [], []))+        PathSelectors selectors -> pure (Right ([], selectors, []))+        PathCohort -> classifyCohort path+    classifyCohort path+      | path == "cohort/active.project" || ".project" `Text.isSuffixOf` Text.pack path =+          pure (Right ([everything (Text.pack path)], [], [Warning "cohort-project-change" ("cohort project changed without a safe descriptor mapping: " <> Text.pack path)]))+      | ".json" `Text.isSuffixOf` Text.pack path = do+          oldBytes <- gitShow root (revision <> ":" <> Text.pack path)+          newBytes <- ByteString.readFile (root </> path)+          pure do+            old <- oldBytes >>= decodeDescriptor+            new <- decodeDescriptor newBytes+            let (changes, warnings) = diffCohorts graph (packagesFromDescriptor old) (packagesFromDescriptor new)+            Right (changes, [], warnings)+      | otherwise = pure (Right ([everything (Text.pack path)], [], [Warning "unknown-cohort-change" ("cannot map cohort change " <> Text.pack path)]))+    everything detail = Change (ComponentRef (ComponentId "all") Nothing) Everything detail++changesFromUpstream :: ComponentGraph -> UpstreamDiff -> IO (Either Text ([Change], Int))+changesFromUpstream graph input = do+  result <- git input.path ["diff", "--name-only", Text.unpack input.revisionA, Text.unpack input.revisionB]+  pure do+    paths <- result+    let mapped = fmap (mapUpstreamPath graph input.repository) paths+        changes = nub [change | Just change <- mapped]+        ignored = length [() | Nothing <- mapped]+    Right (changes, ignored)++mapUpstreamPath :: ComponentGraph -> Text -> FilePath -> Maybe Change+mapUpstreamPath graph repository path = do+  componentValue <- bestComponent+  let subMatches = [(prefix, subcomponentValue.id) | subcomponentValue <- componentValue.subcomponents, prefix <- subcomponentValue.pathPrefixes, prefix `isPrefixOf` path]+      reference = case subMatches of+        [] -> ComponentRef componentValue.id Nothing+        _ -> ComponentRef componentValue.id (Just (snd (maximumBy (comparing (length . fst)) subMatches)))+  pure (Change reference Change.UpstreamDiff (repository <> ":" <> Text.pack path))+  where+    candidates = [componentValue | componentValue <- graph.components, componentValue.repository == repository, any (`isPrefixOf` path) componentValue.sourceRoots]+    bestComponent = case candidates of [] -> Nothing; _ -> Just (maximumBy (comparing (maximum . (0 :) . fmap length . (.sourceRoots))) candidates)++longestRule :: ComponentGraph -> FilePath -> Maybe RepositoryPathRule+longestRule graph path = case filter ((`isPrefixOf` path) . (.prefix)) graph.repositoryPaths of+  [] -> Nothing+  rules -> Just (maximumBy (comparing (length . (.prefix))) rules)++git :: FilePath -> [String] -> IO (Either Text [FilePath])+git root arguments = do+  (code, output, err) <- readProcessWithExitCode "git" (["-C", root] <> arguments) ""+  pure case code of+    ExitSuccess -> Right (filter (not . null) (lines output))+    _ -> Left (Text.strip (Text.pack err))++gitShow :: FilePath -> Text -> IO (Either Text ByteString.ByteString)+gitShow root objectName = do+  (code, output, err) <- readProcessWithExitCode "git" ["-C", root, "show", Text.unpack objectName] ""+  pure case code of ExitSuccess -> Right (Text.Encoding.encodeUtf8 (Text.pack output)); _ -> Left (Text.strip (Text.pack err))++decodeDescriptor :: ByteString.ByteString -> Either Text CohortDescriptor+decodeDescriptor = either (Left . Text.pack) Right . Aeson.eitherDecodeStrict'
+ src/Kenshou/Plan/Components.hs view
@@ -0,0 +1,347 @@+{-# LANGUAGE FieldSelectors #-}+{-# LANGUAGE TemplateHaskell #-}++module Kenshou.Plan.Components+  ( ComponentId (..),+    SubId (..),+    ComponentRef (..),+    EdgeKind (..),+    ComponentKind (..),+    ChangeImplies (..),+    Subcomponent (..),+    DependencyEdge (..),+    Component (..),+    PathEffect (..),+    RepositoryPathRule (..),+    ComponentGraph (..),+    Affected (..),+    GraphProblem (..),+    GraphError (..),+    selftestGraph,+    selftestGraphBytes,+    loadGraphFile,+    decodeGraph,+    validateGraph,+    parseRef,+    renderRef,+    dependents,+    graphDigest,+    componentById,+    selectorsForRef,+    minPolicyForRef,+  )+where++import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.FileEmbed (embedFile)+import Data.Graph (SCC (..), stronglyConnComp)+import Data.List (group, nub, sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Sequence (Seq (..), (|>))+import Data.Sequence qualified as Seq+import Data.Set (Set)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Plan.Selector (Selector, parseSelector, renderSelector)++newtype ComponentId = ComponentId {unComponentId :: Text}+  deriving stock (Eq, Ord, Show)++newtype SubId = SubId {unSubId :: Text}+  deriving stock (Eq, Ord, Show)++data ComponentRef = ComponentRef {component :: ComponentId, sub :: Maybe SubId}+  deriving stock (Eq, Ord, Show)++data EdgeKind = Build | Runtime deriving stock (Eq, Ord, Show)++data ComponentKind = Library | Migrations | Service | Harness | Assembly+  deriving stock (Eq, Ord, Show)++newtype ChangeImplies = ChangeImplies {minDimensionPolicy :: Text}+  deriving stock (Eq, Show)++data Subcomponent = Subcomponent+  { id :: SubId,+    pathPrefixes :: [FilePath],+    selectors :: [Selector],+    uses :: [SubId],+    changeImplies :: Maybe ChangeImplies+  }+  deriving stock (Eq, Show)++data DependencyEdge = DependencyEdge+  { to :: ComponentId,+    kind :: EdgeKind,+    toSubs :: [SubId],+    fromSubs :: [SubId]+  }+  deriving stock (Eq, Show)++data Component = Component+  { id :: ComponentId,+    kind :: ComponentKind,+    uri :: Maybe Text,+    repository :: Text,+    sourceRoots :: [FilePath],+    packages :: [Text],+    selectors :: [Selector],+    changeImplies :: Maybe ChangeImplies,+    verifiedAgainst :: Text,+    subcomponents :: [Subcomponent],+    dependsOn :: [DependencyEdge]+  }+  deriving stock (Eq, Show)++data PathEffect = PathAll | PathNothing | PathCohort | PathSelectors [Selector]+  deriving stock (Eq, Show)++data RepositoryPathRule = RepositoryPathRule {prefix :: FilePath, effect :: PathEffect}+  deriving stock (Eq, Show)++data ComponentGraph = ComponentGraph+  { schema :: Text,+    ignoredPackages :: [Text],+    components :: [Component],+    repositoryPaths :: [RepositoryPathRule]+  }+  deriving stock (Eq, Show)++data Affected = Affected+  { ref :: ComponentRef,+    origin :: ComponentRef,+    path :: [ComponentRef],+    distance :: Int+  }+  deriving stock (Eq, Show)++newtype GraphProblem = GraphProblem {problemText :: Text}+  deriving stock (Eq, Ord, Show)++newtype GraphError = GraphError {errorText :: Text}+  deriving stock (Eq, Show)++selftestGraphBytes :: ByteString+selftestGraphBytes = $(embedFile "data/components.json")++selftestGraph :: Either GraphError ComponentGraph+selftestGraph = decodeGraph selftestGraphBytes++loadGraphFile :: FilePath -> IO (Either GraphError ComponentGraph)+loadGraphFile path = decodeGraph <$> ByteString.readFile path++decodeGraph :: ByteString -> Either GraphError ComponentGraph+decodeGraph bytes = case eitherDecodeStrict' bytes of+  Left err -> Left (GraphError (Text.pack err))+  Right graph -> case validateGraph graph of+    [] -> Right graph+    problems -> Left (GraphError (Text.intercalate "; " (fmap (.problemText) problems)))++validateGraph :: ComponentGraph -> [GraphProblem]+validateGraph graph =+  schemaProblems+    <> duplicateComponentProblems+    <> duplicatePackageProblems+    <> concatMap validateComponent graph.components+    <> cycleProblems+  where+    componentIds = Set.fromList (fmap (.id) graph.components)+    schemaProblems = [GraphProblem ("unsupported component graph schema " <> graph.schema) | graph.schema /= "kenshou.component-graph/v1"]+    duplicateComponentProblems = duplicateProblems "component" (fmap (unComponentId . (.id)) graph.components)+    duplicatePackageProblems = duplicateProblems "package" (concatMap (.packages) graph.components)+    validateComponent componentValue =+      duplicateProblems ("subcomponent of " <> unComponentId componentValue.id) (fmap (unSubId . (.id)) componentValue.subcomponents)+        <> concatMap (validateEdge componentValue) componentValue.dependsOn+        <> concatMap (validateSub componentValue) componentValue.subcomponents+    validateEdge componentValue edge =+      [GraphProblem (unComponentId componentValue.id <> ": unknown dependency " <> unComponentId edge.to) | edge.to `Set.notMember` componentIds]+        <> case componentById graph edge.to of+          Nothing -> []+          Just target ->+            [ GraphProblem (unComponentId componentValue.id <> ": unknown target subcomponent " <> unSubId subId)+            | subId <- edge.toSubs,+              subId `notElem` fmap (.id) target.subcomponents+            ]+        <> [ GraphProblem (unComponentId componentValue.id <> ": unknown source subcomponent " <> unSubId subId)+           | subId <- edge.fromSubs,+             subId `notElem` fmap (.id) componentValue.subcomponents+           ]+    validateSub componentValue subcomponentValue =+      [ GraphProblem (unComponentId componentValue.id <> ": " <> unSubId subcomponentValue.id <> " uses unknown subcomponent " <> unSubId used)+      | used <- subcomponentValue.uses,+        used `notElem` fmap (.id) componentValue.subcomponents+      ]+    cycleProblems =+      [ GraphProblem ("component dependency cycle: " <> Text.intercalate " -> " (sort (fmap (unComponentId . (.id)) members)))+      | CyclicSCC members <- stronglyConnComp [(componentValue, componentValue.id, fmap (.to) componentValue.dependsOn) | componentValue <- graph.components]+      ]++duplicateProblems :: Text -> [Text] -> [GraphProblem]+duplicateProblems label values =+  [GraphProblem ("duplicate " <> label <> " " <> value) | entries@(value : _) <- group (sort values), length entries > 1]++componentById :: ComponentGraph -> ComponentId -> Maybe Component+componentById graph wanted = Map.lookup wanted (Map.fromList [(componentValue.id, componentValue) | componentValue <- graph.components])++parseRef :: ComponentGraph -> Text -> Either Text ComponentRef+parseRef graph raw = case Text.splitOn ":" raw of+  [componentText] -> do+    let componentId = ComponentId componentText+    case componentById graph componentId of+      Nothing -> Left ("unknown component " <> raw)+      Just _ -> Right (ComponentRef componentId Nothing)+  [componentText, subText] -> do+    let componentId = ComponentId componentText+        subId = SubId subText+    componentValue <- maybe (Left ("unknown component " <> componentText)) Right (componentById graph componentId)+    if subId `elem` fmap (.id) componentValue.subcomponents+      then Right (ComponentRef componentId (Just subId))+      else Left ("unknown subcomponent " <> raw)+  _ -> Left ("invalid component reference " <> raw)++renderRef :: ComponentRef -> Text+renderRef reference = unComponentId reference.component <> maybe "" ((":" <>) . unSubId) reference.sub++dependents :: ComponentGraph -> Set ComponentRef -> Map ComponentRef Affected+dependents graph origins = go initialQueue initialMap+  where+    seeded = concatMap expandOrigin (Set.toAscList origins)+    expandOrigin originRef@(ComponentRef componentId Nothing) =+      (originRef, Affected originRef originRef [originRef] 0)+        : case componentById graph componentId of+          Nothing -> []+          Just componentValue ->+            [ let subRef = ComponentRef componentId (Just subcomponentValue.id)+               in (subRef, Affected subRef originRef [originRef, subRef] 0)+            | subcomponentValue <- componentValue.subcomponents+            ]+    expandOrigin originRef = [(originRef, Affected originRef originRef [originRef] 0)]+    initialMap = Map.fromListWith chooseShorter seeded+    initialQueue = Seq.fromList (Map.keys initialMap)+    go Empty results = results+    go (current :<| rest) results =+      let currentAffected = results Map.! current+          candidates = fmap (mkAffected currentAffected) (neighbors current)+          (results', queue') = foldl insertCandidate (results, rest) candidates+       in go queue' results'+    mkAffected previous next =+      Affected next previous.origin (previous.path <> [next]) (previous.distance + if next.component == previous.ref.component then 0 else 1)+    insertCandidate (results, queue) candidate = case Map.lookup candidate.ref results of+      Just existing | existing.distance <= candidate.distance -> (results, queue)+      _ -> (Map.insert candidate.ref candidate results, queue |> candidate.ref)+    chooseShorter left right = if left.distance <= right.distance then left else right+    neighbors target = nub (siblingUsers target <> edgeUsers target)+    siblingUsers (ComponentRef componentId (Just subId)) = case componentById graph componentId of+      Nothing -> []+      Just componentValue -> [ComponentRef componentId (Just sibling.id) | sibling <- componentValue.subcomponents, subId `elem` sibling.uses]+    siblingUsers (ComponentRef componentId Nothing) = case componentById graph componentId of+      Nothing -> []+      Just componentValue+        | null componentValue.selectors -> fmap (ComponentRef componentId . Just . (.id)) componentValue.subcomponents+        | otherwise -> []+    edgeUsers target = concatMap (sourceFor target) graph.components+    sourceFor target source = concatMap (edgeSource target source) source.dependsOn+    edgeSource target source edge+      | edge.to /= target.component = []+      | not (targetMatches target edge) = []+      | null edge.fromSubs = [ComponentRef source.id Nothing]+      | otherwise = fmap (ComponentRef source.id . Just) edge.fromSubs+    targetMatches (ComponentRef _ Nothing) _ = True+    targetMatches (ComponentRef _ (Just subId)) edge = null edge.toSubs || subId `elem` edge.toSubs++selectorsForRef :: ComponentGraph -> ComponentRef -> [Selector]+selectorsForRef graph (ComponentRef componentId Nothing) = maybe [] (.selectors) (componentById graph componentId)+selectorsForRef graph (ComponentRef componentId (Just subId)) = do+  componentValue <- maybe [] pure (componentById graph componentId)+  subcomponentValue <- filter ((== subId) . (.id)) componentValue.subcomponents+  subcomponentValue.selectors++minPolicyForRef :: ComponentGraph -> ComponentRef -> Maybe Text+minPolicyForRef graph (ComponentRef componentId Nothing) = do+  componentValue <- componentById graph componentId+  implies <- componentValue.changeImplies+  pure implies.minDimensionPolicy+minPolicyForRef graph (ComponentRef componentId (Just subId)) = do+  componentValue <- componentById graph componentId+  subcomponentValue <- case filter ((== subId) . (.id)) componentValue.subcomponents of [] -> Nothing; value : _ -> Just value+  implies <- subcomponentValue.changeImplies+  pure implies.minDimensionPolicy++graphDigest :: ComponentGraph -> Text+graphDigest = sha256Hex . canonicalEncode . toJSON++instance FromJSON ComponentGraph where+  parseJSON = withObject "ComponentGraph" \value -> ComponentGraph <$> value .: "schema" <*> value .:? "ignoredPackages" .!= [] <*> value .: "components" <*> value .: "repositoryPaths"++instance ToJSON ComponentGraph where+  toJSON graph = object ["schema" .= graph.schema, "ignoredPackages" .= graph.ignoredPackages, "components" .= graph.components, "repositoryPaths" .= graph.repositoryPaths]++instance FromJSON Component where+  parseJSON = withObject "Component" \value ->+    Component <$> value .: "id" <*> value .: "kind" <*> value .:? "uri" <*> value .: "repository" <*> value .:? "sourceRoots" .!= [] <*> value .:? "packages" .!= [] <*> parseSelectors value <*> value .:? "changeImplies" <*> value .:? "verifiedAgainst" .!= "" <*> value .:? "subcomponents" .!= [] <*> value .:? "dependsOn" .!= []++instance ToJSON Component where+  toJSON componentValue = object ["id" .= componentValue.id, "kind" .= componentValue.kind, "uri" .= componentValue.uri, "repository" .= componentValue.repository, "sourceRoots" .= componentValue.sourceRoots, "packages" .= componentValue.packages, "selectors" .= fmap renderSelector componentValue.selectors, "changeImplies" .= componentValue.changeImplies, "verifiedAgainst" .= componentValue.verifiedAgainst, "subcomponents" .= componentValue.subcomponents, "dependsOn" .= componentValue.dependsOn]++instance FromJSON Subcomponent where+  parseJSON = withObject "Subcomponent" \value -> Subcomponent <$> value .: "id" <*> value .:? "pathPrefixes" .!= [] <*> parseSelectors value <*> value .:? "uses" .!= [] <*> value .:? "changeImplies"++instance ToJSON Subcomponent where+  toJSON subcomponentValue = object ["id" .= subcomponentValue.id, "pathPrefixes" .= subcomponentValue.pathPrefixes, "selectors" .= fmap renderSelector subcomponentValue.selectors, "uses" .= subcomponentValue.uses, "changeImplies" .= subcomponentValue.changeImplies]++parseSelectors :: Object -> Parser [Selector]+parseSelectors value = do+  raw <- value .:? "selectors" .!= []+  traverse (either (fail . Text.unpack) pure . parseSelector) raw++instance FromJSON DependencyEdge where+  parseJSON = withObject "DependencyEdge" \value -> DependencyEdge <$> value .: "to" <*> value .: "kind" <*> value .:? "toSubs" .!= [] <*> value .:? "fromSubs" .!= []++instance ToJSON DependencyEdge where+  toJSON edge = object ["to" .= edge.to, "kind" .= edge.kind, "toSubs" .= edge.toSubs, "fromSubs" .= edge.fromSubs]++instance FromJSON RepositoryPathRule where+  parseJSON = withObject "RepositoryPathRule" \value -> RepositoryPathRule <$> value .: "prefix" <*> value .: "effect"++instance ToJSON RepositoryPathRule where toJSON rule = object ["prefix" .= rule.prefix, "effect" .= rule.effect]++instance FromJSON PathEffect where+  parseJSON = withObject "PathEffect" \value ->+    value .: "kind" >>= \case+      ("all" :: Text) -> pure PathAll+      "nothing" -> pure PathNothing+      "cohort" -> pure PathCohort+      "selectors" -> PathSelectors <$> parseSelectors value+      other -> fail ("unknown path effect " <> Text.unpack other)++instance ToJSON PathEffect where+  toJSON PathAll = object ["kind" .= ("all" :: Text)]+  toJSON PathNothing = object ["kind" .= ("nothing" :: Text)]+  toJSON PathCohort = object ["kind" .= ("cohort" :: Text)]+  toJSON (PathSelectors selectors) = object ["kind" .= ("selectors" :: Text), "selectors" .= fmap renderSelector selectors]++instance FromJSON ChangeImplies where parseJSON = withObject "ChangeImplies" \value -> ChangeImplies <$> value .: "minDimensionPolicy"++instance ToJSON ChangeImplies where toJSON value = object ["minDimensionPolicy" .= value.minDimensionPolicy]++instance FromJSON ComponentId where parseJSON = withText "ComponentId" (pure . ComponentId)++instance ToJSON ComponentId where toJSON = toJSON . unComponentId++instance FromJSON SubId where parseJSON = withText "SubId" (pure . SubId)++instance ToJSON SubId where toJSON = toJSON . unSubId++instance FromJSON EdgeKind where parseJSON = withText "EdgeKind" \case "build" -> pure Build; "runtime" -> pure Runtime; other -> fail ("unknown edge kind " <> Text.unpack other)++instance ToJSON EdgeKind where toJSON Build = String "build"; toJSON Runtime = String "runtime"++instance FromJSON ComponentKind where parseJSON = withText "ComponentKind" \case "library" -> pure Library; "migrations" -> pure Migrations; "service" -> pure Service; "harness" -> pure Harness; "assembly" -> pure Assembly; other -> fail ("unknown component kind " <> Text.unpack other)++instance ToJSON ComponentKind where toJSON Library = String "library"; toJSON Migrations = String "migrations"; toJSON Service = String "service"; toJSON Harness = String "harness"; toJSON Assembly = String "assembly"
+ src/Kenshou/Plan/Components/Check.hs view
@@ -0,0 +1,121 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Components.Check+  ( PackageName (..),+    EdgeDrift (..),+    LintFinding (..),+    checkAgainstPlanJson,+    lintAgainstCatalog,+    isDriftError,+    isLintError,+  )+where++import Data.Aeson+import Data.ByteString qualified as ByteString+import Data.List (nub, sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set (Set)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (ScenarioId (..), renderScenarioId, selftestLayer)+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Components+import Kenshou.Plan.Selector (Selector, matches, renderSelector)++newtype PackageName = PackageName Text+  deriving stock (Eq, Ord, Show)++data EdgeDrift+  = MissingEdge ComponentId ComponentId [PackageName]+  | ExtraEdge ComponentId ComponentId+  | NotInBuildPlan PackageName+  deriving stock (Eq, Ord, Show)++data LintFinding+  = OrphanScenario ScenarioId+  | DeadSelector ComponentRef Selector+  deriving stock (Eq, Show)++data PlanUnit = PlanUnit {unitId :: Text, package :: PackageName, componentName :: Maybe Text, dependencies :: [Text]}++newtype CabalPlan = CabalPlan {units :: [PlanUnit]}++instance FromJSON CabalPlan where parseJSON = withObject "CabalPlan" \value -> CabalPlan <$> value .: "install-plan"++instance FromJSON PlanUnit where+  parseJSON = withObject "PlanUnit" \value ->+    PlanUnit <$> value .: "id" <*> (PackageName <$> value .: "pkg-name") <*> value .:? "component-name" <*> value .:? "depends" .!= []++checkAgainstPlanJson :: ComponentGraph -> FilePath -> IO (Either Text [EdgeDrift])+checkAgainstPlanJson graph path = do+  bytes <- ByteString.readFile path+  pure case eitherDecodeStrict' bytes of+    Left err -> Left (Text.pack err)+    Right plan -> Right (compareEdges graph plan)++compareEdges :: ComponentGraph -> CabalPlan -> [EdgeDrift]+compareEdges graph plan =+  fmap (uncurry missingEdgeWithPackages) missing+    <> fmap (uncurry ExtraEdge) extra+    <> fmap (NotInBuildPlan . PackageName) absentPackages+  where+    libraryUnits = filter ((== Just "lib") . (.componentName)) plan.units+    idToPackage = Map.fromList [(unit.unitId, unit.package) | unit <- plan.units]+    packageToComponent = Map.fromList [(PackageName package, componentValue.id) | componentValue <- graph.components, package <- componentValue.packages]+    presentPackages = Set.fromList (fmap (.package) libraryUnits)+    presentComponents = Set.fromList (Map.elems (Map.restrictKeys packageToComponent presentPackages))+    observed = Set.fromList do+      unit <- libraryUnits+      source <- maybe [] pure (Map.lookup unit.package packageToComponent)+      dependencyId <- unit.dependencies+      dependencyPackage <- maybe [] pure (Map.lookup dependencyId idToPackage)+      target <- maybe [] pure (Map.lookup dependencyPackage packageToComponent)+      if source == target then [] else pure (source, target)+    declared = Set.fromList [(componentValue.id, edge.to) | componentValue <- graph.components, edge <- componentValue.dependsOn, edge.kind == Build, componentValue.id `Set.member` presentComponents, edge.to `Set.member` presentComponents]+    missing = Set.toAscList (observed `Set.difference` declared)+    extra = Set.toAscList (declared `Set.difference` observed)+    absentPackages = sort [package | componentValue <- graph.components, package <- componentValue.packages, PackageName package `Set.notMember` presentPackages]+    directPackages source target =+      nub+        [ dependencyPackage+        | unit <- libraryUnits,+          Map.lookup unit.package packageToComponent == Just source,+          dependencyId <- unit.dependencies,+          Just dependencyPackage <- [Map.lookup dependencyId idToPackage],+          Map.lookup dependencyPackage packageToComponent == Just target+        ]+    missingEdgeWithPackages source target = MissingEdge source target (directPackages source target)++lintAgainstCatalog :: ComponentGraph -> [ScenarioInfo] -> [LintFinding]+lintAgainstCatalog graph catalog = orphaned <> dead+  where+    references =+      [ComponentRef componentValue.id Nothing | componentValue <- graph.components]+        <> [ComponentRef componentValue.id (Just subcomponentValue.id) | componentValue <- graph.components, subcomponentValue <- componentValue.subcomponents]+    selectorPairs = [(reference, selector) | reference <- references, selector <- selectorsForRef graph reference]+    orphaned =+      [ OrphanScenario scenario.id+      | scenario <- catalog,+        scenario.id.layer /= selftestLayer,+        not (any ((`matches` scenario.id) . snd) selectorPairs)+      ]+    dead =+      [ DeadSelector reference selector+      | (reference, selector) <- selectorPairs,+        selectorHasRegisteredLayer selector,+        not (any (matches selector . (.id)) catalog)+      ]+    selectorHasRegisteredLayer selector =+      let prefix = Text.takeWhile (/= '/') (renderSelector selector)+       in prefix == "*" || any ((== prefix) . Text.takeWhile (/= '/') . renderScenarioId . (.id)) catalog++isDriftError :: EdgeDrift -> Bool+isDriftError NotInBuildPlan {} = False+isDriftError _ = True++isLintError :: LintFinding -> Bool+isLintError OrphanScenario {} = True+isLintError DeadSelector {} = False
+ src/Kenshou/Plan/Execute.hs view
@@ -0,0 +1,237 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Execute+  ( ExecuteOptions (..),+    executePlan,+  )+where++import Control.Exception (finally)+import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (traverse_)+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.RunSpec (EnvironmentSpec, RunSpec)+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Plan.Summary+import System.Directory+import System.Environment (getExecutablePath, lookupEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Posix.Process (getProcessID)+import System.Posix.Signals (sigKILL, signalProcess)+import System.Process (createProcess, getPid, proc, readProcessWithExitCode, terminateProcess, waitForProcess)+import System.Timeout (timeout)++data ExecuteOptions = ExecuteOptions+  { planFile :: FilePath,+    outDir :: FilePath,+    resume :: Bool,+    failFast :: Bool,+    environment :: Maybe FilePath,+    timeoutFactor :: Maybe Double+  }+  deriving stock (Eq, Show)++data ExecutablePlan = ExecutablePlan {planId :: RunId, runs :: [ExecutableRun]}+  deriving stock (Eq, Show)++data ExecutableRun = ExecutableRun+  { ordinal :: Int,+    runId :: RunId,+    estimateMinutes :: Int,+    scenario :: ScenarioId,+    spec :: RunSpec+  }+  deriving stock (Eq, Show)++data ResultSummary = ResultSummary {outcome :: Outcome, blocking :: Bool, knownDefect :: Bool}+  deriving stock (Eq, Show)++executePlan :: ExecuteOptions -> IO PlanSummary+executePlan options = do+  planBytes <- ByteString.readFile options.planFile+  plan <- either (ioError . userError . Text.unpack . ("invalid run plan: " <>) . Text.pack) pure (eitherDecodeStrict' planBytes)+  environmentOverride <- traverse loadEnvironment options.environment+  createDirectoryIfMissing True options.outDir+  preparePlanCopy options planBytes+  acquireLock options.outDir+  runWithLock plan planBytes environmentOverride `finally` removeLock options.outDir+  where+    runWithLock plan _planBytes environmentOverride = do+      createDirectoryIfMissing True (options.outDir </> "specs")+      summary <- loadOrCreateSummary options plan+      executeEntries options plan environmentOverride summary++loadEnvironment :: FilePath -> IO EnvironmentSpec+loadEnvironment path = do+  bytes <- ByteString.readFile path+  either (ioError . userError . ("invalid environment document: " <>)) pure (eitherDecodeStrict' bytes)++preparePlanCopy :: ExecuteOptions -> ByteString -> IO ()+preparePlanCopy options bytes = do+  let destination = options.outDir </> "run-plan.json"+  exists <- doesFileExist destination+  if exists+    then do+      existing <- ByteString.readFile destination+      if not options.resume+        then ioError (userError "execution output already contains a plan; use --resume")+        else if sha256Hex existing /= sha256Hex bytes then ioError (userError "resume plan digest does not match run-plan.json") else pure ()+    else ByteString.writeFile destination bytes++acquireLock :: FilePath -> IO ()+acquireLock outDir = do+  let path = outDir </> ".execute.lock"+  exists <- doesFileExist path+  if exists+    then do+      contents <- readFile path+      live <- processAlive contents+      if live then ioError (userError ("executor lock is held by pid " <> contents)) else writePid path+    else writePid path+  where+    writePid path = getProcessID >>= writeFile path . show+    processAlive raw = case reads raw of+      [(pid, _)] -> do+        (code, _, _) <- readProcessWithExitCode "kill" ["-0", show (pid :: Int)] ""+        pure (code == ExitSuccess)+      _ -> pure False++removeLock :: FilePath -> IO ()+removeLock outDir = do+  let path = outDir </> ".execute.lock"+  exists <- doesFileExist path+  if exists then removeFile path else pure ()++loadOrCreateSummary :: ExecuteOptions -> ExecutablePlan -> IO PlanSummary+loadOrCreateSummary options plan = do+  let path = options.outDir </> "plan-summary.json"+  exists <- doesFileExist path+  if exists && options.resume+    then do+      bytes <- ByteString.readFile path+      either (ioError . userError . ("invalid plan summary: " <>)) pure (eitherDecodeStrict' bytes)+    else pure (emptySummary plan.planId [(run.ordinal, run.scenario) | run <- plan.runs])++executeEntries :: ExecuteOptions -> ExecutablePlan -> Maybe EnvironmentSpec -> PlanSummary -> IO PlanSummary+executeEntries options plan environmentOverride = go plan.runs+  where+    summaryPath = options.outDir </> "plan-summary.json"+    go [] summary = writeSummary summaryPath (finalizeSummary summary) >> pure (finalizeSummary summary)+    go (run : rest) summary = case findEntry run.ordinal summary.entries of+      Just entry | entry.status == Completed -> go rest summary+      _ -> do+        let priorAttempts = maybe [] (.attempts) (findEntry run.ordinal summary.entries)+        attemptId <- if null priorAttempts then pure run.runId else newRunId+        startedAt <- getCurrentTime+        let attempt = Attempt attemptId startedAt Nothing Nothing False Nothing+            running = updateEntry run.ordinal (\entry -> entry {status = Running, attempts = entry.attempts <> [attempt]}) summary+        writeSummary summaryPath running+        writeSpec options environmentOverride run+        childCode <- spawnRun options run attemptId+        finishedAt <- getCurrentTime+        result <- readResult options attemptId childCode+        let completedAttempt = attempt {finishedAt = Just finishedAt, outcome = Just result.outcome, knownDefect = result.knownDefect, childExitCode = Just (exitCodeInt childCode)}+            completed = finalizeSummary (updateEntry run.ordinal (\entry -> entry {status = Completed, attempts = priorAttempts <> [completedAttempt]}) running)+        writeSummary summaryPath completed+        if options.failFast && result.blocking && result.outcome == Failed then pure completed else go rest completed++writeSpec :: ExecuteOptions -> Maybe EnvironmentSpec -> ExecutableRun -> IO ()+writeSpec options environmentOverride run = LazyByteString.writeFile path (encode effective)+  where+    path = options.outDir </> "specs" </> pad run.ordinal <> ".json"+    effective = maybe run.spec (replaceEnvironment run.spec) environmentOverride+    pad value = let rendered = show value in replicate (4 - min 4 (length rendered)) '0' <> rendered++replaceEnvironment :: RunSpec -> EnvironmentSpec -> RunSpec+replaceEnvironment spec environment =+  RunSpec.RunSpec+    spec.runId+    spec.scenario+    spec.scenarioRevision+    spec.knobs+    spec.dimensions+    spec.seed+    spec.phases+    spec.timeoutSeconds+    environment+    spec.cohortExpectation+    spec.comparison+    spec.labels++spawnRun :: ExecuteOptions -> ExecutableRun -> RunId -> IO ExitCode+spawnRun options run runId = do+  executable <- getExecutablePath+  cellFingerprint <- lookupEnv "KENSHOU_CELL_FINGERPRINT"+  let specPath = options.outDir </> "specs" </> pad run.ordinal <> ".json"+      arguments = ["run", "--spec", specPath, "--out", options.outDir, "--run-id", Text.unpack (renderRunId runId)] <> maybe [] (\path -> ["--cell-fingerprint", path]) cellFingerprint+  (_, _, _, processHandle) <- createProcess (proc executable arguments)+  case options.timeoutFactor of+    Nothing -> waitForProcess processHandle+    Just factor -> do+      result <- timeout (floor (factor * fromIntegral run.estimateMinutes * 60 * 1000000)) (waitForProcess processHandle)+      case result of+        Just code -> pure code+        Nothing -> do+          terminateProcess processHandle+          stopped <- timeout 30000000 (waitForProcess processHandle)+          case stopped of+            Just _ -> pure (ExitFailure 124)+            Nothing -> do+              traverse_ (signalProcess sigKILL) =<< getPid processHandle+              _ <- waitForProcess processHandle+              pure (ExitFailure 124)+  where+    pad value = let rendered = show value in replicate (4 - min 4 (length rendered)) '0' <> rendered++readResult :: ExecuteOptions -> RunId -> ExitCode -> IO ResultSummary+readResult options runId _childCode = do+  let path = options.outDir </> Text.unpack (renderRunId runId) </> "run-result.json"+  exists <- doesFileExist path+  if not exists+    then pure (ResultSummary Errored True False)+    else do+      bytes <- ByteString.readFile path+      pure (either (const (ResultSummary Errored True False)) (\value -> value) (eitherDecodeStrict' bytes))++findEntry :: Int -> [SummaryEntry] -> Maybe SummaryEntry+findEntry ordinal = foldr (\entry found -> if entry.ordinal == ordinal then Just entry else found) Nothing++updateEntry :: Int -> (SummaryEntry -> SummaryEntry) -> PlanSummary -> PlanSummary+updateEntry ordinal update summary = summary {entries = fmap (\entry -> if entry.ordinal == ordinal then update entry else entry) summary.entries}++exitCodeInt :: ExitCode -> Int+exitCodeInt ExitSuccess = 0+exitCodeInt (ExitFailure value) = value++instance FromJSON ExecutablePlan where+  parseJSON = withObject "ExecutablePlan" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.run-plan/v1" :: Text) then fail "unsupported run-plan schema" else pure ()+    ExecutablePlan <$> value .: "planId" <*> value .: "runs"++instance FromJSON ExecutableRun where+  parseJSON = withObject "ExecutableRun" \value -> do+    ordinal <- value .: "ordinal"+    runId <- value .: "runId"+    estimateMinutes <- value .: "estimateMinutes"+    spec <- value .: "spec"+    let scenario = spec.scenario+    pure ExecutableRun {ordinal, runId, estimateMinutes, scenario, spec}++instance FromJSON ResultSummary where+  parseJSON = withObject "ResultSummary" \value -> do+    outcome <- value .: "outcome"+    blocking <- value .:? "blocking" .!= True+    defect <- value .:? "knownDefect" :: Parser (Maybe Value)+    pure (ResultSummary outcome blocking (isJust defect))
+ src/Kenshou/Plan/Matrix.hs view
@@ -0,0 +1,167 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Matrix+  ( DimensionPolicy (..),+    KnobPolicy (..),+    Config (..),+    MatrixSkip (..),+    expandScenario,+    pairwiseCover,+  )+where++import Data.List (maximumBy, nub, sortOn)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Ord (Down (..), comparing)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Dimension+import Kenshou.Core.Id (Kind (..), ScenarioId (..))+import Kenshou.Core.Knob+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Change (Selected (..))+import Kenshou.Plan.Policy++data Config = Config+  { knobs :: Map KnobName KnobValue,+    dimensions :: Map Text Text,+    isDefault :: Bool+  }+  deriving stock (Eq, Show)++data MatrixSkip = MatrixSkip {reason :: Text, detail :: Text}+  deriving stock (Eq, Show)++expandScenario :: PlanPolicy -> Selected -> ([Config], [MatrixSkip])+expandScenario policy selected = case dimensionRows of+  Left problem -> ([], [problem])+  Right rows -> case knobRows of+    Left problem -> ([], [problem])+    Right (defaultKnobs, variants) ->+      ( [Config defaultKnobs row (index == 0) | (index, row) <- zip [0 :: Int ..] rows]+          <> [Config variant (headOrEmpty rows) False | variant <- variants],+        []+      )+  where+    scenario = selected.scenario+    effectivePolicy = max policy.dimensionPolicy (maybe DefaultOnly parseMinimum selected.minPolicy)+    dimensionRows = expandDimensions effectivePolicy policy.pinnedDimensions scenario+    effectiveKnobPolicy = max policy.knobPolicy (maybe KnobDefaults parseMinimumKnob selected.minKnobPolicy)+    knobRows = expandKnobs (policy {knobPolicy = effectiveKnobPolicy}) scenario+    parseMinimum value = maybe DefaultOnly (\policyValue -> policyValue) (parseDimensionPolicy value)+    parseMinimumKnob value = maybe KnobDefaults (\policyValue -> policyValue) (parseKnobPolicy value)+    headOrEmpty [] = Map.empty+    headOrEmpty (first : _) = first++expandDimensions :: DimensionPolicy -> [(Text, Text)] -> ScenarioInfo -> Either MatrixSkip [Map Text Text]+expandDimensions policy pins scenario+  | scenario.id.kind == Benchmark && not durableSupported = Left (MatrixSkip "benchmark-without-durable" "benchmark does not support pg.durability=durable")+  | otherwise = do+      pinned <- traverse validatePin pins+      let valueSets = fmap (applyPin pinned) supported+          effectiveSets = if scenario.id.kind == Benchmark then fmap forceDurable valueSets else valueSets+          defaults = Map.fromList [(name, def) | (name, _, def) <- effectiveSets]+          rows = case policy of+            DefaultOnly -> [defaults]+            TelemetryCorners -> telemetryCorners effectiveSets defaults+            Pairwise -> pairwiseCover [(name, values) | (name, values, _) <- effectiveSets]+            Full -> productRows [(name, values) | (name, values, _) <- effectiveSets]+      pure rows+  where+    supported = supportRows scenario.dimensions+    durableSupported = case scenario.dimensions.pgDurability of+      Supported support -> PgDurable `elem` support.values+      NotApplicable -> True+    validatePin (name, value) = case lookup name [(rowName, values) | (rowName, values, _) <- supported] of+      Nothing -> Left (MatrixSkip "unsupported-dimension" (name <> " is not applicable"))+      Just values | value `elem` values -> Right (name, value)+      Just _ -> Left (MatrixSkip "unsupported-dimension" (name <> "=" <> value <> " is not supported"))+    applyPin pinnedValues row@(name, _, _) = case lookup name pinnedValues of+      Nothing -> row+      Just value -> (name, [value], value)+    forceDurable ("pg.durability", _, _) = ("pg.durability", ["durable"], "durable")+    forceDurable row = row++supportRows :: DimensionSupport -> [(Text, [Text], Text)]+supportRows support =+  concat+    [ supportRow "telemetry.tracing" renderTracing support.tracing,+      supportRow "telemetry.metrics" renderMetrics support.metrics,+      supportRow "pg.durability" renderDurability support.pgDurability,+      supportRow "pg.version" renderVersion support.pgVersion+    ]+  where+    supportRow _ _ NotApplicable = []+    supportRow name render (Supported value) = [(name, fmap render (NonEmpty.toList value.values), render value.def)]++telemetryCorners :: [(Text, [Text], Text)] -> Map Text Text -> [Map Text Text]+telemetryCorners rows defaults = nub [set tracing metrics defaults | tracing <- tracingValues, metrics <- metricsValues]+  where+    values name fallback = maybe fallback (\(items, def) -> nub [def, preferred name items def]) (lookupRow name)+    lookupRow name = case [(items, def) | (rowName, items, def) <- rows, rowName == name] of [] -> Nothing; first : _ -> Just first+    tracingValues = values "telemetry.tracing" [""]+    metricsValues = values "telemetry.metrics" [""]+    preferred "telemetry.tracing" items def = firstSupported ["sdk-inmemory", "sdk-otlp", "noop"] items def+    preferred "telemetry.metrics" items def = firstSupported ["serve-scraped", "serve", "collect"] items def+    preferred _ _ def = def+    firstSupported preferences items def = case filter (`elem` items) preferences of [] -> def; first : _ -> first+    set tracing metrics = setIfPresent "telemetry.metrics" metrics . setIfPresent "telemetry.tracing" tracing+    setIfPresent _ "" valuesMap = valuesMap+    setIfPresent name value valuesMap = Map.insert name value valuesMap++productRows :: [(Text, [Text])] -> [Map Text Text]+productRows = foldr step [Map.empty]+  where+    step (name, values) rows = [Map.insert name value row | value <- values, row <- rows]++pairwiseCover :: [(Text, [Text])] -> [Map Text Text]+pairwiseCover dimensions+  | length dimensions < 2 = take 1 (productRows dimensions)+  | otherwise = go uncovered [] candidates+  where+    candidates = sortOn renderRow (productRows dimensions)+    uncovered = Set.unions (fmap rowPairs candidates)+    go remaining chosen available+      | Set.null remaining = reverse chosen+      | otherwise = case available of+          [] -> reverse chosen+          _ ->+            let best = maximumBy (comparing (\row -> (Set.size (Set.intersection remaining (rowPairs row)), Down (renderRow row)))) available+             in go (remaining `Set.difference` rowPairs best) (best : chosen) (filter (/= best) available)+    rowPairs row =+      Set.fromList+        [ ((leftName, leftValue), (rightName, rightValue))+        | (leftIndex, (leftName, leftValue)) <- zip [0 :: Int ..] (Map.toAscList row),+          (rightIndex, (rightName, rightValue)) <- zip [0 :: Int ..] (Map.toAscList row),+          leftIndex < rightIndex+        ]+    renderRow = Text.pack . show . Map.toAscList++expandKnobs :: PlanPolicy -> ScenarioInfo -> Either MatrixSkip (Map KnobName KnobValue, [Map KnobName KnobValue])+expandKnobs policy scenario = case resolveKnobs scenario.knobs supplied of+  Left problems -> Left (MatrixSkip "unsupported-knob" (Text.intercalate "; " (fmap renderProblem (NonEmpty.toList problems))))+  Right resolved -> Right (base, variants)+    where+      base = resolvedKnobsMap resolved+      variants+        | policy.knobPolicy == KnobDefaults = []+        | otherwise =+            [ Map.insert spec.name value base+            | spec <- scenario.knobs,+              length spec.variants <= 8,+              value <- spec.variants,+              value /= Map.findWithDefault spec.def spec.name base+            ]+  where+    declaredNames = fmap (renderKnobName . (.name)) scenario.knobs+    supplied =+      [ (spec.name, RawText value)+      | (name, value) <- policy.pinnedKnobs,+        name `elem` declaredNames,+        spec <- scenario.knobs,+        renderKnobName spec.name == name+      ]+    renderProblem (KnobError message) = message
+ src/Kenshou/Plan/Policy.hs view
@@ -0,0 +1,100 @@+module Kenshou.Plan.Policy+  ( DimensionPolicy (..),+    KnobPolicy (..),+    PlanPolicy (..),+    defaultPlanPolicy,+    mergePolicy,+    renderDimensionPolicy,+    parseDimensionPolicy,+    renderKnobPolicy,+    parseKnobPolicy,+  )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set (Set)+import Data.Set qualified as Set+import Data.Text (Text)+import Kenshou.Core.Id (Kind (..), Seed, renderKind)+import Kenshou.Core.RunSpec (SpecPlacement (..))+import Kenshou.Core.Scenario (Tier (..), renderTier)++data DimensionPolicy = DefaultOnly | TelemetryCorners | Pairwise | Full+  deriving stock (Eq, Ord, Show, Enum, Bounded)++data KnobPolicy = KnobDefaults | KnobDeclaredVariants+  deriving stock (Eq, Ord, Show)++data PlanPolicy = PlanPolicy+  { maxTier :: Tier,+    kinds :: Set Kind,+    placement :: SpecPlacement,+    dimensionPolicy :: DimensionPolicy,+    knobPolicy :: KnobPolicy,+    trials :: Int,+    budgetMinutes :: Maybe Int,+    tierMinutes :: Map Tier Int,+    seed :: Seed,+    pinnedKnobs :: [(Text, Text)],+    pinnedDimensions :: [(Text, Text)]+  }+  deriving stock (Eq, Show)++defaultPlanPolicy :: Seed -> PlanPolicy+defaultPlanPolicy seed =+  PlanPolicy+    { maxTier = TierStandard,+      kinds = Set.fromList [Correctness, Concurrency, Benchmark, Soak],+      placement = RunLocal,+      dimensionPolicy = DefaultOnly,+      knobPolicy = KnobDefaults,+      trials = 3,+      budgetMinutes = Nothing,+      tierMinutes = Map.fromList [(TierSmoke, 1), (TierStandard, 10), (TierExtended, 60), (TierSoak, 240)],+      seed,+      pinnedKnobs = [],+      pinnedDimensions = []+    }++-- | Policies are concrete after parsing. A later policy therefore wins, while+-- pins are accumulated so suite defaults can be refined on the command line.+mergePolicy :: PlanPolicy -> PlanPolicy -> PlanPolicy+mergePolicy _base override = override++renderDimensionPolicy :: DimensionPolicy -> Text+renderDimensionPolicy DefaultOnly = "default-only"+renderDimensionPolicy TelemetryCorners = "telemetry-corners"+renderDimensionPolicy Pairwise = "pairwise"+renderDimensionPolicy Full = "full"++parseDimensionPolicy :: Text -> Maybe DimensionPolicy+parseDimensionPolicy value = lookup value [(renderDimensionPolicy policy, policy) | policy <- [minBound .. maxBound]]++renderKnobPolicy :: KnobPolicy -> Text+renderKnobPolicy KnobDefaults = "defaults"+renderKnobPolicy KnobDeclaredVariants = "declared-variants"++parseKnobPolicy :: Text -> Maybe KnobPolicy+parseKnobPolicy "defaults" = Just KnobDefaults+parseKnobPolicy "declared-variants" = Just KnobDeclaredVariants+parseKnobPolicy _ = Nothing++instance ToJSON PlanPolicy where+  toJSON policy =+    object+      [ "maxTier" .= renderTier policy.maxTier,+        "kinds" .= fmap renderKind (Set.toAscList policy.kinds),+        "placement" .= policy.placement,+        "dimensionPolicy" .= renderDimensionPolicy policy.dimensionPolicy,+        "knobPolicy" .= renderKnobPolicy policy.knobPolicy,+        "trials" .= policy.trials,+        "trialOrdering" .= ("alternating/v1" :: Text),+        "budgetMinutes" .= policy.budgetMinutes,+        "tierMinutes" .= object [Key.fromText (renderTier tier) .= minutes | (tier, minutes) <- Map.toAscList policy.tierMinutes],+        "seed" .= policy.seed,+        "set" .= policy.pinnedKnobs,+        "dimensions" .= policy.pinnedDimensions+      ]
+ src/Kenshou/Plan/RunPlan.hs view
@@ -0,0 +1,270 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.RunPlan+  ( PlanInputs (..),+    PlanContext (..),+    TrialInfo (..),+    PlannedRun (..),+    SkipReason (..),+    Skipped (..),+    SkeletonRun (..),+    PlanSkeleton (..),+    RunPlan (..),+    buildPlan,+    stampPlan,+  )+where++import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.List (sortOn)+import Data.List.NonEmpty (NonEmpty)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import Kenshou.Core.Id+import Kenshou.Core.Knob (RawKnob (..))+import Kenshou.Core.RunSpec+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Core.Scenario (Placement (..), renderPlacement, renderTier)+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Change+import Kenshou.Plan.Matrix+import Kenshou.Plan.Policy+import System.Random.SplitMix (nextWord64)++newtype PlanInputs = PlanInputs {value :: Value}+  deriving stock (Eq, Show)++data PlanContext = PlanContext+  { suite :: Maybe Text,+    graphDigest :: Text,+    cohortName :: Text,+    cohortPlanHash :: Text,+    inputs :: PlanInputs,+    changes :: [Change],+    warnings :: [Text]+  }+  deriving stock (Eq, Show)++data TrialInfo = TrialInfo {group :: Text, arm :: Text, index :: Int, of_ :: Int}+  deriving stock (Eq, Show)++data PlannedRun = PlannedRun+  { ordinal :: Int,+    runId :: RunId,+    estimateMinutes :: Int,+    reasons :: NonEmpty Reason,+    trial :: Maybe TrialInfo,+    spec :: RunSpec+  }+  deriving stock (Eq, Show)++data SkipReason+  = SkipTier+  | SkipKind+  | SkipPlacement+  | SkipExcluded+  | SkipUnsupportedDimension+  | SkipBenchmarkWithoutDurable+  | SkipUnsupportedKnob+  | SkipOverBudget+  deriving stock (Eq, Ord, Show)++data Skipped = Skipped {scenario :: ScenarioId, reason :: SkipReason, detail :: Text}+  deriving stock (Eq, Show)++data SkeletonRun = SkeletonRun+  { estimateMinutes :: Int,+    reasons :: NonEmpty Reason,+    trial :: Maybe TrialInfo,+    spec :: RunSpec+  }+  deriving stock (Eq, Show)++data PlanSkeleton = PlanSkeleton+  { context :: PlanContext,+    policy :: PlanPolicy,+    runs :: [SkeletonRun],+    skipped :: [Skipped],+    estimateMinutes :: Int+  }+  deriving stock (Eq, Show)++data RunPlan = RunPlan+  { planId :: RunId,+    createdAt :: UTCTime,+    context :: PlanContext,+    policy :: PlanPolicy,+    runs :: [PlannedRun],+    skipped :: [Skipped],+    estimateMinutes :: Int+  }+  deriving stock (Eq, Show)++buildPlan :: PlanContext -> PlanPolicy -> [Selected] -> PlanSkeleton+buildPlan context policy selected =+  PlanSkeleton+    { context,+      policy,+      runs = kept,+      skipped = initialSkipped <> budgetSkipped,+      estimateMinutes = sum (fmap (.estimateMinutes) kept)+    }+  where+    ordered = sortOn selectedKey selected+    expanded = fmap expandOne ordered+    candidates = concatMap fst expanded+    initialSkipped = concatMap snd expanded+    (kept, budgetSkipped) = applyBudget policy.budgetMinutes candidates+    selectedKey value =+      ( minimum (fmap (.distance) (NonEmpty.toList value.reasons)),+        value.scenario.tier,+        kindRank value.scenario.id.kind,+        renderScenarioId value.scenario.id+      )+    kindRank Correctness = (0 :: Int)+    kindRank Concurrency = 1+    kindRank Benchmark = 2+    kindRank Soak = 3+    expandOne selectedValue+      | selectedValue.scenario.tier > policy.maxTier = ([], [skip SkipTier ("tier " <> renderTier selectedValue.scenario.tier <> " exceeds " <> renderTier policy.maxTier)])+      | selectedValue.scenario.id.kind `Set.notMember` policy.kinds = ([], [skip SkipKind "scenario kind is not enabled"])+      | not (placementMatches policy.placement selectedValue.scenario.placement) = ([], [skip SkipPlacement ("scenario placement is " <> renderPlacement selectedValue.scenario.placement)])+      | otherwise = case expandScenario policy selectedValue of+          ([], matrixSkips) -> ([], fmap (matrixSkip selectedValue.scenario.id) matrixSkips)+          (configs, _) -> (makeRuns selectedValue configs, [])+      where+        skip reason detail = Skipped selectedValue.scenario.id reason detail+    matrixSkip scenarioId problem = Skipped scenarioId (matrixReason problem.reason) problem.detail+    matrixReason "benchmark-without-durable" = SkipBenchmarkWithoutDurable+    matrixReason "unsupported-dimension" = SkipUnsupportedDimension+    matrixReason _ = SkipUnsupportedKnob+    makeRuns selectedValue configs+      | selectedValue.scenario.id.kind == Benchmark = concatMap makeTrial [0 .. policy.trials - 1]+      | otherwise = fmap (makeRun selectedValue Nothing 0) configs+      where+        makeTrial trialIndex =+          [ makeRun selectedValue (Just (TrialInfo (renderScenarioId selectedValue.scenario.id) ("arm-" <> Text.pack (show armIndex)) trialIndex policy.trials)) trialIndex config+          | (armIndex, config) <- zip [0 :: Int ..] (if even trialIndex then configs else reverse configs)+          ]+    makeRun selectedValue trial trialIndex config =+      SkeletonRun+        { estimateMinutes = Map.findWithDefault 10 selectedValue.scenario.tier policy.tierMinutes,+          reasons = selectedValue.reasons,+          trial,+          spec =+            (minimalRunSpec selectedValue.scenario.id)+              { RunSpec.scenarioRevision = Just selectedValue.scenario.revision,+                RunSpec.knobs = [(name, RawJson (toJSON value)) | (name, value) <- Map.toAscList config.knobs],+                RunSpec.dimensions = Map.toAscList config.dimensions,+                RunSpec.seed = Just (derivedSeed policy.seed selectedValue.scenario.id trialIndex),+                RunSpec.environment = defaultEnvironment policy.placement,+                RunSpec.cohortExpectation = Just (CohortExpectation (Just context.cohortName) context.cohortPlanHash),+                RunSpec.comparison = fmap toComparison trial+              }+        }+    toComparison trial = ComparisonMembership trial.group trial.arm trial.index (trial.index + 1)++applyBudget :: Maybe Int -> [SkeletonRun] -> ([SkeletonRun], [Skipped])+applyBudget Nothing runs = (runs, [])+applyBudget (Just budget) runs = go 0 [] [] (groupRuns runs)+  where+    go _ kept skipped [] = (reverse kept, reverse skipped)+    go used kept skipped ([] : rest) = go used kept skipped rest+    go used kept skipped (group@(first : _) : rest)+      | used + groupCost <= budget = go (used + groupCost) (reverse group <> kept) skipped rest+      | otherwise =+          let scenarioId = first.spec.scenario+              item = Skipped scenarioId SkipOverBudget ("group costs " <> Text.pack (show groupCost) <> " minutes with " <> Text.pack (show (budget - used)) <> " remaining")+           in go used kept (item : skipped) rest+      where+        groupCost = sum (fmap (.estimateMinutes) group)+    groupRuns [] = []+    groupRuns (first : rest) =+      let (same, later) = span ((== first.spec.scenario) . (.scenario) . (.spec)) rest+       in (first : same) : groupRuns later++stampPlan :: PlanSkeleton -> IO RunPlan+stampPlan skeleton = do+  planId <- newRunId+  createdAt <- getCurrentTime+  runs <- traverse stamp (zip [1 :: Int ..] skeleton.runs)+  pure+    RunPlan+      { planId,+        createdAt,+        context = skeleton.context,+        policy = skeleton.policy,+        runs,+        skipped = skeleton.skipped,+        estimateMinutes = skeleton.estimateMinutes+      }+  where+    stamp (ordinal, candidate) = do+      runId <- newRunId+      pure+        PlannedRun+          { ordinal,+            runId,+            estimateMinutes = candidate.estimateMinutes,+            reasons = candidate.reasons,+            trial = candidate.trial,+            spec = candidate.spec+          }++derivedSeed :: Seed -> ScenarioId -> Int -> Seed+derivedSeed seed scenario trialIndex = case mkSeed (word `mod` 9007199254740992) of+  Right value -> value+  Left _ -> seed+  where+    (word, _) = nextWord64 (deriveGen seed (renderScenarioId scenario <> ":" <> Text.pack (show trialIndex)))++placementMatches :: SpecPlacement -> Placement -> Bool+placementMatches RunLocal PlaceLocal = True+placementMatches RunLocal PlaceEither = True+placementMatches RunOnCell PlaceCell = True+placementMatches RunOnCell PlaceEither = True+placementMatches _ _ = False++defaultEnvironment :: SpecPlacement -> EnvironmentSpec+defaultEnvironment placement = EnvironmentSpec placement Nothing Nothing Map.empty Map.empty Nothing++instance ToJSON RunPlan where+  toJSON plan =+    object+      [ "schema" .= ("kenshou.run-plan/v1" :: Text),+        "planId" .= plan.planId,+        "createdAt" .= plan.createdAt,+        "suite" .= plan.context.suite,+        "graphDigest" .= plan.context.graphDigest,+        "cohort" .= object ["name" .= plan.context.cohortName, "planHash" .= plan.context.cohortPlanHash],+        "inputs" .= plan.context.inputs.value,+        "policy" .= plan.policy,+        "changes" .= plan.context.changes,+        "runs" .= plan.runs,+        "skipped" .= plan.skipped,+        "warnings" .= plan.context.warnings,+        "estimateMinutes" .= plan.estimateMinutes+      ]++instance ToJSON PlannedRun where+  toJSON run = object ["ordinal" .= run.ordinal, "runId" .= run.runId, "estimateMinutes" .= run.estimateMinutes, "reasons" .= run.reasons, "trial" .= run.trial, "spec" .= run.spec]++instance ToJSON TrialInfo where+  toJSON trial = object ["group" .= trial.group, "arm" .= trial.arm, "index" .= trial.index, "of" .= trial.of_]++instance ToJSON Skipped where+  toJSON skipped = object ["scenario" .= skipped.scenario, "reason" .= renderSkipReason skipped.reason, "detail" .= skipped.detail]++renderSkipReason :: SkipReason -> Text+renderSkipReason SkipTier = "tier"+renderSkipReason SkipKind = "kind"+renderSkipReason SkipPlacement = "placement"+renderSkipReason SkipExcluded = "excluded"+renderSkipReason SkipUnsupportedDimension = "unsupported-dimension"+renderSkipReason SkipBenchmarkWithoutDurable = "benchmark-without-durable"+renderSkipReason SkipUnsupportedKnob = "unsupported-knob"+renderSkipReason SkipOverBudget = "over-budget"
+ src/Kenshou/Plan/Selector.hs view
@@ -0,0 +1,23 @@+module Kenshou.Plan.Selector+  ( Selector,+    parseSelector,+    renderSelector,+    matches,+  )+where++import Data.Text (Text)+import Kenshou.Core.Id (ScenarioId)+import Kenshou.Core.Selector (ScenarioSelector)+import Kenshou.Core.Selector qualified as Core++type Selector = ScenarioSelector++parseSelector :: Text -> Either Text Selector+parseSelector = Core.parseSelector++renderSelector :: Selector -> Text+renderSelector = Core.renderSelector++matches :: Selector -> ScenarioId -> Bool+matches = Core.matchesSelector
+ src/Kenshou/Plan/Suite.hs view
@@ -0,0 +1,148 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Suite+  ( SuiteMode (..),+    SuiteExclusion (..),+    SuiteSelection (..),+    PolicyPatch (..),+    Suite (..),+    readSuite,+    decodeSuite,+    suitePolicy,+    applySuite,+  )+where++import Control.Applicative ((<|>))+import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.Bifunctor (first)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.List.NonEmpty qualified as NonEmpty+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (Seed, parseKind)+import Kenshou.Core.RunSpec (SpecPlacement (..))+import Kenshou.Core.Scenario (Tier (..))+import Kenshou.Plan.Catalog (ScenarioInfo)+import Kenshou.Plan.Change+import Kenshou.Plan.Policy+import Kenshou.Plan.Policy qualified as Policy+import Kenshou.Plan.Selector++data SuiteMode = SuiteAll | SuiteChanged deriving stock (Eq, Show)++data SuiteExclusion = SuiteExclusion {select :: Selector, why :: Text}+  deriving stock (Eq, Show)++data SuiteSelection = SuiteSelection+  { mode :: SuiteMode,+    always :: [Selector],+    exclude :: [SuiteExclusion]+  }+  deriving stock (Eq, Show)++data PolicyPatch = PolicyPatch+  { maxTier :: Maybe Tier,+    kinds :: Maybe [Text],+    placement :: Maybe SpecPlacement,+    dimensionPolicy :: Maybe DimensionPolicy,+    knobPolicy :: Maybe KnobPolicy,+    trials :: Maybe Int,+    budgetMinutes :: Maybe Int+  }+  deriving stock (Eq, Show)++data Suite = Suite+  { name :: Text,+    description :: Text,+    selection :: SuiteSelection,+    policy :: PolicyPatch,+    directlyChanged :: Maybe PolicyPatch+  }+  deriving stock (Eq, Show)++readSuite :: FilePath -> IO (Either Text Suite)+readSuite path = decodeSuite <$> ByteString.readFile path++decodeSuite :: ByteString -> Either Text Suite+decodeSuite = first Text.pack . eitherDecodeStrict'++suitePolicy :: Seed -> Suite -> Either Text PlanPolicy+suitePolicy seed suite = applyPatch suite.policy (defaultPlanPolicy seed)++applySuite :: Suite -> [ScenarioInfo] -> [Selected] -> [Selected]+applySuite suite catalog changed = applyDirectlyChanged (applySelectors [] exclusions (base <> alwaysSelected))+  where+    base = case suite.selection.mode of SuiteAll -> selectAll catalog; SuiteChanged -> changed+    alwaysSelected = selectBySelectors Everything ("suite " <> suite.name <> " always") suite.selection.always catalog+    exclusions = fmap (.select) suite.selection.exclude+    applyDirectlyChanged = case suite.directlyChanged of+      Nothing -> id+      Just patch -> fmap (raiseDirect patch)+    raiseDirect patch selected+      | minimum (fmap (.distance) (NonEmpty.toList selected.reasons)) == 0 =+          selected+            { minPolicy = fmap (renderDimensionPolicy . max (maybe DefaultOnly parseMinimum selected.minPolicy)) patch.dimensionPolicy <|> selected.minPolicy,+              minKnobPolicy = fmap (renderKnobPolicy . max (maybe KnobDefaults parseMinimumKnob selected.minKnobPolicy)) patch.knobPolicy <|> selected.minKnobPolicy+            }+      | otherwise = selected+    parseMinimum value = maybe DefaultOnly (\parsed -> parsed) (parseDimensionPolicy value)+    parseMinimumKnob value = maybe KnobDefaults (\parsed -> parsed) (parseKnobPolicy value)++applyPatch :: PolicyPatch -> PlanPolicy -> Either Text PlanPolicy+applyPatch patch base = do+  kinds <- maybe (Right base.kinds) (fmap Set.fromList . traverse parseKind) patch.kinds+  pure+    base+      { Policy.maxTier = maybe base.maxTier (\value -> value) patch.maxTier,+        Policy.kinds = kinds,+        Policy.placement = maybe base.placement (\value -> value) patch.placement,+        Policy.dimensionPolicy = maybe base.dimensionPolicy (\value -> value) patch.dimensionPolicy,+        Policy.knobPolicy = maybe base.knobPolicy (\value -> value) patch.knobPolicy,+        Policy.trials = maybe base.trials (\value -> value) patch.trials,+        Policy.budgetMinutes = patch.budgetMinutes <|> base.budgetMinutes+      }++instance FromJSON Suite where+  parseJSON = withObject "Suite" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.suite/v1" :: Text) then fail "unsupported suite schema" else pure ()+    Suite <$> value .: "name" <*> value .: "description" <*> value .: "selection" <*> value .: "policy" <*> value .:? "directlyChanged"++instance FromJSON SuiteSelection where+  parseJSON = withObject "SuiteSelection" \value -> SuiteSelection <$> (value .: "mode" >>= parseMode) <*> (value .:? "always" .!= [] >>= traverse parseSelectorValue) <*> value .:? "exclude" .!= []+    where+      parseMode ("all" :: Text) = pure SuiteAll+      parseMode "changed" = pure SuiteChanged+      parseMode other = fail ("unknown suite mode " <> Text.unpack other)++instance FromJSON SuiteExclusion where+  parseJSON = withObject "SuiteExclusion" \value -> SuiteExclusion <$> (value .: "select" >>= parseSelectorValue) <*> value .: "why"++instance FromJSON PolicyPatch where+  parseJSON = withObject "PolicyPatch" \value ->+    PolicyPatch+      <$> (value .:? "maxTier" >>= traverse parseTier)+      <*> value .:? "kinds"+      <*> (value .:? "placement" >>= traverse parsePlacement)+      <*> (value .:? "dimensionPolicy" >>= traverse parseDimension)+      <*> (value .:? "knobPolicy" >>= traverse parseKnob)+      <*> value .:? "trials"+      <*> value .:? "budgetMinutes"+    where+      parseTier "smoke" = pure TierSmoke+      parseTier "standard" = pure TierStandard+      parseTier "extended" = pure TierExtended+      parseTier "soak" = pure TierSoak+      parseTier other = fail ("unknown tier " <> Text.unpack other)+      parsePlacement "local" = pure RunLocal+      parsePlacement "cell" = pure RunOnCell+      parsePlacement other = fail ("unknown placement " <> Text.unpack other)+      parseDimension raw = maybe (fail ("unknown dimension policy " <> Text.unpack raw)) pure (parseDimensionPolicy raw)+      parseKnob raw = maybe (fail ("unknown knob policy " <> Text.unpack raw)) pure (parseKnobPolicy raw)++parseSelectorValue :: Text -> Parser Selector+parseSelectorValue = either (fail . Text.unpack) pure . parseSelector
+ src/Kenshou/Plan/Summary.hs view
@@ -0,0 +1,127 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Summary+  ( EntryStatus (..),+    Attempt (..),+    SummaryEntry (..),+    PlanSummary (..),+    emptySummary,+    worstOutcome,+    summaryExitCode,+    finalizeSummary,+    writeSummary,+  )+where++import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId, ScenarioId)+import Kenshou.Core.Outcome (Outcome (..), outcomeExitCode, renderOutcome)+import System.Directory (renameFile)+import System.Exit (ExitCode (..))++data EntryStatus = Pending | Running | Completed deriving stock (Eq, Show)++data Attempt = Attempt+  { runId :: RunId,+    startedAt :: UTCTime,+    finishedAt :: Maybe UTCTime,+    outcome :: Maybe Outcome,+    knownDefect :: Bool,+    childExitCode :: Maybe Int+  }+  deriving stock (Eq, Show)++data SummaryEntry = SummaryEntry+  { ordinal :: Int,+    scenario :: ScenarioId,+    status :: EntryStatus,+    attempts :: [Attempt]+  }+  deriving stock (Eq, Show)++data PlanSummary = PlanSummary+  { planId :: RunId,+    entries :: [SummaryEntry],+    counts :: Map Text Int,+    worst :: Outcome,+    exitCode :: Int+  }+  deriving stock (Eq, Show)++emptySummary :: RunId -> [(Int, ScenarioId)] -> PlanSummary+emptySummary planId entries = finalizeSummary (PlanSummary planId [SummaryEntry ordinal scenario Pending [] | (ordinal, scenario) <- entries] Map.empty Passed 0)++worstOutcome :: PlanSummary -> Outcome+worstOutcome summary = case blocking of+  [] -> Passed+  values -> foldl1 worse values+  where+    blocking = concatMap entryOutcome summary.entries+    entryOutcome entry+      | entry.status /= Completed = [Errored]+      | otherwise = case reverse entry.attempts of+          attempt : _ | attempt.knownDefect -> []+          attempt : _ -> maybe [Errored] pure attempt.outcome+          [] -> [Errored]+    worse left right = if rank left >= rank right then left else right+    rank Passed = (0 :: Int)+    rank Inconclusive = 1+    rank InfrastructureFailure = 2+    rank Errored = 3+    rank Failed = 4++summaryExitCode :: PlanSummary -> ExitCode+summaryExitCode summary = case outcomeExitCode (worstOutcome summary) of 0 -> ExitSuccess; value -> ExitFailure value++finalizeSummary :: PlanSummary -> PlanSummary+finalizeSummary summary = summary {counts = outcomeCounts, worst, exitCode = outcomeExitCode worst}+  where+    worst = worstOutcome summary+    latest = [attempt | entry <- summary.entries, attempt <- take 1 (reverse entry.attempts)]+    outcomes = [outcome | attempt <- latest, not attempt.knownDefect, Just outcome <- [attempt.outcome]]+    defectCount = length (filter (.knownDefect) latest)+    outcomeCounts = Map.fromListWith (+) ([(renderOutcome outcome, 1) | outcome <- outcomes] <> [("known-defect", defectCount) | defectCount > 0])++writeSummary :: FilePath -> PlanSummary -> IO ()+writeSummary path summary = do+  let temporary = path <> ".tmp"+  LazyByteString.writeFile temporary (encode (finalizeSummary summary))+  renameFile temporary path++instance ToJSON PlanSummary where+  toJSON summary = object ["schema" .= ("kenshou.plan-summary/v1" :: Text), "planId" .= summary.planId, "entries" .= summary.entries, "counts" .= summary.counts, "worstOutcome" .= summary.worst, "exitCode" .= summary.exitCode]++instance FromJSON PlanSummary where+  parseJSON = withObject "PlanSummary" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.plan-summary/v1" :: Text) then fail "unsupported plan summary schema" else pure ()+    PlanSummary <$> value .: "planId" <*> value .: "entries" <*> value .:? "counts" .!= Map.empty <*> value .:? "worstOutcome" .!= Passed <*> value .:? "exitCode" .!= 0++instance ToJSON SummaryEntry where+  toJSON entry = object ["ordinal" .= entry.ordinal, "scenario" .= entry.scenario, "status" .= renderStatus entry.status, "attempts" .= entry.attempts]++instance FromJSON SummaryEntry where+  parseJSON = withObject "SummaryEntry" \value -> SummaryEntry <$> value .: "ordinal" <*> value .: "scenario" <*> (value .: "status" >>= parseStatus) <*> value .:? "attempts" .!= []++instance ToJSON Attempt where+  toJSON attempt = object ["runId" .= attempt.runId, "startedAt" .= attempt.startedAt, "finishedAt" .= attempt.finishedAt, "outcome" .= attempt.outcome, "knownDefect" .= attempt.knownDefect, "childExitCode" .= attempt.childExitCode]++instance FromJSON Attempt where+  parseJSON = withObject "Attempt" \value -> Attempt <$> value .: "runId" <*> value .: "startedAt" <*> value .:? "finishedAt" <*> value .:? "outcome" <*> value .:? "knownDefect" .!= False <*> value .:? "childExitCode"++renderStatus :: EntryStatus -> Text+renderStatus Pending = "pending"+renderStatus Running = "running"+renderStatus Completed = "completed"++parseStatus :: Text -> Parser EntryStatus+parseStatus "pending" = pure Pending+parseStatus "running" = pure Running+parseStatus "completed" = pure Completed+parseStatus other = fail ("unknown entry status " <> show other)
+ test/Kenshou/ConsumerBaseline.hs view
@@ -0,0 +1,25 @@+-- | The first consumer's performance contracts, read from+-- ../fixtures/consumer-baseline/perf-contracts.json. The captured preparation in+-- that directory pins the consumer's reviewed benchmark revisions, so validating it+-- needs the same contracts the consumer supplies.+module Kenshou.ConsumerBaseline (consumerContracts) where++import Data.Aeson (eitherDecodeFileStrict', withObject, (.:))+import Data.Aeson.Types (Parser, Value, parseEither)+import Data.Text (Text)+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)++consumerContracts :: IO PerfContracts+consumerContracts = do+  document <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/perf-contracts.json" >>= either fail pure+  (reviewed, knob) <- either fail pure (parseEither contractsParser (document :: Value))+  countKnob <- either (fail . show) pure (mkKnobName knob)+  pure noPerfContracts {reviewedPhaseContracts = reviewed, steadyCountKnob = Just countKnob}+  where+    contractsParser :: Value -> Parser ([(Text, Int)], Text)+    contractsParser = withObject "PerfContracts" \value -> do+      entries <- value .: "reviewedPhaseContracts"+      reviewed <- traverse (withObject "ReviewedPhaseContract" \entry -> (,) <$> entry .: "scenario" <*> entry .: "revision") entries+      knob <- value .: "steadyCountKnob"+      pure (reviewed, knob)
+ test/Kenshou/PerfCalibrationSpec.hs view
@@ -0,0 +1,80 @@+module Kenshou.PerfCalibrationSpec (spec) where++import Data.Aeson+import Data.Either (isLeft)+import Data.Text qualified as Text+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Perf.Calibration+import Test.Hspec++spec :: Spec+spec = describe "performance calibration admission" do+  it "accepts twenty independently replayed slots and preserves frozen protocol roundtrips" do+    (definition, acceptance, aa, slowdown) <- fixture+    validateCalibration now now definition acceptance aa slowdown `shouldBe` Right 90+    eitherDecode (encode definition) `shouldBe` Right definition+    eitherDecode (encode acceptance) `shouldBe` Right acceptance+  it "never drops a completed statistical outcome to replace it with a better result" do+    (definition, acceptance, aa, slowdown) <- fixture+    replacementId <- newRunId+    let slot = first acceptance.aa+        replaced = slot {receipts = slot.receipts <> [CalibrationReceipt "gs://fixture/replacement/receipt.json" "sha256:replacement"]}+        statisticalRetry = first aa <> [(first (first aa)) {sessionId = replacementId}]+    validateCalibration now now definition acceptance {aa = replaced : drop 1 acceptance.aa} (statisticalRetry : drop 1 aa) slowdown `shouldSatisfy` isLeft+  it "requires at least nineteen expected outcomes and zero opposite outcomes" do+    (definition, acceptance, aa, slowdown) <- fixture+    let change verdict = map (\trial -> trial {verdict})+    validateCalibration now now definition acceptance (change "inconclusive" (first aa) : drop 1 aa) slowdown `shouldBe` Right 90+    validateCalibration now now definition acceptance (change "regression" (first aa) : drop 1 aa) slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance aa (change "pass" (first slowdown) : drop 1 slowdown) `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (map (change "inconclusive") (take 2 aa) <> drop 2 aa) slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (change "not-run" (first aa) : drop 1 aa) slowdown `shouldSatisfy` isLeft+  it "invalidates expired, drifted, duplicate, missing and runtime-blocked batches" do+    (definition, acceptance, aa, slowdown) <- fixture+    validateCalibration (addUTCTime 3601 now) now definition acceptance aa slowdown `shouldSatisfy` isLeft+    validateCalibration now (addUTCTime 1 now) definition acceptance aa slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (drop 1 aa) slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (first aa : first aa : drop 2 aa) slowdown `shouldSatisfy` isLeft+    let corrupt :: (VerifiedCalibrationTrial -> VerifiedCalibrationTrial) -> [[VerifiedCalibrationTrial]]+        corrupt change = map change (first aa) : drop 1 aa+    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialHealthPolicySha256 = "relaxed-health-limits"})) slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialMachineProfile = "different-cell-image"})) slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialToolchainSha256 = "changed"})) slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialHarnessSemanticsSha256 = "changed"})) slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition acceptance (corrupt (\trial -> trial {runtimeBlockers = ["known exit-zero defect"]})) slowdown `shouldSatisfy` isLeft+  it "retains infrastructure retries under the frozen cap and includes their time" do+    (definition, acceptance, aa, slowdown) <- fixture+    extra <- newRunId+    let failed = (first (first aa)) {sessionId = extra, infrastructureState = "failed", verdict = "not-run", elapsedSeconds = 850}+        original = first acceptance.aa+        slot = original {receipts = CalibrationReceipt "gs://fixture/failed/receipt.json" "sha256:failed" : original.receipts}+        revised = acceptance {aa = slot : drop 1 acceptance.aa}+        trials = (failed : first aa) : drop 1 aa+    validateCalibration now now definition {infrastructureRetryCap = 0} revised trials slowdown `shouldSatisfy` isLeft+    validateCalibration now now definition revised trials slowdown `shouldBe` Right 90+    let late = map (map (\trial -> trial {elapsedSeconds = 901})) aa+    validateCalibration now now definition acceptance late slowdown `shouldSatisfy` isLeft++fixture :: IO (CalibrationDefinition, CalibrationAcceptance, [[VerifiedCalibrationTrial]], [[VerifiedCalibrationTrial]])+fixture = do+  identifier <- newRunId+  let seeds = [1 .. 20]+      definition = CalibrationDefinition identifier "mori://fixture/runtime" "core" "synthetic-machine" (digest "d") (digest "a") (digest "b") (digest "c") (Text.replicate 40 "a") (Text.replicate 40 "b") "delay at operation boundary" "op.latency.p99" 0.5 1000000 seeds 1 (addUTCTime 3600 now)+      slot control seed = CalibrationSlot seed [CalibrationReceipt ("gs://fixture/" <> control <> "/" <> Text.pack (show seed) <> "/receipt.json") "sha256:receipt"]+      acceptance = CalibrationAcceptance identifier (map (slot "aa") seeds) (map (slot "slowdown") seeds)+      trial control verdict seed = do+        session <- newRunId+        pure [VerifiedCalibrationTrial session seed control definition.project definition.baselineRevision (if control == "aa" then definition.baselineRevision else definition.slowdownRevision) definition.profile definition.machineProfile definition.healthPolicySha256 definition.workloadSha256 definition.toolchainSha256 definition.harnessSemanticsSha256 verdict "healthy" [] now 90]+  aa <- traverse (trial "aa" "pass") seeds+  slowdown <- traverse (trial "none" "regression") seeds+  pure (definition, acceptance, aa, slowdown)+  where+    digest letter = "sha256:" <> Text.replicate 64 letter++now :: UTCTime+now = UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0)++first :: [a] -> a+first (value : _) = value+first [] = error "empty synthetic calibration fixture"
+ test/Kenshou/PerfPreparedSpec.hs view
@@ -0,0 +1,171 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.PerfPreparedSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseEither)+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified+import Kenshou.ConsumerBaseline qualified as ConsumerBaseline+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort qualified as Cohort+import Kenshou.Core.Manifest qualified as Manifest+import Kenshou.Perf.Artifacts+import Kenshou.Perf.Calibration qualified as Calibration+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)+import Kenshou.Perf.Prepared hiding (plan)+import Kenshou.Perf.Source qualified as Source+import System.Directory (createFileLink)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance preparation integrity" do+  it "validates the pinned synthetic protocol fixture and rejects changed build keys" do+    consumerContracts <- loadConsumerContracts+    prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+    validatePrepared consumerContracts prepared `shouldBe` Right ()+    let baselineArm = (prepared :: PerfPrepared).baseline+    validatePrepared consumerContracts prepared {baseline = baselineArm {buildKey = "sha256:changed"}} `shouldSatisfy` isLeft+    decoded <- right (eitherDecode (encode prepared) :: Either String PerfPrepared)+    toJSON decoded `shouldBe` toJSON prepared+  it "admits resolved-cohort identities, replays legacy identities and rejects a rehashed alias" do+    consumerContracts <- loadConsumerContracts+    prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+    (request, template, baselineSource, candidateSource) <- right (parseEither (withObject "Inputs" \fields -> (,,,) <$> fields .: "request" <*> fields .: "template" <*> fields .: "baselineSource" <*> fields .: "candidateSource") (prepared :: PerfPrepared).inputs)+    baselineDescriptor <- right (Source.descriptorForSnapshot request template "baseline" baselineSource)+    candidateDescriptor <- right (Source.descriptorForSnapshot request template "candidate" candidateSource)+    let bind descriptor arm = do+          let digest = Data.Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor)))+          lock <- case arm.sourceLock of+            Object fields -> pure (Object (KeyMap.insert "cohort" (toJSON descriptor.descriptorName) (KeyMap.insert "descriptorSha256" (String digest) fields)))+            _ -> fail "fixture source lock is not an object"+          payload <- case arm.payload of+            Object fields -> do+              original <- maybe (fail "fixture payload has no identity") pure (KeyMap.lookup "cohortIdentity" fields)+              identity <- case fromJSON original of Error problem -> fail problem; Success value -> pure value+              let changed = (identity :: Cohort.CohortIdentity) {Cohort.identityCohort = descriptor.descriptorName, Cohort.identityDescriptorSha256 = digest}+              pure (Object (KeyMap.insert "cohortIdentity" (toJSON changed) (KeyMap.insert "flakeAttr" (String ("packages.x86_64-linux.kenshou-" <> Cohort.unCohortName descriptor.descriptorName)) fields)))+            _ -> fail "fixture payload is not an object"+          pure arm {descriptor = descriptor, sourceLock = lock, payload = payload, buildKey = buildKeyFor prepared.harnessRevision descriptor lock arm.buildInventory arm.boundsInventory}+    baseline <- bind baselineDescriptor prepared.baseline+    candidate <- bind candidateDescriptor prepared.candidate+    let modern = prepared {baseline = baseline, candidate = candidate}+    validatePrepared consumerContracts modern `shouldBe` Right ()+    validatePrepared consumerContracts prepared `shouldBe` Right ()+    validatePrepared consumerContracts modern {candidate = prepared.candidate} `shouldSatisfy` isLeft+    alias <- bind baselineDescriptor {Cohort.descriptorName = Cohort.CohortName "perf-unrelated-alias", Cohort.descriptorProjectFile = "perf-unrelated-alias.project"} baseline+    validatePrepared consumerContracts modern {baseline = alias, candidate = alias} `shouldSatisfy` isLeft+  it "binds actual inventories to the planning toolchain while allowing catalog-only promotion" do+    consumerContracts <- loadConsumerContracts+    prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+    let inventory = (prepared :: PerfPrepared).baseline.buildInventory+    case (prepared.inputs, inventory.toolchain) of+      (Object inputs, Object toolchain) -> do+        let expected = Object (KeyMap.delete "harnessRevision" toolchain)+            pinned = prepared {inputs = Object (KeyMap.insert "buildToolchain" expected inputs)}+            changed = inventory {toolchain = Object (KeyMap.insert "harnessRevision" (String "another catalog commit") toolchain)}+        validatePrepared consumerContracts pinned `shouldBe` Right ()+        Calibration.toolchainDigest inventory `shouldBe` Calibration.toolchainDigest changed+        validatePrepared consumerContracts prepared {inputs = Object (KeyMap.insert "buildToolchain" (object ["strictBounds" .= False]) inputs)} `shouldSatisfy` isLeft+      _ -> expectationFailure "invalid synthetic inventory fixture"+  it "rejects bound lifting inside a resealed inventory even when its toolchain claims strict bounds" do+    consumerContracts <- loadConsumerContracts+    prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+    let arm = (prepared :: PerfPrepared).baseline+        corrupt = arm.buildInventory {packages = [entry {settings = object ["jailbreak" .= True, "configureFlags" .= ([] :: [Text])]} | entry <- arm.buildInventory.packages]}+        changed = arm {buildInventory = corrupt, buildKey = buildKeyFor prepared.harnessRevision arm.descriptor arm.sourceLock corrupt arm.boundsInventory}+    validatePrepared consumerContracts prepared {baseline = changed, candidate = changed} `shouldSatisfy` isLeft+  it "allows only target source/version changes; target flags still must match" do+    compareInventories ["runtime"] inventory inventory {packages = [runtime {version = "2", source = String "candidate"}, dependency]} `shouldBe` Right ()+    compareInventories ["runtime"] inventory inventory {packages = [runtime {flags = Map.singleton "feature" True}, dependency]} `shouldSatisfy` isLeft+    compareInventories ["runtime"] inventory inventory {packages = [runtime {settings = String "different"}, dependency]} `shouldSatisfy` isLeft+  it "rejects unrelated source, version, toolchain and package-set drift" do+    mapM_+      (\changed -> compareInventories ["runtime"] inventory changed `shouldSatisfy` isLeft)+      [inventory {packages = [runtime, dependency {source = String "other"}]}, inventory {packages = [runtime, dependency {version = "2"}]}, inventory {toolchain = String "other"}, inventory {packages = [runtime]}]+  it "rejects missing targets and ambiguous package identities" do+    compareInventories ["missing"] inventory inventory `shouldSatisfy` isLeft+    compareInventories ["runtime"] inventory inventory {packages = [runtime, dependency, dependency]} `shouldSatisfy` isLeft+  it "identifies harness sources by commit and includes transitive flags/revisions" do+    resolved <- right (cabalInventory "/workspace/harness" revision plan)+    length resolved.packages `shouldBe` 2+    map (.flags) resolved.packages `shouldContain` [Map.singleton "feature" True]+    cabalInventory "/workspace/other" revision plan `shouldSatisfy` isLeft+    cabalInventory "/workspace/harness" "main" plan `shouldSatisfy` isLeft+    eitherDecode (encode resolved) `shouldBe` Right resolved+  it "renders a standalone bounds project and rejects blanket exceptions" do+    descriptor <- syntheticCohort+    project <- right (renderBoundsProject noPerfContracts ["/harness/kenshou-core/kenshou-core.cabal"] descriptor)+    project `shouldNotContainText` "active.project"+    renderBoundsProject noPerfContracts ["/harness/package.cabal"] descriptor {Cohort.descriptorAllowNewer = ["all"]} `shouldSatisfy` isLeft+    renderBoundsProject noPerfContracts ["/unsafe path/package.cabal"] descriptor `shouldSatisfy` isLeft+  it "appends the suite's lane stanzas for the cohorts it selects" do+    descriptor <- syntheticCohort+    let laneName = Cohort.CohortName "acme-lane"+        contracts = noPerfContracts {laneStanzas = \selected -> if selected.descriptorName == laneName then ["package acme-suite", "  flags: -extras"] else []}+    lane <- right (renderBoundsProject contracts ["/harness/acme-suite/acme-suite.cabal"] descriptor {Cohort.descriptorName = laneName})+    lane `shouldSatisfy` Data.Text.isInfixOf "package acme-suite\n  flags: -extras"+    full <- right (renderBoundsProject contracts ["/harness/acme-suite/acme-suite.cabal"] descriptor)+    full `shouldNotContainText` "flags: -extras"+  it "verifies the complete sealed file set and refuses an omitted file" $ withSystemTempDirectory "perf-seal" \root -> do+    writeFile (root </> "input.json") "{}"+    sealPerfDirectory root `shouldReturn` Right ()+    verifyPerfManifest root `shouldReturn` Right ()+    writeFile (root </> "unlisted.json") "{}"+    verifyPerfManifest root >>= (`shouldSatisfy` isLeft)+  it "refuses duplicate and escaping paths before opening a listed file" $ withSystemTempDirectory "perf-seal" \root -> do+    writeFile (root </> "input.json") "{}"+    sealPerfDirectory root `shouldReturn` Right ()+    original <- eitherDecodeFileStrict' (root </> "manifest.json") >>= right+    encodeFile (root </> "manifest.json") (original :: Manifest.Manifest) {Manifest.files = original.files <> original.files}+    verifyPerfManifest root >>= (`shouldSatisfy` isLeft)+    encodeFile (root </> "manifest.json") original {Manifest.files = map (\file -> file {Manifest.path = "../outside.json"}) original.files}+    verifyPerfManifest root >>= (`shouldSatisfy` isLeft)+  it "refuses symbolic links when sealing or verifying evidence" $ withSystemTempDirectory "perf-seal" \root -> do+    writeFile (root </> "input.json") "{}"+    sealPerfDirectory root `shouldReturn` Right ()+    createFileLink (root </> "input.json") (root </> "link.json")+    sealPerfDirectory root >>= (`shouldSatisfy` isLeft)+    verifyPerfManifest root >>= (`shouldSatisfy` isLeft)++runtime, dependency :: InventoryPackage+runtime = InventoryPackage "runtime" "1" (String "baseline") Map.empty (object [])+dependency = InventoryPackage "dependency" "1" (String "fixed") Map.empty (object [])++inventory :: BuildInventory+inventory = BuildInventory "nix" "ghc-9.12.4" "x86_64-linux" (String "pins") [dependency, runtime]++revision :: Text+revision = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"++plan :: Value+plan =+  object+    [ "compiler-id" .= ("ghc-9.12.4" :: Text),+      "cabal-version" .= ("3.16" :: Text),+      "os" .= ("linux" :: Text),+      "arch" .= ("x86_64" :: Text),+      "install-plan"+        .= [ object ["type" .= ("configured" :: Text), "pkg-name" .= ("kenshou-core" :: Text), "pkg-version" .= ("0.1" :: Text), "pkg-src" .= object ["type" .= ("local" :: Text), "path" .= ("/workspace/harness/kenshou-core" :: Text)]],+             object ["type" .= ("configured" :: Text), "pkg-name" .= ("dependency" :: Text), "pkg-version" .= ("1" :: Text), "pkg-src" .= object ["type" .= ("repo-tar" :: Text)], "flags" .= Map.singleton ("feature" :: Text) True, "pkg-revision" .= (1 :: Int), "pkg-cabal-sha256" .= ("description" :: Text), "pkg-src-sha256" .= ("archive" :: Text)]+           ]+    ]++right :: (Show e) => Either e a -> IO a+right = either (\problem -> expectationFailure (show problem) >> fail "expected Right") pure++shouldNotContainText :: Text -> Text -> Expectation+shouldNotContainText actual needle = actual `shouldSatisfy` (not . Data.Text.isInfixOf needle)++loadConsumerContracts :: IO PerfContracts+loadConsumerContracts = ConsumerBaseline.consumerContracts++syntheticCohort :: IO Cohort.CohortDescriptor+syntheticCohort = ByteString.readFile "test/fixtures/perf/cohort.json" >>= right . eitherDecodeStrict'
+ test/Kenshou/PerfResultSpec.hs view
@@ -0,0 +1,116 @@+module Kenshou.PerfResultSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Perf.Plan+import Kenshou.Perf.Plan qualified as Plan+import Kenshou.Perf.Result+import Kenshou.Perf.Result qualified as Result+import Test.Hspec++spec :: Spec+spec = describe "performance aggregate results" do+  it "decodes the synthetic outcome fixtures with their distinct exit meanings" do+    let cases = [("pass", 0), ("incomplete", 3), ("runtime-failure", 1), ("regression", 1), ("inconclusive", 3), ("infrastructure-failure", 4), ("not-applicable", 0)]+    mapM_+      ( \(name, code) -> do+          result <- eitherDecodeFileStrict' ((if name == "not-applicable" then "test/golden/" else "../fixtures/consumer-baseline/kenshou-core/test/golden/") <> "perf-result." <> name <> ".fixture.json") >>= either fail pure+          (result :: PerfResult).exitCode `shouldBe` code+      )+      cases+  it "keeps calibration passes non-gating and distinguishes requested from change scope" do+    plan <- fixture+    result <- accepted (aggregateResult plan digest True (map (configuration "pass") plan.selected))+    result.exitCode `shouldBe` 0+    result.gatingEligible `shouldBe` False+    result.purpose `shouldBe` "calibration"+    result.changeCoverage `shouldBe` plan.changeCoverage+    (eitherDecode (encode result) :: Either String PerfResult) `shouldBe` Right result+  it "retains report references and measured stages while rejecting invalid metadata" do+    plan <- fixture+    result <- accepted (aggregateResult plan digest True (map (configuration "pass") plan.selected))+    let reported = result {stageSeconds = Map.singleton "totalAtReportAssembly" 17.5, artifactReferences = [ArtifactReference "aggregate-manifest" "gs://fixture-evidence/performance/manifest.json" Nothing]}+    (eitherDecode (encode reported) :: Either String PerfResult) `shouldBe` Right reported+    validateResult reported {stageSeconds = Map.singleton "negative" (-1)} `shouldSatisfy` isLeft+    validateResult reported {artifactReferences = [ArtifactReference "prepared" "gs://fixture-evidence/performance/prepared.json" (Just "mutable")]} `shouldSatisfy` isLeft+  it "uses 4 before 1 before 3 while preserving blockers and omissions" do+    plan <- fixture+    let (first, second) = twoProfiles plan+        blocker = (configuration "inconclusive" first) {runtimeFailures = ["known defect reproduced with exit zero"], comparison = Nothing}+        failure = (configuration "inconclusive" second) {infrastructureState = "failed", validPairs = 0, comparison = Nothing}+    result <- accepted (aggregateResult plan digest True [blocker, failure])+    result.exitCode `shouldBe` 4+    result.blockingRuntimeFailures `shouldBe` ["known defect reproduced with exit zero"]+    regression <- accepted (aggregateResult plan {Plan.coverage = "incomplete"} digest True [configuration "regression" first])+    regression.exitCode `shouldBe` 1+    regression.coverage `shouldBe` "incomplete"+    incomplete <- accepted (aggregateResult plan digest True [configuration "pass" first])+    incomplete.exitCode `shouldBe` 3+  it "does not turn an unresolved publication failure into success" do+    plan <- fixture+    result <- accepted (aggregateResult plan digest False (map (configuration "pass") plan.selected))+    result.exitCode `shouldBe` 4+    result.performanceVerdict `shouldBe` "inconclusive"+  it "keeps a statistical pass separate from a blocking runtime defect" do+    plan <- fixture+    let (first, second) = twoProfiles plan+        blocked = (configuration "pass" first) {runtimeFailures = ["known-defect-reproduced"]}+    result <- accepted (aggregateResult plan digest True [blocked, configuration "pass" second])+    result.performanceVerdict `shouldBe` "pass"+    result.exitCode `shouldBe` 1+    result.gatingEligible `shouldBe` False+  it "retains recovered invalid attempts without making healthy evidence fail" do+    plan <- fixture+    let results = [(configuration "pass" entry) {attempts = [object ["state" .= ("infrastructure-invalid" :: Text), "reason" .= ("reset failed" :: Text)]]} | entry <- plan.selected]+    result <- accepted (aggregateResult plan digest True results)+    result.exitCode `shouldBe` 0+  it "rejects a forged gate, missing pairs and inconsistent retained verdict" do+    plan <- fixture+    result <- accepted (aggregateResult plan digest True (map (configuration "pass") plan.selected))+    let first = fst (twoProfiles plan)+    validateResult result {Result.gatingEligible = True} `shouldSatisfy` isLeft+    aggregateResult plan digest True [(configuration "pass" first) {validPairs = 4}] `shouldSatisfy` isLeft+    aggregateResult plan digest True [(configuration "pass" first) {comparison = Just (comparisonValue "regression" 5)}] `shouldSatisfy` isLeft+  it "requires workload and pair counts from the pinned selection" do+    plan <- fixture+    let first = fst (twoProfiles plan)+    aggregateResult plan digest True [(configuration "pass" first) {Result.workloadSha256 = "sha256:" <> "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}] `shouldSatisfy` isLeft+    aggregateResult plan digest True [(configuration "pass" first) {requiredPairs = 6, validPairs = 6, comparison = Just (comparisonValue "pass" 6)}] `shouldSatisfy` isLeft+  it "uses not-applicable only for a proved unexecuted scope" do+    plan <- fixture+    let noWork = plan {selected = [], coverage = "not-applicable", changeCoverage = "not-applicable", gatingEligible = False}+    result <- accepted (aggregateResult noWork digest True [])+    result.performanceVerdict `shouldBe` "not-applicable"+    result.exitCode `shouldBe` 0+    result.gatingEligible `shouldBe` False+  it "replays across analysis IDs, clocks and hosts but retains metric and evidence identity" do+    let core = object ["schema" .= ("kenshou.comparison/v1" :: Text), "design" .= ("abba" :: Text), "baselineRuns" .= ["run-a" :: Text], "candidateRuns" .= ["run-b" :: Text], "algorithm" .= object [], "policy" .= object [], "variedFactors" .= ["control" :: Text], "pairCount" .= (5 :: Int), "metrics" .= object ["latency" .= (10 :: Int)], "reasons" .= ([] :: [Text]), "verdict" .= ("pass" :: Text), "exitCode" .= (0 :: Int)]+        change key value (Object fields) = Object (KeyMap.insert key value fields)+        change _ _ value = value+        original = change "comparisonId" (String "original") (change "harnessRevision" (String "host-a") core)+        replay = change "comparisonId" (String "replay") (change "finishedAt" (String "later") (change "harnessRevision" (String "host-b") core))+    semanticComparison original `shouldBe` semanticComparison replay+    semanticComparison (change "metrics" (object ["latency" .= (11 :: Int)]) replay) `shouldNotBe` semanticComparison original++twoProfiles :: PerfPlan -> (PlannedProfile, PlannedProfile)+twoProfiles plan = case plan.selected of+  [first, second] -> (first, second)+  _ -> error "synthetic plan fixture must contain exactly two profiles"++fixture :: IO PerfPlan+fixture = eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either (fail . show) pure++digest :: Text+digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"++configuration :: Text -> PlannedProfile -> ConfigurationResult+configuration verdict entry = ConfigurationResult entry.profile.name entry.workloadSha256 verdict "healthy" [] entry.profile.minimumPairs entry.profile.minimumPairs [] (Just (comparisonValue verdict entry.profile.minimumPairs))++comparisonValue :: Text -> Int -> Value+comparisonValue verdict count = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= verdict, "pairCount" .= count]++accepted :: Either Text a -> IO a+accepted = either (fail . show) pure
+ test/Kenshou/PerfSpec.hs view
@@ -0,0 +1,230 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.PerfSpec (spec) where++import Control.Monad (forM_)+import Data.Aeson+import Data.ByteString qualified as ByteString+import Data.Either (isLeft)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Cohort qualified as Cohort+import Kenshou.Core.Dimension+import Kenshou.Core.Id (parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Core.Scenario (Placement (..), Tier (..))+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)+import Kenshou.Perf.Plan qualified as Perf+import Kenshou.Perf.Request qualified as Request+import Kenshou.Perf.Source qualified as Source+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Change+import Kenshou.Plan.Components qualified as Graph+import Test.Hspec hiding (context)++spec :: Spec+spec = do+  describe "performance requests" do+    it "normalizes defaults and round-trips explicit request fields" do+      eitherDecode (encode request) `shouldBe` Right request+    it "binds scope, purpose, control and budgets to request identity" do+      map Request.requestDigest [request {Request.coverageMode = "affected"}, request {Request.purpose = "gating"}, request {Request.control = "none"}, request {Request.budgetMinutes = 31}]+        `shouldSatisfy` all (/= Request.requestDigest request)+    it "sorts set fields without changing the digest" do+      Request.requestDigest request {Request.profiles = reverse request.profiles} `shouldBe` Request.requestDigest request+    it "rejects abbreviated, mutable, uppercase and placeholder revisions" do+      mapM_ (\revision -> Request.validateRequest request {Request.baselineRevision = revision} `shouldSatisfy` isLeft) ["main", "HEAD", "BASE_SHA", "abcd", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]+    it "rejects cross-project artifact URIs and malformed canonical project URIs" do+      mapM_ (\project -> Request.validateRequest request {Request.project = project} `shouldSatisfy` isLeft) ["acme", "mori://shinzui/acme/packages/acme-core", "mori://shinzui/../acme"]+    it "rejects A/A with distinct revisions and A/A without explicit profiles" do+      Request.validateRequest request {Request.candidateRevision = shaB} `shouldSatisfy` isLeft+      Request.validateRequest request {Request.profiles = []} `shouldSatisfy` isLeft+    it "rejects duplicate target IDs, empty scope and invalid budgets" do+      mapM_+        (\value -> Request.validateRequest value `shouldSatisfy` isLeft)+        [request {Request.targetComponents = ["acme", "acme"]}, request {Request.profiles = []}, request {Request.budgetMinutes = 0}, request {Request.waitSeconds = -1}]+    it "rejects unknown wire fields instead of silently ignoring a typo" do+      let malformed = object ["schema" .= ("kenshou.perf-request/v1" :: Text), "purpos" .= ("calibration" :: Text)]+      (eitherDecode (encode malformed) :: Either String Request.PerfRequest) `shouldSatisfy` isLeft+  describe "performance source identities for generated cohorts" do+    it "reuses an unchanged cohort across candidate revisions, calibration seeds and measurement scope" do+      descriptor <- template+      manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")+      metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.3\n")+      snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])+      baseline <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)+      let otherCandidate = request {Request.control = "none", Request.candidateRevision = shaB}+          revised = [request {Request.seed = request.seed + 1}, request {Request.cells = ["beta"]}, request {Request.budgetMinutes = 31}, request {Request.waitSeconds = 0}, request {Request.profiles = take 1 request.profiles}, request {Request.purpose = "gating"}, otherCandidate, otherCandidate {Request.candidateRevision = Text.replicate 40 "c"}]+      forM_ revised \selection -> Source.descriptorForSnapshot selection descriptor "baseline" snapshot `shouldBe` Right baseline+      Source.descriptorForSnapshot otherCandidate {Request.baselineRevision = shaB, Request.candidateRevision = shaA} descriptor "candidate" snapshot `shouldBe` Right baseline+      different <- right (Source.sourceSnapshot otherCandidate descriptor upstream shaB [manifest, metrics])+      Source.descriptorForSnapshot otherCandidate descriptor "candidate" different `shouldNotBe` Right baseline+    it "invalidates cohort identity when compiler, index state or fixed dependency inputs change" do+      descriptor <- template+      manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")+      metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.3\n")+      snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])+      original <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)+      let altered = descriptor {Cohort.descriptorComponents = map (\component -> if component.componentId == Cohort.ComponentId "acme" then component else component {Cohort.componentPackages = map (\pin -> pin {Cohort.pinVersion = "99.0.0"}) component.componentPackages}) descriptor.descriptorComponents}+      forM_ [descriptor {Cohort.descriptorCompiler = "ghc-9.14.1"}, descriptor {Cohort.descriptorIndexState = "2026-10-11T00:00:00Z"}, altered] \changed -> Source.descriptorForSnapshot request changed "baseline" snapshot `shouldNotBe` Right original+    it "reads manifest version and package directory independently in each revision" do+      descriptor <- template+      manifest <- right (Source.parsePackageManifest "moved/acme-core.cabal" "name: acme-core\nversion: 0.9.0.4\n")+      metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.4\n")+      snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])+      generated <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)+      otherArm <- right (Source.descriptorForSnapshot request descriptor "candidate" snapshot)+      generated `shouldBe` otherArm+      target <- first (filter ((== Cohort.ComponentId "acme") . (.componentId)) generated.descriptorComponents)+      target.componentSource `shouldBe` Cohort.GitSource upstream shaA+      map (.pinVersion) target.componentPackages `shouldBe` ["0.9.0.4", "0.9.0.4"]+      map (.pinSubdir) target.componentPackages `shouldBe` [Just "moved", Just "acme-metrics"]+      filter ((/= Cohort.ComponentId "acme") . (.componentId)) generated.descriptorComponents+        `shouldBe` filter ((/= Cohort.ComponentId "acme") . (.componentId)) descriptor.descriptorComponents+    it "rejects unknown descriptor IDs and targets owned by a different project" do+      descriptor <- template+      Source.targetPackages request {Request.targetComponents = ["unknown"]} descriptor `shouldSatisfy` isLeft+      Source.targetPackages request {Request.targetComponents = ["acme-quill-adapter"]} descriptor `shouldSatisfy` isLeft+    it "rejects missing and duplicate packages at a requested revision" do+      descriptor <- template+      manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")+      Source.sourceSnapshot request descriptor upstream shaA [manifest] `shouldSatisfy` isLeft+      Source.sourceSnapshot request descriptor upstream shaA [manifest, manifest] `shouldSatisfy` isLeft+    it "rejects mutable template source pins" do+      descriptor <- template+      let components = map (\component -> component {Cohort.componentSource = Cohort.GitSource upstream "main"}) descriptor.descriptorComponents+      Source.targetPackages request descriptor {Cohort.descriptorComponents = components} `shouldSatisfy` isLeft+    it "rejects forged wire paths and manifest digests" do+      manifest <- right (Source.parsePackageManifest "safe/package.cabal" "name: package\nversion: 1.0\n")+      (eitherDecode (encode manifest {Source.path = "../outside.cabal"}) :: Either String Source.PackageManifest) `shouldSatisfy` isLeft+      (eitherDecode (encode manifest {Source.sha256 = "sha256:fake"}) :: Either String Source.PackageManifest) `shouldSatisfy` isLeft+    it "rejects ambiguous identity and paths that escape generated inputs" do+      Source.parsePackageManifest "../bad.cabal" "name: bad\nversion: 1.0\n" `shouldSatisfy` isLeft+      Source.parsePackageManifest "bad.cabal" "name: bad\nname: other\nversion: 1.0\n" `shouldSatisfy` isLeft+  describe "bounded performance planning" do+    it "retains explicit A/A profiles in calibration mode with an empty diff" do+      (graph, profiles, catalog) <- context+      plan <- right (Perf.planPerformance syntheticContracts request graph catalog profiles [] [] [] policyDigests Map.empty)+      length plan.selected `shouldBe` 2+      plan.coverage `shouldBe` "complete"+      plan.gatingEligible `shouldBe` False+      plan.estimateSeconds `shouldBe` 1540+      eitherDecode (encode plan) `shouldBe` Right plan+      map (.reasons) plan.selected `shouldSatisfy` all (not . null)+    it "cannot admit a catalog's unverified calibration claim to a gating plan" do+      (graph, profiles, catalog) <- context+      let claimed = profiles {Perf.profiles = map (\profile -> profile {Perf.calibration = Just (object ["calibrated" .= True])}) profiles.profiles}+      plan <- right (Perf.planPerformance syntheticContracts request {Request.purpose = "gating"} graph catalog claimed [] [] [] policyDigests Map.empty)+      plan.coverage `shouldBe` "incomplete"+      map snd plan.excluded `shouldBe` ["uncalibrated", "uncalibrated"]+      plan.gatingEligible `shouldBe` False+    it "binds workload identity without making calibration promotion self-referential" do+      (_, profiles, _) <- context+      profile <- first profiles.profiles+      Perf.workloadDigest profile "sha256:policy" `shouldBe` Perf.workloadDigest profile {Perf.calibration = Just (String "evidence")} "sha256:policy"+      Perf.workloadDigest profile "sha256:policy" `shouldNotBe` Perf.workloadDigest profile {Perf.scenarioRevision = 3} "sha256:policy"+    it "defers an entire configuration when the paired estimate exceeds budget" do+      (graph, profiles, catalog) <- context+      plan <- right (Perf.planPerformance syntheticContracts request {Request.budgetMinutes = 13} graph catalog profiles [] [] [] policyDigests Map.empty)+      length plan.selected `shouldBe` 1+      map snd plan.excluded `shouldBe` ["over-budget"]+      plan.coverage `shouldBe` "incomplete"+    it "reports transitive dependent and uncovered correctness paths" do+      (graph, profiles, catalog) <- context+      reference <- right (Graph.parseRef graph "acme-core:contract")+      let changes = [Change reference Named "core contract changed"]+      plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none", Request.coverageMode = "affected", Request.profiles = []} graph catalog profiles changes [] [] policyDigests Map.empty)+      plan.affectedScenarios `shouldContain` ["acme/quill-adapter/benchmark/end-to-end-throughput-latency"]+      plan.uncoveredScenarios `shouldContain` ["acme/metrics/correctness/test"]+      plan.coverage `shouldBe` "incomplete"+    it "separates selected-profile completion from whole-change coverage" do+      (graph, profiles, catalog) <- context+      reference <- right (Graph.parseRef graph "acme-core:contract")+      plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none"} graph catalog profiles [Change reference Named "core changed"] [] [] policyDigests Map.empty)+      plan.coverage `shouldBe` "complete"+      plan.changeCoverage `shouldBe` "incomplete"+    it "does not certify a telemetry change with disabled telemetry" do+      (graph, profiles, catalog) <- context+      reference <- right (Graph.parseRef graph "acme-core:telemetry")+      plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none", Request.coverageMode = "affected", Request.profiles = []} graph catalog profiles [Change reference Named "telemetry changed"] [] [] policyDigests Map.empty)+      plan.uncoveredScenarios `shouldContain` ["acme/core-ordering/benchmark/concurrency-sweep"]+      plan.coverage `shouldBe` "incomplete"+    it "selects conservatively for unmapped build inputs" do+      (graph, _, _) <- context+      let (changes, unknown, documentation) = Perf.conservativeChanges graph "acme" ["cabal.project", "docs/guide.md"] []+      map (.source) changes `shouldBe` [Everything]+      unknown `shouldBe` ["cabal.project"]+      documentation `shouldBe` ["docs/guide.md"]+    it "requires a repository-scoped documentation proof for not-applicable" do+      (graph, profiles, catalog) <- context+      let normal = request {Request.control = "none", Request.profiles = [], Request.coverageMode = "affected"}+      empty <- right (Perf.planPerformance syntheticContracts normal graph catalog profiles [] [] [] policyDigests Map.empty)+      empty.coverage `shouldBe` "incomplete"+      proved <- right (Perf.planPerformance syntheticContracts normal graph catalog profiles [] [] ["docs/guide.md"] policyDigests Map.empty)+      proved.coverage `shouldBe` "not-applicable"+      proved.gatingEligible `shouldBe` False+    it "rejects phases the pinned scenario revision cannot honor" do+      (_, profiles, catalog) <- context+      profile <- first profiles.profiles+      Perf.validateProfile syntheticContracts catalog profile {Perf.warmupSeconds = 1} `shouldSatisfy` isLeft+      Perf.validateProfile syntheticContracts catalog profile {Perf.steadyCount = 2000} `shouldSatisfy` isLeft+      Perf.validateProfile syntheticContracts catalog profile {Perf.timeoutSeconds = 181} `shouldSatisfy` isLeft+    it "rejects unknown profile names and raw sample suppression" do+      (graph, profiles, catalog) <- context+      Perf.planPerformance syntheticContracts request {Request.profiles = ["missing"]} graph catalog profiles [] [] [] policyDigests Map.empty `shouldSatisfy` isLeft+      profile <- first profiles.profiles+      Perf.validateProfile syntheticContracts catalog profile {Perf.knobs = Map.insert "measure.raw-samples" (String "off") profile.knobs} `shouldSatisfy` isLeft++shaA, shaB, upstream :: Text+shaA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+shaB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+upstream = "https://github.com/shinzui/acme.git"++request :: Request.PerfRequest+request = Request.PerfRequest "mori://shinzui/acme" shaA shaA "cohort/released.json" ["acme"] "change-performance" "selected-profiles" "calibration" "aa" ["acme-core-quick", "acme-quill-quick"] 20261010 ["alpha"] 90 30++policyDigests :: Map.Map Text Text+policyDigests = Map.singleton "policies/default.json" "sha256:policy"++context :: IO (Graph.ComponentGraph, Perf.ProfileCatalog, [ScenarioInfo])+context = do+  graph <- Graph.loadGraphFile "test/fixtures/plan/components.synthetic.json" >>= right+  profiles <- load "catalog.v1.json"+  catalog <- traverse makeScenario [(profile.scenario, profile.scenarioRevision, True) | profile <- profiles.profiles]+  correctness <- makeScenario ("acme/metrics/correctness/test", 1, False)+  pure (graph, profiles, catalog <> [correctness])+  where+    makeScenario (name, revision, benchmark) = do+      identifier <- right (parseScenarioId name)+      let dimensions = if name == "acme/core-ordering/benchmark/concurrency-sweep" then allTelemetryArms noDimensions else postgresDimensions (PgDurable :| []) (Pg18 :| [Pg17]) noDimensions+      knobs <- traverse makeKnob [("bench.arm", KnobText, VText "configured"), ("bench.rate", KnobInt, VInt 200), ("acme.messages", KnobInt, VInt 1000), ("measure.raw-samples", KnobText, VText "full")]+      pure (ScenarioInfo identifier revision "fixture" TierStandard PlaceEither (if benchmark then knobs else []) dimensions Nothing)+    makeKnob (name, kind, def) = do+      key <- right (mkKnobName name)+      pure (KnobSpec key "fixture" kind def AnyValue [])++template :: IO Cohort.CohortDescriptor+template = load "cohort.json"++load :: (FromJSON a) => FilePath -> IO a+load path = ByteString.readFile ("test/fixtures/perf/" <> path) >>= right . eitherDecodeStrict'++right :: (Show e) => Either e a -> IO a+right (Right value) = pure value+right (Left problem) = expectationFailure (show problem) >> fail "expected Right"++first :: [a] -> IO a+first (value : _) = pure value+first [] = expectationFailure "expected a nonempty fixture" >> fail "empty fixture"++-- The two synthetic benchmark revisions the catalog's profiles pin implement+-- count-bound steady work; their steady count is the acme.messages knob.+syntheticContracts :: PerfContracts+syntheticContracts =+  noPerfContracts+    { reviewedPhaseContracts = [("acme/core-ordering/benchmark/concurrency-sweep", 2), ("acme/quill-adapter/benchmark/end-to-end-throughput-latency", 1)],+      steadyCountKnob = either (const Nothing) Just (mkKnobName "acme.messages")+    }
+ test/Kenshou/PlanSpec.hs view
@@ -0,0 +1,336 @@+module Kenshou.PlanSpec (spec) where++import Data.Aeson qualified as Aeson+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Dimension+import Kenshou.Core.Id (Kind (..), mkSeed, parseRunId, parseScenarioId, renderScenarioId)+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.RunSpec (SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Tier (..))+import Kenshou.Plan.Catalog (ScenarioInfo (..), readCatalogFile)+import Kenshou.Plan.Change+import Kenshou.Plan.Change.Cohort+import Kenshou.Plan.Change.Git qualified as Git+import Kenshou.Plan.Components+import Kenshou.Plan.Components.Check+import Kenshou.Plan.Matrix+import Kenshou.Plan.Policy+import Kenshou.Plan.Policy qualified as Policy+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlanSkeleton (..), SkipReason (..), Skipped (..), buildPlan)+import Kenshou.Plan.Selector+import Kenshou.Plan.Suite+import Kenshou.Plan.Summary qualified as Summary+import System.Directory (createDirectoryIfMissing)+import System.FilePath (takeDirectory, (</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (callProcess)+import Test.Hspec++spec :: Spec+spec = do+  describe "Kenshou.Plan.Selector" do+    it "round-trips and matches recursive selectors" do+      selector <- expectRight (parseSelector "acme/quill-adapter/**")+      scenario <- expectRight (parseScenarioId "acme/quill-adapter/correctness/finalize")+      renderSelector selector `shouldBe` "acme/quill-adapter/**"+      matches selector scenario `shouldBe` True+    it "rejects recursive wildcards before the final position" do+      parseSelector "acme/**/correctness/*" `shouldBe` Left "** is permitted only as the final segment"++  describe "Kenshou.Plan.Catalog" do+    it "decodes the planned scenario catalog" do+      result <- readCatalogFile "test/fixtures/plan/catalog-planned.json"+      catalog <- expectRight result+      length catalog `shouldSatisfy` (> 40)++  describe "Kenshou.Plan.Components" do+    it "validates the embedded graph" do+      graph <- syntheticGraph >>= expectRight+      validateGraph graph `shouldBe` []+    it "computes the quill dependent closure" do+      graph <- syntheticGraph >>= expectRight+      origin <- expectRight (parseRef graph "quill-hs")+      let affected = dependents graph (Set.singleton origin)+          present text = either (const False) (`Map.member` affected) (parseRef graph text)+      present "acme-quill-adapter" `shouldBe` True+      present "flux-quill" `shouldBe` True+      present "kenshou-harness" `shouldBe` True+      present "runtime-assembly" `shouldBe` True+    it "keeps acme runner changes away from flux contract consumers" do+      graph <- syntheticGraph >>= expectRight+      origin <- expectRight (parseRef graph "acme-core:runner")+      let affected = dependents graph (Set.singleton origin)+          present text = either (const False) (`Map.member` affected) (parseRef graph text)+      present "flux-quill" `shouldBe` True+      present "acme-metrics" `shouldBe` True+      present "acme-tally-adapter" `shouldBe` True+      present "runtime-assembly" `shouldBe` True+      present "flux" `shouldBe` False+      present "flux:router" `shouldBe` False+    it "routes acme contract changes to the three flux consumers" do+      graph <- syntheticGraph >>= expectRight+      origin <- expectRight (parseRef graph "acme-core:contract")+      let affected = dependents graph (Set.singleton origin)+          present text = either (const False) (`Map.member` affected) (parseRef graph text)+      present "flux:process-manager" `shouldBe` True+      present "flux:router" `shouldBe` True+      present "flux:inbox" `shouldBe` True+      present "flux:command" `shouldBe` False+      present "flux:outbox" `shouldBe` False+      present "flux:workflow" `shouldBe` False++  describe "Kenshou.Plan.Components.Check" do+    it "resolves shortened unit ids and finds a planted missing edge" do+      graph <- syntheticGraph >>= expectRight+      result <- checkAgainstPlanJson graph "test/fixtures/plan/plan-missing-edge.json"+      findings <- expectRight result+      findings `shouldSatisfy` any isMissing++  describe "Kenshou.Plan.Change" do+    it "selects only the QUILL family and its dependents" do+      (graph, catalog) <- fixtureInputs+      changed <- named graph "quill-hs"+      selectedIds (selectScenarios graph catalog [changed])+        `shouldBe` expectedIds catalog ["quill/", "acme/quill-adapter/", "flux/queue/", "runtime/", "selftest/kernel/correctness/postgres-roundtrip"]+    it "selects all Tally, Flux, runtime, and kernel evidence for tally-store" do+      (graph, catalog) <- fixtureInputs+      changed <- named graph "tally-store"+      selectedIds (selectScenarios graph catalog [changed])+        `shouldBe` expectedIds catalog ["tally/", "acme/tally-adapter/", "flux/", "runtime/", "selftest/kernel/"]+    it "distinguishes acme runner changes from contract changes" do+      (graph, catalog) <- fixtureInputs+      runner <- named graph "acme-core:runner"+      contract <- named graph "acme-core:contract"+      selectedIds (selectScenarios graph catalog [runner])+        `shouldBe` expectedIds catalog ["acme/", "brook/", "flux/queue/", "runtime/"]+      let contractIds = selectedIds (selectScenarios graph catalog [contract])+      contractIds `shouldSatisfy` Set.member "flux/process-manager/correctness/react"+      contractIds `shouldSatisfy` Set.member "flux/router/correctness/route"+      contractIds `shouldSatisfy` Set.member "flux/inbox/correctness/deduplicate"+      contractIds `shouldSatisfy` Set.notMember "flux/command/correctness/decide"+      contractIds `shouldSatisfy` Set.notMember "flux/outbox/concurrency/publish"+      contractIds `shouldSatisfy` Set.notMember "flux/workflow/correctness/resume"+    it "maps an outbox-only upstream diff and ignores upstream documentation" do+      (graph, catalog) <- fixtureInputs+      withSystemTempDirectory "kenshou-upstream" \repository -> do+        initialiseRepository repository+        writeRepoFile repository "flux/src/Flux/Outbox/Schema.hs" "module Schema where\n"+        writeRepoFile repository "docs/readme.md" "initial\n"+        commitAll repository "initial"+        writeRepoFile repository "flux/src/Flux/Outbox/Schema.hs" "module Schema where\nvalue = 1\n"+        commitAll repository "outbox"+        result <- Git.changesFromUpstream graph (Git.UpstreamDiff "flux" repository "HEAD^" "HEAD")+        (changes, ignored) <- expectRight result+        ignored `shouldBe` 0+        selectedIds (selectScenarios graph catalog changes)+          `shouldBe` expectedIds catalog ["flux/outbox/", "runtime/"]+        writeRepoFile repository "docs/readme.md" "changed\n"+        commitAll repository "docs"+        docsResult <- Git.changesFromUpstream graph (Git.UpstreamDiff "flux" repository "HEAD^" "HEAD")+        (docsChanges, docsIgnored) <- expectRight docsResult+        docsChanges `shouldBe` []+        docsIgnored `shouldBe` 1+    it "maps --since repository paths to selectors and ignores docs" do+      (graph, catalog) <- fixtureInputs+      withSystemTempDirectory "kenshou-since" \repository -> do+        initialiseRepository repository+        writeRepoFile repository "kenshou-tally/x" "initial\n"+        writeRepoFile repository "docs/z" "initial\n"+        commitAll repository "initial"+        writeRepoFile repository "kenshou-tally/x" "changed\n"+        result <- Git.changesSince graph repository "HEAD"+        (changes, selectors, warnings) <- expectRight result+        changes `shouldBe` []+        warnings `shouldBe` []+        selectedIds (selectBySelectors Since "repository path" selectors catalog)+          `shouldBe` expectedIds catalog ["tally/"]+    it "maps a cohort package difference and fails safe for unknown packages" do+      (graph, _catalog) <- fixtureInputs+      let old = Map.fromList [("tally-store", FromHackage "0.8.0.1")]+          new = Map.fromList [("tally-store", FromHackage "0.8.0.2"), ("unmapped", FromHackage "1")]+          (changes, warnings) = diffCohorts graph old new+      fmap (.ref) changes `shouldContain` [ComponentRef (ComponentId "tally-store") Nothing]+      fmap (.source) changes `shouldContain` [Everything]+      fmap (.code) warnings `shouldContain` ["unmapped-package"]++  describe "Kenshou.Plan.Matrix" do+    it "covers every pair in at most twenty rows" do+      let dimensions = [("a", ["0", "1", "2", "3"]), ("b", ["0", "1", "2", "3"]), ("c", ["0", "1"]), ("d", ["0", "1"])]+          rows = pairwiseCover dimensions+      length rows `shouldSatisfy` (<= 20)+      pairSet rows `shouldBe` pairSet (fullRows dimensions)+    it "raises an observability change to telemetry corners" do+      selected <- matrixSelected Correctness (Just "telemetry-corners")+      seed <- expectRight (mkSeed 42)+      let (configs, skipped) = expandScenario (defaultPlanPolicy seed) selected+      skipped `shouldBe` []+      length configs `shouldBe` 4+    it "never emits fsync-off for a benchmark" do+      selected <- matrixSelected Benchmark Nothing+      seed <- expectRight (mkSeed 42)+      let policy = (defaultPlanPolicy seed :: PlanPolicy) {Policy.placement = RunOnCell}+          (configs, skipped) = expandScenario policy selected+      skipped `shouldBe` []+      fmap (Map.lookup "pg.durability" . (.dimensions)) configs `shouldSatisfy` all (== Just "durable")+    it "admits a benchmark without PostgreSQL without inventing a durability dimension" do+      selected <- matrixSelected Benchmark Nothing+      seed <- expectRight (mkSeed 42)+      let scenario = (selected.scenario :: ScenarioInfo) {dimensions = noDimensions}+          policy = (defaultPlanPolicy seed :: PlanPolicy) {Policy.placement = RunOnCell}+          (configs, skipped) = expandScenario policy (selected {scenario = scenario})+      skipped `shouldBe` []+      configs `shouldSatisfy` (not . null)+      fmap (Map.lookup "pg.durability" . (.dimensions)) configs `shouldSatisfy` all (== Nothing)++  describe "Kenshou.Plan.RunPlan" do+    it "keeps benchmark trial groups whole under a budget" do+      selected <- matrixSelected Benchmark Nothing+      seed <- expectRight (mkSeed 42)+      let policy = (defaultPlanPolicy seed :: PlanPolicy) {Policy.placement = RunOnCell, Policy.budgetMinutes = Just 2}+          skeleton = buildPlan testPlanContext policy [selected]+      skeleton.runs `shouldBe` []+      fmap (.reason) skeleton.skipped `shouldContain` [SkipOverBudget]++  describe "Kenshou.Plan.Suite" do+    it "decodes every checked-in suite" do+      let names = ["smoke", "change", "nightly", "weekly-soak", "release"] :: [Text]+      results <- traverse (readSuite . ("../suites/" <>) . (<> ".json") . Text.unpack) names+      suites <- traverse expectRight results+      fmap (.name) suites `shouldBe` names+    it "raises knob policy only for directly selected scenarios" do+      suite <- readSuite "../suites/change.json" >>= expectRight+      (_, catalog) <- fixtureInputs+      scenario <- maybe (expectationFailure "fixture has no scenario with knobs" >> fail "unreachable") pure (find (not . null . (.knobs)) catalog)+      selector <- expectRight (parseSelector "**")+      let change = Change (ComponentRef (ComponentId "fixture") Nothing) Named "fixture"+          selected distance = Selected scenario (Reason change [change.ref] selector distance :| []) Nothing Nothing+          direct = applySuite suite [scenario] [selected 0]+          dependent = applySuite suite [scenario] [selected 1]+      fmap (.minKnobPolicy) direct `shouldBe` [Just "declared-variants"]+      fmap (.minKnobPolicy) dependent `shouldBe` [Nothing]++  describe "Kenshou.Plan.Summary" do+    it "treats reproduced known defects as non-blocking" do+      runId <- expectRight (parseRunId "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011")+      scenarioId <- expectRight (parseScenarioId "selftest/kernel/correctness/known-defect")+      let attempt = Summary.Attempt runId (read "2026-09-21 00:00:00 UTC") (Just (read "2026-09-21 00:00:01 UTC")) (Just Failed) True (Just 0)+          summary = Summary.PlanSummary runId [Summary.SummaryEntry 1 scenarioId Summary.Completed [attempt]] Map.empty Passed 0+      Summary.worstOutcome summary `shouldBe` Passed+    it "counts an unfinished entry as errored" do+      runId <- expectRight (parseRunId "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011")+      scenarioId <- expectRight (parseScenarioId "selftest/kernel/correctness/always-pass")+      let summary = Summary.PlanSummary runId [Summary.SummaryEntry 1 scenarioId Summary.Pending []] Map.empty Passed 0+      Summary.worstOutcome summary `shouldBe` Errored+  where+    isMissing (MissingEdge (ComponentId "quill-hs") (ComponentId "tally-store") _) = True+    isMissing _ = False++fixtureInputs :: IO (ComponentGraph, [ScenarioInfo])+fixtureInputs = do+  graph <- syntheticGraph >>= expectRight+  catalog <- readCatalogFile "test/fixtures/plan/catalog-planned.json" >>= expectRight+  pure (graph, catalog)++named :: ComponentGraph -> Text -> IO Change+named graph reference = do+  parsed <- expectRight (parseRef graph reference)+  pure (Change parsed Named ("--changed " <> reference))++selectedIds :: [Selected] -> Set.Set Text+selectedIds = Set.fromList . fmap (renderScenarioId . (.id) . (.scenario))++expectedIds :: [ScenarioInfo] -> [Text] -> Set.Set Text+expectedIds catalog prefixes =+  Set.fromList+    [ identifier+    | scenario <- catalog,+      let identifier = renderScenarioId scenario.id,+      any (`Text.isPrefixOf` identifier) prefixes+    ]++initialiseRepository :: FilePath -> IO ()+initialiseRepository repository = do+  callProcess "git" ["-C", repository, "init", "-q"]+  callProcess "git" ["-C", repository, "config", "user.email", "kenshou@example.invalid"]+  callProcess "git" ["-C", repository, "config", "user.name", "Kenshou Test"]++writeRepoFile :: FilePath -> FilePath -> String -> IO ()+writeRepoFile repository relative contents = do+  createDirectoryIfMissing True (repository </> takeDirectory relative)+  writeFile (repository </> relative) contents++commitAll :: FilePath -> String -> IO ()+commitAll repository message = do+  callProcess "git" ["-C", repository, "add", "."]+  callProcess "git" ["-C", repository, "commit", "-q", "-m", message]++matrixSelected :: Kind -> Maybe Text -> IO Selected+matrixSelected kind minimumPolicy = do+  scenarioId <- expectRight (parseScenarioId ("tally/append/" <> kindText kind <> "/matrix"))+  selector <- expectRight (parseSelector "tally/**")+  let scenario =+        ScenarioInfo+          { id = scenarioId,+            revision = 1,+            summary = "matrix fixture",+            tier = TierSmoke,+            placement = if kind == Benchmark then PlaceCell else PlaceEither,+            knobs = [],+            dimensions = allMatrixDimensions,+            knownDefect = Nothing+          }+      change = Change (ComponentRef (ComponentId "tally-store") Nothing) Named "fixture"+  pure (Selected scenario (Reason change [change.ref] selector 0 :| []) minimumPolicy Nothing)+  where+    kindText Correctness = "correctness"+    kindText Concurrency = "concurrency"+    kindText Benchmark = "benchmark"+    kindText Soak = "soak"++allMatrixDimensions :: DimensionSupport+allMatrixDimensions =+  DimensionSupport+    (Supported (Support (TracingOff :| [TracingNoop, TracingSdkInMemory, TracingSdkOtlp]) TracingOff))+    (Supported (Support (MetricsOff :| [MetricsCollect, MetricsServe, MetricsServeScraped]) MetricsOff))+    (Supported (Support (PgFsyncOff :| [PgDurable]) PgFsyncOff))+    (Supported (Support (Pg17 :| [Pg18]) Pg18))++pairSet :: [Map.Map Text Text] -> Set.Set ((Text, Text), (Text, Text))+pairSet rows =+  Set.fromList+    [ (left, right)+    | row <- rows,+      (leftIndex, left) <- zip [0 :: Int ..] (Map.toAscList row),+      (rightIndex, right) <- zip [0 :: Int ..] (Map.toAscList row),+      leftIndex < rightIndex+    ]++fullRows :: [(Text, [Text])] -> [Map.Map Text Text]+fullRows = foldr (\(name, values) rows -> [Map.insert name value row | value <- values, row <- rows]) [Map.empty]++testPlanContext :: PlanContext+testPlanContext =+  PlanContext+    { suite = Nothing,+      graphDigest = "sha256:test",+      cohortName = "released",+      cohortPlanHash = "sha256:test",+      inputs = PlanInputs (Aeson.object []),+      changes = [],+      warnings = []+    }++expectRight :: (Show problem) => Either problem value -> IO value+expectRight (Right value) = pure value+expectRight (Left problem) = expectationFailure (show problem) >> fail "unreachable"++-- A component graph with the shapes the planner must handle (libraries with+-- subcomponents and uses edges, dependsOn edges with fromSubs and toSubs, a+-- changeImplies policy, and repository paths) under synthetic names.+syntheticGraph :: IO (Either GraphError ComponentGraph)+syntheticGraph = loadGraphFile "test/fixtures/plan/components.synthetic.json"
+ test/Kenshou/ResourceSpec.hs view
@@ -0,0 +1,51 @@+module Kenshou.ResourceSpec (spec) where++import Data.Aeson (FromJSON, Value (..), eitherDecodeFileStrict', object, toJSON, (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Core.Canonical (canonicalEncode)+import Kenshou.Core.Compat (renderResourceInputs)+import Kenshou.Core.Resource.Types (ResourceName, mkResourceName, renderResourceName)+import Kenshou.Core.Role (WorkerInit (..))+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Core.RunSpec qualified as RunSpec+import Test.Hspec++spec :: Spec+spec = describe "named resources" do+  it "validates resource names" do+    renderResourceName <$> mkResourceName "object-store" `shouldBe` Right "object-store"+    mapM_ (\bad -> isLeft (mkResourceName bad) `shouldBe` True) ["", "Object", "9lives", "has space", "x_y"]+  it "round-trips a run specification with a resource map to identical canonical JSON" do+    raw <- decodeFile "test/golden/resources/run-spec.resources.json" :: IO Value+    decoded <- decodeFile "test/golden/resources/run-spec.resources.json" :: IO RunSpec+    Map.keys decoded.environment.resources `shouldBe` [name "broker", name "object-store"]+    canonicalEncode (toJSON decoded) `shouldBe` canonicalEncode raw+  it "round-trips a worker initialization with a resource map to identical canonical JSON" do+    raw <- decodeFile "test/golden/resources/worker-init.resources.json" :: IO Value+    decoded <- decodeFile "test/golden/resources/worker-init.resources.json" :: IO WorkerInit+    Map.keys decoded.resources `shouldBe` [name "broker", name "object-store"]+    canonicalEncode (toJSON decoded) `shouldBe` canonicalEncode raw+  it "omits an empty resource map from the run specification but always writes it for workers" do+    decoded <- decodeFile "test/golden/resources/run-spec.resources.json" :: IO RunSpec+    let emptied = decoded {environment = decoded.environment {RunSpec.resources = Map.empty}}+    environmentKeys (toJSON emptied) `shouldSatisfy` notElem "resources"+    worker <- decodeFile "test/golden/resources/worker-init.resources.json" :: IO WorkerInit+    case toJSON worker {resources = Map.empty} of+      Object fields -> KeyMap.lookup "resources" fields `shouldBe` Just (object [])+      _ -> expectationFailure "worker-init is not an object"+  it "renders resources under one compatibility member, omitted when empty" do+    let value = object ["bucket" .= ("fixture" :: Text)]+    object (renderResourceInputs (Map.singleton (name "object-store") value)) `shouldBe` object ["resources" .= object ["object-store" .= value]]+    object (renderResourceInputs Map.empty) `shouldBe` object []+  where+    name text = either (error . show) id (mkResourceName text) :: ResourceName+    environmentKeys (Object fields) = case KeyMap.lookup "environment" fields of+      Just (Object environment) -> KeyMap.keys environment+      _ -> []+    environmentKeys _ = []++decodeFile :: (FromJSON a) => FilePath -> IO a+decodeFile path = eitherDecodeFileStrict' path >>= either fail pure
+ test/Main.hs view
@@ -0,0 +1,582 @@+{-# LANGUAGE FieldSelectors #-}++module Main (main) where++import Control.Exception (bracket_)+import Data.Aeson (FromJSON, Value (..), eitherDecode, eitherDecodeFileStrict', encode, encodeFile, object, toJSON, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.Either (isLeft, isRight)+import Data.Foldable (toList)+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Database.PostgreSQL.Migrate (migrationComponent, sqlMigration)+import Kenshou.Core.Bundle (LayerBundle (..), ListFilter (..), RegistryError (..), ScenarioRule (..), allScenarios, emptyListFilter, mkBundle, mkRegistry, registryMigrationSets, requirePgVersionsFor, selectScenarios)+import Kenshou.Core.Bundle qualified as Bundle+import Kenshou.Core.Canonical (canonicalEncode)+import Kenshou.Core.Cohort+import Kenshou.Core.Compat (comparisonKey, compatInputs, seriesKey)+import Kenshou.Core.Context (Environment (..), RunContext (..))+import Kenshou.Core.Dimension+import Kenshou.Core.Env (EnvRequirements (..), MigrationSet (..), MigrationSetName (..), PostgresRequirement (..), noEnvironment, requiresResource)+import Kenshou.Core.Env.Migration (composePlan, orderMigrationSets)+import Kenshou.Core.Env.Postgres (EnvError (..), PgSettingsSnapshot (..), PostgresEnv (..), ServerControl (..), StopMode (..), withPostgresEnv, withPostgresEnvNamed)+import Kenshou.Core.Id+import Kenshou.Core.Knob+import Kenshou.Core.Log (nullLogger)+import Kenshou.Core.Manifest (verifyManifest, writeManifest)+import Kenshou.Core.Outcome (Outcome (Passed))+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.Resource (ProvisionedResource (..), ResourceProvider (..), mkResourceName)+import Kenshou.Core.Run (RunOutput (..), RunnerConfig (..), defectDisposition, executeRun)+import Kenshou.Core.RunResult (KnownDefectStatus (..))+import Kenshou.Core.RunSpec+import Kenshou.Core.RunSpec.Resolve (resolveRunSpec)+import Kenshou.Core.Scenario (CohortScope (..), KnownDefect (..), PackageCondition (..), Placement (..), Scenario (..), ScenarioReport (..), Tier (..), failedWith)+import Kenshou.Core.Selector (matchesSelector, parseSelector)+import Kenshou.Core.Selftest qualified as Selftest+import Kenshou.Core.Selftest.Schema (selftestMigrationSet, selftestSchema)+import Kenshou.PerfCalibrationSpec qualified+import Kenshou.PerfPreparedSpec qualified+import Kenshou.PerfResultSpec qualified+import Kenshou.PerfSpec qualified+import Kenshou.PlanSpec qualified+import Kenshou.ResourceSpec qualified+import System.Environment (lookupEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (readProcessWithExitCode)+import Test.Hspec+  ( Spec,+    describe,+    expectationFailure,+    hspec,+    it,+    pendingWith,+    shouldBe,+    shouldContain,+    shouldNotBe,+    shouldReturn,+  )++main :: IO ()+main = hspec do+  Kenshou.PerfCalibrationSpec.spec+  Kenshou.PerfResultSpec.spec+  descriptorSpec+  hashSpec+  mismatchSpec+  activeCohortSpec+  identifierSpec+  layerIdentifierSpec+  selectorSpec+  registrySpec+  openRegistrySpec+  resourceProvisioningSpec+  knownDefectGroupSpec+  knobSpec+  dimensionSpec+  runSpecSpec+  compatibilitySpec+  manifestSpec+  goldenSpec+  postgresEnvironmentSpec+  migrationPlanSpec+  Kenshou.PlanSpec.spec+  Kenshou.ResourceSpec.spec+  Kenshou.PerfPreparedSpec.spec+  Kenshou.PerfSpec.spec++-- The consumer's own layer strings are checked against its sealed records in the+-- kenshou-evidence consumer compatibility tests; these cases use synthetic names.+layerIdentifierSpec :: Spec+layerIdentifierSpec = describe "layer identifiers" do+  it "renders the self-test layer" do+    renderLayer selftestLayer `shouldBe` "selftest"+  it "accepts any valid segment and renders scenario identifiers in any layer" do+    renderLayer <$> parseLayer "acme" `shouldBe` Right "acme"+    renderScenarioId <$> parseScenarioId "acme/widget/correctness/roundtrip" `shouldBe` Right "acme/widget/correctness/roundtrip"+  it "rejects invalid segments" do+    isLeft (parseLayer "Acme") `shouldBe` True+    isLeft (parseLayer "") `shouldBe` True++descriptorSpec :: Spec+descriptorSpec = describe "cohort identity" do+  it "matches the stable JSON golden" do+    descriptor <- decodeFixture "descriptor-released.json"+    plan <- decodeFixture "plan-released.json"+    golden <- decodeFixture "cohort-identity.golden.json"+    case identityFromPlan descriptor "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" plan of+      Left err -> expectationFailure (show err)+      Right identity -> toJSON identity `shouldBe` (golden :: Value)+  it "round-trips a Nix resolver while old Cabal records omit it" do+    let cabalIdentity = identityWith []+        nixIdentity = cabalIdentity {identityResolver = Just "nix"}+    eitherDecode (encode cabalIdentity) `shouldBe` Right cabalIdentity+    eitherDecode (encode nixIdentity) `shouldBe` Right nixIdentity++hashSpec :: Spec+hashSpec = describe "planHash" do+  it "ignores install-plan order and local checkout paths" do+    baseline <- hashFixture "plan-released.json"+    reordered <- hashFixture "plan-reordered.json"+    moved <- hashFixture "plan-local-change.json"+    reordered `shouldBe` baseline+    moved `shouldBe` baseline++  it "changes with a package version or git revision" do+    baseline <- hashFixture "plan-released.json"+    versionChanged <- hashFixture "plan-version-change.json"+    commitChanged <- hashFixture "plan-commit-change.json"+    versionChanged `shouldNotBe` baseline+    commitChanged `shouldNotBe` baseline++mismatchSpec :: Spec+mismatchSpec = describe "checkCohort" do+  it "reports every mismatch constructor" do+    let descriptor = onePackageDescriptor+        missing = identityWith []+        wrongVersion = identityWith [ResolvedPackage "foo" "2.0.0" (FromHackage Nothing)]+        wrongSource = identityWith [ResolvedPackage "foo" "1.0.0" (FromGit "https://example.com/foo.git" "abc" Nothing)]+        localSource = identityWith [ResolvedPackage "foo" "1.0.0" (FromLocalPath "/tmp/foo")]+        wrongCohort = (identityWith [ResolvedPackage "foo" "1.0.0" (FromHackage Nothing)]) {identityCohort = CohortName "head"}+        wrongCompiler = (identityWith [ResolvedPackage "foo" "1.0.0" (FromHackage Nothing)]) {identityCompiler = "ghc-9.10.3"}+    checkCohort descriptor missing `shouldContain` [MissingPackage "foo"]+    checkCohort descriptor wrongVersion `shouldContain` [VersionMismatch "foo" "1.0.0" "2.0.0"]+    checkCohort descriptor wrongSource `shouldContain` [SourceMismatch "foo" HackageSource (FromGit "https://example.com/foo.git" "abc" Nothing)]+    checkCohort descriptor localSource `shouldContain` [LocalPathSource "foo" "/tmp/foo"]+    checkCohort descriptor wrongCohort `shouldContain` [CohortNameMismatch (CohortName "released") (CohortName "head")]+    checkCohort descriptor wrongCompiler `shouldContain` [CompilerMismatch "ghc-9.12.4" "ghc-9.10.3"]++activeCohortSpec :: Spec+activeCohortSpec = describe "activeCohortName" do+  it "accepts exactly one cohort import" do+    result <- activeCohortName (fixtures <> "/valid-project")+    result `shouldBe` Right (CohortName "released")++  it "rejects extra lines" do+    result <- activeCohortName (fixtures <> "/invalid-project")+    case result of+      Left (CohortInvalidActive _) -> pure ()+      other -> expectationFailure ("expected CohortInvalidActive, got " <> show other)++identifierSpec :: Spec+identifierSpec = describe "scenario identifiers" do+  it "round-trips canonical identifiers" do+    let rendered = "acme/append/concurrency/expected-version-race"+    fmap renderScenarioId (parseScenarioId rendered) `shouldBe` Right rendered++  it "rejects malformed identifiers" do+    parseScenarioId "acme/append/correctness" `shouldBe` Left "scenario identifier must have four segments: \"acme/append/correctness\""+    parseScenarioId "ACME/append/correctness/example" `shouldBe` Left "invalid layer \"ACME\": invalid segment \"ACME\""++  it "generates canonical UUIDv7 run ids" do+    generated <- newRunId+    parseRunId (renderRunId generated) `shouldBe` Right generated++selectorSpec :: Spec+selectorSpec = describe "scenario selectors" do+  it "supports one-segment and recursive wildcards" do+    scenario <- expectRight (parseScenarioId "selftest/kernel/correctness/always-pass")+    one <- expectRight (parseSelector "*/*/correctness/*")+    recursive <- expectRight (parseSelector "selftest/**")+    matchesSelector one scenario `shouldBe` True+    matchesSelector recursive scenario `shouldBe` True++-- A suite that is not the kernel's: its own layer, migration set, and rule.+openRegistrySpec :: Spec+openRegistrySpec = describe "open registry" do+  let layer = either (error . Text.unpack) id (mkLayerId "acme")+      acmeSet = syntheticSet "acme" []+      roundtrip pgVersions =+        Scenario+          { id = either (error . show) id (parseScenarioId "acme/widget/correctness/roundtrip"),+            revision = 1,+            summary = "Round-trips a widget.",+            tier = TierSmoke,+            placement = PlaceEither,+            knobs = [],+            dimensions = postgresDimensions (PgDurable :| []) pgVersions noDimensions,+            phases = zeroPhases,+            requires = noEnvironment {postgres = Just (PostgresRequirement [MigrationSetName "acme"] [] False)},+            knownDefect = Nothing,+            run = const (pure (ScenarioReport Passed Nothing []))+          }+      bundleWith scenario = (mkBundle layer [scenario] []) {migrations = [acmeSet], rules = [requirePgVersionsFor (MigrationSetName "acme") (Pg18 :| [])]}+      registryErrors bundles = either (fmap (\(RegistryError message) -> message) . toList) (const []) (mkRegistry bundles)+  it "accepts a bundle for a new layer with its own migration set and rule" do+    registry <- expectRight (mkRegistry [Selftest.bundle, bundleWith (roundtrip (Pg18 :| []))])+    Map.keys (registryMigrationSets registry) `shouldBe` [MigrationSetName "acme", MigrationSetName "kenshou-selftest"]+    fmap (renderScenarioId . (.id)) (selectScenarios registry emptyListFilter {layers = [layer]}) `shouldBe` ["acme/widget/correctness/roundtrip"]+  it "rejects a scenario that violates a contributed rule with the rule's message" do+    registryErrors [bundleWith (roundtrip (Pg17 :| [Pg18]))] `shouldBe` ["acme/widget/correctness/roundtrip: acme scenarios require PostgreSQL 18"]+  it "applies a rule to scenarios of every bundle" do+    let rule = ScenarioRule "no-selftest" (\scenario -> ["rejected" | renderLayer scenario.id.layer == "selftest"])+    length (registryErrors [Selftest.bundle, (bundleWith (roundtrip (Pg18 :| []))) {rules = [rule]}]) `shouldBe` length Selftest.bundle.scenarios+  it "rejects duplicate and undefined migration sets" do+    registryErrors [bundleWith (roundtrip (Pg18 :| [])), (mkBundle layer [] []) {migrations = [acmeSet]}] `shouldBe` ["duplicate migration set: acme"]+    registryErrors [(bundleWith (roundtrip (Pg18 :| []))) {migrations = []}] `shouldBe` ["acme/widget/correctness/roundtrip: no registered bundle defines migration set acme"]++-- A suite resource with a stub provider that records when it is acquired and released.+resourceProvisioningSpec :: Spec+resourceProvisioningSpec = describe "resource provisioning" do+  let layer = either (error . Text.unpack) id (mkLayerId "acme")+      stub = either (error . Text.unpack) id (mkResourceName "stub")+      provider :: (Text -> IO ()) -> ResourceProvider+      provider record =+        ResourceProvider+          { name = stub,+            summary = "Stub resource",+            normalize = Right . fromMaybe (object ["mode" .= ("default" :: Text)]),+            withLocal = \_ value action ->+              bracket_ (record "acquire") (record "release") (action (ProvisionedResource (object ["spec" .= value]) (object ["kind" .= ("stub" :: Text)]))),+            cell = Nothing+          }+      roundtrip :: (Text -> IO ()) -> Scenario+      roundtrip record =+        Scenario+          { id = either (error . show) id (parseScenarioId "acme/widget/correctness/uses-stub"),+            revision = 1,+            summary = "Uses the stub resource.",+            tier = TierSmoke,+            placement = PlaceEither,+            knobs = [],+            dimensions = noDimensions,+            phases = zeroPhases,+            requires = requiresResource stub,+            knownDefect = Nothing,+            run = \context -> do+              record "run"+              pure (if fmap (.workerValue) (Map.lookup stub context.env.resources) == Just (object ["spec" .= object ["mode" .= ("default" :: Text)]]) then ScenarioReport Passed Nothing [] else failedWith ["no-stub"] "the stub resource was not provisioned")+          }+      bundleWith record = (mkBundle layer [roundtrip record] []) {Bundle.resources = [provider record]}+      registryErrors bundles = either (fmap (\(RegistryError message) -> message) . toList) (const []) (mkRegistry bundles)+      ignore = const (pure ())+  it "rejects a duplicate provider and an unprovided resource" do+    registryErrors [bundleWith ignore, (mkBundle layer [] []) {Bundle.resources = [provider ignore]}] `shouldBe` ["duplicate resource provider: stub"]+    registryErrors [(bundleWith ignore) {Bundle.resources = []}] `shouldBe` ["scenario requires unprovided resource stub: acme/widget/correctness/uses-stub"]+  it "normalizes the value, provisions around the run, and records the fingerprint" $ withSystemTempDirectory "kenshou-resource" \root -> do+    events <- newIORef ([] :: [Text])+    let record event = modifyIORef' events (<> [event])+    registry <- expectRight (mkRegistry [bundleWith record])+    scenarioId <- expectRight (parseScenarioId "acme/widget/correctness/uses-stub")+    output <- executeRun (RunnerConfig registry root (identityWith []) Nothing False False []) (minimalRunSpec scenarioId) >>= expectRight+    readIORef events `shouldReturn` ["acquire", "run", "release"]+    runSpec <- decodeFile (output.directory </> "run-spec.json")+    runResult <- decodeFile (output.directory </> "run-result.json")+    path ["environment", "resources", "stub"] runSpec `shouldBe` Just (object ["mode" .= ("default" :: Text)])+    path ["fingerprint", "resources", "stub"] runResult `shouldBe` Just (object ["kind" .= ("stub" :: Text)])+    path ["outcome"] runResult `shouldBe` Just (String "passed")+  where+    decodeFile file = eitherDecodeFileStrict' file >>= either fail pure :: IO Value+    path [] value = Just value+    path (key : rest) (Object fields) = KeyMap.lookup (Key.fromText key) fields >>= path rest+    path _ _ = Nothing++registrySpec :: Spec+registrySpec = describe "scenario registry" do+  it "sorts the kernel self-tests by identifier" do+    case mkRegistry [Selftest.bundle] of+      Left errors -> expectationFailure (show errors)+      Right registry ->+        fmap (renderScenarioId . (.id)) (allScenarios registry)+          `shouldBe` [ "selftest/kernel/concurrency/worker-echo",+                       "selftest/kernel/correctness/always-fail",+                       "selftest/kernel/correctness/always-pass",+                       "selftest/kernel/correctness/errors",+                       "selftest/kernel/correctness/known-defect",+                       "selftest/kernel/correctness/outcome",+                       "selftest/kernel/correctness/postgres-roundtrip"+                     ]++knownDefectGroupSpec :: Spec+knownDefectGroupSpec = describe "grouped known defects" do+  let base = case filter (\item -> renderScenarioId item.id == "selftest/kernel/correctness/known-defect") Selftest.bundle.scenarios of+        [item] -> item+        _ -> error "missing known-defect self-test"+      first = KnownDefect "mori://example/bugs/1" "first" ["first-failure"] AllCohorts+      second = KnownDefect "mori://example/bugs/2" "second" ["second-failure"] AllCohorts+      scenario = base {knownDefect = Just (KnownDefectGroup (first :| [second]))}+      cohort = identityWith []+      disposition strict labels = defectDisposition strict cohort scenario (failedWith labels "grouped defect probe")+  it "accepts either or both independently reproduced defects" do+    let check labels = case disposition False labels of+          (Just (KnownDefectGroup _, DefectReproduced), False, 0) -> pure ()+          other -> expectationFailure (show other)+    check ["first-failure"]+    check ["second-failure"]+    check ["first-failure", "second-failure"]+  it "blocks unrelated failures and strict known-defect runs" do+    case disposition False ["second-failure", "new-failure"] of+      (Just (_, DefectDifferentFailure), True, 1) -> pure ()+      other -> expectationFailure (show other)+    case disposition True ["second-failure"] of+      (Just (_, DefectReproduced), False, 1) -> pure ()+      other -> expectationFailure (show other)+  it "does not excuse a failed result without a failure label" do+    case disposition False [] of+      (Just (_, DefectDifferentFailure), True, 1) -> pure ()+      other -> expectationFailure (show other)+  it "ignores labels owned only by a defect outside the resolved cohort" do+    let scoped = KnownDefect "mori://example/bugs/2" "second" ["second-failure"] (OnlyWhen (ResolvedFromGit "foo" :| []))+        scopedScenario = base {knownDefect = Just (KnownDefectGroup (first :| [scoped]))}+        result = defectDisposition False (identityWith [ResolvedPackage "foo" "1.0.0" (FromHackage Nothing)]) scopedScenario (failedWith ["second-failure"] "scoped defect probe")+    case result of+      (Just (_, DefectDifferentFailure), True, 1) -> pure ()+      other -> expectationFailure (show other)++knobSpec :: Spec+knobSpec = describe "knob resolution" do+  let workers = knobName "selftest.workers"+      enabled = knobName "selftest.enabled"+      declarations =+        [ KnobSpec workers "Worker count" KnobInt (VInt 2) (IntRange 1 8) [VInt 1, VInt 8],+          KnobSpec enabled "Enable work" KnobBool (VBool False) AnyValue []+        ]+  it "fills defaults and parses typed command-line values" do+    resolved <- expectRight (resolveKnobs declarations [(workers, RawText "4"), (enabled, RawText "true")])+    knobInt resolved workers `shouldBe` (4 :: Int64)+    knobBool resolved enabled `shouldBe` True+  it "reports duplicates, unknown names and range violations together" do+    let unknown = knobName "selftest.unknown"+        result = resolveKnobs declarations [(workers, RawText "9"), (workers, RawText "1"), (unknown, RawText "x")]+    case result of+      Left problems -> length problems `shouldBe` 3+      Right _ -> expectationFailure "expected knob errors"++dimensionSpec :: Spec+dimensionSpec = describe "dimension resolution" do+  let support = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| []) noDimensions+  it "fills every applicable default" do+    resolveDimensions support [] `shouldBe` Right (Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18))+  it "rejects unsupported and inapplicable values" do+    case resolveDimensions support [("pg.version", "17"), ("telemetry.tracing", "off")] of+      Left problems -> length problems `shouldBe` 2+      Right _ -> expectationFailure "expected dimension errors"++runSpecSpec :: Spec+runSpecSpec = describe "run specification" do+  it "decodes a minimal document and emits a round-trippable effective document" do+    input <- case eitherDecode "{\"schema\":\"kenshou.run-spec/v1\",\"scenario\":\"selftest/kernel/correctness/always-pass\",\"seed\":7}" of+      Left err -> expectationFailure err >> fail "unreachable"+      Right value -> pure value+    registry <- expectRight (mkRegistry [Selftest.bundle])+    resolution <- resolveRunSpec registry input+    (_, effective) <- expectRight resolution+    eitherDecode (encode effective) `shouldBe` Right (toJSON effective :: Value)+  it "rejects a non-v7 run id while decoding" do+    let document = "{\"schema\":\"kenshou.run-spec/v1\",\"scenario\":\"selftest/kernel/correctness/always-pass\",\"runId\":\"550e8400-e29b-41d4-a716-446655440000\"}"+    case (eitherDecode document :: Either String RunSpec) of+      Left err -> err `shouldContain` "UUIDv7"+      Right _ -> expectationFailure "expected UUIDv7 rejection"++compatibilitySpec :: Spec+compatibilitySpec = describe "compatibility keys" do+  it "exclude run identity and cohort from comparison keys, but include the cohort in series keys" do+    registry <- expectRight (mkRegistry [Selftest.bundle])+    scenarioId <- expectRight (parseScenarioId "selftest/kernel/correctness/always-pass")+    firstResolution <- resolveRunSpec registry (minimalRunSpec scenarioId)+    secondResolution <- resolveRunSpec registry (minimalRunSpec scenarioId)+    (_, firstSpec) <- expectRight firstResolution+    (_, secondSpec) <- expectRight secondResolution+    let firstCohort = identityWith []+        secondCohort = firstCohort {identityPlanHash = PlanHash "sha256:other"}+        firstInputs = compatInputs firstSpec Nothing [] firstCohort+        repeatedInputs = compatInputs secondSpec Nothing [] firstCohort+        otherCohortInputs = compatInputs firstSpec Nothing [] secondCohort+    comparisonKey firstInputs `shouldBe` comparisonKey repeatedInputs+    comparisonKey firstInputs `shouldBe` comparisonKey otherCohortInputs+    seriesKey firstInputs `shouldNotBe` seriesKey otherCohortInputs++knobName :: Text -> KnobName+knobName = either (error . show) id . mkKnobName++manifestSpec :: Spec+manifestSpec = describe "artifact manifests" do+  it "detects a changed file" $ withSystemTempDirectory "kenshou-manifest" \directory -> do+    runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+    ByteString.writeFile (directory <> "/run-spec.json") "{}\n"+    manifest <- writeManifest directory runId Map.empty+    encodeFile (directory <> "/manifest.json") manifest+    verifyManifest directory `shouldReturn` Right ()+    ByteString.appendFile (directory <> "/run-spec.json") "x"+    verification <- verifyManifest directory+    case verification of+      Left _ -> pure ()+      Right () -> expectationFailure "expected a digest mismatch"++goldenSpec :: Spec+goldenSpec = describe "JSON goldens" do+  mapM_ roundTrips ["run-spec.minimal.json", "run-spec.effective.json", "run-spec.external.json", "run-result.passed.json", "run-result.known-defect.json", "manifest.json", "scenario-list.json"]+  where+    roundTrips name = it ("round-trips " <> name <> " canonically") do+      original <- decodeGolden name+      redecoded <- case eitherDecode (encode original) of+        Left err -> expectationFailure err >> fail "unreachable"+        Right value -> pure value+      canonicalEncode original `shouldBe` canonicalEncode (redecoded :: Value)+    decodeGolden name = do+      result <- eitherDecodeFileStrict' ("test/golden/" <> name)+      case result of+        Left err -> expectationFailure err >> fail "unreachable"+        Right value -> pure (value :: Value)++postgresEnvironmentSpec :: Spec+postgresEnvironmentSpec = describe "PostgreSQL environments" do+  it "migrates, clones, restarts, and crash-restarts PostgreSQL 18" $ withSystemTempDirectory "kenshou-pg18" \directory -> do+    runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+    let requirement = PostgresRequirement [selftestSchema] [] True+        expectedSchemas = "kenshou_selftest,pgmigrate"+        dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18)+    result <- withPostgresEnv selftestSets nullLogger directory runId requirement (PostgresEphemeral []) dimensions \environment -> do+      schemas <- psqlTest environment.connectionString "select string_agg(schema_name, ',' order by schema_name) from information_schema.schemata where schema_name in ('kenshou_selftest','pgmigrate')"+      clone <- environment.newDatabase "clone"+      clonedSchemas <- psqlTest clone "select string_agg(schema_name, ',' order by schema_name) from information_schema.schemata where schema_name in ('kenshou_selftest','pgmigrate')"+      case environment.control of+        Nothing -> expectationFailure "expected server control"+        Just control -> do+          control.restartServer+          psqlTest environment.connectionString "select 1" `shouldReturn` Right "1"+          control.stopServer StopImmediate+          control.startServer+          psqlTest environment.connectionString "select 1" `shouldReturn` Right "1"+      pure (schemas, clonedSchemas, Map.lookup "fsync" environment.snapshot.settings)+    result `shouldBe` Right (Right expectedSchemas, Right expectedSchemas, Just "off")++  it "creates and drops fresh databases in external mode" $ withSystemTempDirectory "kenshou-pg-external" \directory -> do+    runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+    let dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18)+        outerRequirement = PostgresRequirement [] [] False+        externalRequirement = PostgresRequirement [selftestSchema] [("shared_preload_libraries", "'pg_stat_statements'")] False+    outer <- withPostgresEnv selftestSets nullLogger directory runId outerRequirement (PostgresEphemeral [("shared_preload_libraries", "'pg_stat_statements'")]) dimensions \server -> do+      inner <- withPostgresEnv selftestSets nullLogger directory runId externalRequirement (PostgresExternal (ConnLiteral server.adminConnectionString)) dimensions (\environment -> psqlTest environment.connectionString "select to_regnamespace('kenshou_selftest') is not null")+      missing <- withPostgresEnv selftestSets nullLogger directory runId (PostgresRequirement [] [("shared_preload_libraries", "'missing_extension'")] False) (PostgresExternal (ConnLiteral server.adminConnectionString)) dimensions (const (pure ()))+      missing `shouldBe` Left (EnvError "external PostgreSQL is missing required preloaded libraries: missing_extension")+      remaining <- psqlTest server.adminConnectionString "select count(*) from pg_database where datname like 'kenshou_01997f3a5b7c7%'"+      pure (inner, remaining)+    outer `shouldBe` Right (Right (Right "t"), Right "0")++  it "keeps two named contexts of one run apart on a shared external server" $ withSystemTempDirectory "kenshou-pg-shared" \directory -> do+    runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e56")+    let dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18)+        requirement = PostgresRequirement [] [] False+    outer <- withPostgresEnv selftestSets nullLogger directory runId requirement (PostgresEphemeral []) dimensions \server -> do+      let external = PostgresExternal (ConnLiteral server.adminConnectionString)+      nested <- withPostgresEnvNamed selftestSets False (Just "shop") nullLogger directory runId requirement external dimensions \shop ->+        withPostgresEnvNamed selftestSets False (Just "warehouse") nullLogger directory runId requirement external dimensions \warehouse ->+          pure (shop.databaseName /= warehouse.databaseName)+      remaining <- psqlTest server.adminConnectionString "select count(*) from pg_database where datname like 'kenshou_01997f3a5b7c7%'"+      pure (nested, remaining)+    outer `shouldBe` Right (Right (Right True), Right "0")++  it "selects PostgreSQL 17 and applies a composed plan" do+    available <- lookupEnv "KENSHOU_PG17_BIN"+    case available of+      Nothing -> pendingWith "KENSHOU_PG17_BIN is not set"+      Just _ -> withSystemTempDirectory "kenshou-pg17" \directory -> do+        runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+        let requirement = PostgresRequirement [selftestSchema] [] False+            dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg17)+        result <- withPostgresEnv selftestSets nullLogger directory runId requirement (PostgresEphemeral []) dimensions (pure . (.snapshot.serverVersionNum))+        case result of+          Left err -> expectationFailure (show err)+          Right version -> (version >= 170000 && version < 180000) `shouldBe` True++psqlTest :: Text -> Text -> IO (Either Text Text)+psqlTest connection query = do+  (code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+  pure case code of ExitSuccess -> Right (Text.strip (Text.pack output)); _ -> Left (Text.strip (Text.pack err))++onePackageDescriptor :: CohortDescriptor+onePackageDescriptor =+  CohortDescriptor+    "kenshou.cohort/v1"+    (CohortName "released")+    "fixture"+    "2026-09-20"+    "ghc-9.12.4"+    "2026-09-20T13:44:47Z"+    "cohort/released.project"+    [ComponentSpec (ComponentId "foo") "mori://example/foo" HackageSource [PackagePin "foo" "1.0.0" Nothing]]+    []+    []++identityWith :: [ResolvedPackage] -> CohortIdentity+identityWith packages =+  CohortIdentity+    (CohortName "released")+    "ghc-9.12.4"+    "3.16.1.0"+    "linux"+    "x86_64"+    (Just "2026-09-20T13:44:47Z")+    (PlanHash "sha256:fixture")+    "fixture"+    [ResolvedComponent (ComponentId "foo") "mori://example/foo" packages]+    Nothing++hashFixture :: FilePath -> IO PlanHash+hashFixture name = do+  value <- decodeFixture name+  case planHash value of+    Left err -> expectationFailure (show err) >> fail "unreachable"+    Right result -> pure result++decodeFixture :: (FromJSON value) => FilePath -> IO value+decodeFixture name = do+  result <- eitherDecodeFileStrict' (fixtures <> "/" <> name)+  case result of+    Left err -> expectationFailure err >> fail "unreachable"+    Right value -> pure value++fixtures :: FilePath+fixtures = "test/fixtures"++expectRight :: (Show problem) => Either problem value -> IO value+expectRight (Right value) = pure value+expectRight (Left problem) = expectationFailure (show problem) >> fail "unreachable"++migrationPlanSpec :: Spec+migrationPlanSpec = describe "migration plan composition" do+  let alpha = syntheticSet "alpha" []+      beta = syntheticSet "beta" [MigrationSetName "alpha"]+      gamma = syntheticSet "gamma" [MigrationSetName "beta"]+      unavailable = MigrationSet (MigrationSetName "delta") [] (Left "not built")+      cyclic = syntheticSet "epsilon" [MigrationSetName "epsilon"]+      sets = Map.fromList [(set.name, set) | set <- [alpha, beta, gamma, unavailable, cyclic]]+      order names = fmap (fmap (.name)) (orderMigrationSets sets (fmap MigrationSetName names))+  it "composes the self-test migration set" do+    isRight (composePlan selftestSets [selftestSchema]) `shouldBe` True+  it "keeps the declared order and inserts required sets first, transitively" do+    order ["gamma"] `shouldBe` Right (fmap MigrationSetName ["alpha", "beta", "gamma"])+    order ["beta", "alpha"] `shouldBe` Right (fmap MigrationSetName ["alpha", "beta"])+    order ["alpha", "gamma", "beta"] `shouldBe` Right (fmap MigrationSetName ["alpha", "beta", "gamma"])+    isRight (composePlan sets [MigrationSetName "gamma"]) `shouldBe` True+  it "rejects an empty selection, an undefined set, an unavailable set, and a cycle" do+    isLeft (composePlan sets []) `shouldBe` True+    isLeft (composePlan sets [MigrationSetName "missing"]) `shouldBe` True+    isLeft (composePlan sets [MigrationSetName "delta"]) `shouldBe` True+    isLeft (order ["epsilon"]) `shouldBe` True++-- A set whose one migration creates a schema of the same name.+syntheticSet :: Text -> [MigrationSetName] -> MigrationSet+syntheticSet name requires =+  MigrationSet+    { name = MigrationSetName name,+      requires,+      component = either (Left . Text.pack . show) Right do+        migration <- sqlMigration "create-schema" (Text.Encoding.encodeUtf8 ("create schema " <> name <> ";\n"))+        migrationComponent name (Set.fromList [dependency | MigrationSetName dependency <- requires]) (migration :| [])+    }++selftestSets :: Map.Map MigrationSetName MigrationSet+selftestSets = Map.singleton selftestSchema selftestMigrationSet
+ test/fixtures/cohort-identity.golden.json view
@@ -0,0 +1,19 @@+{+  "schema": "kenshou.cohort-identity/v1",+  "cohort": "released",+  "compiler": "ghc-9.12.4",+  "cabalVersion": "3.16.1.0",+  "os": "linux",+  "arch": "x86_64",+  "indexState": "2026-09-20T13:44:47Z",+  "planHash": "sha256:857e6ff76844ccdf1903dc62d6518fe59b64e34d4bc626873efe2534934f1cc7",+  "descriptorSha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",+  "components": [+    { "id": "foo", "moriUri": "mori://example/foo", "packages": [+      { "name": "foo", "version": "1.0.0", "source": { "type": "hackage", "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } }+    ] },+    { "id": "bar", "moriUri": "mori://example/bar", "packages": [+      { "name": "bar", "version": "2.0.0", "source": { "type": "git", "location": "https://example.com/bar.git", "rev": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }+    ] }+  ]+}
+ test/fixtures/descriptor-released.json view
@@ -0,0 +1,29 @@+{+  "schema": "kenshou.cohort/v1",+  "name": "released",+  "description": "Trimmed unit-test cohort.",+  "verifiedAt": "2026-09-20",+  "compiler": "ghc-9.12.4",+  "indexState": "2026-09-20T13:44:47Z",+  "projectFile": "cohort/released.project",+  "components": [+    {+      "id": "foo",+      "moriUri": "mori://example/foo",+      "source": { "type": "hackage" },+      "packages": [{ "name": "foo", "version": "1.0.0" }]+    },+    {+      "id": "bar",+      "moriUri": "mori://example/bar",+      "source": {+        "type": "git",+        "location": "https://example.com/bar.git",+        "rev": "0123456789abcdef0123456789abcdef01234567"+      },+      "packages": [{ "name": "bar", "version": "2.0.0", "subdir": "bar" }]+    }+  ],+  "allowNewer": [],+  "exceptions": []+}
+ test/fixtures/invalid-project/cohort/active.project view
@@ -0,0 +1,2 @@+import: released.project+import: head.project
+ test/fixtures/perf/catalog.v1.json view
@@ -0,0 +1,109 @@+{+  "schema": "kenshou.perf-catalog/v1",+  "profiles": [+    {+      "name": "acme-core-quick",+      "revision": 1,+      "scenario": "acme/core-ordering/benchmark/concurrency-sweep",+      "scenarioRevision": 2,+      "knobs": {+        "bench.arm": "configured",+        "bench.rate": 200,+        "acme.messages": 1000,+        "measure.raw-samples": "full"+      },+      "dimensions": {+        "telemetry.tracing": "off",+        "telemetry.metrics": "off"+      },+      "workloadPurpose": "fixed-load-latency",+      "warmupSeconds": 0,+      "steadyCount": 1000,+      "cooldownSeconds": 0,+      "timeoutSeconds": 180,+      "cachePolicy": "cold-reset",+      "policy": "policies/default.json",+      "requiredMetrics": [+        {+          "name": "op.publish-to-finalize.throughput",+          "unit": "ops/s",+          "minimumSteadySamples": 1000+        },+        {+          "name": "op.publish-to-finalize.latency.p50",+          "unit": "ns",+          "minimumSteadySamples": 1000+        },+        {+          "name": "op.publish-to-finalize.latency.p99",+          "unit": "ns",+          "minimumSteadySamples": 1000+        }+      ],+      "capabilities": [+        "cold-reset",+        "full-samples"+      ],+      "machineProfile": "quick-cell-pending-calibration",+      "trialSeconds": 15,+      "resetSeconds": 30,+      "sealSeconds": 10,+      "minimumPairs": 5,+      "replacementPairs": 2,+      "calibration": null+    },+    {+      "name": "acme-quill-quick",+      "revision": 1,+      "scenario": "acme/quill-adapter/benchmark/end-to-end-throughput-latency",+      "scenarioRevision": 1,+      "knobs": {+        "bench.arm": "adapter",+        "bench.rate": 200,+        "acme.messages": 1000,+        "measure.raw-samples": "full"+      },+      "dimensions": {+        "pg.durability": "durable",+        "pg.version": "18"+      },+      "workloadPurpose": "fixed-load-latency",+      "warmupSeconds": 0,+      "steadyCount": 1000,+      "cooldownSeconds": 0,+      "timeoutSeconds": 180,+      "cachePolicy": "cold-reset",+      "policy": "policies/default.json",+      "requiredMetrics": [+        {+          "name": "op.publish-to-finalize.throughput",+          "unit": "ops/s",+          "minimumSteadySamples": 1000+        },+        {+          "name": "op.publish-to-finalize.latency.p50",+          "unit": "ns",+          "minimumSteadySamples": 1000+        },+        {+          "name": "op.publish-to-finalize.latency.p99",+          "unit": "ns",+          "minimumSteadySamples": 1000+        }+      ],+      "capabilities": [+        "cold-reset",+        "full-samples",+        "postgresql-18",+        "quill"+      ],+      "machineProfile": "quick-cell-pending-calibration",+      "trialSeconds": 15,+      "resetSeconds": 30,+      "sealSeconds": 10,+      "minimumPairs": 5,+      "replacementPairs": 2,+      "calibration": null+    }+  ]+}
+ test/fixtures/perf/cohort.json view
@@ -0,0 +1,89 @@+{+  "schema": "kenshou.cohort/v1",+  "name": "released",+  "description": "Historical Hackage runtime baseline verified on 2026-09-20.",+  "verifiedAt": "2026-09-20",+  "compiler": "ghc-9.12.4",+  "indexState": "2026-09-20T13:44:47Z",+  "projectFile": "cohort/released.project",+  "components": [+    { "id": "keiki", "moriUri": "mori://shinzui/keiki", "source": { "type": "hackage" }, "packages": [+      { "name": "keiki", "version": "0.9.1.0" },+      { "name": "keiki-codec-json", "version": "0.9.1.0" }+    ] },+    { "id": "tally", "moriUri": "mori://shinzui/tally", "source": { "type": "hackage" }, "packages": [+      { "name": "tally-store", "version": "0.8.0.1" },+      { "name": "tally-store-migrations", "version": "0.4.0.0" },+      { "name": "tally-otel", "version": "0.2.0.8" },+      { "name": "tally-metrics", "version": "0.1.0.8" },+      { "name": "tally-cli", "version": "0.2.0.6" },+      { "name": "acme-tally-adapter", "version": "0.5.1.2" }+    ] },+    { "id": "flux", "moriUri": "mori://shinzui/flux", "source": { "type": "hackage" }, "packages": [+      { "name": "flux", "version": "0.17.0.0" },+      { "name": "flux-core", "version": "0.17.0.0" },+      { "name": "flux-quill", "version": "0.17.0.0" },+      { "name": "flux-migrations", "version": "0.17.0.0" },+      { "name": "flux-ops", "version": "0.17.0.0" },+      { "name": "flux-test-support", "version": "0.17.0.0" }+    ] },+    { "id": "acme", "moriUri": "mori://shinzui/acme", "source": { "type": "hackage" }, "packages": [+      { "name": "acme-core", "version": "0.9.0.3" },+      { "name": "acme-metrics", "version": "0.9.0.3" }+    ] },+    { "id": "acme-quill-adapter", "moriUri": "mori://shinzui/acme-quill-adapter", "source": { "type": "hackage" }, "packages": [+      { "name": "acme-quill-adapter", "version": "0.16.0.0" }+    ] },+    { "id": "acme-brook-adapter", "moriUri": "mori://shinzui/acme-brook-adapter", "source": { "type": "hackage" }, "packages": [+      { "name": "acme-brook-adapter", "version": "0.9.0.1" }+    ] },+    { "id": "brook-effectful", "moriUri": "mori://shinzui/brook-effectful", "source": { "type": "hackage" }, "packages": [+      { "name": "brook-effectful", "version": "0.3.1.0" }+    ] },+    { "id": "hw-brook-streamly", "moriUri": "mori://shinzui/hw-brook-streamly", "source": { "type": "hackage" }, "packages": [+      { "name": "hw-brook-streamly", "version": "0.2.0.0" }+    ] },+    { "id": "hw-brook-client", "moriUri": "mori://haskell-works/hw-brook-client", "source": { "type": "hackage" }, "packages": [+      { "name": "hw-brook-client", "version": "5.3.0" }+    ] },+    { "id": "quill-hs", "moriUri": "mori://shinzui/quill-hs", "source": { "type": "hackage" }, "packages": [+      { "name": "quill-core", "version": "0.6.1.0" },+      { "name": "quill-hasql", "version": "0.6.1.0" },+      { "name": "quill-effectful", "version": "0.6.1.0" },+      { "name": "quill-config", "version": "0.6.1.0" },+      { "name": "quill-migration", "version": "0.6.1.0" }+    ] },+    { "id": "pg-migrate", "moriUri": "mori://shinzui/pg-migrate", "source": { "type": "hackage" }, "packages": [+      { "name": "pg-migrate", "version": "1.1.0.0" },+      { "name": "pg-migrate-embed", "version": "1.1.0.0" },+      { "name": "pg-migrate-cli", "version": "1.1.0.0" },+      { "name": "pg-migrate-import-codd", "version": "1.1.0.0" },+      { "name": "pg-migrate-import-hasql-migration", "version": "1.1.0.0" }+    ] },+    { "id": "ephemeral-pg", "moriUri": "mori://shinzui/ephemeral-pg", "source": { "type": "hackage" }, "packages": [+      { "name": "ephemeral-pg", "version": "0.3.1.0" }+    ] },+    { "id": "effectful", "moriUri": "mori://effectful/effectful", "source": { "type": "hackage" }, "packages": [+      { "name": "effectful", "version": "2.6.1.0" },+      { "name": "effectful-core", "version": "2.6.1.0" }+    ] },+    { "id": "hs-opentelemetry", "moriUri": "mori://iand675/hs-opentelemetry", "source": { "type": "hackage" }, "packages": [+      { "name": "hs-opentelemetry-api", "version": "1.0.0.0" },+      { "name": "hs-opentelemetry-sdk", "version": "1.0.0.0" },+      { "name": "hs-opentelemetry-propagator-w3c", "version": "1.0.0.0" },+      { "name": "hs-opentelemetry-exporter-in-memory", "version": "1.0.0.0" },+      { "name": "hs-opentelemetry-exporter-otlp", "version": "1.0.0.0" },+      { "name": "hs-opentelemetry-exporter-prometheus", "version": "1.0.0.0" },+      { "name": "hs-opentelemetry-otlp", "version": "1.0.0.0" },+      { "name": "hs-opentelemetry-semantic-conventions", "version": "1.40.0.0" }+    ] }+  ],+  "allowNewer": ["flux-test-support:ephemeral-pg"],+  "exceptions": [+    { "package": "tally-store-migrations", "selected": "0.4.0.0", "latestObserved": "0.5.0.0", "reason": "flux-migrations 0.17.0.0 bounds ^>=0.4.0.0", "recheckAfter": "2026-12-20" },+    { "package": "pg-migrate", "selected": "1.1.0.0", "latestObserved": "1.2.0.0", "reason": "flux-migrations and quill-migration bound ^>=1.1.0.0", "recheckAfter": "2026-12-20" },+    { "package": "effectful", "selected": "2.6.1.0", "latestObserved": "2.7.1.0", "reason": "flux 0.17.0.0 requires effectful <2.7", "recheckAfter": "2026-12-20" },+    { "package": "aeson", "selected": "2.2.5.1", "latestObserved": "2.3.2.0", "reason": "resolved transitively by the released runtime bounds", "recheckAfter": "2026-12-20" },+    { "package": "hw-brook-client", "selected": "5.3.0", "latestObserved": "5.3.0", "reason": "the head cohort uses a fork with unreleased correctness fixes", "recheckAfter": "2026-12-20" }+  ]+}
+ test/fixtures/plan-commit-change.json view
@@ -0,0 +1,8 @@+{+  "compiler-id": "ghc-9.12.4", "cabal-version": "3.16.1.0", "os": "linux", "arch": "x86_64",+  "install-plan": [+    { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+    { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+    { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "fedcba9876543210fedcba9876543210fedcba98", "subdir": "bar" } }, "flags": {} }+  ]+}
+ test/fixtures/plan-local-change.json view
@@ -0,0 +1,9 @@+{+  "compiler-id": "ghc-9.12.4", "cabal-version": "3.16.1.0", "os": "linux", "arch": "x86_64",+  "install-plan": [+    { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+    { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+    { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }, "flags": {} },+    { "type": "configured", "style": "local", "id": "kenshou-core", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0", "pkg-src": { "type": "local", "path": "/another/checkout/kenshou-core" }, "flags": {} }+  ]+}
+ test/fixtures/plan-released.json view
@@ -0,0 +1,34 @@+{+  "compiler-id": "ghc-9.12.4",+  "cabal-version": "3.16.1.0",+  "os": "linux",+  "arch": "x86_64",+  "install-plan": [+    { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+    {+      "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0",+      "pkg-src": { "type": "repo-tar" },+      "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+      "flags": { "fast": true }+    },+    {+      "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0",+      "pkg-src": {+        "type": "source-repo",+        "source-repo": {+          "type": "git", "location": "https://example.com/bar.git",+          "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar"+        }+      },+      "flags": {}+    },+    {+      "type": "configured", "style": "local", "id": "kenshou-core-lib", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0",+      "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {}+    },+    {+      "type": "configured", "style": "local", "id": "kenshou-core-test", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0",+      "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {}+    }+  ]+}
+ test/fixtures/plan-reordered.json view
@@ -0,0 +1,13 @@+{+  "compiler-id": "ghc-9.12.4",+  "cabal-version": "3.16.1.0",+  "os": "linux",+  "arch": "x86_64",+  "install-plan": [+    { "type": "configured", "style": "local", "id": "kenshou-core-test", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0", "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {} },+    { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }, "flags": {} },+    { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+    { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+    { "type": "configured", "style": "local", "id": "kenshou-core-lib", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0", "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {} }+  ]+}
+ test/fixtures/plan-version-change.json view
@@ -0,0 +1,8 @@+{+  "compiler-id": "ghc-9.12.4", "cabal-version": "3.16.1.0", "os": "linux", "arch": "x86_64",+  "install-plan": [+    { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+    { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.1", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+    { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }, "flags": {} }+  ]+}
+ test/fixtures/plan/catalog-planned.json view
@@ -0,0 +1,64 @@+{+  "schema": "kenshou.scenario-list/v1",+  "suiteVersion": "0.1.0.0",+  "roles": [],+  "scenarios": [+    {"id":"selftest/kernel/concurrency/worker-echo","summary":"Worker echo","tier":"smoke","placement":"either"},+    {"id":"selftest/kernel/correctness/always-fail","summary":"Always fails","tier":"smoke","placement":"either"},+    {"id":"selftest/kernel/correctness/always-pass","summary":"Always passes","tier":"smoke","placement":"either"},+    {"id":"selftest/kernel/correctness/errors","summary":"Always errors","tier":"smoke","placement":"either"},+    {"id":"selftest/kernel/correctness/known-defect","summary":"Known defect","tier":"smoke","placement":"either"},+    {"id":"selftest/kernel/correctness/outcome","summary":"Selected outcome","tier":"smoke","placement":"either"},+    {"id":"selftest/kernel/correctness/postgres-roundtrip","summary":"PostgreSQL roundtrip","tier":"smoke","placement":"either","dimensions":{"pg.durability":{"values":["fsync-off","durable"],"default":"fsync-off"},"pg.version":{"values":["17","18"],"default":"18"}}},++    {"id":"quill/queue/correctness/create-delete","summary":"Queue lifecycle","tier":"smoke","placement":"either"},+    {"id":"quill/send/correctness/send-read","summary":"Send and read","tier":"standard","placement":"either"},+    {"id":"quill/send/benchmark/batch-throughput","summary":"Batch throughput","tier":"standard","placement":"cell","knobs":[{"name":"quill.batch-size","summary":"Batch size","type":"int","default":10,"allowed":{"kind":"one-of","values":[10,100]},"variants":[100]}],"dimensions":{"telemetry.tracing":{"values":["off","sdk-inmemory"],"default":"off"},"telemetry.metrics":{"values":["off","collect"],"default":"off"},"pg.durability":{"values":["durable"],"default":"durable"},"pg.version":{"values":["17","18"],"default":"18"}}},+    {"id":"quill/read/concurrency/visibility","summary":"Visibility concurrency","tier":"standard","placement":"either"},+    {"id":"quill/ack/correctness/archive","summary":"Archive acknowledgement","tier":"smoke","placement":"either"},+    {"id":"quill/vt/correctness/expiry-redelivers","summary":"Visibility timeout","tier":"standard","placement":"either"},+    {"id":"quill/fifo/concurrency/ordering","summary":"FIFO ordering","tier":"standard","placement":"either"},+    {"id":"quill/topics/correctness/routing","summary":"Topic routing","tier":"standard","placement":"either"},+    {"id":"quill/notify/correctness/wakeup","summary":"Notify wakeup","tier":"standard","placement":"either"},+    {"id":"quill/config/correctness/roundtrip","summary":"Configuration","tier":"smoke","placement":"either"},+    {"id":"quill/effectful/correctness/interpreter","summary":"Effect interpreter","tier":"smoke","placement":"either"},++    {"id":"tally/append/correctness/expected-version","summary":"Expected version","tier":"smoke","placement":"either"},+    {"id":"tally/append/benchmark/throughput","summary":"Append throughput","tier":"standard","placement":"cell","knobs":[{"name":"tally.batch-size","summary":"Batch size","type":"int","default":1,"allowed":{"kind":"one-of","values":[1,10,100]},"variants":[10,100]}],"dimensions":{"telemetry.tracing":{"values":["off","sdk-inmemory","sdk-otlp"],"default":"off"},"telemetry.metrics":{"values":["off","collect","serve-scraped"],"default":"off"},"pg.durability":{"values":["durable"],"default":"durable"},"pg.version":{"values":["17","18"],"default":"18"}}},+    {"id":"tally/read/correctness/stream","summary":"Read stream","tier":"smoke","placement":"either"},+    {"id":"tally/subscription/concurrency/checkpoint","summary":"Subscription checkpoint","tier":"standard","placement":"either"},+    {"id":"tally/consumer-group/concurrency/partition","summary":"Consumer group partition","tier":"standard","placement":"either"},+    {"id":"tally/dead-letter/correctness/retry","summary":"Dead letter retry","tier":"standard","placement":"either"},+    {"id":"tally/lifecycle/correctness/restart","summary":"Lifecycle restart","tier":"standard","placement":"local"},+    {"id":"tally/transaction/correctness/atomicity","summary":"Transaction atomicity","tier":"standard","placement":"either"},+    {"id":"tally/notifier/concurrency/wakeup","summary":"Notifier wakeup","tier":"standard","placement":"either"},+    {"id":"tally/retention/correctness/prune","summary":"Retention","tier":"extended","placement":"either"},+    {"id":"tally/metrics/correctness/truthful","summary":"Metrics truthful","tier":"standard","placement":"either","dimensions":{"telemetry.metrics":{"values":["off","collect","serve","serve-scraped"],"default":"off"}}},+    {"id":"tally/otel/correctness/spans","summary":"Tracing spans","tier":"standard","placement":"either","dimensions":{"telemetry.tracing":{"values":["off","noop","sdk-inmemory","sdk-otlp"],"default":"off"}}},++    {"id":"acme/core-runner/correctness/lifecycle","summary":"Runner lifecycle","tier":"standard","placement":"either"},+    {"id":"acme/core-ordering/concurrency/ordered","summary":"Ordering","tier":"standard","placement":"either"},+    {"id":"acme/core-batch/correctness/batches","summary":"Batching","tier":"standard","placement":"either"},+    {"id":"acme/quill-adapter/correctness/finalize","summary":"QUILL adapter","tier":"standard","placement":"either"},+    {"id":"acme/tally-adapter/correctness/checkpoint","summary":"Tally adapter","tier":"standard","placement":"either"},+    {"id":"acme/metrics/correctness/truthful","summary":"Metrics","tier":"standard","placement":"either"},++    {"id":"brook/adapter/correctness/delivery","summary":"Brook delivery","tier":"standard","placement":"either"},+    {"id":"brook/adapter/concurrency/rebalance","summary":"Brook rebalance","tier":"extended","placement":"either"},+    {"id":"brook/producer/benchmark/throughput","summary":"Brook producer throughput","tier":"standard","placement":"cell","dimensions":{"pg.durability":{"values":["durable"],"default":"durable"}}},++    {"id":"flux/command/correctness/decide","summary":"Command processing","tier":"smoke","placement":"either"},+    {"id":"flux/process-manager/correctness/react","summary":"Process manager","tier":"standard","placement":"either"},+    {"id":"flux/router/correctness/route","summary":"Router","tier":"standard","placement":"either"},+    {"id":"flux/outbox/concurrency/publish","summary":"Outbox","tier":"standard","placement":"either"},+    {"id":"flux/inbox/correctness/deduplicate","summary":"Inbox","tier":"standard","placement":"either"},+    {"id":"flux/queue/correctness/job","summary":"Job queue","tier":"standard","placement":"either"},+    {"id":"flux/workflow/correctness/resume","summary":"Workflow","tier":"standard","placement":"either"},+    {"id":"flux/timer/concurrency/claim","summary":"Timer","tier":"standard","placement":"either"},+    {"id":"flux/shard/concurrency/lease","summary":"Shard lease","tier":"standard","placement":"either"},+    {"id":"flux/wake/correctness/push","summary":"Push wake","tier":"standard","placement":"either"},++    {"id":"runtime/end-to-end/correctness/two-contexts","summary":"End to end","tier":"extended","placement":"either"},+    {"id":"runtime/failure-matrix/soak/recovery","summary":"Failure recovery soak","tier":"soak","placement":"cell"}+  ]+}
+ test/fixtures/plan/components.synthetic.json view
@@ -0,0 +1,89 @@+{+  "schema": "kenshou.component-graph/v1",+  "ignoredPackages": ["pg-migrate-test-support", "tally-test-support", "flux-dsl"],+  "components": [+    {"id":"postgresql","kind":"service","uri":null,"repository":"postgresql","sourceRoots":[],"packages":[],"selectors":[],"verifiedAgainst":"PostgreSQL 17 and 18","dependsOn":[]},+    {"id":"pg-migrate","kind":"library","uri":"mori://shinzui/pg-migrate","repository":"pg-migrate","sourceRoots":["src/","pg-migrate.cabal"],"packages":["pg-migrate","pg-migrate-embed","pg-migrate-cli","pg-migrate-import-codd","pg-migrate-import-hasql-migration"],"selectors":[],"verifiedAgainst":"pg-migrate 1.1.0.0","dependsOn":[{"to":"postgresql","kind":"runtime"}]},+    {"id":"ephemeral-pg","kind":"harness","uri":"mori://shinzui/ephemeral-pg","repository":"ephemeral-pg","sourceRoots":["src/","ephemeral-pg.cabal"],"packages":["ephemeral-pg"],"selectors":["selftest/kernel/**"],"verifiedAgainst":"ephemeral-pg 0.3.1.0","dependsOn":[{"to":"postgresql","kind":"runtime"}]},+    {"id":"hs-opentelemetry","kind":"library","uri":"mori://iand675/hs-opentelemetry","repository":"hs-opentelemetry","sourceRoots":[],"packages":["hs-opentelemetry-api","hs-opentelemetry-sdk","hs-opentelemetry-exporter-otlp","hs-opentelemetry-exporter-in-memory","hs-opentelemetry-propagator-w3c","hs-opentelemetry-semantic-conventions"],"selectors":["selftest/telemetry/**"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"hs-opentelemetry 1.0 family","dependsOn":[]},+    {"id":"quill-hs","kind":"library","uri":"mori://shinzui/quill-hs","repository":"quill-hs","sourceRoots":["quill-core/","quill-hasql/","quill-effectful/","quill-config/","quill-migration/"],"packages":["quill-core","quill-hasql","quill-effectful","quill-config","quill-migration"],"selectors":["quill/**"],"verifiedAgainst":"quill-hs 0.6.1.0","dependsOn":[{"to":"pg-migrate","kind":"build"},{"to":"hs-opentelemetry","kind":"build"},{"to":"postgresql","kind":"runtime"}]},+    {"id":"tally-store-migrations","kind":"migrations","uri":"mori://shinzui/tally/packages/tally-store-migrations","repository":"tally","sourceRoots":["tally-store-migrations/"],"packages":["tally-store-migrations"],"selectors":[],"verifiedAgainst":"tally-store-migrations 0.4.0.0","dependsOn":[{"to":"pg-migrate","kind":"build"},{"to":"postgresql","kind":"runtime"}]},+    {"id":"tally-store","kind":"library","uri":"mori://shinzui/tally/packages/tally-store","repository":"tally","sourceRoots":["tally-store/"],"packages":["tally-store"],"selectors":["tally/**"],"verifiedAgainst":"tally-store 0.8.0.1","dependsOn":[{"to":"tally-store-migrations","kind":"runtime"},{"to":"postgresql","kind":"runtime"}]},+    {"id":"tally-cli","kind":"library","uri":"mori://shinzui/tally/packages/tally-cli","repository":"tally","sourceRoots":["tally-cli/"],"packages":["tally-cli"],"selectors":[],"verifiedAgainst":"tally-cli 0.2.0.6","dependsOn":[{"to":"tally-store","kind":"build"}]},+    {"id":"tally-otel","kind":"library","uri":"mori://shinzui/tally/packages/tally-otel","repository":"tally","sourceRoots":["tally-otel/"],"packages":["tally-otel"],"selectors":["tally/otel/**"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"tally-otel 0.2.0.8","dependsOn":[{"to":"tally-store","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+    {"id":"tally-metrics","kind":"library","uri":"mori://shinzui/tally/packages/tally-metrics","repository":"tally","sourceRoots":["tally-metrics/"],"packages":["tally-metrics"],"selectors":["tally/metrics/**","flux/outbox/correctness/telemetry-contract","flux/outbox/benchmark/**","flux/inbox/benchmark/**"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"tally-metrics 0.1.0.8","dependsOn":[{"to":"tally-store","kind":"build"},{"to":"tally-cli","kind":"build"}]},+    {"id":"acme-core","kind":"library","uri":"mori://shinzui/acme/packages/acme-core","repository":"acme","sourceRoots":["acme-core/src/","acme-core/acme-core.cabal"],"packages":["acme-core"],"selectors":[],"verifiedAgainst":"acme-core 0.9.0.3","changeImplies":null,+      "subcomponents":[+        {"id":"contract","pathPrefixes":["acme-core/src/Acme/Core.hs","acme-core/src/Acme/Core/","acme-core/src/Acme/Adapter.hs","acme-core/src/Acme/Adapter/","acme-core/src/Acme/Handler.hs","acme-core/src/Acme/Policy.hs"],"selectors":["acme/**"],"uses":[]},+        {"id":"telemetry","pathPrefixes":["acme-core/src/Acme/Telemetry.hs","acme-core/src/Acme/Telemetry/"],"selectors":["acme/**"],"uses":[],"changeImplies":{"minDimensionPolicy":"telemetry-corners"}},+        {"id":"runner","pathPrefixes":["acme-core/src/Acme/App.hs","acme-core/src/Acme/Internal/","acme-core/src/Acme/Batch.hs","acme-core/src/Acme/Stream.hs"],"selectors":["acme/**","brook/**"],"uses":["contract","telemetry"]}+      ],"dependsOn":[{"to":"hs-opentelemetry","kind":"build"}]},+    {"id":"acme-metrics","kind":"library","uri":"mori://shinzui/acme/packages/acme-metrics","repository":"acme","sourceRoots":["acme-metrics/"],"packages":["acme-metrics"],"selectors":["acme/metrics/**","flux/queue/correctness/worker-metrics-contract"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"acme-metrics 0.9.0.3","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract","runner"]}]},+    {"id":"acme-quill-adapter","kind":"library","uri":"mori://shinzui/acme-quill-adapter","repository":"acme-quill-adapter","sourceRoots":["src/","acme-quill-adapter.cabal"],"packages":["acme-quill-adapter"],"selectors":["acme/quill-adapter/**"],"verifiedAgainst":"acme-quill-adapter 0.16.0.0","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract","telemetry"]},{"to":"quill-hs","kind":"build"}]},+    {"id":"acme-tally-adapter","kind":"library","uri":"mori://shinzui/tally/packages/acme-tally-adapter","repository":"tally","sourceRoots":["acme-tally-adapter/"],"packages":["acme-tally-adapter"],"selectors":["acme/tally-adapter/**"],"verifiedAgainst":"acme-tally-adapter 0.5.1.2","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract","telemetry","runner"]},{"to":"tally-store","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+    {"id":"hw-brook-client","kind":"library","uri":"mori://haskell-works/hw-brook-client","repository":"hw-brook-client","sourceRoots":[],"packages":["hw-brook-client"],"selectors":["brook/**"],"verifiedAgainst":"hw-brook-client 5.3.0","dependsOn":[]},+    {"id":"hw-brook-streamly","kind":"library","uri":"mori://shinzui/hw-brook-streamly","repository":"hw-brook-streamly","sourceRoots":["src/"],"packages":["hw-brook-streamly"],"selectors":["brook/**"],"verifiedAgainst":"hw-brook-streamly 0.2.0.0","dependsOn":[{"to":"hw-brook-client","kind":"build"}]},+    {"id":"brook-effectful","kind":"library","uri":"mori://shinzui/brook-effectful","repository":"brook-effectful","sourceRoots":["src/"],"packages":["brook-effectful"],"selectors":["brook/**"],"verifiedAgainst":"brook-effectful 0.3.1.0","dependsOn":[{"to":"hw-brook-client","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+    {"id":"acme-brook-adapter","kind":"library","uri":"mori://shinzui/acme-brook-adapter","repository":"acme-brook-adapter","sourceRoots":["src/"],"packages":["acme-brook-adapter"],"selectors":["brook/**"],"verifiedAgainst":"acme-brook-adapter 0.9.0.1","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract"]},{"to":"brook-effectful","kind":"build"},{"to":"hw-brook-streamly","kind":"build"},{"to":"hw-brook-client","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+    {"id":"keiki","kind":"library","uri":"mori://shinzui/keiki","repository":"keiki","sourceRoots":["keiki/","keiki-codec-json/"],"packages":["keiki","keiki-codec-json"],"selectors":[],"verifiedAgainst":"keiki 0.9.1.0","dependsOn":[]},+    {"id":"flux-core","kind":"library","uri":"mori://shinzui/flux/packages/flux-core","repository":"flux","sourceRoots":["flux-core/"],"packages":["flux-core"],"selectors":[],"verifiedAgainst":"flux-core 0.17.0.0","dependsOn":[{"to":"keiki","kind":"build"},{"to":"tally-store","kind":"build"}]},+    {"id":"flux","kind":"library","uri":"mori://shinzui/flux/packages/flux","repository":"flux","sourceRoots":["flux/src/","flux/flux.cabal"],"packages":["flux"],"selectors":[],"verifiedAgainst":"flux 0.17.0.0",+      "subcomponents":[+        {"id":"command","pathPrefixes":["flux/src/Flux/Command.hs","flux/src/Flux/Command/","flux/src/Flux/ReplayAudit.hs"],"selectors":["flux/command/**"],"uses":["snapshot","telemetry"]},+        {"id":"snapshot","pathPrefixes":["flux/src/Flux/Snapshot.hs","flux/src/Flux/Snapshot/"],"selectors":["flux/snapshot/**"],"uses":[]},+        {"id":"projection","pathPrefixes":["flux/src/Flux/Projection.hs","flux/src/Flux/Projection/","flux/src/Flux/ReadModel.hs","flux/src/Flux/ReadModel/"],"selectors":["flux/projection/**","flux/command/**"],"uses":["command","telemetry"]},+        {"id":"process-manager","pathPrefixes":["flux/src/Flux/ProcessManager.hs","flux/src/Flux/ProcessManager/"],"selectors":["flux/process-manager/**"],"uses":["command","projection","timer","telemetry"]},+        {"id":"router","pathPrefixes":["flux/src/Flux/Router.hs","flux/src/Flux/Router/"],"selectors":["flux/router/**"],"uses":["command","process-manager","projection","telemetry"]},+        {"id":"outbox","pathPrefixes":["flux/src/Flux/Outbox.hs","flux/src/Flux/Outbox/"],"selectors":["flux/outbox/**"],"uses":["telemetry"]},+        {"id":"inbox","pathPrefixes":["flux/src/Flux/Inbox.hs","flux/src/Flux/Inbox/"],"selectors":["flux/inbox/**"],"uses":["command","process-manager","telemetry"]},+        {"id":"workflow","pathPrefixes":["flux/src/Flux/Workflow.hs","flux/src/Flux/Workflow/"],"selectors":["flux/workflow/**"],"uses":["timer","wake","snapshot","telemetry"]},+        {"id":"timer","pathPrefixes":["flux/src/Flux/Timer.hs","flux/src/Flux/Timer/"],"selectors":["flux/timer/**"],"uses":["telemetry"]},+        {"id":"shard","pathPrefixes":["flux/src/Flux/Subscription/"],"selectors":["flux/shard/**"],"uses":[]},+        {"id":"wake","pathPrefixes":["flux/src/Flux/Wake.hs"],"selectors":["flux/wake/**"],"uses":[]},+        {"id":"telemetry","pathPrefixes":["flux/src/Flux/Telemetry.hs"],"selectors":["flux/**"],"uses":[],"changeImplies":{"minDimensionPolicy":"telemetry-corners"}}+      ],"dependsOn":[{"to":"flux-core","kind":"build"},{"to":"keiki","kind":"build"},{"to":"tally-store","kind":"build"},{"to":"hs-opentelemetry","kind":"build"},{"to":"acme-core","kind":"build","toSubs":["contract"],"fromSubs":["process-manager","router","inbox"]},{"to":"flux-migrations","kind":"runtime"}]},+    {"id":"flux-quill","kind":"library","uri":"mori://shinzui/flux/packages/flux-quill","repository":"flux","sourceRoots":["flux-quill/src/","flux-quill/flux-quill.cabal"],"packages":["flux-quill"],"selectors":["flux/queue/**"],"verifiedAgainst":"flux-quill 0.17.0.0","dependsOn":[{"to":"flux-core","kind":"build"},{"to":"quill-hs","kind":"build"},{"to":"acme-quill-adapter","kind":"build"},{"to":"acme-core","kind":"build","toSubs":["contract","runner","telemetry"]},{"to":"hs-opentelemetry","kind":"build"}]},+    {"id":"flux-migrations","kind":"migrations","uri":"mori://shinzui/flux/packages/flux-migrations","repository":"flux","sourceRoots":["flux-migrations/"],"packages":["flux-migrations"],"selectors":[],"verifiedAgainst":"flux-migrations 0.17.0.0","dependsOn":[{"to":"tally-store-migrations","kind":"build"},{"to":"pg-migrate","kind":"build"},{"to":"postgresql","kind":"runtime"}]},+    {"id":"flux-test-support","kind":"harness","uri":"mori://shinzui/flux/packages/flux-test-support","repository":"flux","sourceRoots":["flux-test-support/"],"packages":["flux-test-support"],"selectors":["selftest/kernel/**"],"verifiedAgainst":"flux-test-support 0.17.0.0","dependsOn":[{"to":"ephemeral-pg","kind":"build"},{"to":"flux-migrations","kind":"build"},{"to":"tally-store","kind":"build"},{"to":"tally-store-migrations","kind":"build"},{"to":"pg-migrate","kind":"build"}]},+    {"id":"kenshou-harness","kind":"harness","uri":"mori://shinzui/flux-runtime-kenshou/packages/kenshou-core","repository":"flux-runtime-kenshou","sourceRoots":["kenshou-core/"],"packages":["kenshou-core"],"selectors":["selftest/kernel/correctness/postgres-roundtrip"],"verifiedAgainst":"kenshou-core 0.1.0.0","subcomponents":[{"id":"kernel","pathPrefixes":[],"selectors":["selftest/kernel/**"],"uses":[]}],"dependsOn":[{"to":"pg-migrate","kind":"build"},{"to":"tally-store-migrations","kind":"build"},{"to":"flux-migrations","kind":"build"},{"to":"quill-hs","kind":"build"},{"to":"ephemeral-pg","kind":"build"},{"to":"tally-store","kind":"runtime","fromSubs":["kernel"]}]},+    {"id":"flux-ops","kind":"library","uri":"mori://shinzui/flux/packages/flux-ops","repository":"flux","sourceRoots":["flux-ops/src/","flux-ops/flux-ops.cabal"],"packages":["flux-ops"],"selectors":["runtime/ops/**"],"verifiedAgainst":"flux-ops 0.17.0.0","dependsOn":[{"to":"flux","kind":"build"},{"to":"flux-migrations","kind":"build"},{"to":"flux-quill","kind":"build"},{"to":"tally-store","kind":"build"}]},+    {"id":"runtime-assembly","kind":"assembly","uri":"mori://shinzui/flux-runtime-kenshou/packages/kenshou-runtime","repository":"flux-runtime-kenshou","sourceRoots":["kenshou-runtime/"],"packages":["kenshou-runtime"],"selectors":["runtime/**"],"verifiedAgainst":"kenshou-runtime 0.1.0.0","dependsOn":[{"to":"flux","kind":"build"},{"to":"flux-core","kind":"build"},{"to":"keiki","kind":"build"},{"to":"tally-store","kind":"build"},{"to":"brook-effectful","kind":"build"},{"to":"hw-brook-client","kind":"build"},{"to":"kenshou-harness","kind":"build"},{"to":"flux-quill","kind":"build"},{"to":"acme-core","kind":"build"},{"to":"acme-brook-adapter","kind":"build"},{"to":"flux-ops","kind":"build"},{"to":"hs-opentelemetry","kind":"build"},{"to":"tally-otel","kind":"build"},{"to":"quill-hs","kind":"build"},{"to":"flux-migrations","kind":"runtime"}]}+  ],+  "repositoryPaths": [+    {"prefix":"kenshou-core/src/Kenshou/Plan/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"kenshou-core/data/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"kenshou-cli/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"kenshou-remote/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"kenshou-evidence/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"schemas/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"suites/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"policies/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+    {"prefix":"kenshou-quill/","effect":{"kind":"selectors","selectors":["quill/**"]}},+    {"prefix":"kenshou-tally/","effect":{"kind":"selectors","selectors":["tally/**"]}},+    {"prefix":"kenshou-acme/","effect":{"kind":"selectors","selectors":["acme/**"]}},+    {"prefix":"kenshou-brook/","effect":{"kind":"selectors","selectors":["brook/**","runtime/**"]}},+    {"prefix":"kenshou-flux/","effect":{"kind":"selectors","selectors":["flux/**","runtime/**"]}},+    {"prefix":"kenshou-runtime/","effect":{"kind":"selectors","selectors":["runtime/**"]}},+    {"prefix":"cohort/","effect":{"kind":"cohort"}},+    {"prefix":"kenshou-core/","effect":{"kind":"all"}},+    {"prefix":"kenshou-measure/","effect":{"kind":"all"}},+    {"prefix":"kenshou-check/","effect":{"kind":"all"}},+    {"prefix":"kenshou-diagnose/","effect":{"kind":"all"}},+    {"prefix":"kenshou-telemetry/","effect":{"kind":"all"}},+    {"prefix":"cabal.project","effect":{"kind":"all"}},+    {"prefix":"flake.","effect":{"kind":"all"}},+    {"prefix":"nix/","effect":{"kind":"all"}},+    {"prefix":"docs/","effect":{"kind":"nothing"}},+    {"prefix":"agents/","effect":{"kind":"nothing"}},+    {"prefix":".seihou/","effect":{"kind":"nothing"}},+    {"prefix":".github/","effect":{"kind":"nothing"}},+    {"prefix":".claude/","effect":{"kind":"nothing"}},+    {"prefix":".agents/","effect":{"kind":"nothing"}},+    {"prefix":"mori/","effect":{"kind":"nothing"}},+    {"prefix":"README.md","effect":{"kind":"nothing"}},+    {"prefix":"mori.dhall","effect":{"kind":"nothing"}},+    {"prefix":"mina.kdl","effect":{"kind":"nothing"}},+    {"prefix":"Justfile","effect":{"kind":"nothing"}},+    {"prefix":"process-compose.yaml","effect":{"kind":"nothing"}}+  ]+}
+ test/fixtures/plan/plan-missing-edge.json view
@@ -0,0 +1,6 @@+{+  "install-plan": [+    {"id":"krk-str-0.8.0.1-8c49cfa0","pkg-name":"tally-store","pkg-version":"0.8.0.1","component-name":"lib","depends":[]},+    {"id":"quill-hsql-0.6.1.0-c3f074ba","pkg-name":"quill-hasql","pkg-version":"0.6.1.0","component-name":"lib","depends":["krk-str-0.8.0.1-8c49cfa0"]}+  ]+}
+ test/fixtures/valid-project/cohort/active.project view
@@ -0,0 +1,1 @@+import: released.project
+ test/golden/manifest.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.artifact-manifest/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","createdAt":"2026-09-20T17:20:03.902Z","algorithm":"sha256","files":[{"path":"run-spec.json","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","bytes":82,"mediaType":"application/json"}]}
+ test/golden/perf-calibration-acceptance.synthetic.json view
@@ -0,0 +1,368 @@+{+  "schema": "kenshou.perf-calibration-acceptance/v1",+  "batchId": "0199a3c3-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "aa": [+    {+      "seed": 1,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/1/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 2,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/2/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 3,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/3/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 4,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/4/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 5,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/5/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 6,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/6/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 7,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/7/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 8,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/8/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 9,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/9/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 10,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/10/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 11,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/11/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 12,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/12/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 13,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/13/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 14,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/14/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 15,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/15/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 16,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/16/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 17,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/17/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 18,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/18/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 19,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/19/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 20,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/aa/20/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    }+  ],+  "slowdown": [+    {+      "seed": 1,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/1/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 2,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/2/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 3,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/3/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 4,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/4/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 5,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/5/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 6,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/6/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 7,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/7/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 8,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/8/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 9,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/9/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 10,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/10/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 11,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/11/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 12,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/12/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 13,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/13/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 14,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/14/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 15,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/15/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 16,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/16/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 17,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/17/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 18,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/18/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 19,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/19/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    },+    {+      "seed": 20,+      "receipts": [+        {+          "uri": "gs://fixture-evidence/slowdown/20/receipt.json",+          "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+        }+      ]+    }+  ]+}
+ test/golden/perf-calibration-definition.synthetic.json view
@@ -0,0 +1,41 @@+{+  "schema": "kenshou.perf-calibration-definition/v1",+  "batchId": "0199a3c3-7b1e-7c44-9d0a-3f5e2a61b7c9",+  "project": "mori://fixture/runtime",+  "profile": "synthetic-core",+  "workloadSha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+  "toolchainSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+  "harnessSemanticsSha256": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",+  "baselineRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+  "slowdownRevision": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+  "injectionMechanism": "Synthetic codec control only; no runtime evidence.",+  "targetMetric": "op.latency.p99",+  "targetRelativeEffect": 0.5,+  "targetAbsoluteEffect": 1000000,+  "seeds": [+    1,+    2,+    3,+    4,+    5,+    6,+    7,+    8,+    9,+    10,+    11,+    12,+    13,+    14,+    15,+    16,+    17,+    18,+    19,+    20+  ],+  "infrastructureRetryCap": 1,+  "expiresAt": "2026-10-12T00:00:00Z",+  "machineProfile": "synthetic-machine-no-live-evidence",+  "healthPolicySha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+}
+ test/golden/perf-calibration-reference.synthetic.json view
@@ -0,0 +1,6 @@+{+  "definitionUri": "gs://fixture-evidence/definition.json",+  "definitionSha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+  "acceptanceUri": "gs://fixture-evidence/acceptance.json",+  "acceptanceSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+}
+ test/golden/perf-result.not-applicable.fixture.json view
@@ -0,0 +1,17 @@+{+  "schema": "kenshou.perf-result/v1",+  "requestSha256": "sha256:7a0072d3819fddd8d6ef0278182b4a696a7006922e4f93610d3ff3d8338f59ac",+  "preparedSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+  "purpose": "calibration",+  "coverage": "not-applicable",+  "changeCoverage": "incomplete",+  "performanceVerdict": "not-applicable",+  "infrastructureState": "healthy",+  "blockingRuntimeFailures": [],+  "gatingEligible": false,+  "configurations": [],+  "exclusions": [],+  "uncoveredScenarios": [],+  "evidenceVerified": true,+  "exitCode": 0+}
+ test/golden/plan-summary.minimal.json view
@@ -0,0 +1,24 @@+{+  "schema": "kenshou.plan-summary/v1",+  "planId": "0199a3f2-7c1e-7b8a-9d41-3e5f6a7b8c9d",+  "entries": [+    {+      "ordinal": 1,+      "scenario": "selftest/kernel/correctness/always-pass",+      "status": "completed",+      "attempts": [+        {+          "runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011",+          "startedAt": "2026-09-21T14:18:40Z",+          "finishedAt": "2026-09-21T14:18:41Z",+          "outcome": "passed",+          "knownDefect": false,+          "childExitCode": 0+        }+      ]+    }+  ],+  "counts": {"passed": 1},+  "worstOutcome": "passed",+  "exitCode": 0+}
+ test/golden/resources/run-spec.resources.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/postgres-roundtrip","scenarioRevision":1,"knobs":{"selftest.events":3},"dimensions":{"pg.durability":"durable","pg.version":"18"},"seed":7,"phases":{"warmUpSeconds":0,"steadySeconds":0,"drainSeconds":0},"timeoutSeconds":120,"environment":{"placement":"local","machineProfile":"local/darwin-aarch64","postgres":{"mode":"ephemeral","settings":{}},"telemetry":null,"resources":{"broker":{"backend":"external","brokers":["10.0.0.5:9092"]},"object-store":{"bucket":"fixture"}}},"cohortExpectation":null,"comparison":null,"labels":{}}
+ test/golden/resources/worker-init.resources.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.worker-init/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/postgres-roundtrip","role":"selftest/worker","instance":"worker-0","seed":7,"knobs":{},"dimensions":{},"postgres":null,"resources":{"broker":{"backend":"external","brokers":["10.0.0.5:9092"]},"object-store":{"bucket":"fixture"}},"telemetry":null,"outDir":"/tmp/out","args":{}}
+ test/golden/run-plan.minimal.json view
@@ -0,0 +1,44 @@+{+  "schema": "kenshou.run-plan/v1",+  "planId": "0199a3f2-7c1e-7b8a-9d41-3e5f6a7b8c9d",+  "createdAt": "2026-09-21T14:03:22Z",+  "suite": null,+  "graphDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",+  "cohort": {"name": "released", "planHash": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},+  "inputs": {"changed": ["kenshou-harness"]},+  "policy": {+    "maxTier": "standard",+    "kinds": ["correctness"],+    "placement": "local",+    "dimensionPolicy": "default-only",+    "knobPolicy": "defaults",+    "trials": 3,+    "seed": 42+  },+  "changes": [{"component": "kenshou-harness", "source": "named", "detail": "--changed kenshou-harness"}],+  "runs": [+    {+      "ordinal": 1,+      "runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011",+      "estimateMinutes": 1,+      "reasons": [{"component": "kenshou-harness", "source": "named", "via": ["kenshou-harness"], "selector": "selftest/kernel/correctness/postgres-roundtrip", "distance": 0}],+      "trial": null,+      "spec": {+        "schema": "kenshou.run-spec/v1",+        "runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011",+        "scenario": "selftest/kernel/correctness/postgres-roundtrip",+        "scenarioRevision": 1,+        "knobs": {},+        "dimensions": {},+        "seed": 42,+        "environment": {"placement": "local"},+        "cohortExpectation": null,+        "comparison": null,+        "labels": {}+      }+    }+  ],+  "skipped": [],+  "warnings": [],+  "estimateMinutes": 1+}
+ test/golden/run-result.known-defect.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-result/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e56","scenario":"selftest/kernel/correctness/known-defect","scenarioRevision":1,"outcome":"failed","blocking":false,"exitCode":0,"seed":7,"spec":{"path":"run-spec.json","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"cohort":{},"fingerprint":{},"compatibility":{"algorithm":"kenshou.compat-key/v1","comparisonKey":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","seriesKey":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","inputs":{}},"summaries":{},"observations":[]}
+ test/golden/run-result.passed.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-result/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/always-pass","scenarioRevision":1,"outcome":"passed","blocking":false,"exitCode":0,"seed":7,"spec":{"path":"run-spec.json","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"cohort":{},"fingerprint":{},"compatibility":{"algorithm":"kenshou.compat-key/v1","comparisonKey":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","seriesKey":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","inputs":{}},"summaries":{},"observations":[]}
+ test/golden/run-spec.effective.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/postgres-roundtrip","scenarioRevision":1,"knobs":{"selftest.events":3},"dimensions":{"pg.durability":"durable","pg.version":"18"},"seed":7,"phases":{"warmUpSeconds":0,"steadySeconds":0,"drainSeconds":0},"timeoutSeconds":120,"environment":{"placement":"local","machineProfile":"local/darwin-aarch64","postgres":{"mode":"ephemeral","settings":{}},"telemetry":null},"cohortExpectation":null,"comparison":null,"labels":{}}
+ test/golden/run-spec.external.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","scenario":"selftest/kernel/correctness/postgres-roundtrip","environment":{"placement":"local","postgres":{"mode":"external","connectionStringEnv":"KENSHOU_PG_URL"}}}
+ test/golden/run-spec.minimal.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","scenario":"selftest/kernel/correctness/always-pass"}
+ test/golden/scenario-list.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.scenario-list/v1","suiteVersion":"0.1.0.0","scenarios":[{"id":"selftest/kernel/correctness/always-pass","layer":"selftest","component":"kernel","kind":"correctness","name":"always-pass","revision":1,"summary":"Always reports passed.","tier":"smoke","placement":"either","knownDefect":null}],"roles":[{"name":"selftest/echo","summary":"Echoes custom worker messages."}]}
+ test/golden/worker-messages.jsonl view
@@ -0,0 +1,4 @@+{"v":1,"type":"ready"}+{"v":1,"type":"custom","name":"echo","payload":{"n":1}}+{"v":1,"type":"stop","graceMillis":2000}+{"v":1,"type":"done","detail":null}