kenshou-core (empty) → 0.1.0.0
raw patch · 96 files changed
+10796/−0 lines, 96 filesdep +aesondep +basedep +base16-bytestring
Dependencies added: aeson, base, base16-bytestring, bytestring, containers, cryptohash-sha256, directory, ephemeral-pg, file-embed, filepath, hasql, hedgehog, hspec, hspec-hedgehog, kenshou-core, mmzk-typeid, optparse-applicative, pg-migrate, process, random, scientific, settei, settei-optparse-applicative, settei-yaml, splitmix, temporary, text, time, unix, uuid, vector
Files
- CHANGELOG.md +17/−0
- LICENSE +28/−0
- data/components.json +333/−0
- kenshou-core.cabal +169/−0
- src/Kenshou/Core/Bundle.hs +272/−0
- src/Kenshou/Core/Canonical.hs +39/−0
- src/Kenshou/Core/Cli.hs +128/−0
- src/Kenshou/Core/Cli/Config.hs +36/−0
- src/Kenshou/Core/Cohort.hs +636/−0
- src/Kenshou/Core/Compat.hs +118/−0
- src/Kenshou/Core/Context.hs +135/−0
- src/Kenshou/Core/Contribution.hs +97/−0
- src/Kenshou/Core/Dimension.hs +163/−0
- src/Kenshou/Core/Env.hs +61/−0
- src/Kenshou/Core/Env/Migration.hs +54/−0
- src/Kenshou/Core/Env/Postgres.hs +318/−0
- src/Kenshou/Core/Fingerprint.hs +118/−0
- src/Kenshou/Core/Id.hs +174/−0
- src/Kenshou/Core/Knob.hs +177/−0
- src/Kenshou/Core/Log.hs +57/−0
- src/Kenshou/Core/Manifest.hs +94/−0
- src/Kenshou/Core/Outcome.hs +55/−0
- src/Kenshou/Core/Phase.hs +44/−0
- src/Kenshou/Core/Resource.hs +37/−0
- src/Kenshou/Core/Resource/Types.hs +71/−0
- src/Kenshou/Core/Role.hs +142/−0
- src/Kenshou/Core/Role/Dispatch.hs +45/−0
- src/Kenshou/Core/Role/Spawn.hs +66/−0
- src/Kenshou/Core/Run.hs +169/−0
- src/Kenshou/Core/RunResult.hs +114/−0
- src/Kenshou/Core/RunSpec.hs +223/−0
- src/Kenshou/Core/RunSpec/Resolve.hs +106/−0
- src/Kenshou/Core/Scenario.hs +119/−0
- src/Kenshou/Core/Selector.hs +40/−0
- src/Kenshou/Core/Selftest.hs +151/−0
- src/Kenshou/Core/Selftest/Schema.hs +26/−0
- src/Kenshou/Core/Version.hs +9/−0
- src/Kenshou/Perf/Artifacts.hs +64/−0
- src/Kenshou/Perf/Calibration.hs +185/−0
- src/Kenshou/Perf/Contracts.hs +40/−0
- src/Kenshou/Perf/Plan.hs +345/−0
- src/Kenshou/Perf/Prepared.hs +352/−0
- src/Kenshou/Perf/Request.hs +126/−0
- src/Kenshou/Perf/Result.hs +192/−0
- src/Kenshou/Perf/Source.hs +173/−0
- src/Kenshou/Plan/Catalog.hs +208/−0
- src/Kenshou/Plan/Change.hs +131/−0
- src/Kenshou/Plan/Change/Cohort.hs +87/−0
- src/Kenshou/Plan/Change/Git.hs +128/−0
- src/Kenshou/Plan/Components.hs +347/−0
- src/Kenshou/Plan/Components/Check.hs +121/−0
- src/Kenshou/Plan/Execute.hs +237/−0
- src/Kenshou/Plan/Matrix.hs +167/−0
- src/Kenshou/Plan/Policy.hs +100/−0
- src/Kenshou/Plan/RunPlan.hs +270/−0
- src/Kenshou/Plan/Selector.hs +23/−0
- src/Kenshou/Plan/Suite.hs +148/−0
- src/Kenshou/Plan/Summary.hs +127/−0
- test/Kenshou/ConsumerBaseline.hs +25/−0
- test/Kenshou/PerfCalibrationSpec.hs +80/−0
- test/Kenshou/PerfPreparedSpec.hs +171/−0
- test/Kenshou/PerfResultSpec.hs +116/−0
- test/Kenshou/PerfSpec.hs +230/−0
- test/Kenshou/PlanSpec.hs +336/−0
- test/Kenshou/ResourceSpec.hs +51/−0
- test/Main.hs +582/−0
- test/fixtures/cohort-identity.golden.json +19/−0
- test/fixtures/descriptor-released.json +29/−0
- test/fixtures/invalid-project/cohort/active.project +2/−0
- test/fixtures/perf/catalog.v1.json +109/−0
- test/fixtures/perf/cohort.json +89/−0
- test/fixtures/plan-commit-change.json +8/−0
- test/fixtures/plan-local-change.json +9/−0
- test/fixtures/plan-released.json +34/−0
- test/fixtures/plan-reordered.json +13/−0
- test/fixtures/plan-version-change.json +8/−0
- test/fixtures/plan/catalog-planned.json +64/−0
- test/fixtures/plan/components.synthetic.json +89/−0
- test/fixtures/plan/plan-missing-edge.json +6/−0
- test/fixtures/valid-project/cohort/active.project +1/−0
- test/golden/manifest.json +1/−0
- test/golden/perf-calibration-acceptance.synthetic.json +368/−0
- test/golden/perf-calibration-definition.synthetic.json +41/−0
- test/golden/perf-calibration-reference.synthetic.json +6/−0
- test/golden/perf-result.not-applicable.fixture.json +17/−0
- test/golden/plan-summary.minimal.json +24/−0
- test/golden/resources/run-spec.resources.json +1/−0
- test/golden/resources/worker-init.resources.json +1/−0
- test/golden/run-plan.minimal.json +44/−0
- test/golden/run-result.known-defect.json +1/−0
- test/golden/run-result.passed.json +1/−0
- test/golden/run-spec.effective.json +1/−0
- test/golden/run-spec.external.json +1/−0
- test/golden/run-spec.minimal.json +1/−0
- test/golden/scenario-list.json +1/−0
- test/golden/worker-messages.jsonl +4/−0
+ CHANGELOG.md view
@@ -0,0 +1,17 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of the scenario kernel: run specifications, results, worker roles, knobs, dimensions, phases, cohort identity, change-aware planning, and the performance session protocol.+- Open layer identifiers (`LayerId`, `mkLayerId`) replace the closed `Layer` type, and a suite registers its layers as `LayerBundle` values built with `mkBundle`.+- Suite-supplied migration sets (`MigrationSet`) compose into one pg-migrate ledger per run database.+- Named resources (`ResourceProvider`, `requiresResource`) replace the broker-shaped environment fields; documents written in the legacy shape remain readable.+- Bundles contribute oracles, advisory-lock labels, pool methodology, routing rules, and their evidence subject.+- `Program`, `defaultProgram`, and `runCli` let an executable supply its bundles, commands, component graph, and performance contracts.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.+- The kernel no longer depends on any runtime migration library; its own self-test schema is the `kenshou-selftest` migration set.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+ may be used to endorse or promote products derived from this software without+ specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ data/components.json view
@@ -0,0 +1,333 @@+{+ "schema": "kenshou.component-graph/v1",+ "ignoredPackages": [+ "pg-migrate-test-support"+ ],+ "components": [+ {+ "id": "postgresql",+ "kind": "service",+ "uri": null,+ "repository": "postgresql",+ "sourceRoots": [],+ "packages": [],+ "selectors": [],+ "verifiedAgainst": "PostgreSQL 17 and 18",+ "dependsOn": []+ },+ {+ "id": "pg-migrate",+ "kind": "library",+ "uri": "mori://shinzui/pg-migrate",+ "repository": "pg-migrate",+ "sourceRoots": [+ "src/",+ "pg-migrate.cabal"+ ],+ "packages": [+ "pg-migrate"+ ],+ "selectors": [],+ "verifiedAgainst": "pg-migrate 1.1.0.0",+ "dependsOn": [+ {+ "to": "postgresql",+ "kind": "runtime"+ }+ ]+ },+ {+ "id": "ephemeral-pg",+ "kind": "harness",+ "uri": "mori://shinzui/ephemeral-pg",+ "repository": "ephemeral-pg",+ "sourceRoots": [+ "src/",+ "ephemeral-pg.cabal"+ ],+ "packages": [+ "ephemeral-pg"+ ],+ "selectors": [+ "selftest/kernel/**"+ ],+ "verifiedAgainst": "ephemeral-pg 0.3.1.0",+ "dependsOn": [+ {+ "to": "postgresql",+ "kind": "runtime"+ }+ ]+ },+ {+ "id": "hs-opentelemetry",+ "kind": "library",+ "uri": "mori://iand675/hs-opentelemetry",+ "repository": "hs-opentelemetry",+ "sourceRoots": [],+ "packages": [+ "hs-opentelemetry-api",+ "hs-opentelemetry-sdk",+ "hs-opentelemetry-exporter-otlp",+ "hs-opentelemetry-propagator-w3c",+ "hs-opentelemetry-semantic-conventions"+ ],+ "selectors": [+ "selftest/telemetry/**"+ ],+ "changeImplies": {+ "minDimensionPolicy": "telemetry-corners"+ },+ "verifiedAgainst": "hs-opentelemetry 1.0 family",+ "dependsOn": []+ },+ {+ "id": "kenshou-harness",+ "kind": "harness",+ "uri": "mori://shinzui/kenshou/packages/kenshou-core",+ "repository": "kenshou",+ "sourceRoots": [+ "kenshou-core/"+ ],+ "packages": [+ "kenshou-core"+ ],+ "selectors": [+ "selftest/**"+ ],+ "verifiedAgainst": "kenshou-core 0.1.0.0",+ "subcomponents": [+ {+ "id": "kernel",+ "pathPrefixes": [],+ "selectors": [+ "selftest/kernel/**"+ ],+ "uses": []+ }+ ],+ "dependsOn": [+ {+ "to": "pg-migrate",+ "kind": "build"+ },+ {+ "to": "ephemeral-pg",+ "kind": "build"+ }+ ]+ }+ ],+ "repositoryPaths": [+ {+ "prefix": "kenshou-core/src/Kenshou/Plan/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "kenshou-core/data/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "kenshou-cli/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "kenshou-remote/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "kenshou-evidence/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "schemas/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "suites/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "policies/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "cohort/",+ "effect": {+ "kind": "cohort"+ }+ },+ {+ "prefix": "kenshou-core/",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "kenshou-measure/",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "kenshou-check/",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "kenshou-diagnose/",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "kenshou-telemetry/",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "cabal.project",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "flake.",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "nix/",+ "effect": {+ "kind": "all"+ }+ },+ {+ "prefix": "docs/",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": "agents/",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": ".seihou/",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": ".github/",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": ".claude/",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": ".agents/",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": "README.md",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": "mori.dhall",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": "mina.kdl",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": "justfile",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": "process-compose.yaml",+ "effect": {+ "kind": "nothing"+ }+ },+ {+ "prefix": "kenshou-synthetic/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ },+ {+ "prefix": "fixtures/",+ "effect": {+ "kind": "selectors",+ "selectors": [+ "selftest/**"+ ]+ }+ }+ ]+}
+ kenshou-core.cabal view
@@ -0,0 +1,169 @@+cabal-version: 3.0+name: kenshou-core+version: 0.1.0.0+synopsis: Scenario kernel and protocol for the Kenshou verification harness+description: Shared contracts and utilities for the Kenshou verification harness.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+ data/components.json+ test/fixtures/*.json+ test/fixtures/invalid-project/cohort/*.project+ test/fixtures/perf/*.json+ test/fixtures/plan/*.json+ test/fixtures/valid-project/cohort/*.project+ test/golden/*.json+ test/golden/*.jsonl+ test/golden/resources/*.json++common warnings+ ghc-options:+ -Wall+ -Wcompat+ -Widentities+ -Wincomplete-record-updates+ -Wincomplete-uni-patterns+ -Wpartial-fields+ -Wredundant-constraints++ default-language: GHC2024+ default-extensions:+ BlockArguments+ DuplicateRecordFields+ ImportQualifiedPost+ NoFieldSelectors+ OverloadedLabels+ OverloadedRecordDot+ OverloadedStrings++library+ import: warnings+ hs-source-dirs: src+ exposed-modules:+ Kenshou.Core.Bundle+ Kenshou.Core.Canonical+ Kenshou.Core.Cli+ Kenshou.Core.Cli.Config+ Kenshou.Core.Cohort+ Kenshou.Core.Compat+ Kenshou.Core.Context+ Kenshou.Core.Contribution+ Kenshou.Core.Dimension+ Kenshou.Core.Env+ Kenshou.Core.Env.Migration+ Kenshou.Core.Env.Postgres+ Kenshou.Core.Fingerprint+ Kenshou.Core.Id+ Kenshou.Core.Knob+ Kenshou.Core.Log+ Kenshou.Core.Manifest+ Kenshou.Core.Outcome+ Kenshou.Core.Phase+ Kenshou.Core.Resource+ Kenshou.Core.Resource.Types+ Kenshou.Core.Role+ Kenshou.Core.Role.Dispatch+ Kenshou.Core.Role.Spawn+ Kenshou.Core.Run+ Kenshou.Core.RunResult+ Kenshou.Core.RunSpec+ Kenshou.Core.RunSpec.Resolve+ Kenshou.Core.Scenario+ Kenshou.Core.Selector+ Kenshou.Core.Selftest+ Kenshou.Core.Selftest.Schema+ Kenshou.Core.Version+ Kenshou.Perf.Artifacts+ Kenshou.Perf.Calibration+ Kenshou.Perf.Contracts+ Kenshou.Perf.Plan+ Kenshou.Perf.Prepared+ Kenshou.Perf.Request+ Kenshou.Perf.Result+ Kenshou.Perf.Source+ Kenshou.Plan.Catalog+ Kenshou.Plan.Change+ Kenshou.Plan.Change.Cohort+ Kenshou.Plan.Change.Git+ Kenshou.Plan.Components+ Kenshou.Plan.Components.Check+ Kenshou.Plan.Execute+ Kenshou.Plan.Matrix+ Kenshou.Plan.Policy+ Kenshou.Plan.RunPlan+ Kenshou.Plan.Selector+ Kenshou.Plan.Suite+ Kenshou.Plan.Summary++ other-modules: Paths_kenshou_core+ autogen-modules: Paths_kenshou_core+ build-depends:+ aeson >=2.2 && <2.3,+ base >=4.21 && <4.22,+ base16-bytestring >=1.0 && <1.1,+ bytestring >=0.12 && <0.13,+ containers >=0.7 && <0.8,+ cryptohash-sha256 >=0.11 && <0.12,+ directory >=1.3 && <1.4,+ ephemeral-pg >=0.3.1 && <0.4,+ file-embed >=0.0.16 && <0.1,+ filepath >=1.5 && <1.6,+ hasql >=1.10 && <1.11,+ mmzk-typeid >=0.7 && <0.8,+ optparse-applicative >=0.19 && <0.20,+ pg-migrate ^>=1.1.0.0 || ^>=1.2.0.0,+ process >=1.6 && <1.7,+ random >=1.2.1 && <1.4,+ scientific >=0.3 && <0.4,+ settei ^>=0.2.0.0,+ settei-optparse-applicative ^>=0.2.0.0,+ settei-yaml ^>=0.2.0.0,+ splitmix >=0.1 && <0.2,+ text >=2.1 && <2.2,+ time >=1.14 && <1.15,+ unix >=2.8 && <2.9,+ uuid >=1.3 && <1.4,+ vector >=0.13 && <0.14,++test-suite kenshou-core-test+ import: warnings+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Main.hs+ other-modules:+ Kenshou.ConsumerBaseline+ Kenshou.PerfCalibrationSpec+ Kenshou.PerfPreparedSpec+ Kenshou.PerfResultSpec+ Kenshou.PerfSpec+ Kenshou.PlanSpec+ Kenshou.ResourceSpec++ build-depends:+ aeson >=2.2 && <2.3,+ base >=4.21 && <4.22,+ bytestring,+ containers,+ directory >=1.3 && <1.4,+ filepath,+ hedgehog >=1.4 && <1.8,+ hspec >=2.11 && <2.12,+ hspec-hedgehog >=0.0 && <0.4,+ kenshou-core,+ pg-migrate,+ process >=1.6 && <1.7,+ temporary,+ text >=2.1 && <2.2,+ time >=1.14 && <1.15,++source-repository head+ type: git+ location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Core/Bundle.hs view
@@ -0,0 +1,272 @@+module Kenshou.Core.Bundle+ ( LayerBundle (..),+ mkBundle,+ lookupBundle,+ lookupResource,+ registrySubjects,+ registryRoutingRules,+ ScenarioRule (..),+ requirePgVersionsFor,+ Registry,+ registryMigrationSets,+ RegistryError (..),+ ListFilter (..),+ emptyListFilter,+ mkRegistry,+ allScenarios,+ allRoles,+ lookupScenario,+ lookupRole,+ selectScenarios,+ scenarioListDocument,+ scenarioListDocumentFor,+ )+where++import Data.Aeson (Value, object, toJSON, (.=))+import Data.Foldable (toList)+import Data.List (group, sort, sortOn)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Contribution (AdvisoryLabel, OracleSpec (..), PoolMethodology, RoutingRule)+import Kenshou.Core.Dimension (DimensionSupport (..), PgDurability (..), PgVersion (..), Support (..), Supported (..))+import Kenshou.Core.Env (EnvRequirements (..), MigrationSet (..), MigrationSetName, PostgresRequirement (..), mkMigrationSetName, renderMigrationSetName)+import Kenshou.Core.Id (Kind (..), LayerId, ScenarioId (..), renderKind, renderLayer, renderScenarioId, unSegment)+import Kenshou.Core.Knob (KnobSpec (..), renderKnobName)+import Kenshou.Core.Resource (ResourceName, ResourceProvider (..), renderResourceName)+import Kenshou.Core.Role (RoleName, WorkerRole (..), renderRoleName)+import Kenshou.Core.Scenario (KnownDefect (..), Placement, Scenario (..), Tier, individualKnownDefects, renderPlacement, renderTier)+import Kenshou.Core.Selector (ScenarioSelector, matchesSelector)+import Kenshou.Core.Version (suiteVersion)++-- | What a suite registers with the kernel. Build one with 'mkBundle' and set the+-- optional contributions with record update, so new fields never break a suite.+data LayerBundle = LayerBundle+ { layer :: LayerId,+ scenarios :: [Scenario],+ roles :: [WorkerRole],+ -- | Migration sets this bundle defines. Any scenario in the registry may+ -- require them by name.+ migrations :: [MigrationSet],+ -- | Checks this bundle contributes. 'mkRegistry' applies every bundle's rules+ -- to every scenario in the registry.+ rules :: [ScenarioRule],+ -- | Providers for named resources. Any scenario in the registry may require a+ -- resource a bundle provides.+ resources :: [ResourceProvider],+ -- | SQL oracles the suite's scenarios reconcile, by name.+ oracles :: [OracleSpec],+ -- | Advisory-lock labels a stall capture of the suite's scenarios names.+ advisoryLabels :: [AdvisoryLabel],+ -- | The connection-pool size the suite recommends, when it has one.+ methodology :: Maybe PoolMethodology,+ -- | The canonical Mori URI that evidence records of this layer name as their+ -- subject.+ evidenceSubject :: Maybe Text,+ -- | Cell capabilities the suite's runs need.+ routingRules :: [RoutingRule]+ }++mkBundle :: LayerId -> [Scenario] -> [WorkerRole] -> LayerBundle+mkBundle layer scenarios roles = LayerBundle {layer, scenarios, roles, migrations = [], rules = [], resources = [], oracles = [], advisoryLabels = [], methodology = Nothing, evidenceSubject = Nothing, routingRules = []}++-- | A named registry check. Each message 'check' returns becomes a 'RegistryError'+-- prefixed with the scenario identifier.+data ScenarioRule = ScenarioRule+ { name :: Text,+ check :: Scenario -> [Text]+ }++-- | Require that a scenario using the migration set supports only the given+-- PostgreSQL versions, for a set whose migrations need a particular server.+requirePgVersionsFor :: MigrationSetName -> NonEmpty PgVersion -> ScenarioRule+requirePgVersionsFor set versions =+ ScenarioRule+ { name = renderMigrationSetName set <> "-postgres-versions",+ check = \scenario ->+ [ renderMigrationSetName set <> " scenarios require PostgreSQL " <> Text.intercalate " or " (fmap versionText (toList versions))+ | Just requirement <- [scenario.requires.postgres],+ set `elem` requirement.schemas,+ not (supportsOnly scenario.dimensions.pgVersion)+ ]+ }+ where+ supportsOnly (Supported support) = all (`elem` versions) support.values+ supportsOnly NotApplicable = False+ versionText Pg17 = "17"+ versionText Pg18 = "18"++newtype Registry = Registry [LayerBundle]++newtype RegistryError = RegistryError Text+ deriving stock (Eq, Show)++data ListFilter = ListFilter+ { selectors :: [ScenarioSelector],+ layers :: [LayerId],+ kinds :: [Kind],+ maxTier :: Maybe Tier,+ placements :: [Placement]+ }++emptyListFilter :: ListFilter+emptyListFilter = ListFilter [] [] [] Nothing []++mkRegistry :: [LayerBundle] -> Either (NonEmpty RegistryError) Registry+mkRegistry bundles = case errors of+ [] -> Right (Registry bundles)+ first : rest -> Left (first :| rest)+ where+ scenarios = concatMap (.scenarios) bundles+ roles = concatMap (.roles) bundles+ errors =+ [RegistryError ("scenario layer differs from bundle: " <> renderScenarioId scenario.id) | bundle <- bundles, scenario <- bundle.scenarios, scenario.id.layer /= bundle.layer]+ <> duplicateErrors "scenario" (fmap (renderScenarioId . (.id)) scenarios)+ <> duplicateErrors "role" (fmap (renderRoleName . (.name)) roles)+ <> [RegistryError ("role layer differs from bundle: " <> renderRoleName role.name) | bundle <- bundles, role <- bundle.roles, not ((renderLayer bundle.layer <> "/") `Text.isPrefixOf` renderRoleName role.name)]+ <> concatMap validateScenario scenarios+ <> migrationErrors bundles+ <> resourceErrors bundles+ <> [RegistryError ("evidence subject must be a canonical Mori URI: " <> subject) | bundle <- bundles, Just subject <- [bundle.evidenceSubject], not ("mori://" `Text.isPrefixOf` subject)]+ <> concat [duplicateErrors ("oracle in " <> renderLayer bundle.layer) (fmap (.name) bundle.oracles) | bundle <- bundles]+ <> [RegistryError (renderScenarioId scenario.id <> ": " <> message) | rule <- concatMap (.rules) bundles, scenario <- scenarios, message <- rule.check scenario]++-- Every set and required name is a valid pg-migrate component name, no two bundles+-- define the same set, and every set a scenario or another set requires is defined.+migrationErrors :: [LayerBundle] -> [RegistryError]+migrationErrors bundles =+ [RegistryError problem | name <- definedNames <> requiredNames, Left problem <- [mkMigrationSetName (renderMigrationSetName name)]]+ <> duplicateErrors "migration set" (fmap renderMigrationSetName definedNames)+ <> [ RegistryError (renderScenarioId scenario.id <> ": no registered bundle defines migration set " <> renderMigrationSetName name)+ | scenario <- concatMap (.scenarios) bundles,+ name <- scenarioSets scenario,+ name `notElem` definedNames+ ]+ <> [ RegistryError ("migration set " <> renderMigrationSetName set.name <> " requires undefined set " <> renderMigrationSetName name)+ | set <- sets,+ name <- set.requires,+ name `notElem` definedNames+ ]+ where+ sets = concatMap (.migrations) bundles+ definedNames = fmap (.name) sets+ requiredNames = concatMap scenarioSets (concatMap (.scenarios) bundles) <> concatMap (.requires) sets+ scenarioSets scenario = concatMap (.schemas) (maybe [] pure scenario.requires.postgres <> fmap snd scenario.requires.extraPostgres)++-- No two bundles provide the same resource, and every resource a scenario requires+-- has a provider.+resourceErrors :: [LayerBundle] -> [RegistryError]+resourceErrors bundles =+ duplicateErrors "resource provider" (fmap (renderResourceName . (.name)) providers)+ <> [ RegistryError ("scenario requires unprovided resource " <> renderResourceName required <> ": " <> renderScenarioId scenario.id)+ | scenario <- concatMap (.scenarios) bundles,+ required <- scenario.requires.resources,+ required `notElem` fmap (.name) providers+ ]+ where+ providers = concatMap (.resources) bundles++-- | The first registered bundle of a layer. Several bundles may share a layer;+-- suite contributions are read from the first.+lookupBundle :: Registry -> LayerId -> Maybe LayerBundle+lookupBundle (Registry bundles) wanted = case filter ((== wanted) . (.layer)) bundles of+ bundle : _ -> Just bundle+ [] -> Nothing++-- | The evidence subject each layer's first bundle that declares one names.+registrySubjects :: Registry -> Map.Map LayerId Text+registrySubjects (Registry bundles) = Map.fromListWith (\_ first -> first) [(bundle.layer, subject) | bundle <- bundles, Just subject <- [bundle.evidenceSubject]]++-- | Every bundle's routing rules, in registry order.+registryRoutingRules :: Registry -> [RoutingRule]+registryRoutingRules (Registry bundles) = concatMap (.routingRules) bundles++lookupResource :: Registry -> ResourceName -> Maybe ResourceProvider+lookupResource (Registry bundles) wanted = case [provider | bundle <- bundles, provider <- bundle.resources, provider.name == wanted] of+ provider : _ -> Just provider+ [] -> Nothing++registryMigrationSets :: Registry -> Map.Map MigrationSetName MigrationSet+registryMigrationSets (Registry bundles) = Map.fromList [(set.name, set) | bundle <- bundles, set <- bundle.migrations]++duplicateErrors :: Text -> [Text] -> [RegistryError]+duplicateErrors label values = [RegistryError ("duplicate " <> label <> ": " <> item) | items@(item : _) <- group (sort values), length items > 1]++validateScenario :: Scenario -> [RegistryError]+validateScenario scenario =+ [RegistryError (renderScenarioId scenario.id <> ": revision must be positive") | scenario.revision < 1]+ <> [RegistryError (renderScenarioId scenario.id <> ": duplicate knob " <> name) | name <- duplicateNames]+ <> postgresErrors+ <> benchmarkErrors+ <> knownDefectErrors+ where+ duplicateNames = [item | items@(item : _) <- group (sort (fmap (renderKnobName . (.name)) scenario.knobs)), length items > 1]+ hasPostgres = isJust scenario.requires.postgres+ pgApplicable = case (scenario.dimensions.pgDurability, scenario.dimensions.pgVersion) of+ (NotApplicable, NotApplicable) -> False+ _ -> True+ postgresErrors =+ [RegistryError (renderScenarioId scenario.id <> ": PostgreSQL dimensions disagree with environment requirement") | hasPostgres /= pgApplicable]+ benchmarkErrors = case (scenario.id.kind, scenario.requires.postgres, scenario.dimensions.pgDurability) of+ (Benchmark, Just _, Supported support) | PgFsyncOff `elem` support.values -> [RegistryError (renderScenarioId scenario.id <> ": benchmark supports fsync-off")]+ _ -> []+ knownDefectErrors =+ [ RegistryError (renderScenarioId scenario.id <> ": invalid known-defect reference")+ | defect <- maybe [] individualKnownDefects scenario.knownDefect,+ not ("mori://" `Text.isPrefixOf` defect.reference || "https://" `Text.isPrefixOf` defect.reference)+ ]++allScenarios :: Registry -> [Scenario]+allScenarios (Registry bundles) = sortOn (renderScenarioId . (.id)) (concatMap (.scenarios) bundles)++allRoles :: Registry -> [WorkerRole]+allRoles (Registry bundles) = sortOn (renderRoleName . (.name)) (concatMap (.roles) bundles)++lookupScenario :: Registry -> ScenarioId -> Maybe Scenario+lookupScenario registry wanted = Map.lookup wanted (Map.fromList [(scenario.id, scenario) | scenario <- allScenarios registry])++lookupRole :: Registry -> RoleName -> Maybe WorkerRole+lookupRole registry wanted = Map.lookup wanted (Map.fromList [(role.name, role) | role <- allRoles registry])++selectScenarios :: Registry -> ListFilter -> [Scenario]+selectScenarios registry filterValue = filter selected (allScenarios registry)+ where+ selected scenario =+ (null filterValue.selectors || any (`matchesSelector` scenario.id) filterValue.selectors)+ && (null filterValue.layers || scenario.id.layer `elem` filterValue.layers)+ && (null filterValue.kinds || scenario.id.kind `elem` filterValue.kinds)+ && maybe True (scenario.tier <=) filterValue.maxTier+ && (null filterValue.placements || scenario.placement `elem` filterValue.placements)++scenarioListDocument :: Registry -> Value+scenarioListDocument registry = scenarioListDocumentFor (allScenarios registry) (allRoles registry)++scenarioListDocumentFor :: [Scenario] -> [WorkerRole] -> Value+scenarioListDocumentFor scenarios roles =+ object+ [ "schema" .= ("kenshou.scenario-list/v1" :: Text),+ "suiteVersion" .= suiteVersion,+ "scenarios" .= fmap scenarioValue scenarios,+ "roles" .= fmap roleValue roles+ ]+ where+ scenarioValue scenario =+ object+ [ "id" .= renderScenarioId scenario.id,+ "layer" .= renderLayer scenario.id.layer,+ "component" .= unSegment scenario.id.component,+ "kind" .= renderKind scenario.id.kind,+ "name" .= unSegment scenario.id.name,+ "revision" .= scenario.revision,+ "summary" .= scenario.summary,+ "tier" .= renderTier scenario.tier,+ "placement" .= renderPlacement scenario.placement,+ "knownDefect" .= fmap defectReferences scenario.knownDefect+ ]+ roleValue role = object ["name" .= renderRoleName role.name, "summary" .= role.summary]+ defectReferences defect = case individualKnownDefects defect of+ [single] -> toJSON single.reference+ many -> toJSON (fmap (.reference) many)
+ src/Kenshou/Core/Canonical.hs view
@@ -0,0 +1,39 @@+module Kenshou.Core.Canonical (canonicalEncode, sha256Hex) where++import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson (Value (..), encode)+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString (ByteString)+import Data.ByteString.Base16 qualified as Base16+import Data.ByteString.Builder (Builder, char8, lazyByteString, toLazyByteString)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (sortOn)+import Data.Text (Text)+import Data.Text.Encoding qualified as Text+import Data.Vector qualified as Vector++canonicalEncode :: Value -> ByteString+canonicalEncode = LazyByteString.toStrict . toLazyByteString . render+ where+ render Null = "null"+ render (Bool True) = "true"+ render (Bool False) = "false"+ render (Number number) = lazyByteString (encode number)+ render (String value) = lazyByteString (encode value)+ render (Array values) = enclosed '[' ']' (fmap render (Vector.toList values))+ render (Object values) =+ enclosed+ '{'+ '}'+ [ lazyByteString (encode (Key.toText key)) <> char8 ':' <> render value+ | (key, value) <- sortOn (Text.encodeUtf8 . Key.toText . fst) (KeyMap.toList values)+ ]++ enclosed :: Char -> Char -> [Builder] -> Builder+ enclosed open close values = char8 open <> separated values <> char8 close+ separated [] = mempty+ separated (value : values) = value <> foldMap (char8 ',' <>) values++sha256Hex :: ByteString -> Text+sha256Hex = ("sha256:" <>) . Text.decodeUtf8 . Base16.encode . SHA256.hash
+ src/Kenshou/Core/Cli.hs view
@@ -0,0 +1,128 @@+module Kenshou.Core.Cli+ ( Program (..),+ defaultProgram,+ CliEnv (..),+ CliGroup (..),+ CliCommand (..),+ HelpTopic (..),+ InputSource (..),+ parseInputSource,+ readInputSource,+ runCli,+ exitWithOutcome,+ )+where++import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle, Registry, mkRegistry)+import Kenshou.Core.Outcome (Outcome, outcomeExitCode)+import Kenshou.Perf.Contracts (PerfContracts, noPerfContracts)+import Kenshou.Plan.Components (selftestGraphBytes)+import Options.Applicative+import System.Exit (ExitCode (..))+import System.IO (hPutStrLn, stderr, stdin, stdout)++-- | Everything an executable supplies: its name and help header, the bundles it+-- registers, its commands and help topics, the component graph that change-aware+-- planning reads when no graph file is given, and its performance contracts.+data Program = Program+ { name :: String,+ header :: String,+ bundles :: [LayerBundle],+ commands :: [CliCommand],+ topics :: [HelpTopic],+ componentGraph :: ByteString,+ perf :: PerfContracts+ }++-- | A program named @kenshou@ with no bundles or commands, the self-test component+-- graph, and no performance contracts. Executables extend it with record update.+defaultProgram :: Program+defaultProgram =+ Program+ { name = "kenshou",+ header = "kenshou - run verification scenarios",+ bundles = [],+ commands = [],+ topics = [],+ componentGraph = selftestGraphBytes,+ perf = noPerfContracts+ }++data CliEnv = CliEnv+ { registry :: Registry,+ programName :: Text,+ componentGraph :: ByteString,+ perf :: PerfContracts+ }++data CliGroup = Discovery | Execution | Analysis | Evidence | Maintenance | Internal deriving stock (Eq, Ord, Show, Enum, Bounded)++data CliCommand = CliCommand+ { name :: String,+ description :: String,+ group :: CliGroup,+ hidden :: Bool,+ parser :: Parser (CliEnv -> IO ExitCode)+ }++data HelpTopic = HelpTopic {name :: Text, description :: Text, content :: Text}++data InputSource = InputFile FilePath | InputStdin deriving stock (Eq, Show)++parseInputSource :: String -> InputSource+parseInputSource "-" = InputStdin+parseInputSource path = InputFile path++readInputSource :: InputSource -> IO ByteString+readInputSource InputStdin = ByteString.hGetContents stdin+readInputSource (InputFile path) = ByteString.readFile path++runCli :: Program -> [String] -> IO ExitCode+runCli program arguments = case mkRegistry program.bundles of+ Left problems -> do+ mapM_ (hPutStrLn stderr . show) (NonEmpty.toList problems)+ pure (ExitFailure 4)+ Right registry -> case execParserPure defaultPrefs (parserInfo program) arguments of+ Success action -> action (CliEnv registry (Text.pack program.name) program.componentGraph program.perf)+ Failure failure -> do+ let (message, code) = renderFailure failure program.name+ case code of+ ExitSuccess -> hPutStrLn stdout message >> pure ExitSuccess+ ExitFailure _ -> hPutStrLn stderr message >> pure (ExitFailure 2)+ CompletionInvoked completion -> execCompletion completion program.name >>= putStr >> pure ExitSuccess++parserInfo :: Program -> ParserInfo (CliEnv -> IO ExitCode)+parserInfo program = info (commandParser program.commands <**> helper) (fullDesc <> header program.header)++commandParser :: [CliCommand] -> Parser (CliEnv -> IO ExitCode)+commandParser commands = visibleParser <|> hiddenParser+ where+ visibleCommands = filter (not . (.hidden)) commands+ hiddenCommands = filter (.hidden) commands+ populatedGroups = filter (not . null . commandsIn) [Discovery .. Internal]+ commandsIn groupValue = filter ((== groupValue) . (.group)) visibleCommands+ visibleParser = foldr1 (<|>) (fmap groupParser populatedGroups)+ hiddenParser = subparser (foldMap hiddenCommand hiddenCommands)+ groupParser groupValue = subparser (groupModifier groupValue <> foldMap commandModifier (commandsIn groupValue))+ groupModifier Internal = mempty+ groupModifier groupValue = commandGroup (groupLabel groupValue)+ commandModifier commandValue = command commandValue.name (info (commandValue.parser <**> helper) (progDesc commandValue.description))+ hiddenCommand commandValue = command commandValue.name (info (commandValue.parser <**> helper) (progDesc commandValue.description)) <> internal++groupLabel :: CliGroup -> String+groupLabel Discovery = "Discovery"+groupLabel Execution = "Execution"+groupLabel Analysis = "Analysis"+groupLabel Evidence = "Evidence"+groupLabel Maintenance = "Maintenance"+groupLabel Internal = "Internal"++exitWithOutcome :: Outcome -> ExitCode+exitWithOutcome outcome = case outcomeExitCode outcome of+ 0 -> ExitSuccess+ code -> ExitFailure code
+ src/Kenshou/Core/Cli/Config.hs view
@@ -0,0 +1,36 @@+module Kenshou.Core.Cli.Config+ ( ConfigInputs (..),+ configInputsParser,+ loadConfigSources,+ )+where++import Data.List.NonEmpty qualified as NonEmpty+import Data.Maybe (catMaybes)+import Data.Text (Text)+import Data.Text qualified as Text+import Options.Applicative (Parser)+import Options.Applicative qualified as Options+import Settei (Source)+import Settei.Optparse (DiagnosticMode, configPathOptions, diagnosticModeOptions)+import Settei.Yaml (readYamlSource, renderYamlErrorsText, yamlSourceOptions)++data ConfigInputs = ConfigInputs+ { paths :: [FilePath],+ namedSources :: [Source],+ diagnostic :: DiagnosticMode+ }++configInputsParser :: Parser [Maybe Source] -> Parser ConfigInputs+configInputsParser namedParser =+ ConfigInputs+ <$> Options.parserOptionGroup "Configuration" configPathOptions+ <*> (catMaybes <$> Options.parserOptionGroup "Configuration" namedParser)+ <*> Options.parserOptionGroup "Configuration diagnostics" diagnosticModeOptions++loadConfigSources :: ConfigInputs -> IO (Either Text [Source])+loadConfigSources inputs = do+ loaded <- sequence [readYamlSource (yamlSourceOptions ("configuration " <> Text.pack (show index))) path | (index, path) <- zip [1 :: Int ..] inputs.paths]+ pure $ case traverse (either (Left . renderYamlErrorsText) Right) loaded of+ Left err -> Left err+ Right sources -> Right (sources <> inputs.namedSources)
+ src/Kenshou/Core/Cohort.hs view
@@ -0,0 +1,636 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Core.Cohort+ ( CohortName (..),+ ComponentId (..),+ PlanHash (..),+ CohortDescriptor (..),+ ComponentSpec (..),+ PackagePin (..),+ SourceSpec (..),+ PinException (..),+ loadCohortDescriptor,+ CohortIdentity (..),+ ResolvedComponent (..),+ ResolvedPackage (..),+ PackageSource (..),+ CohortSource (..),+ CohortError (..),+ activeCohortName,+ resolveCohortIdentity,+ identityFromPlan,+ planHash,+ CohortMismatch (..),+ checkCohort,+ renderCohortIdentity,+ )+where++import Control.Exception (IOException, displayException, try)+import Crypto.Hash.SHA256 qualified as SHA256+import Data.Aeson+ ( FromJSON (parseJSON),+ Object,+ ToJSON (toJSON),+ Value (..),+ eitherDecodeStrict',+ object,+ withObject,+ withText,+ (.!=),+ (.:),+ (.:?),+ (.=),+ )+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.Char (isAlphaNum)+import Data.List (sort, sortOn)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, fromMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Data.Word (Word8)+import Numeric (showHex)+import System.FilePath ((</>))++newtype CohortName = CohortName {unCohortName :: Text}+ deriving stock (Eq, Ord, Show)++newtype ComponentId = ComponentId {unComponentId :: Text}+ deriving stock (Eq, Ord, Show)++newtype PlanHash = PlanHash {unPlanHash :: Text}+ deriving stock (Eq, Ord, Show)++data SourceSpec+ = HackageSource+ | GitSource Text Text+ deriving stock (Eq, Show)++data PackagePin = PackagePin+ { pinName :: Text,+ pinVersion :: Text,+ pinSubdir :: Maybe Text+ }+ deriving stock (Eq, Show)++data ComponentSpec = ComponentSpec+ { componentId :: ComponentId,+ componentMoriUri :: Text,+ componentSource :: SourceSpec,+ componentPackages :: [PackagePin]+ }+ deriving stock (Eq, Show)++data PinException = PinException+ { exceptionPackage :: Text,+ exceptionSelected :: Text,+ exceptionLatestObserved :: Text,+ exceptionReason :: Text,+ exceptionRecheckAfter :: Text+ }+ deriving stock (Eq, Show)++data CohortDescriptor = CohortDescriptor+ { descriptorSchema :: Text,+ descriptorName :: CohortName,+ descriptorDescription :: Text,+ descriptorVerifiedAt :: Text,+ descriptorCompiler :: Text,+ descriptorIndexState :: Text,+ descriptorProjectFile :: FilePath,+ descriptorComponents :: [ComponentSpec],+ descriptorAllowNewer :: [Text],+ descriptorExceptions :: [PinException]+ }+ deriving stock (Eq, Show)++data PackageSource+ = FromHackage (Maybe Text)+ | FromGit Text Text (Maybe Text)+ | FromBoot+ | FromLocalPath FilePath+ deriving stock (Eq, Show)++data ResolvedPackage = ResolvedPackage+ { resolvedPackageName :: Text,+ resolvedPackageVersion :: Text,+ resolvedPackageSource :: PackageSource+ }+ deriving stock (Eq, Show)++data ResolvedComponent = ResolvedComponent+ { resolvedComponentId :: ComponentId,+ resolvedComponentMoriUri :: Text,+ resolvedComponentPackages :: [ResolvedPackage]+ }+ deriving stock (Eq, Show)++data CohortIdentity = CohortIdentity+ { identityCohort :: CohortName,+ identityCompiler :: Text,+ identityCabalVersion :: Text,+ identityOs :: Text,+ identityArch :: Text,+ identityIndexState :: Maybe Text,+ identityPlanHash :: PlanHash,+ identityDescriptorSha256 :: Text,+ identityComponents :: [ResolvedComponent],+ identityResolver :: Maybe Text+ }+ deriving stock (Eq, Show)++data CohortSource+ = FromProject FilePath (Maybe FilePath) (Maybe FilePath)+ | FromIdentityFile FilePath+ deriving stock (Eq, Show)++data CohortError+ = CohortIoError Text+ | CohortDecodeError Text+ | CohortPlanError Text+ | CohortInvalidActive Text+ deriving stock (Eq, Show)++data CohortMismatch+ = CohortNameMismatch CohortName CohortName+ | CompilerMismatch Text Text+ | MissingPackage Text+ | VersionMismatch Text Text Text+ | SourceMismatch Text SourceSpec PackageSource+ | LocalPathSource Text FilePath+ deriving stock (Eq, Show)++instance ToJSON CohortName where+ toJSON = toJSON . unCohortName++instance FromJSON CohortName where+ parseJSON = withText "CohortName" (pure . CohortName)++instance ToJSON ComponentId where+ toJSON = toJSON . unComponentId++instance FromJSON ComponentId where+ parseJSON = withText "ComponentId" (pure . ComponentId)++instance ToJSON PlanHash where+ toJSON = toJSON . unPlanHash++instance FromJSON PlanHash where+ parseJSON = withText "PlanHash" (pure . PlanHash)++instance ToJSON SourceSpec where+ toJSON HackageSource = object ["type" .= ("hackage" :: Text)]+ toJSON (GitSource gitLocation gitRevision) =+ object ["type" .= ("git" :: Text), "location" .= gitLocation, "rev" .= gitRevision]++instance FromJSON SourceSpec where+ parseJSON = withObject "SourceSpec" \value -> do+ sourceType <- value .: "type"+ case (sourceType :: Text) of+ "hackage" -> pure HackageSource+ "git" -> GitSource <$> value .: "location" <*> value .: "rev"+ other -> fail ("unknown cohort source type: " <> Text.unpack other)++instance ToJSON PackagePin where+ toJSON PackagePin {pinName, pinVersion, pinSubdir} =+ object (["name" .= pinName, "version" .= pinVersion] <> maybe [] (pure . ("subdir" .=)) pinSubdir)++instance FromJSON PackagePin where+ parseJSON = withObject "PackagePin" \value ->+ PackagePin <$> value .: "name" <*> value .: "version" <*> value .:? "subdir"++instance ToJSON ComponentSpec where+ toJSON ComponentSpec {componentId, componentMoriUri, componentSource, componentPackages} =+ object+ [ "id" .= componentId,+ "moriUri" .= componentMoriUri,+ "source" .= componentSource,+ "packages" .= componentPackages+ ]++instance FromJSON ComponentSpec where+ parseJSON = withObject "ComponentSpec" \value ->+ ComponentSpec+ <$> value .: "id"+ <*> value .: "moriUri"+ <*> value .: "source"+ <*> value .: "packages"++instance ToJSON PinException where+ toJSON PinException {exceptionPackage, exceptionSelected, exceptionLatestObserved, exceptionReason, exceptionRecheckAfter} =+ object+ [ "package" .= exceptionPackage,+ "selected" .= exceptionSelected,+ "latestObserved" .= exceptionLatestObserved,+ "reason" .= exceptionReason,+ "recheckAfter" .= exceptionRecheckAfter+ ]++instance FromJSON PinException where+ parseJSON = withObject "PinException" \value ->+ PinException+ <$> value .: "package"+ <*> value .: "selected"+ <*> value .: "latestObserved"+ <*> value .: "reason"+ <*> value .: "recheckAfter"++instance ToJSON CohortDescriptor where+ toJSON CohortDescriptor {descriptorSchema, descriptorName, descriptorDescription, descriptorVerifiedAt, descriptorCompiler, descriptorIndexState, descriptorProjectFile, descriptorComponents, descriptorAllowNewer, descriptorExceptions} =+ object+ [ "schema" .= descriptorSchema,+ "name" .= descriptorName,+ "description" .= descriptorDescription,+ "verifiedAt" .= descriptorVerifiedAt,+ "compiler" .= descriptorCompiler,+ "indexState" .= descriptorIndexState,+ "projectFile" .= descriptorProjectFile,+ "components" .= descriptorComponents,+ "allowNewer" .= descriptorAllowNewer,+ "exceptions" .= descriptorExceptions+ ]++instance FromJSON CohortDescriptor where+ parseJSON = withObject "CohortDescriptor" \value -> do+ descriptorSchema <- value .: "schema"+ if descriptorSchema /= ("kenshou.cohort/v1" :: Text)+ then fail ("unsupported cohort descriptor schema: " <> Text.unpack descriptorSchema)+ else+ CohortDescriptor descriptorSchema+ <$> value .: "name"+ <*> value .: "description"+ <*> value .: "verifiedAt"+ <*> value .: "compiler"+ <*> value .: "indexState"+ <*> value .: "projectFile"+ <*> value .: "components"+ <*> value .:? "allowNewer" .!= []+ <*> value .:? "exceptions" .!= []++instance ToJSON PackageSource where+ toJSON (FromHackage packageSha256) = object ["type" .= ("hackage" :: Text), "sha256" .= packageSha256]+ toJSON (FromGit packageLocation packageRevision packageSubdir) =+ object+ [ "type" .= ("git" :: Text),+ "location" .= packageLocation,+ "rev" .= packageRevision,+ "subdir" .= packageSubdir+ ]+ toJSON FromBoot = object ["type" .= ("boot" :: Text)]+ toJSON (FromLocalPath path) = object ["type" .= ("local" :: Text), "path" .= path]++instance FromJSON PackageSource where+ parseJSON = withObject "PackageSource" \value -> do+ sourceType <- value .: "type"+ case (sourceType :: Text) of+ "hackage" -> FromHackage <$> value .:? "sha256"+ "git" -> FromGit <$> value .: "location" <*> value .: "rev" <*> value .:? "subdir"+ "boot" -> pure FromBoot+ "local" -> FromLocalPath <$> value .: "path"+ other -> fail ("unknown resolved package source type: " <> Text.unpack other)++instance ToJSON ResolvedPackage where+ toJSON ResolvedPackage {resolvedPackageName, resolvedPackageVersion, resolvedPackageSource} =+ object ["name" .= resolvedPackageName, "version" .= resolvedPackageVersion, "source" .= resolvedPackageSource]++instance FromJSON ResolvedPackage where+ parseJSON = withObject "ResolvedPackage" \value ->+ ResolvedPackage <$> value .: "name" <*> value .: "version" <*> value .: "source"++instance ToJSON ResolvedComponent where+ toJSON ResolvedComponent {resolvedComponentId, resolvedComponentMoriUri, resolvedComponentPackages} =+ object ["id" .= resolvedComponentId, "moriUri" .= resolvedComponentMoriUri, "packages" .= resolvedComponentPackages]++instance FromJSON ResolvedComponent where+ parseJSON = withObject "ResolvedComponent" \value ->+ ResolvedComponent <$> value .: "id" <*> value .: "moriUri" <*> value .: "packages"++instance ToJSON CohortIdentity where+ toJSON CohortIdentity {identityCohort, identityCompiler, identityCabalVersion, identityOs, identityArch, identityIndexState, identityPlanHash, identityDescriptorSha256, identityComponents, identityResolver} =+ object+ ( [ "schema" .= ("kenshou.cohort-identity/v1" :: Text),+ "cohort" .= identityCohort,+ "compiler" .= identityCompiler,+ "cabalVersion" .= identityCabalVersion,+ "os" .= identityOs,+ "arch" .= identityArch,+ "indexState" .= identityIndexState,+ "planHash" .= identityPlanHash,+ "descriptorSha256" .= identityDescriptorSha256,+ "components" .= identityComponents+ ]+ <> maybe [] (\resolver -> ["resolver" .= resolver]) identityResolver+ )++instance FromJSON CohortIdentity where+ parseJSON = withObject "CohortIdentity" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.cohort-identity/v1" :: Text)+ then fail ("unsupported cohort identity schema: " <> Text.unpack schema)+ else do+ resolver <- value .:? "resolver"+ case (resolver :: Maybe Text) of+ Just "cabal" -> pure ()+ Just "nix" -> pure ()+ Nothing -> pure ()+ Just other -> fail ("unsupported cohort resolver: " <> Text.unpack other)+ CohortIdentity+ <$> value .: "cohort"+ <*> value .: "compiler"+ <*> value .: "cabalVersion"+ <*> value .: "os"+ <*> value .: "arch"+ <*> value .:? "indexState"+ <*> value .: "planHash"+ <*> value .: "descriptorSha256"+ <*> value .: "components"+ <*> pure resolver++loadCohortDescriptor :: FilePath -> IO (Either CohortError CohortDescriptor)+loadCohortDescriptor path = do+ bytesResult <- readBytes path+ pure (bytesResult >>= decodeBytes path)++activeCohortName :: FilePath -> IO (Either CohortError CohortName)+activeCohortName projectDir = do+ bytesResult <- readBytes (projectDir </> "cohort" </> "active.project")+ pure do+ bytes <- bytesResult+ text <- either (Left . CohortDecodeError . Text.pack . displayException) Right (Text.Encoding.decodeUtf8' bytes)+ case Text.lines text of+ [line]+ | Just project <- Text.stripPrefix "import: " line,+ Just name <- Text.stripSuffix ".project" project,+ not (Text.null name),+ Text.all validNameCharacter name ->+ Right (CohortName name)+ _ -> Left (CohortInvalidActive "cohort/active.project must contain exactly one line: import: <name>.project")+ where+ validNameCharacter character = isAlphaNum character || character == '-'++resolveCohortIdentity :: CohortSource -> IO (Either CohortError CohortIdentity)+resolveCohortIdentity (FromIdentityFile path) = do+ bytesResult <- readBytes path+ pure (bytesResult >>= decodeBytes path)+resolveCohortIdentity (FromProject sourceProjectDir sourcePlanJson sourceDescriptor) = do+ nameResult <- activeCohortName sourceProjectDir+ case nameResult of+ Left err -> pure (Left err)+ Right cohortName@(CohortName name) -> do+ let descriptorPath = fromMaybe (sourceProjectDir </> "cohort" </> Text.unpack name <> ".json") sourceDescriptor+ planPath = fromMaybe (sourceProjectDir </> "dist-newstyle" </> "cache" </> "plan.json") sourcePlanJson+ descriptorBytesResult <- readBytes descriptorPath+ planBytesResult <- readBytes planPath+ pure do+ descriptorBytes <- descriptorBytesResult+ planBytes <- planBytesResult+ descriptor <- decodeBytes descriptorPath descriptorBytes+ if descriptorName descriptor /= cohortName+ then Left (CohortDecodeError "active cohort name does not match descriptor name")+ else do+ planValue <- decodeBytes planPath planBytes+ identityFromPlan descriptor (sha256Hex descriptorBytes) planValue++identityFromPlan :: CohortDescriptor -> Text -> Value -> Either CohortError CohortIdentity+identityFromPlan descriptor descriptorSha value = do+ plan <- parsePlan value+ identityPlanHash <- planHash value+ let unitsByName = Map.fromListWith preferRuntimeUnit [(planUnitName unit, unit) | unit <- planUnits plan]+ identityComponents <- traverse (resolveComponent unitsByName) (descriptorComponents descriptor)+ pure+ CohortIdentity+ { identityCohort = descriptorName descriptor,+ identityCompiler = planCompiler plan,+ identityCabalVersion = planCabalVersion plan,+ identityOs = planOs plan,+ identityArch = planArch plan,+ identityIndexState = Just (descriptorIndexState descriptor),+ identityPlanHash,+ identityDescriptorSha256 = descriptorSha,+ identityComponents,+ identityResolver = Nothing+ }+ where+ preferRuntimeUnit left right+ | planUnitStyle left == Just "local" = right+ | otherwise = left++planHash :: Value -> Either CohortError PlanHash+planHash value = do+ plan <- parsePlan value+ renderedUnits <- traverse renderPlanUnit (filter ((/= Just "local") . planUnitStyle) (planUnits plan))+ let material = Text.intercalate "\n" ("compiler " <> planCompiler plan : sort (Set.toList (Set.fromList renderedUnits)))+ pure (PlanHash ("sha256:" <> sha256Hex (Text.Encoding.encodeUtf8 material)))++checkCohort :: CohortDescriptor -> CohortIdentity -> [CohortMismatch]+checkCohort descriptor identity = identityMismatches <> concatMap checkComponent (descriptorComponents descriptor)+ where+ identityMismatches =+ [CohortNameMismatch (descriptorName descriptor) (identityCohort identity) | descriptorName descriptor /= identityCohort identity]+ <> [CompilerMismatch (descriptorCompiler descriptor) (identityCompiler identity) | descriptorCompiler descriptor /= identityCompiler identity]++ resolvedByName =+ Map.fromList+ [ (resolvedPackageName package, package)+ | component <- identityComponents identity,+ package <- resolvedComponentPackages component+ ]++ checkComponent component = concatMap (checkPackage (componentSource component)) (componentPackages component)++ checkPackage expectedSource pin =+ case Map.lookup (pinName pin) resolvedByName of+ Nothing -> [MissingPackage (pinName pin)]+ Just actual -> versionMismatch pin actual <> sourceMismatch expectedSource pin actual++ versionMismatch pin actual+ | pinVersion pin == resolvedPackageVersion actual = []+ | otherwise = [VersionMismatch (pinName pin) (pinVersion pin) (resolvedPackageVersion actual)]++ sourceMismatch _ pin ResolvedPackage {resolvedPackageSource = FromLocalPath path} = [LocalPathSource (pinName pin) path]+ sourceMismatch HackageSource _ ResolvedPackage {resolvedPackageSource = FromHackage _} = []+ sourceMismatch expected@HackageSource pin ResolvedPackage {resolvedPackageSource = actual} = [SourceMismatch (pinName pin) expected actual]+ sourceMismatch expected@(GitSource gitLocation gitRevision) pin ResolvedPackage {resolvedPackageSource = actual@(FromGit packageLocation packageRevision _)}+ | normaliseGitLocation gitLocation == normaliseGitLocation packageLocation,+ gitRevision == packageRevision =+ []+ | otherwise = [SourceMismatch (pinName pin) expected actual]+ sourceMismatch expected@(GitSource _ _) pin ResolvedPackage {resolvedPackageSource = actual} = [SourceMismatch (pinName pin) expected actual]++renderCohortIdentity :: CohortIdentity -> Text+renderCohortIdentity identity =+ Text.unlines+ ( [ "cohort " <> unCohortName (identityCohort identity),+ "compiler " <> identityCompiler identity <> " " <> fromMaybe "cabal" (identityResolver identity) <> " " <> identityCabalVersion identity <> " " <> identityOs identity <> "/" <> identityArch identity,+ "index " <> fromMaybe "-" (identityIndexState identity),+ "plan " <> unPlanHash (identityPlanHash identity)+ ]+ <> map renderComponent (identityComponents identity)+ )+ where+ renderComponent component =+ Text.justifyLeft 22 ' ' (unComponentId (resolvedComponentId component))+ <> Text.intercalate ", " (map renderPackage (resolvedComponentPackages component))+ renderPackage package =+ resolvedPackageName package+ <> " "+ <> resolvedPackageVersion package+ <> " ["+ <> renderSource (resolvedPackageSource package)+ <> "]"+ renderSource (FromHackage _) = "hackage"+ renderSource (FromGit _ packageRevision _) = "git " <> Text.take 12 packageRevision+ renderSource FromBoot = "boot"+ renderSource FromLocalPath {} = "local"++data Plan = Plan+ { planCompiler :: Text,+ planCabalVersion :: Text,+ planOs :: Text,+ planArch :: Text,+ planUnits :: [PlanUnit]+ }++data PlanUnit = PlanUnit+ { planUnitType :: Text,+ planUnitStyle :: Maybe Text,+ planUnitName :: Text,+ planUnitVersion :: Text,+ planUnitSource :: Maybe Object,+ planUnitSha256 :: Maybe Text,+ planUnitFlags :: Map Text Bool+ }++parsePlan :: Value -> Either CohortError Plan+parsePlan (Object root) = do+ planCompiler <- requiredText "compiler-id" root+ planCabalVersion <- requiredText "cabal-version" root+ planOs <- requiredText "os" root+ planArch <- requiredText "arch" root+ unitValues <- requiredArray "install-plan" root+ planUnits <- traverse parseUnit unitValues+ pure Plan {planCompiler, planCabalVersion, planOs, planArch, planUnits}+parsePlan _ = Left (CohortPlanError "plan.json root must be an object")++parseUnit :: Value -> Either CohortError PlanUnit+parseUnit (Object unit) = do+ planUnitType <- requiredText "type" unit+ planUnitName <- requiredText "pkg-name" unit+ planUnitVersion <- requiredText "pkg-version" unit+ let planUnitStyle = optionalText "style" unit+ planUnitSource = optionalObject "pkg-src" unit+ planUnitSha256 = optionalText "pkg-src-sha256" unit+ planUnitFlags = fromMaybe Map.empty (optionalFlags "flags" unit)+ pure PlanUnit {planUnitType, planUnitStyle, planUnitName, planUnitVersion, planUnitSource, planUnitSha256, planUnitFlags}+parseUnit _ = Left (CohortPlanError "install-plan entries must be objects")++resolveComponent :: Map Text PlanUnit -> ComponentSpec -> Either CohortError ResolvedComponent+resolveComponent units component = do+ resolvedComponentPackages <- catMaybes <$> traverse resolvePackage (componentPackages component)+ pure+ ResolvedComponent+ { resolvedComponentId = componentId component,+ resolvedComponentMoriUri = componentMoriUri component,+ resolvedComponentPackages+ }+ where+ resolvePackage pin = case Map.lookup (pinName pin) units of+ Nothing -> pure Nothing+ Just unit -> Just . ResolvedPackage (planUnitName unit) (planUnitVersion unit) <$> packageSource unit++renderPlanUnit :: PlanUnit -> Either CohortError Text+renderPlanUnit unit = do+ source <- packageSource unit+ let flags =+ if Map.null (planUnitFlags unit)+ then "-"+ else+ Text.intercalate+ ","+ [ (if enabled then "+" else "-") <> name+ | (name, enabled) <- sortOn fst (Map.toList (planUnitFlags unit))+ ]+ pure (planUnitName unit <> " " <> planUnitVersion unit <> " " <> renderPlanSource source <> " " <> flags)++packageSource :: PlanUnit -> Either CohortError PackageSource+packageSource PlanUnit {planUnitType = "pre-existing"} = Right FromBoot+packageSource PlanUnit {planUnitSource = Nothing} = Right FromBoot+packageSource PlanUnit {planUnitSource = Just source, planUnitSha256} = do+ sourceType <- requiredText "type" source+ case sourceType of+ "repo-tar" -> Right (FromHackage planUnitSha256)+ "local" -> FromLocalPath . Text.unpack <$> requiredText "path" source+ "source-repo" -> parseSourceRepo source+ other -> Left (CohortPlanError ("unsupported plan package source: " <> other))++parseSourceRepo :: Object -> Either CohortError PackageSource+parseSourceRepo source = do+ let repo = fromMaybe source (optionalObject "source-repo" source)+ packageLocation <- requiredText "location" repo+ packageRevision <- requiredText "tag" repo+ let packageSubdir = optionalText "subdir" repo+ pure (FromGit packageLocation packageRevision packageSubdir)++renderPlanSource :: PackageSource -> Text+renderPlanSource (FromHackage packageSha256) = "hackage:" <> fromMaybe "-" packageSha256+renderPlanSource (FromGit packageLocation packageRevision packageSubdir) =+ "git:" <> packageLocation <> "@" <> packageRevision <> "#" <> fromMaybe "-" packageSubdir+renderPlanSource FromBoot = "boot"+renderPlanSource (FromLocalPath path) = "local:" <> Text.pack path++requiredText :: Text -> Object -> Either CohortError Text+requiredText key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+ Just (String value) -> Right value+ _ -> Left (CohortPlanError ("plan field " <> key <> " must be text"))++optionalText :: Text -> Object -> Maybe Text+optionalText key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+ Just (String value) -> Just value+ _ -> Nothing++optionalObject :: Text -> Object -> Maybe Object+optionalObject key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+ Just (Object value) -> Just value+ _ -> Nothing++requiredArray :: Text -> Object -> Either CohortError [Value]+requiredArray key objectValue = case KeyMap.lookup (Key.fromText key) objectValue of+ Just (Array values) -> Right (foldr (:) [] values)+ _ -> Left (CohortPlanError ("plan field " <> key <> " must be an array"))++optionalFlags :: Text -> Object -> Maybe (Map Text Bool)+optionalFlags key objectValue = do+ flags <- optionalObject key objectValue+ traverse valueToBool (Map.fromList [(Key.toText name, value) | (name, value) <- KeyMap.toList flags])+ where+ valueToBool (Bool value) = Just value+ valueToBool _ = Nothing++readBytes :: FilePath -> IO (Either CohortError ByteString)+readBytes path = do+ result <- try @IOException (ByteString.readFile path)+ pure (either (\err -> Left (CohortIoError (Text.pack (path <> ": " <> displayException err)))) Right result)++decodeBytes :: (FromJSON value) => FilePath -> ByteString -> Either CohortError value+decodeBytes path bytes =+ either (\err -> Left (CohortDecodeError (Text.pack (path <> ": " <> err)))) Right (eitherDecodeStrict' bytes)++sha256Hex :: ByteString -> Text+sha256Hex = Text.pack . concatMap renderByte . ByteString.unpack . SHA256.hash+ where+ renderByte :: Word8 -> String+ renderByte byte = case showHex byte "" of+ [digit] -> ['0', digit]+ digits -> digits++normaliseGitLocation :: Text -> Text+normaliseGitLocation location =+ let withoutSlash = Text.dropWhileEnd (== '/') location+ in fromMaybe withoutSlash (Text.stripSuffix ".git" withoutSlash)
+ src/Kenshou/Core/Compat.hs view
@@ -0,0 +1,118 @@+module Kenshou.Core.Compat+ ( CompatField (..),+ CompatInputs (..),+ compatInputs,+ comparisonKey,+ seriesKey,+ compatibleExcept,+ renderResourceInputs,+ )+where++import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.Types (Pair)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))+import Kenshou.Core.Dimension (DimensionName (..), Dimensions (..))+import Kenshou.Core.Env (MigrationSetName, renderMigrationSetName)+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot)+import Kenshou.Core.Knob (KnobName)+import Kenshou.Core.Knob qualified as Knob+import Kenshou.Core.Resource.Types (ResourceName, legacyKafkaName, renderResourceName)+import Kenshou.Core.RunSpec (EffectiveRunSpec (..), EnvironmentSpec (..))+import Kenshou.Core.Version (suiteVersion)++data CompatField+ = CfSuiteVersion+ | CfScenario+ | CfKnob KnobName+ | CfDimension DimensionName+ | CfPhases+ | CfMachineProfile+ | CfPostgresProfile+ | CfSchemas+ | CfCohort+ | CfResource ResourceName+ deriving stock (Eq, Show)++data CompatInputs = CompatInputs+ { suiteVersion :: Text,+ spec :: EffectiveRunSpec,+ postgres :: Maybe PgSettingsSnapshot,+ schemas :: [MigrationSetName],+ cohortPlanHash :: Text+ }+ deriving stock (Eq, Show)++compatInputs :: EffectiveRunSpec -> Maybe PgSettingsSnapshot -> [MigrationSetName] -> CohortIdentity -> CompatInputs+compatInputs spec postgres schemas cohort = CompatInputs suiteVersion spec postgres schemas (unPlanHash cohort.identityPlanHash)++comparisonKey :: CompatInputs -> Text+comparisonKey = sha256Hex . canonicalEncode . comparisonValue++seriesKey :: CompatInputs -> Text+seriesKey = sha256Hex . canonicalEncode . toJSON++compatibleExcept :: [CompatField] -> CompatInputs -> CompatInputs -> Either (NonEmpty CompatField) ()+compatibleExcept allowed left right = case filter (`notElem` allowed) (differences left right) of+ [] -> Right ()+ first : rest -> Left (first :| rest)++instance ToJSON CompatInputs where+ toJSON = compatibilityValue True++comparisonValue :: CompatInputs -> Value+comparisonValue = compatibilityValue False++compatibilityValue :: Bool -> CompatInputs -> Value+compatibilityValue includeCohort inputs =+ object $+ [ "suiteVersion" .= inputs.suiteVersion,+ "scenario" .= inputs.spec.scenario,+ "scenarioRevision" .= inputs.spec.scenarioRevision,+ "knobs" .= inputs.spec.knobs,+ "dimensions" .= inputs.spec.dimensions,+ "phases" .= inputs.spec.phases,+ "machineProfile" .= inputs.spec.environment.machineProfile,+ "postgresProfile" .= inputs.postgres,+ "schemas" .= fmap renderMigrationSetName inputs.schemas+ ]+ <> renderResourceInputs inputs.spec.environment.resources+ <> ["cohortPlanHash" .= inputs.cohortPlanHash | includeCohort]++-- | compat-key/v1 shim: keeps Kafka comparison keys stable for recorded baselines.+-- The resource that documents once carried as a top-level field renders there; every+-- other resource renders under @resources@, which is omitted when empty.+renderResourceInputs :: Map ResourceName Value -> [Pair]+renderResourceInputs resources =+ maybe [] (pure . (Key.fromText (renderResourceName legacyKafkaName) .=)) (Map.lookup legacyKafkaName resources)+ <> ["resources" .= others | not (Map.null others)]+ where+ others = Map.delete legacyKafkaName resources++differences :: CompatInputs -> CompatInputs -> [CompatField]+differences left right =+ [CfSuiteVersion | left.suiteVersion /= right.suiteVersion]+ <> [CfScenario | left.spec.scenario /= right.spec.scenario || left.spec.scenarioRevision /= right.spec.scenarioRevision]+ <> [CfPhases | left.spec.phases /= right.spec.phases]+ <> [CfMachineProfile | left.spec.environment.machineProfile /= right.spec.environment.machineProfile]+ <> [CfPostgresProfile | left.postgres /= right.postgres]+ <> [CfCohort | left.cohortPlanHash /= right.cohortPlanHash]+ <> [CfSchemas | left.schemas /= right.schemas]+ <> [CfDimension DimTracing | left.spec.dimensions.tracing /= right.spec.dimensions.tracing]+ <> [CfDimension DimMetrics | left.spec.dimensions.metrics /= right.spec.dimensions.metrics]+ <> [CfDimension DimPgDurability | left.spec.dimensions.pgDurability /= right.spec.dimensions.pgDurability]+ <> [CfDimension DimPgVersion | left.spec.dimensions.pgVersion /= right.spec.dimensions.pgVersion]+ <> [CfKnob name | name <- Set.toAscList knobNames, Map.lookup name leftKnobs /= Map.lookup name rightKnobs]+ <> [CfResource name | name <- Set.toAscList resourceNames, Map.lookup name left.spec.environment.resources /= Map.lookup name right.spec.environment.resources]+ where+ resourceNames = Map.keysSet left.spec.environment.resources <> Map.keysSet right.spec.environment.resources+ leftKnobs = Knob.resolvedKnobsMap left.spec.knobs+ rightKnobs = Knob.resolvedKnobsMap right.spec.knobs+ knobNames = Map.keysSet leftKnobs <> Map.keysSet rightKnobs
+ src/Kenshou/Core/Context.hs view
@@ -0,0 +1,135 @@+module Kenshou.Core.Context+ ( SummarySection (..),+ ArtifactDir (..),+ Observation (..),+ Environment (..),+ RunContext (..),+ RunState,+ PhaseTiming (..),+ InfrastructureError (..),+ newRunState,+ readRunState,+ requirePostgres,+ genFor,+ withPhase,+ putSummary,+ observe,+ artifactPath,+ declareMediaType,+ )+where++import Control.Exception (Exception, bracket_)+import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.IORef+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime, getCurrentTime)+import Data.Word (Word64)+import GHC.Clock (getMonotonicTimeNSec)+import GHC.Stack (HasCallStack)+import Kenshou.Core.Dimension (Dimensions)+import Kenshou.Core.Env.Postgres (PostgresEnv)+import Kenshou.Core.Id (RunId, ScenarioId, Seed, deriveGen)+import Kenshou.Core.Knob (ResolvedKnobs)+import Kenshou.Core.Log (Logger, Severity)+import Kenshou.Core.Phase (PhaseName, PhasePlan)+import Kenshou.Core.Resource.Types (ProvisionedResource, ResourceName)+import Kenshou.Core.RunSpec (ComparisonMembership, EnvironmentSpec)+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.Random.SplitMix (SMGen)++data SummarySection = Measurements | Verdicts | Diagnosis | Telemetry deriving stock (Eq, Ord, Show)++data ArtifactDir = SamplesDir | SeriesDir | VerdictsDir | DiagnosisDir | LogsDir deriving stock (Eq, Ord, Show)++data Observation = Observation {source :: Text, severity :: Severity, message :: Text, at :: UTCTime} deriving stock (Eq, Show)++data Environment = Environment {postgres :: Maybe PostgresEnv, extraPostgres :: Map Text PostgresEnv, resources :: Map ResourceName ProvisionedResource}++data PhaseTiming = PhaseTiming+ { name :: PhaseName,+ startedAt :: UTCTime,+ endedAt :: UTCTime,+ startedMonotonicNs :: Word64,+ endedMonotonicNs :: Word64+ }+ deriving stock (Eq, Show)++instance ToJSON PhaseTiming where+ toJSON timing = object ["name" .= timing.name, "startedAt" .= timing.startedAt, "endedAt" .= timing.endedAt, "startedMonotonicNs" .= timing.startedMonotonicNs, "endedMonotonicNs" .= timing.endedMonotonicNs]++data RunState = RunState+ { summaries :: IORef (Map SummarySection (Map Text Value)),+ observations :: IORef [Observation],+ phaseTimings :: IORef [PhaseTiming],+ mediaTypes :: IORef (Map FilePath Text)+ }++data RunContext = RunContext+ { runId :: RunId,+ scenario :: ScenarioId,+ knobs :: ResolvedKnobs,+ dimensions :: Dimensions,+ seed :: Seed,+ phases :: PhasePlan,+ env :: Environment,+ environmentSpec :: EnvironmentSpec,+ comparison :: Maybe ComparisonMembership,+ outDir :: FilePath,+ logger :: Logger,+ state :: RunState+ }++newtype InfrastructureError = InfrastructureError Text deriving stock (Eq, Show)++instance Exception InfrastructureError++newRunState :: IO RunState+newRunState = RunState <$> newIORef Map.empty <*> newIORef [] <*> newIORef [] <*> newIORef Map.empty++readRunState :: RunState -> IO (Map SummarySection (Map Text Value), [Observation], [PhaseTiming], Map FilePath Text)+readRunState state = (,,,) <$> readIORef state.summaries <*> readIORef state.observations <*> readIORef state.phaseTimings <*> readIORef state.mediaTypes++requirePostgres :: (HasCallStack) => RunContext -> PostgresEnv+requirePostgres context = maybe (error "requirePostgres: scenario did not declare PostgreSQL") id context.env.postgres++genFor :: RunContext -> Text -> SMGen+genFor context = deriveGen context.seed++withPhase :: RunContext -> PhaseName -> IO value -> IO value+withPhase context phase action = do+ startedAt <- getCurrentTime+ startedMono <- getMonotonicTimeNSec+ action `finallyRecord` do+ endedAt <- getCurrentTime+ endedMono <- getMonotonicTimeNSec+ modifyIORef' context.state.phaseTimings (<> [PhaseTiming phase startedAt endedAt startedMono endedMono])+ where+ finallyRecord body cleanup = bracket_ (pure ()) cleanup body++putSummary :: RunContext -> SummarySection -> Text -> Value -> IO ()+putSummary context section key value = modifyIORef' context.state.summaries (Map.alter (Just . Map.insert key value . maybe Map.empty id) section)++observe :: RunContext -> Text -> Severity -> Text -> IO ()+observe context source severity message = do+ at <- getCurrentTime+ modifyIORef' context.state.observations (<> [Observation source severity message at])++artifactPath :: RunContext -> ArtifactDir -> FilePath -> IO FilePath+artifactPath context directory name = do+ let root = context.outDir </> artifactDir directory+ createDirectoryIfMissing True root+ pure (root </> name)++declareMediaType :: RunContext -> FilePath -> Text -> IO ()+declareMediaType context path mediaType = modifyIORef' context.state.mediaTypes (Map.insert path mediaType)++artifactDir :: ArtifactDir -> FilePath+artifactDir SamplesDir = "samples"+artifactDir SeriesDir = "series"+artifactDir VerdictsDir = "verdicts"+artifactDir DiagnosisDir = "diagnosis"+artifactDir LogsDir = "logs"
+ src/Kenshou/Core/Contribution.hs view
@@ -0,0 +1,97 @@+-- | Data a suite contributes through its layer bundle for the toolkits to consume:+-- SQL oracles the correctness toolkit runs, advisory-lock labels stall diagnoses+-- name, and a connection-pool methodology. The types live in the kernel because+-- 'Kenshou.Core.Bundle.LayerBundle' carries them; the toolkits re-export them from+-- the modules that use them.+module Kenshou.Core.Contribution+ ( OracleSpec (..),+ AdvisoryKeySpec (..),+ AdvisoryLabel (..),+ PoolMethodology (..),+ RuleCondition (..),+ RoutingRule (..),+ )+where++import Control.Applicative ((<|>))+import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), object, withObject, (.:), (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Plan.Selector (Selector, parseSelector)++-- | A named SQL query whose rows a correctness scenario reconciles.+data OracleSpec = OracleSpec {name :: !Text, sql :: !Text}+ deriving stock (Eq, Show)++-- | How PostgreSQL derives an advisory-lock key from a label's text.+data AdvisoryKeySpec = HashTextExtended !Text | HashText !Text | LiteralKey !Int64+ deriving stock (Eq, Show)++-- | A human label for an advisory lock, so a stall capture names who holds it.+data AdvisoryLabel = AdvisoryLabel {label :: !Text, key :: !AdvisoryKeySpec}+ deriving stock (Eq, Show)++-- | The pool size a suite recommends for its scenarios and the range its+-- measurements justify.+data PoolMethodology = PoolMethodology {recommendedPoolSize :: !Int, poolRange :: !(Int, Int)}+ deriving stock (Eq, Show)++-- | Which runs a routing rule applies to: a knob with a given text value, or the+-- scenarios a selector matches.+data RuleCondition = KnobEquals Text Text | ScenarioMatches Selector+ deriving stock (Eq, Show)++-- | A cell capability a run needs, with the reason; cell routing and preparation+-- refuse a cell that reports the capability absent.+data RoutingRule = RoutingRule+ { condition :: !RuleCondition,+ requires :: !Text,+ why :: !Text+ }+ deriving stock (Eq, Show)++instance FromJSON RoutingRule where+ parseJSON = withObject "cell routing rule" \fields -> do+ condition <- fields .: "when" >>= parseCondition+ capability <- fields .: "requires"+ explanation <- fields .: "why"+ if Text.null capability || Text.null explanation then fail "cell routing rule requires a capability and explanation" else pure ()+ pure (RoutingRule condition capability explanation)+ where+ parseCondition = withObject "cell routing condition" \fields ->+ case (KeyMap.lookup "knob" fields, KeyMap.lookup "scenario" fields) of+ (Just (String knob), Nothing) -> do+ _ <- either (fail . Text.unpack) pure (mkKnobName knob)+ KnobEquals knob <$> fields .: "equals"+ (Nothing, Just (String patternText)) -> ScenarioMatches <$> either (fail . Text.unpack) pure (parseSelector patternText)+ _ -> fail "cell routing condition must select one knob or scenario"++instance ToJSON OracleSpec where+ toJSON spec = object ["name" .= spec.name, "sql" .= spec.sql]++instance FromJSON OracleSpec where+ parseJSON = withObject "OracleSpec" \value -> OracleSpec <$> value .: "name" <*> value .: "sql"++instance ToJSON AdvisoryKeySpec where+ toJSON (HashTextExtended text) = object ["hashTextExtended" .= text]+ toJSON (HashText text) = object ["hashText" .= text]+ toJSON (LiteralKey key) = object ["literal" .= key]++instance FromJSON AdvisoryKeySpec where+ parseJSON = withObject "AdvisoryKeySpec" \value ->+ (HashTextExtended <$> value .: "hashTextExtended") <|> (HashText <$> value .: "hashText") <|> (LiteralKey <$> value .: "literal")++instance ToJSON AdvisoryLabel where+ toJSON advisory = object ["label" .= advisory.label, "key" .= advisory.key]++instance FromJSON AdvisoryLabel where+ parseJSON = withObject "AdvisoryLabel" \value -> AdvisoryLabel <$> value .: "label" <*> value .: "key"++instance ToJSON PoolMethodology where+ toJSON methodology = object ["recommendedPoolSize" .= methodology.recommendedPoolSize, "poolRange" .= methodology.poolRange]++instance FromJSON PoolMethodology where+ parseJSON = withObject "PoolMethodology" \value -> PoolMethodology <$> value .: "recommendedPoolSize" <*> value .: "poolRange"
+ src/Kenshou/Core/Dimension.hs view
@@ -0,0 +1,163 @@+module Kenshou.Core.Dimension+ ( TracingArm (..),+ MetricsArm (..),+ PgDurability (..),+ PgVersion (..),+ DimensionName (..),+ Support (..),+ Supported (..),+ DimensionSupport (..),+ Dimensions (..),+ noDimensions,+ emptyDimensions,+ allTelemetryArms,+ postgresDimensions,+ resolveDimensions,+ renderDimensions,+ renderDimensionName,+ renderTracing,+ parseTracing,+ renderMetrics,+ parseMetrics,+ renderDurability,+ parseDurability,+ renderVersion,+ parseVersion,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:?), (.=))+import Data.Aeson.Key qualified as Key+import Data.List (group, sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text++data TracingArm = TracingOff | TracingNoop | TracingSdkInMemory | TracingSdkOtlp deriving stock (Eq, Ord, Show)++data MetricsArm = MetricsOff | MetricsCollect | MetricsServe | MetricsServeScraped deriving stock (Eq, Ord, Show)++data PgDurability = PgFsyncOff | PgDurable deriving stock (Eq, Ord, Show)++data PgVersion = Pg17 | Pg18 deriving stock (Eq, Ord, Show)++data DimensionName = DimTracing | DimMetrics | DimPgDurability | DimPgVersion deriving stock (Eq, Ord, Show)++data Support a = Support {values :: NonEmpty a, def :: a} deriving stock (Eq, Show)++data Supported a = NotApplicable | Supported (Support a) deriving stock (Eq, Show)++data DimensionSupport = DimensionSupport+ { tracing :: Supported TracingArm,+ metrics :: Supported MetricsArm,+ pgDurability :: Supported PgDurability,+ pgVersion :: Supported PgVersion+ }+ deriving stock (Eq, Show)++data Dimensions = Dimensions+ { tracing :: Maybe TracingArm,+ metrics :: Maybe MetricsArm,+ pgDurability :: Maybe PgDurability,+ pgVersion :: Maybe PgVersion+ }+ deriving stock (Eq, Show)++noDimensions :: DimensionSupport+noDimensions = DimensionSupport NotApplicable NotApplicable NotApplicable NotApplicable++emptyDimensions :: Dimensions+emptyDimensions = Dimensions Nothing Nothing Nothing Nothing++allTelemetryArms :: DimensionSupport -> DimensionSupport+allTelemetryArms support = support {tracing = Supported (Support (TracingOff :| [TracingNoop, TracingSdkInMemory, TracingSdkOtlp]) TracingOff), metrics = Supported (Support (MetricsOff :| [MetricsCollect, MetricsServe, MetricsServeScraped]) MetricsOff)}++postgresDimensions :: NonEmpty PgDurability -> NonEmpty PgVersion -> DimensionSupport -> DimensionSupport+postgresDimensions durabilities versions support = support {pgDurability = Supported (Support durabilities (NonEmpty.head durabilities)), pgVersion = Supported (Support versions (NonEmpty.head versions))}++resolveDimensions :: DimensionSupport -> [(Text, Text)] -> Either (NonEmpty Text) Dimensions+resolveDimensions support assignments = case errors of+ [] -> Right (Dimensions tracing metrics durability version)+ first : rest -> Left (first :| rest)+ where+ names = fmap fst assignments+ duplicateErrors = [name <> ": assigned more than once" | values@(name : _) <- group (sort names), length values > 1]+ unknownErrors = [name <> ": unknown dimension" | name <- names, name `notElem` fmap renderDimensionName [DimTracing, DimMetrics, DimPgDurability, DimPgVersion]]+ lookupValue name = lookup (renderDimensionName name) assignments+ (tracing, tracingErrors) = resolveOne DimTracing renderTracing parseTracing support.tracing (lookupValue DimTracing)+ (metrics, metricsErrors) = resolveOne DimMetrics renderMetrics parseMetrics support.metrics (lookupValue DimMetrics)+ (durability, durabilityErrors) = resolveOne DimPgDurability renderDurability parseDurability support.pgDurability (lookupValue DimPgDurability)+ (version, versionErrors) = resolveOne DimPgVersion renderVersion parseVersion support.pgVersion (lookupValue DimPgVersion)+ errors = duplicateErrors <> unknownErrors <> tracingErrors <> metricsErrors <> durabilityErrors <> versionErrors++resolveOne :: (Eq a) => DimensionName -> (a -> Text) -> (Text -> Maybe a) -> Supported a -> Maybe Text -> (Maybe a, [Text])+resolveOne _ _ _ NotApplicable Nothing = (Nothing, [])+resolveOne name _ _ NotApplicable (Just _) = (Nothing, [renderDimensionName name <> ": not applicable to this scenario"])+resolveOne _ _ _ (Supported support) Nothing = (Just support.def, [])+resolveOne name _render parse (Supported support) (Just raw) = case parse raw of+ Nothing -> (Nothing, [renderDimensionName name <> ": unknown value \"" <> raw <> "\""])+ Just value | value `elem` support.values -> (Just value, [])+ Just _ -> (Nothing, [renderDimensionName name <> ": value \"" <> raw <> "\" is not supported"])++renderDimensions :: Dimensions -> [(Text, Text)]+renderDimensions dimensions =+ concat+ [ maybe [] (pure . (renderDimensionName DimTracing,) . renderTracing) dimensions.tracing,+ maybe [] (pure . (renderDimensionName DimMetrics,) . renderMetrics) dimensions.metrics,+ maybe [] (pure . (renderDimensionName DimPgDurability,) . renderDurability) dimensions.pgDurability,+ maybe [] (pure . (renderDimensionName DimPgVersion,) . renderVersion) dimensions.pgVersion+ ]++renderDimensionName :: DimensionName -> Text+renderDimensionName DimTracing = "telemetry.tracing"+renderDimensionName DimMetrics = "telemetry.metrics"+renderDimensionName DimPgDurability = "pg.durability"+renderDimensionName DimPgVersion = "pg.version"++renderTracing :: TracingArm -> Text+renderTracing TracingOff = "off"+renderTracing TracingNoop = "noop"+renderTracing TracingSdkInMemory = "sdk-inmemory"+renderTracing TracingSdkOtlp = "sdk-otlp"++parseTracing :: Text -> Maybe TracingArm+parseTracing value = lookup value [(renderTracing item, item) | item <- [TracingOff, TracingNoop, TracingSdkInMemory, TracingSdkOtlp]]++renderMetrics :: MetricsArm -> Text+renderMetrics MetricsOff = "off"+renderMetrics MetricsCollect = "collect"+renderMetrics MetricsServe = "serve"+renderMetrics MetricsServeScraped = "serve-scraped"++parseMetrics :: Text -> Maybe MetricsArm+parseMetrics value = lookup value [(renderMetrics item, item) | item <- [MetricsOff, MetricsCollect, MetricsServe, MetricsServeScraped]]++renderDurability :: PgDurability -> Text+renderDurability PgFsyncOff = "fsync-off"+renderDurability PgDurable = "durable"++parseDurability :: Text -> Maybe PgDurability+parseDurability value = lookup value [(renderDurability item, item) | item <- [PgFsyncOff, PgDurable]]++renderVersion :: PgVersion -> Text+renderVersion Pg17 = "17"+renderVersion Pg18 = "18"++parseVersion :: Text -> Maybe PgVersion+parseVersion value = lookup value [(renderVersion item, item) | item <- [Pg17, Pg18]]++instance ToJSON Dimensions where+ toJSON dimensions = object [Key.fromText name .= value | (name, value) <- renderDimensions dimensions]++instance FromJSON Dimensions where+ parseJSON = withObject "Dimensions" \value ->+ Dimensions+ <$> optionalArm value "telemetry.tracing" parseTracing+ <*> optionalArm value "telemetry.metrics" parseMetrics+ <*> optionalArm value "pg.durability" parseDurability+ <*> optionalArm value "pg.version" parseVersion+ where+ optionalArm objectValue name parse = do+ raw <- objectValue .:? Key.fromText name+ traverse (maybe (fail ("unknown dimension value for " <> Text.unpack name)) pure . parse) raw
+ src/Kenshou/Core/Env.hs view
@@ -0,0 +1,61 @@+module Kenshou.Core.Env+ ( MigrationSetName (..),+ mkMigrationSetName,+ renderMigrationSetName,+ MigrationSet (..),+ PostgresRequirement (..),+ EnvRequirements (..),+ noEnvironment,+ requiresResource,+ )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Database.PostgreSQL.Migrate (MigrationComponent, componentName)+import Kenshou.Core.Resource.Types (ResourceName)++-- | The name of a migration set, which is also its pg-migrate component name and the+-- text a scenario's comparison key records. 'Kenshou.Core.Bundle.mkRegistry' rejects+-- names that 'mkMigrationSetName' would reject.+newtype MigrationSetName = MigrationSetName Text+ deriving stock (Eq, Ord, Show)++mkMigrationSetName :: Text -> Either Text MigrationSetName+mkMigrationSetName value = either (\problem -> Left ("invalid migration set name \"" <> value <> "\": " <> Text.pack (show problem))) (const (Right (MigrationSetName value))) (componentName value)++renderMigrationSetName :: MigrationSetName -> Text+renderMigrationSetName (MigrationSetName value) = value++-- | A named group of database migrations a registered bundle contributes. Sets are+-- resolved by name across the whole registry, so a scenario may require a set that+-- another bundle defines. 'requires' names sets that must be applied first. A+-- 'Left' component names a set this build cannot provide; it fails only when a+-- scenario requires it.+data MigrationSet = MigrationSet+ { name :: MigrationSetName,+ requires :: [MigrationSetName],+ component :: Either Text MigrationComponent+ }++data PostgresRequirement = PostgresRequirement+ { schemas :: [MigrationSetName],+ settings :: [(Text, Text)],+ needsServerControl :: Bool+ }+ deriving stock (Eq, Show)++data EnvRequirements = EnvRequirements+ { postgres :: Maybe PostgresRequirement,+ extraPostgres :: [(Text, PostgresRequirement)],+ -- | Named resources the scenario needs, provisioned by registered providers.+ resources :: [ResourceName]+ }+ deriving stock (Eq, Show)++noEnvironment :: EnvRequirements+noEnvironment = EnvRequirements Nothing [] []++-- | An environment that needs one named resource and nothing else.+requiresResource :: ResourceName -> EnvRequirements+requiresResource name = noEnvironment {resources = [name]}
+ src/Kenshou/Core/Env/Migration.hs view
@@ -0,0 +1,54 @@+module Kenshou.Core.Env.Migration+ ( MigrationSetupError (..),+ composePlan,+ orderMigrationSets,+ migrateDatabase,+ )+where++import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Database.PostgreSQL.Migrate (MigrationPlan, defaultRunOptions, migrationPlan, runMigrationPlan)+import Hasql.Connection.Settings qualified as Settings+import Kenshou.Core.Env (MigrationSet (..), MigrationSetName, renderMigrationSetName)++newtype MigrationSetupError = MigrationSetupError Text deriving stock (Eq, Show)++-- | Compose the requested migration sets into one ordered pg-migrate plan, applied+-- as one ledger, in the order 'orderMigrationSets' gives.+composePlan :: Map MigrationSetName MigrationSet -> [MigrationSetName] -> Either MigrationSetupError MigrationPlan+composePlan sets requested = do+ ordered <- orderMigrationSets sets requested+ components <- traverse component ordered+ case components of+ [] -> Left (MigrationSetupError "cannot compose an empty migration plan")+ first : rest -> mapError (migrationPlan (first :| rest))+ where+ component set = either (\problem -> Left (MigrationSetupError (renderMigrationSetName set.name <> ": " <> problem))) Right set.component+ mapError :: (Show problem) => Either problem value -> Either MigrationSetupError value+ mapError = either (Left . MigrationSetupError . Text.pack . show) Right++-- | The sets to apply, in application order. Sets keep the order the scenario+-- declared them; each set's 'requires' are inserted, transitively, before the set+-- that needs them, and a set appears once, at its first position.+orderMigrationSets :: Map MigrationSetName MigrationSet -> [MigrationSetName] -> Either MigrationSetupError [MigrationSet]+orderMigrationSets sets requested = fmap (fmap snd) (foldlM (visit []) [] requested)+ where+ visit path placed wanted+ | wanted `elem` fmap fst placed = Right placed+ | wanted `elem` path = Left (MigrationSetupError ("migration sets require one another in a cycle: " <> render (reverse (wanted : path))))+ | otherwise = case Map.lookup wanted sets of+ Nothing -> Left (MigrationSetupError ("no registered bundle defines migration set " <> renderMigrationSetName wanted))+ Just set -> do+ withRequired <- foldlM (visit (wanted : path)) placed set.requires+ Right (withRequired <> [(wanted, set)])+ render = Text.intercalate " -> " . fmap renderMigrationSetName+ foldlM step initial = foldl (\accumulated item -> accumulated >>= (`step` item)) (Right initial)++migrateDatabase :: Text -> MigrationPlan -> IO (Either MigrationSetupError ())+migrateDatabase connectionString plan = do+ result <- runMigrationPlan defaultRunOptions (Settings.connectionString connectionString) plan+ pure (either (Left . MigrationSetupError . Text.pack . show) (const (Right ())) result)
+ src/Kenshou/Core/Env/Postgres.hs view
@@ -0,0 +1,318 @@+module Kenshou.Core.Env.Postgres+ ( PgSettingsSnapshot (..),+ PostgresMode (..),+ StopMode (..),+ ServerControl (..),+ PostgresEnv (..),+ EnvError (..),+ withPostgresEnv,+ withPostgresEnvKeeping,+ withPostgresEnvNamed,+ )+where++import Control.Exception (finally)+import Control.Monad (unless)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.Char (isAlphaNum, toLower)+import Data.IORef+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Monoid (Last (..))+import Data.Text (Text)+import Data.Text qualified as Text+import EphemeralPg qualified as Pg+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..))+import Kenshou.Core.Env (MigrationSet, MigrationSetName, PostgresRequirement (..))+import Kenshou.Core.Env.Migration (composePlan, migrateDatabase)+import Kenshou.Core.Id (RunId, renderRunId)+import Kenshou.Core.Log (Logger, Severity (Warning), logAt)+import Kenshou.Core.RunSpec (ConnectionSource (..), PostgresSpec (..))+import System.Directory (copyFile, createDirectoryIfMissing, doesFileExist)+import System.Environment (getEnv, lookupEnv, setEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Posix.User (getRealUserID)+import System.Process (readProcessWithExitCode)++data PgSettingsSnapshot = PgSettingsSnapshot+ { serverVersion :: Text,+ serverVersionNum :: Int,+ settings :: Map Text Text,+ superuser :: Bool,+ collation :: Text,+ encoding :: Text+ }+ deriving stock (Eq, Show)++data PostgresMode = PgEphemeral | PgExternal deriving stock (Eq, Show)++data StopMode = StopFast | StopImmediate deriving stock (Eq, Show)++data ServerControl = ServerControl {currentServer :: IO Pg.Database, restartServer :: IO (), stopServer :: StopMode -> IO (), startServer :: IO ()}++data PostgresEnv = PostgresEnv+ { mode :: PostgresMode,+ connectionString :: Text,+ tcpEndpoint :: Maybe (Text, Int),+ adminConnectionString :: Text,+ databaseName :: Text,+ newDatabase :: Text -> IO Text,+ snapshot :: PgSettingsSnapshot,+ control :: Maybe ServerControl+ }++newtype EnvError = EnvError Text deriving stock (Eq, Show)++instance ToJSON PgSettingsSnapshot where+ toJSON snapshot = object ["serverVersion" .= snapshot.serverVersion, "serverVersionNum" .= snapshot.serverVersionNum, "settings" .= snapshot.settings, "superuser" .= snapshot.superuser, "collation" .= snapshot.collation, "encoding" .= snapshot.encoding]++withPostgresEnv :: Map MigrationSetName MigrationSet -> Logger -> FilePath -> RunId -> PostgresRequirement -> PostgresSpec -> Dimensions -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withPostgresEnv sets = withPostgresEnvKeeping sets False++withPostgresEnvKeeping :: Map MigrationSetName MigrationSet -> Bool -> Logger -> FilePath -> RunId -> PostgresRequirement -> PostgresSpec -> Dimensions -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withPostgresEnvKeeping sets keepEnvironment = withPostgresEnvNamed sets keepEnvironment Nothing++-- | Like 'withPostgresEnvKeeping', for a named context of a scenario. On an+-- external server several contexts share one server, so the context name+-- becomes part of every database name the run creates there.+withPostgresEnvNamed :: Map MigrationSetName MigrationSet -> Bool -> Maybe Text -> Logger -> FilePath -> RunId -> PostgresRequirement -> PostgresSpec -> Dimensions -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withPostgresEnvNamed sets keepEnvironment contextName logger runDirectory runId requirement postgresSpec dimensions action = case postgresSpec of+ PostgresEphemeral extraSettings -> withSelectedBinaries dimensions.pgVersion $ do+ createDirectoryIfMissing True (runDirectory </> "logs")+ uid <- getRealUserID+ let temporaryRoot = "/tmp/ephpg-kenshou-" <> show uid+ createDirectoryIfMissing True temporaryRoot+ let durabilitySettings = case dimensions.pgDurability of+ Just PgDurable -> [("fsync", "on"), ("synchronous_commit", "on"), ("full_page_writes", "on")]+ _ -> [("fsync", "off"), ("synchronous_commit", "off"), ("full_page_writes", "off")]+ settings = [("listen_addresses", "'127.0.0.1'"), ("log_min_messages", "'WARNING'"), ("log_line_prefix", "'%m [%p] %a '"), ("logging_collector", "on"), ("log_directory", "'log'"), ("log_filename", "'postgres.log'")] <> durabilitySettings <> requirement.settings <> extraSettings+ config = Pg.defaultConfig {Pg.databaseName = "kenshou_template", Pg.temporaryRoot = Last (Just temporaryRoot), Pg.postgresSettings = settings}+ started <- Pg.startCached config Pg.defaultCacheConfig+ case started of+ Left err -> pure (Left (EnvError (Text.pack (show err))))+ Right database -> do+ databaseRef <- newIORef database+ let cleanup = do+ current <- readIORef databaseRef+ let postgresLog = current.dataDirectory </> "log" </> "postgres.log"+ exists <- doesFileExist postgresLog+ if exists then copyFile postgresLog (runDirectory </> "logs" </> "postgres.log") else pure ()+ Pg.stop current+ if keepEnvironment+ then setup databaseRef database >>= either (pure . Left) (\environment -> logAt logger Warning "leaving ephemeral PostgreSQL running" [] >> fmap Right (action environment))+ else (setup databaseRef database >>= either (pure . Left) (fmap Right . action)) `finally` cleanup+ PostgresExternal source -> do+ connection <- resolveConnection source+ case connection of+ Left err -> pure (Left err)+ Right maintenanceConnection -> do+ initialSnapshot <- snapshotDatabase maintenanceConnection+ case initialSnapshot >>= \snapshot -> validateExternal dimensions snapshot *> validatePreloadedLibraries requirement snapshot of+ Left err -> pure (Left err)+ Right () -> do+ unless (null (filter ((/= "shared_preload_libraries") . fst) requirement.settings)) $ logAt logger Warning "external PostgreSQL settings cannot be applied" []+ withExternalDatabases sets keepEnvironment maintenanceConnection runId contextName requirement action+ where+ setup databaseRef database = do+ migration <- migrateIfNeeded sets (Pg.connectionString database) requirement+ case migration of+ Left err -> pure (Left err)+ Right () -> do+ cloned <- runCommand "createdb" ["-h", database.socketDirectory, "-p", show database.port, "-U", Text.unpack database.user, "-T", "kenshou_template", "kenshou_run"]+ case cloned of+ Left err -> pure (Left err)+ Right () -> do+ let connection = connectionFor database "kenshou_run"+ snapshot <- snapshotDatabase connection+ pure (makeEnvironment databaseRef database connection <$> snapshot)++ makeEnvironment databaseRef database connection snapshot =+ PostgresEnv PgEphemeral connection (Just ("127.0.0.1", fromIntegral database.port)) (Pg.connectionString database) "kenshou_run" (cloneDatabase database) snapshot (Just (serverControl databaseRef))++ cloneDatabase database label = do+ let name = "kenshou_" <> label+ result <- runCommand "createdb" ["-h", database.socketDirectory, "-p", show database.port, "-U", Text.unpack database.user, "-T", "kenshou_template", Text.unpack name]+ either (ioError . userError . show) (const (pure (connectionFor database name))) result++ serverControl databaseRef = ServerControl (readIORef databaseRef) restart stopNow start+ where+ restart = do+ current <- readIORef databaseRef+ Pg.restart current >>= either (ioError . userError . show) (writeIORef databaseRef)+ stopNow mode = do+ current <- readIORef databaseRef+ let modeText = case mode of StopFast -> "fast"; StopImmediate -> "immediate"+ runCommand "pg_ctl" ["-D", current.dataDirectory, "stop", "-m", modeText, "-w"] >>= either (ioError . userError . show) pure+ start = do+ current <- readIORef databaseRef+ Pg.restart current >>= either (ioError . userError . show) (writeIORef databaseRef)++withExternalDatabases :: Map MigrationSetName MigrationSet -> Bool -> Text -> RunId -> Maybe Text -> PostgresRequirement -> (PostgresEnv -> IO value) -> IO (Either EnvError value)+withExternalDatabases sets keepEnvironment maintenanceConnection runId contextName requirement action = do+ counter <- newIORef (0 :: Int)+ created <- newIORef []+ let context = maybe "" (\name -> "_" <> Text.take 16 (Text.map (\character -> if isAlphaNum character then toLower character else '_') name)) contextName+ prefix = Text.take 13 (Text.filter (/= '-') (renderRunId runId)) <> context+ templateName = "kenshou_" <> prefix <> "_template"+ runName = "kenshou_" <> prefix+ remember name = modifyIORef' created (name :)+ create name template = do+ result <- psqlUnit maintenanceConnection ("create database " <> name <> maybe "" (" template " <>) template)+ case result of Left err -> pure (Left err); Right () -> remember name >> pure (Right ())+ cleanup = readIORef created >>= mapM_ (\name -> psqlUnit maintenanceConnection ("drop database if exists " <> name <> " with (force)") >> pure ())+ setup = do+ createdTemplate <- create templateName Nothing+ case createdTemplate of+ Left err -> pure (Left err)+ Right () -> do+ let templateConnection = connectionForExternal maintenanceConnection templateName+ migrated <- migrateIfNeeded sets templateConnection requirement+ case migrated of+ Left err -> pure (Left err)+ Right () -> do+ createdRun <- create runName (Just templateName)+ case createdRun of+ Left err -> pure (Left err)+ Right () -> do+ let runConnection = connectionForExternal maintenanceConnection runName+ snapshot <- snapshotDatabase runConnection+ pure $ fmap (externalEnvironment counter created maintenanceConnection templateName runName runConnection) snapshot+ if keepEnvironment+ then setup >>= either (pure . Left) (fmap Right . action)+ else (setup >>= either (pure . Left) (fmap Right . action)) `finally` cleanup++externalEnvironment :: IORef Int -> IORef [Text] -> Text -> Text -> Text -> Text -> PgSettingsSnapshot -> PostgresEnv+externalEnvironment counter created maintenanceConnection templateName runName connection snapshot =+ PostgresEnv PgExternal connection (externalTcpEndpoint maintenanceConnection) maintenanceConnection runName clone snapshot Nothing+ where+ clone label = do+ number <- atomicModifyIORef' counter (\value -> let next = value + 1 in (next, next))+ let safeLabel = Text.map (\character -> if isAlphaNum character then character else '_') label+ name = "kenshou_" <> safeLabel <> "_" <> Text.pack (show number)+ result <- psqlUnit maintenanceConnection ("create database " <> name <> " template " <> templateName)+ case result of+ Left err -> ioError (userError (show err))+ Right () -> modifyIORef' created (name :) >> pure (connectionForExternal maintenanceConnection name)++resolveConnection :: ConnectionSource -> IO (Either EnvError Text)+resolveConnection (ConnLiteral value) = pure (Right value)+resolveConnection (ConnFromEnv name) = maybe (Left (EnvError ("environment variable is not set: " <> name))) (Right . Text.pack) <$> lookupEnv (Text.unpack name)++migrateIfNeeded :: Map MigrationSetName MigrationSet -> Text -> PostgresRequirement -> IO (Either EnvError ())+migrateIfNeeded _ _ requirement | null requirement.schemas = pure (Right ())+migrateIfNeeded sets connection requirement = case composePlan sets requirement.schemas of+ Left err -> pure (Left (EnvError (Text.pack (show err))))+ Right plan -> fmap (either (Left . EnvError . Text.pack . show) Right) (migrateDatabase connection plan)++connectionFor :: Pg.Database -> Text -> Text+connectionFor database name = Text.unwords ["host=" <> Text.pack database.socketDirectory, "port=" <> Text.pack (show database.port), "dbname=" <> name, "user=" <> database.user]++snapshotDatabase :: Text -> IO (Either EnvError PgSettingsSnapshot)+snapshotDatabase connection = do+ let setting name = psql connection ("show " <> name)+ version <- setting "server_version"+ versionNumber <- setting "server_version_num"+ settingPairs <- traverse (\name -> fmap ((name,) <$>) (setting name)) fingerprintSettings+ collation <- psql connection "select datcollate from pg_database where datname=current_database()"+ encoding <- psql connection "select pg_encoding_to_char(encoding) from pg_database where datname=current_database()"+ superuser <- psql connection "select rolsuper::text from pg_roles where rolname=current_user"+ pure do+ serverVersion <- version+ numberText <- versionNumber+ serverVersionNum <- maybe (Left (EnvError "invalid server_version_num")) Right (readMaybeText numberText)+ resolvedSettings <- sequence settingPairs+ databaseCollation <- collation+ databaseEncoding <- encoding+ isSuperuser <- (== "true") . Text.toCaseFold <$> superuser+ pure (PgSettingsSnapshot serverVersion serverVersionNum (Map.fromList resolvedSettings) isSuperuser databaseCollation databaseEncoding)++fingerprintSettings :: [Text]+fingerprintSettings =+ [ "fsync",+ "synchronous_commit",+ "full_page_writes",+ "wal_level",+ "shared_buffers",+ "effective_cache_size",+ "work_mem",+ "max_connections",+ "checkpoint_timeout",+ "checkpoint_completion_target",+ "max_wal_size",+ "wal_buffers",+ "random_page_cost",+ "autovacuum",+ "shared_preload_libraries"+ ]++validateExternal :: Dimensions -> PgSettingsSnapshot -> Either EnvError ()+validateExternal dimensions snapshot = do+ case dimensions.pgVersion of+ Just Pg17 | snapshot.serverVersionNum < 170000 || snapshot.serverVersionNum >= 180000 -> Left (EnvError "external PostgreSQL does not match pg.version=17")+ Just Pg18 | snapshot.serverVersionNum < 180000 || snapshot.serverVersionNum >= 190000 -> Left (EnvError "external PostgreSQL does not match pg.version=18")+ _ -> Right ()++ let actual name = Text.toCaseFold <$> Map.lookup name snapshot.settings+ case dimensions.pgDurability of+ Just PgFsyncOff | actual "fsync" /= Just "off" || actual "synchronous_commit" /= Just "off" -> Left (EnvError "external PostgreSQL does not match pg.durability=fsync-off")+ Just PgDurable | actual "fsync" /= Just "on" || actual "synchronous_commit" == Just "off" -> Left (EnvError "external PostgreSQL does not match pg.durability=durable")+ _ -> Right ()++validatePreloadedLibraries :: PostgresRequirement -> PgSettingsSnapshot -> Either EnvError ()+validatePreloadedLibraries requirement snapshot =+ case filter (`notElem` available) requested of+ [] -> Right ()+ missing -> Left (EnvError ("external PostgreSQL is missing required preloaded libraries: " <> Text.intercalate ", " missing))+ where+ requested = concatMap (libraries . snd) (filter ((== "shared_preload_libraries") . fst) requirement.settings)+ available = libraries (Map.findWithDefault "" "shared_preload_libraries" snapshot.settings)+ libraries = filter (not . Text.null) . fmap (Text.toCaseFold . Text.strip) . Text.splitOn "," . Text.dropAround (== '\'') . Text.strip++psql :: Text -> Text -> IO (Either EnvError Text)+psql connection query = do+ (code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+ pure case code of ExitSuccess -> Right (Text.strip (Text.pack output)); _ -> Left (EnvError (Text.strip (Text.pack err)))++psqlUnit :: Text -> Text -> IO (Either EnvError ())+psqlUnit connection query = fmap (const ()) <$> psql connection query++connectionForExternal :: Text -> Text -> Text+connectionForExternal maintenanceConnection name = maintenanceConnection <> " dbname=" <> name++externalTcpEndpoint :: Text -> Maybe (Text, Int)+externalTcpEndpoint connection = do+ host <- lookupWord "host"+ portText <- lookupWord "port"+ port <- readMaybeText portText+ pure (host, port)+ where+ pairs = [(key, Text.drop 1 rest) | word <- Text.words connection, let (key, rest) = Text.breakOn "=" word, not (Text.null rest)]+ lookupWord key = lookup key pairs++runCommand :: FilePath -> [String] -> IO (Either EnvError ())+runCommand command arguments = do+ (code, _, err) <- readProcessWithExitCode command arguments ""+ pure case code of ExitSuccess -> Right (); _ -> Left (EnvError (Text.strip (Text.pack err)))++withSelectedBinaries :: Maybe PgVersion -> IO (Either EnvError value) -> IO (Either EnvError value)+withSelectedBinaries version action = do+ (original, validation) <- prepare+ case validation of+ Left err -> restore original >> pure (Left err)+ Right () -> action `finally` restore original+ where+ prepare = do+ original <- getEnv "PATH"+ let (variable, expectedMajor) = case version of Just Pg17 -> ("KENSHOU_PG17_BIN", 17 :: Int); _ -> ("KENSHOU_PG18_BIN", 18)+ binaryDirectory <- lookupEnv variable+ maybe (pure ()) (\directory -> setEnv "PATH" (directory <> ":" <> original)) binaryDirectory+ (code, output, err) <- readProcessWithExitCode "postgres" ["--version"] ""+ let versionText = Text.pack output+ matches = (" " <> Text.pack (show expectedMajor) <> ".") `Text.isInfixOf` versionText+ validation = if code == ExitSuccess && matches then Right () else Left (EnvError (Text.pack (variable <> " is unset or does not select PostgreSQL " <> show expectedMajor <> ": " <> err <> output)))+ pure (original, validation)+ restore = setEnv "PATH"++readMaybeText :: (Read value) => Text -> Maybe value+readMaybeText value = case reads (Text.unpack value) of [(parsed, "")] -> Just parsed; _ -> Nothing
+ src/Kenshou/Core/Fingerprint.hs view
@@ -0,0 +1,118 @@+module Kenshou.Core.Fingerprint+ ( HostFingerprint (..),+ collectHostFingerprint,+ hostValue,+ runtimeValue,+ kenshouValue,+ )+where++import Control.Exception (IOException, try)+import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.ByteString qualified as ByteString+import Data.Text (Text)+import Data.Text qualified as Text+import GHC.Conc (getNumCapabilities)+import GHC.Stats (getRTSStatsEnabled)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Version (suiteVersion)+import System.Environment (getExecutablePath, lookupEnv)+import System.Info qualified as System+import System.Process (readProcess)++data HostFingerprint = HostFingerprint+ { os :: Text,+ arch :: Text,+ kernel :: Text,+ cpuModel :: Maybe Text,+ logicalCores :: Int,+ memoryBytes :: Maybe Integer,+ hostname :: Text,+ ghc :: Text,+ capabilities :: Int,+ threaded :: Bool,+ rtsStats :: Bool,+ kenshouVersion :: Text,+ executableSha256 :: Text,+ revision :: Maybe Text,+ dirty :: Maybe Bool+ }+ deriving stock (Eq, Show)++instance ToJSON HostFingerprint where+ toJSON = hostValue++hostValue :: HostFingerprint -> Value+hostValue fingerprint =+ object+ [ "os" .= fingerprint.os,+ "arch" .= fingerprint.arch,+ "kernel" .= fingerprint.kernel,+ "cpuModel" .= fingerprint.cpuModel,+ "logicalCores" .= fingerprint.logicalCores,+ "memoryBytes" .= fingerprint.memoryBytes,+ "hostname" .= fingerprint.hostname+ ]++runtimeValue :: HostFingerprint -> Value+runtimeValue fingerprint = object ["ghc" .= fingerprint.ghc, "capabilities" .= fingerprint.capabilities, "threaded" .= fingerprint.threaded, "rtsStats" .= fingerprint.rtsStats]++kenshouValue :: HostFingerprint -> Value+kenshouValue fingerprint = object ["version" .= fingerprint.kenshouVersion, "executableSha256" .= fingerprint.executableSha256, "revision" .= fingerprint.revision, "dirty" .= fingerprint.dirty]++collectHostFingerprint :: IO HostFingerprint+collectHostFingerprint = do+ kernel <- commandText "uname" ["-r"]+ hostname <- commandText "hostname" []+ cpuModel <- optionalCommandText (if System.os == "darwin" then "sysctl" else "sh") (if System.os == "darwin" then ["-n", "machdep.cpu.brand_string"] else ["-c", "sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo | head -1"])+ memoryBytes <- collectMemoryBytes+ capabilities <- getNumCapabilities+ rtsStats <- getRTSStatsEnabled+ executable <- getExecutablePath+ bytes <- ByteString.readFile executable+ revision <- firstPresent "KENSHOU_HARNESS_REVISION" (optionalCommandText "git" ["rev-parse", "HEAD"])+ dirtyOverride <- lookupEnv "KENSHOU_HARNESS_DIRTY"+ dirty <- case fmap Text.toCaseFold (Text.pack <$> dirtyOverride) of+ Just "true" -> pure (Just True)+ Just "false" -> pure (Just False)+ Just "1" -> pure (Just True)+ Just "0" -> pure (Just False)+ Just _ -> pure Nothing+ Nothing -> fmap (fmap (not . Text.null)) (optionalCommandText "git" ["status", "--porcelain"])+ pure+ HostFingerprint+ { os = Text.pack System.os,+ arch = Text.pack System.arch,+ kernel,+ cpuModel,+ logicalCores = capabilities,+ memoryBytes,+ hostname,+ ghc = Text.pack (show System.compilerVersion),+ capabilities,+ threaded = True,+ rtsStats,+ kenshouVersion = suiteVersion,+ executableSha256 = sha256Hex bytes,+ revision,+ dirty+ }++collectMemoryBytes :: IO (Maybe Integer)+collectMemoryBytes+ | System.os == "darwin" = optionalCommandText "sysctl" ["-n", "hw.memsize"] >>= pure . (>>= readInteger)+ | otherwise = optionalCommandText "sh" ["-c", "awk '/MemTotal:/ {print $2 * 1024}' /proc/meminfo"] >>= pure . (>>= readInteger)++firstPresent :: String -> IO (Maybe Text) -> IO (Maybe Text)+firstPresent variable fallback = lookupEnv variable >>= maybe fallback (pure . Just . Text.pack)++readInteger :: Text -> Maybe Integer+readInteger input = case reads (Text.unpack input) of [(value, "")] -> Just value; _ -> Nothing++commandText :: FilePath -> [String] -> IO Text+commandText command arguments = Text.strip . Text.pack <$> readProcess command arguments ""++optionalCommandText :: FilePath -> [String] -> IO (Maybe Text)+optionalCommandText command arguments = do+ result <- try (commandText command arguments) :: IO (Either IOException Text)+ pure (either (const Nothing) Just result)
+ src/Kenshou/Core/Id.hs view
@@ -0,0 +1,174 @@+module Kenshou.Core.Id+ ( LayerId,+ Kind (..),+ Segment,+ ScenarioId (..),+ RunId,+ Seed,+ mkLayerId,+ selftestLayer,+ renderLayer,+ parseLayer,+ renderKind,+ parseKind,+ mkSegment,+ unSegment,+ parseScenarioId,+ renderScenarioId,+ newRunId,+ parseRunId,+ renderRunId,+ mkSeed,+ unSeed,+ deriveGen,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), withText)+import Data.Bits (xor)+import Data.ByteString qualified as ByteString+import Data.Char (isAsciiLower, isDigit)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Data.UUID (UUID)+import Data.UUID qualified as UUID+import Data.UUID.V7 qualified as UUID.V7+import Data.Word (Word64)+import System.Random.SplitMix (SMGen, mkSMGen)++-- | The first segment of a scenario identifier: the name of what a registered layer+-- bundle verifies. Any valid segment is a layer; suites choose their own.+newtype LayerId = LayerId Segment+ deriving stock (Eq, Ord, Show)++data Kind = Correctness | Concurrency | Soak | Benchmark+ deriving stock (Eq, Ord, Show, Enum, Bounded)++newtype Segment = Segment Text+ deriving stock (Eq, Ord, Show)++data ScenarioId = ScenarioId+ { layer :: LayerId,+ component :: Segment,+ kind :: Kind,+ name :: Segment+ }+ deriving stock (Eq, Ord, Show)++newtype RunId = RunId UUID+ deriving stock (Eq, Ord, Show)++newtype Seed = Seed Word64+ deriving stock (Eq, Ord, Show)++mkLayerId :: Text -> Either Text LayerId+mkLayerId value = either (\problem -> Left ("invalid layer \"" <> value <> "\": " <> problem)) (Right . LayerId) (mkSegment value)++-- | The layer of the kernel's and toolkits' own self-test scenarios.+selftestLayer :: LayerId+selftestLayer = LayerId (Segment "selftest")++renderLayer :: LayerId -> Text+renderLayer (LayerId segment) = unSegment segment++parseLayer :: Text -> Either Text LayerId+parseLayer = mkLayerId++renderKind :: Kind -> Text+renderKind Correctness = "correctness"+renderKind Concurrency = "concurrency"+renderKind Soak = "soak"+renderKind Benchmark = "benchmark"++parseKind :: Text -> Either Text Kind+parseKind value = maybe (Left ("unknown kind \"" <> value <> "\"")) Right (lookup value pairs)+ where+ pairs = [(renderKind item, item) | item <- [minBound .. maxBound]]++mkSegment :: Text -> Either Text Segment+mkSegment value+ | Text.length value > 48 = Left "segment is longer than 48 characters"+ | Text.null value = Left "segment is empty"+ | valid = Right (Segment value)+ | otherwise = Left ("invalid segment \"" <> value <> "\"")+ where+ chunks = Text.splitOn "-" value+ validChunk chunk =+ not (Text.null chunk)+ && isAsciiLower (Text.head chunk)+ && Text.all (\c -> isAsciiLower c || isDigit c) chunk+ valid = all validChunk chunks++unSegment :: Segment -> Text+unSegment (Segment value) = value++parseScenarioId :: Text -> Either Text ScenarioId+parseScenarioId value = case Text.splitOn "/" value of+ [layerText, componentText, kindText, nameText] ->+ ScenarioId+ <$> parseLayer layerText+ <*> mkSegment componentText+ <*> parseKind kindText+ <*> mkSegment nameText+ _ -> Left ("scenario identifier must have four segments: \"" <> value <> "\"")++renderScenarioId :: ScenarioId -> Text+renderScenarioId value =+ Text.intercalate+ "/"+ [renderLayer value.layer, unSegment value.component, renderKind value.kind, unSegment value.name]++newRunId :: IO RunId+newRunId = RunId <$> UUID.V7.genUUID++parseRunId :: Text -> Either Text RunId+parseRunId value = case UUID.fromText value of+ Nothing -> Left ("invalid UUIDv7 \"" <> value <> "\"")+ Just uuid+ | UUID.toText uuid /= value -> Left "run id must use canonical lowercase UUID text"+ | Text.index value 14 /= '7' -> Left "run id must be UUIDv7"+ | otherwise -> Right (RunId uuid)++renderRunId :: RunId -> Text+renderRunId (RunId uuid) = UUID.toText uuid++mkSeed :: Word64 -> Either Text Seed+mkSeed value+ | value <= 9007199254740991 = Right (Seed value)+ | otherwise = Left "seed must be between 0 and 9007199254740991"++unSeed :: Seed -> Word64+unSeed (Seed value) = value++deriveGen :: Seed -> Text -> SMGen+deriveGen (Seed seed) label = mkSMGen (seed `xor` fnv1a64 (Text.encodeUtf8 label))++fnv1a64 :: ByteString.ByteString -> Word64+fnv1a64 = ByteString.foldl' step 14695981039346656037+ where+ step hash byte = (hash `xor` fromIntegral byte) * 1099511628211++instance ToJSON LayerId where toJSON = toJSON . renderLayer++instance FromJSON LayerId where parseJSON = withText "LayerId" (either (fail . Text.unpack) pure . parseLayer)++instance ToJSON Kind where toJSON = toJSON . renderKind++instance FromJSON Kind where parseJSON = withText "Kind" (either (fail . Text.unpack) pure . parseKind)++instance ToJSON Segment where toJSON = toJSON . unSegment++instance FromJSON Segment where parseJSON = withText "Segment" (either (fail . Text.unpack) pure . mkSegment)++instance ToJSON ScenarioId where toJSON = toJSON . renderScenarioId++instance FromJSON ScenarioId where parseJSON = withText "ScenarioId" (either (fail . Text.unpack) pure . parseScenarioId)++instance ToJSON RunId where toJSON = toJSON . renderRunId++instance FromJSON RunId where parseJSON = withText "RunId" (either (fail . Text.unpack) pure . parseRunId)++instance ToJSON Seed where toJSON = toJSON . unSeed++instance FromJSON Seed where parseJSON value = parseJSON value >>= either (fail . Text.unpack) pure . mkSeed
+ src/Kenshou/Core/Knob.hs view
@@ -0,0 +1,177 @@+module Kenshou.Core.Knob+ ( KnobName,+ KnobType (..),+ KnobValue (..),+ Allowed (..),+ KnobSpec (..),+ RawKnob (..),+ ResolvedKnobs,+ KnobError (..),+ mkKnobName,+ renderKnobName,+ emptyKnobs,+ parseAssignment,+ resolveKnobs,+ resolvedKnobsMap,+ knobBool,+ knobInt,+ knobDouble,+ knobText,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), withObject, withText)+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Char (isAsciiLower, isDigit)+import Data.Int (Int64)+import Data.List (group, sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Scientific qualified as Scientific+import Data.Text (Text)+import Data.Text qualified as Text++newtype KnobName = KnobName Text+ deriving stock (Eq, Ord, Show)++data KnobType = KnobBool | KnobInt | KnobDouble | KnobText+ deriving stock (Eq, Ord, Show)++data KnobValue = VBool Bool | VInt Int64 | VDouble Double | VText Text+ deriving stock (Eq, Show)++data Allowed = AnyValue | OneOf (NonEmpty KnobValue) | IntRange Int64 Int64 | DoubleRange Double Double+ deriving stock (Eq, Show)++data KnobSpec = KnobSpec+ { name :: KnobName,+ summary :: Text,+ knobType :: KnobType,+ def :: KnobValue,+ allowed :: Allowed,+ variants :: [KnobValue]+ }+ deriving stock (Eq, Show)++newtype ResolvedKnobs = ResolvedKnobs (Map KnobName KnobValue)+ deriving stock (Eq, Show)++data RawKnob = RawText Text | RawJson Value deriving stock (Eq, Show)++newtype KnobError = KnobError Text deriving stock (Eq, Show)++mkKnobName :: Text -> Either Text KnobName+mkKnobName value+ | length pieces < 2 = Left "knob name must contain a dot"+ | all validSegment pieces = Right (KnobName value)+ | otherwise = Left ("invalid knob name \"" <> value <> "\"")+ where+ pieces = Text.splitOn "." value+ validSegment piece =+ not (Text.null piece)+ && isAsciiLower (Text.head piece)+ && Text.all (\c -> isAsciiLower c || isDigit c || c == '-') piece++renderKnobName :: KnobName -> Text+renderKnobName (KnobName value) = value++emptyKnobs :: ResolvedKnobs+emptyKnobs = ResolvedKnobs Map.empty++parseAssignment :: Text -> Either KnobError (KnobName, RawKnob)+parseAssignment input = case Text.breakOn "=" input of+ (name, assignment)+ | Text.null assignment -> Left (KnobError "expected NAME=VALUE")+ | otherwise -> case mkKnobName name of+ Left err -> Left (KnobError err)+ Right parsed -> Right (parsed, RawText (Text.drop 1 assignment))++resolveKnobs :: [KnobSpec] -> [(KnobName, RawKnob)] -> Either (NonEmpty KnobError) ResolvedKnobs+resolveKnobs specs supplied = case errors of+ [] -> Right (ResolvedKnobs values)+ first : rest -> Left (first :| rest)+ where+ declarations = Map.fromList [(spec.name, spec) | spec <- specs]+ duplicateNames = [head names | names <- group (sort (fmap fst supplied)), length names > 1]+ unknown = [name | (name, _) <- supplied, Map.notMember name declarations]+ parsed = [(name, resolveOne spec raw) | (name, raw) <- supplied, Just spec <- [Map.lookup name declarations]]+ parseErrors = [KnobError (renderKnobName name <> ": " <> message) | (name, Left message) <- parsed]+ overrides = Map.fromList [(name, value) | (name, Right value) <- parsed]+ values = Map.fromList [(spec.name, Map.findWithDefault spec.def spec.name overrides) | spec <- specs]+ errors =+ [KnobError (renderKnobName name <> ": assigned more than once") | name <- duplicateNames]+ <> [KnobError (renderKnobName name <> ": unknown knob") | name <- unknown]+ <> parseErrors++resolveOne :: KnobSpec -> RawKnob -> Either Text KnobValue+resolveOne spec raw = do+ parsed <- parseByType spec.knobType raw+ if allowedValue spec.allowed parsed then Right parsed else Left "value is outside the allowed set"++parseByType :: KnobType -> RawKnob -> Either Text KnobValue+parseByType KnobBool (RawText "true") = Right (VBool True)+parseByType KnobBool (RawText "false") = Right (VBool False)+parseByType KnobBool (RawJson (Bool value)) = Right (VBool value)+parseByType KnobInt (RawText value) = maybe (Left "expected an integer") (Right . VInt) (readMaybeText value)+parseByType KnobInt (RawJson (Number value)) = maybe (Left "expected an integer") (Right . VInt) (Scientific.toBoundedInteger value)+parseByType KnobDouble (RawText value) = maybe (Left "expected a number") (Right . VDouble) (readMaybeText value)+parseByType KnobDouble (RawJson (Number value)) = Right (VDouble (Scientific.toRealFloat value))+parseByType KnobText (RawText value) = Right (VText value)+parseByType KnobText (RawJson (String value)) = Right (VText value)+parseByType _ _ = Left "value has the wrong JSON type"++readMaybeText :: (Read value) => Text -> Maybe value+readMaybeText value = case reads (Text.unpack value) of [(parsed, "")] -> Just parsed; _ -> Nothing++allowedValue :: Allowed -> KnobValue -> Bool+allowedValue AnyValue _ = True+allowedValue (OneOf values) value = value `elem` values+allowedValue (IntRange low high) (VInt value) = value >= low && value <= high+allowedValue (DoubleRange low high) (VDouble value) = value >= low && value <= high+allowedValue _ _ = False++resolvedKnobsMap :: ResolvedKnobs -> Map KnobName KnobValue+resolvedKnobsMap (ResolvedKnobs values) = values++knobBool :: ResolvedKnobs -> KnobName -> Bool+knobBool knobs name = case Map.lookup name (resolvedKnobsMap knobs) of Just (VBool value) -> value; _ -> error "knobBool: missing or wrong type"++knobInt :: ResolvedKnobs -> KnobName -> Int64+knobInt knobs name = case Map.lookup name (resolvedKnobsMap knobs) of Just (VInt value) -> value; _ -> error "knobInt: missing or wrong type"++knobDouble :: ResolvedKnobs -> KnobName -> Double+knobDouble knobs name = case Map.lookup name (resolvedKnobsMap knobs) of+ Just (VDouble value) -> value+ -- JSON number decoding cannot recover whether an integral value was+ -- originally a decimal knob. Worker init messages use this round trip.+ Just (VInt value) -> fromIntegral value+ _ -> error "knobDouble: missing or wrong type"++knobText :: ResolvedKnobs -> KnobName -> Text+knobText knobs name = case Map.lookup name (resolvedKnobsMap knobs) of Just (VText value) -> value; _ -> error "knobText: missing or wrong type"++instance ToJSON KnobName where toJSON = toJSON . renderKnobName++instance FromJSON KnobName where parseJSON = withText "KnobName" (either (fail . Text.unpack) pure . mkKnobName)++instance ToJSON KnobValue where+ toJSON (VBool value) = Bool value+ toJSON (VInt value) = toJSON value+ toJSON (VDouble value) = toJSON value+ toJSON (VText value) = String value++instance ToJSON ResolvedKnobs where+ toJSON (ResolvedKnobs values) = Object (KeyMap.fromList [(Key.fromText (renderKnobName name), toJSON value) | (name, value) <- Map.toAscList values])++instance FromJSON ResolvedKnobs where+ parseJSON = withObject "ResolvedKnobs" \values -> ResolvedKnobs . Map.fromList <$> traverse parseEntry (KeyMap.toList values)+ where+ parseEntry (key, value) = (,) <$> either (fail . Text.unpack) pure (mkKnobName (Key.toText key)) <*> parseJSON value++instance FromJSON KnobValue where+ parseJSON (Bool value) = pure (VBool value)+ parseJSON (String value) = pure (VText value)+ parseJSON (Number value) = pure (maybe (VDouble (Scientific.toRealFloat value)) VInt (Scientific.toBoundedInteger value))+ parseJSON _ = fail "knob values must be JSON scalars"
+ src/Kenshou/Core/Log.hs view
@@ -0,0 +1,57 @@+module Kenshou.Core.Log+ ( Severity (..),+ Logger,+ withLogger,+ logAt,+ nullLogger,+ )+where++import Data.Aeson (Value, encode, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Text (Text)+import Data.Text.IO qualified as Text.IO+import Data.Time (getCurrentTime)+import Kenshou.Core.Id (RunId, renderRunId)+import System.IO (BufferMode (LineBuffering), Handle, IOMode (AppendMode), hSetBuffering, stderr, withFile)+import System.Posix.Process (getProcessID)++data Severity = Debug | Info | Warning | Error deriving stock (Eq, Ord, Show)++newtype Logger = Logger (Severity -> Text -> [(Text, Value)] -> IO ())++withLogger :: FilePath -> RunId -> (Logger -> IO value) -> IO value+withLogger path runId action = withFile path AppendMode \handle -> do+ hSetBuffering handle LineBuffering+ action (fileLogger handle runId)++fileLogger :: Handle -> RunId -> Logger+fileLogger handle runId = Logger \severity message fields -> do+ now <- getCurrentTime+ pid <- getProcessID+ LazyByteString.hPutStr handle $+ encode+ ( object+ [ "ts" .= now,+ "level" .= severityText severity,+ "msg" .= message,+ "fields" .= object [Key.fromText name .= value | (name, value) <- fields],+ "runId" .= renderRunId runId,+ "process" .= show pid+ ]+ )+ <> "\n"+ Text.IO.hPutStrLn stderr ("[" <> severityText severity <> "] " <> message)++logAt :: Logger -> Severity -> Text -> [(Text, Value)] -> IO ()+logAt (Logger write) = write++nullLogger :: Logger+nullLogger = Logger \_ _ _ -> pure ()++severityText :: Severity -> Text+severityText Debug = "debug"+severityText Info = "info"+severityText Warning = "warning"+severityText Error = "error"
+ src/Kenshou/Core/Manifest.hs view
@@ -0,0 +1,94 @@+module Kenshou.Core.Manifest+ ( ManifestFile (..),+ Manifest (..),+ ManifestProblem (..),+ writeManifest,+ verifyManifest,+ )+where++import Data.Aeson+import Data.ByteString qualified as ByteString+import Data.List (sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (RunId)+import System.Directory (doesDirectoryExist, doesFileExist, listDirectory)+import System.FilePath (makeRelative, takeExtension, (</>))++data ManifestFile = ManifestFile {path :: FilePath, sha256 :: Text, bytes :: Integer, mediaType :: Text} deriving stock (Eq, Show)++data Manifest = Manifest {runId :: RunId, createdAt :: UTCTime, files :: [ManifestFile]} deriving stock (Eq, Show)++newtype ManifestProblem = ManifestProblem Text deriving stock (Eq, Show)++instance ToJSON ManifestFile where+ toJSON file = object ["path" .= file.path, "sha256" .= file.sha256, "bytes" .= file.bytes, "mediaType" .= file.mediaType]++instance FromJSON ManifestFile where+ parseJSON = withObject "ManifestFile" \value -> ManifestFile <$> value .: "path" <*> value .: "sha256" <*> value .: "bytes" <*> value .: "mediaType"++instance ToJSON Manifest where+ toJSON manifest = object ["schema" .= ("kenshou.artifact-manifest/v1" :: Text), "runId" .= manifest.runId, "createdAt" .= manifest.createdAt, "algorithm" .= ("sha256" :: Text), "files" .= manifest.files]++instance FromJSON Manifest where+ parseJSON = withObject "Manifest" \value -> do+ schema <- value .: "schema"+ algorithm <- value .: "algorithm"+ if schema /= ("kenshou.artifact-manifest/v1" :: Text) || algorithm /= ("sha256" :: Text) then fail "unsupported artifact manifest" else pure ()+ Manifest <$> value .: "runId" <*> value .: "createdAt" <*> value .: "files"++writeManifest :: FilePath -> RunId -> Map FilePath Text -> IO Manifest+writeManifest root runId mediaTypes = do+ paths <- filter (/= "manifest.json") . sort <$> regularFiles root+ files <- traverse fileRecord paths+ createdAt <- getCurrentTime+ pure Manifest {runId, createdAt, files}+ where+ fileRecord relativePath = do+ content <- ByteString.readFile (root </> relativePath)+ pure ManifestFile {path = relativePath, sha256 = sha256Hex content, bytes = fromIntegral (ByteString.length content), mediaType = Map.findWithDefault (mediaTypeFor relativePath) relativePath mediaTypes}++verifyManifest :: FilePath -> IO (Either (NonEmpty ManifestProblem) ())+verifyManifest root = do+ decoded <- eitherDecodeFileStrict' (root </> "manifest.json") :: IO (Either String Manifest)+ case decoded of+ Left err -> pure (Left (ManifestProblem ("manifest.json: " <> Text.pack err) :| []))+ Right manifest -> do+ problems <- concat <$> traverse checkFile manifest.files+ pure case problems of [] -> Right (); first : rest -> Left (first :| rest)+ where+ checkFile :: ManifestFile -> IO [ManifestProblem]+ checkFile file = do+ let fullPath = root </> file.path+ exists <- doesFileExist fullPath+ if not exists+ then pure [ManifestProblem (Text.pack file.path <> ": missing")]+ else do+ content <- ByteString.readFile fullPath+ pure [ManifestProblem (Text.pack file.path <> ": digest mismatch") | sha256Hex content /= file.sha256]++regularFiles :: FilePath -> IO [FilePath]+regularFiles root = go root+ where+ go directory = do+ names <- listDirectory directory+ concat <$> traverse (visit directory) names+ visit directory name = do+ let path = directory </> name+ isDirectory <- doesDirectoryExist path+ if isDirectory then go path else pure [makeRelative root path]++mediaTypeFor :: FilePath -> Text+mediaTypeFor path = case takeExtension path of+ ".json" -> "application/json"+ ".jsonl" -> "application/x-ndjson"+ ".csv" -> "text/csv"+ ".log" -> "text/plain"+ ".txt" -> "text/plain"+ _ -> "application/octet-stream"
+ src/Kenshou/Core/Outcome.hs view
@@ -0,0 +1,55 @@+module Kenshou.Core.Outcome+ ( Outcome (..),+ renderOutcome,+ parseOutcome,+ outcomeExitCode,+ usageExitCode,+ worstOutcome,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), withText)+import Data.List (maximumBy)+import Data.List.NonEmpty (NonEmpty)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text++data Outcome = Passed | Failed | Errored | Inconclusive | InfrastructureFailure+ deriving stock (Eq, Show, Enum, Bounded)++renderOutcome :: Outcome -> Text+renderOutcome Passed = "passed"+renderOutcome Failed = "failed"+renderOutcome Errored = "errored"+renderOutcome Inconclusive = "inconclusive"+renderOutcome InfrastructureFailure = "infrastructure-failure"++parseOutcome :: Text -> Either Text Outcome+parseOutcome value = maybe (Left ("unknown outcome \"" <> value <> "\"")) Right (lookup value pairs)+ where+ pairs = [(renderOutcome item, item) | item <- [minBound .. maxBound]]++outcomeExitCode :: Outcome -> Int+outcomeExitCode Passed = 0+outcomeExitCode Failed = 1+outcomeExitCode Inconclusive = 3+outcomeExitCode Errored = 4+outcomeExitCode InfrastructureFailure = 4++usageExitCode :: Int+usageExitCode = 2++worstOutcome :: NonEmpty Outcome -> Outcome+worstOutcome = maximumBy (compare `onRank` rank) . NonEmpty.toList+ where+ onRank comparator project left right = comparator (project left) (project right)+ rank Passed = 0 :: Int+ rank Inconclusive = 1+ rank InfrastructureFailure = 2+ rank Errored = 3+ rank Failed = 4++instance ToJSON Outcome where toJSON = toJSON . renderOutcome++instance FromJSON Outcome where parseJSON = withText "Outcome" (either (fail . Text.unpack) pure . parseOutcome)
+ src/Kenshou/Core/Phase.hs view
@@ -0,0 +1,44 @@+module Kenshou.Core.Phase+ ( PhaseName (..),+ PhasePlan (..),+ zeroPhases,+ renderPhaseName,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, withText, (.:), (.=))+import Data.Text (Text)+import Data.Text qualified as Text++data PhaseName = WarmUp | Steady | Drain+ deriving stock (Eq, Ord, Show)++data PhasePlan = PhasePlan+ { warmUpSeconds :: Double,+ steadySeconds :: Double,+ drainSeconds :: Double+ }+ deriving stock (Eq, Show)++zeroPhases :: PhasePlan+zeroPhases = PhasePlan 0 0 0++renderPhaseName :: PhaseName -> Text+renderPhaseName WarmUp = "warm-up"+renderPhaseName Steady = "steady"+renderPhaseName Drain = "drain"++instance ToJSON PhaseName where toJSON = toJSON . renderPhaseName++instance FromJSON PhaseName where+ parseJSON = withText "PhaseName" \case+ "warm-up" -> pure WarmUp+ "steady" -> pure Steady+ "drain" -> pure Drain+ other -> fail ("unknown phase " <> Text.unpack other)++instance ToJSON PhasePlan where+ toJSON value = object ["warmUpSeconds" .= value.warmUpSeconds, "steadySeconds" .= value.steadySeconds, "drainSeconds" .= value.drainSeconds]++instance FromJSON PhasePlan where+ parseJSON = withObject "PhasePlan" \value -> PhasePlan <$> value .: "warmUpSeconds" <*> value .: "steadySeconds" <*> value .: "drainSeconds"
+ src/Kenshou/Core/Resource.hs view
@@ -0,0 +1,37 @@+-- | Named resources a scenario requires besides PostgreSQL, and the providers that+-- suites register for them. A scenario names its resources in+-- 'Kenshou.Core.Env.EnvRequirements'; a bundle contributes one 'ResourceProvider'+-- per name; the kernel normalizes the specification value, provisions the resource+-- around a local run, hands each worker the provisioned value, and records a+-- fingerprint in the run result.+module Kenshou.Core.Resource+ ( ResourceName,+ mkResourceName,+ renderResourceName,+ ProvisionedResource (..),+ CellBinding (..),+ ResourceProvider (..),+ )+where++import Data.Aeson (Value)+import Data.Text (Text)+import Kenshou.Core.Context (RunContext)+import Kenshou.Core.Resource.Types++data ResourceProvider = ResourceProvider+ { name :: ResourceName,+ summary :: Text,+ -- | Turn the run specification's value, absent when the specification omits+ -- the resource, into the complete validated value the provider acts on. This+ -- is where defaults apply and bad input is rejected; the effective run+ -- specification records the result, so the comparison key hashes it.+ normalize :: Maybe Value -> Either Text Value,+ -- | Acquire the resource for one local run and release it when the action+ -- returns or throws. The run directory in the context is where a provider keeps+ -- its logs.+ withLocal :: forall a. RunContext -> Value -> (ProvisionedResource -> IO a) -> IO a,+ -- | How the resource is bound on a leased cell; 'Nothing' rejects cell placement+ -- for scenarios that require it.+ cell :: Maybe CellBinding+ }
+ src/Kenshou/Core/Resource/Types.hs view
@@ -0,0 +1,71 @@+-- | Names and values of the resources a scenario may require besides PostgreSQL:+-- anything a suite provisions for a run, such as a message broker or an object+-- store. 'Kenshou.Core.Resource' adds the provider contract; these types live apart+-- so the environment and context modules can name them without an import cycle.+module Kenshou.Core.Resource.Types+ ( ResourceName,+ mkResourceName,+ renderResourceName,+ legacyKafkaName,+ ProvisionedResource (..),+ CellBinding (..),+ )+where++import Data.Aeson (FromJSON (..), FromJSONKey (..), FromJSONKeyFunction (..), ToJSON (..), ToJSONKey (..), Value, withText)+import Data.Aeson.Types (toJSONKeyText)+import Data.Char (isAsciiLower, isDigit)+import Data.Text (Text)+import Data.Text qualified as Text++-- | Lowercase ASCII letters, digits, and hyphens, starting with a letter, at most+-- 63 characters, so a name is usable as a JSON object key without escaping.+newtype ResourceName = ResourceName Text+ deriving stock (Eq, Ord, Show)++mkResourceName :: Text -> Either Text ResourceName+mkResourceName value+ | valid = Right (ResourceName value)+ | otherwise = Left ("invalid resource name \"" <> value <> "\"")+ where+ valid =+ Text.length value >= 1+ && Text.length value <= 63+ && isAsciiLower (Text.head value)+ && Text.all (\character -> isAsciiLower character || isDigit character || character == '-') value++renderResourceName :: ResourceName -> Text+renderResourceName (ResourceName value) = value++-- | The resource name that documents written before named resources carried as a+-- top-level field. It exists only for the legacy readers and the compat-key/v1 shim;+-- the coupling allow-list names it.+legacyKafkaName :: ResourceName+legacyKafkaName = ResourceName "kafka"++-- | What a provider hands a run for one resource.+data ProvisionedResource = ProvisionedResource+ { -- | Sent to every worker in @WorkerInit.resources@.+ workerValue :: Value,+ -- | Recorded in the run result fingerprint under @resources.<name>@.+ fingerprint :: Value+ }+ deriving stock (Eq, Show)++-- | How a resource is bound on a leased verification cell.+data CellBinding = CellBinding+ { -- | The capability key a cell must report.+ capability :: Text,+ -- | The value for @cell.submission@ @reset.resources.<name>@, if the cell must+ -- reset the resource before the run.+ reset :: Maybe Value+ }+ deriving stock (Eq, Show)++instance ToJSON ResourceName where toJSON = toJSON . renderResourceName++instance FromJSON ResourceName where parseJSON = withText "ResourceName" (either (fail . Text.unpack) pure . mkResourceName)++instance ToJSONKey ResourceName where toJSONKey = toJSONKeyText renderResourceName++instance FromJSONKey ResourceName where fromJSONKey = FromJSONKeyTextParser (either (fail . Text.unpack) pure . mkResourceName)
+ src/Kenshou/Core/Role.hs view
@@ -0,0 +1,142 @@+module Kenshou.Core.Role+ ( RoleName,+ WorkerRole (..),+ PostgresConnInfo (..),+ WorkerInit (..),+ ControlMessage (..),+ WorkerMessage (..),+ RoleContext (..),+ mkRoleName,+ renderRoleName,+ )+where++import Data.Aeson+import Data.Aeson.Types (Pair, Parser)+import Data.Int (Int64)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Dimension (Dimensions)+import Kenshou.Core.Id (RunId, ScenarioId, Seed)+import Kenshou.Core.Knob (ResolvedKnobs)+import Kenshou.Core.Log (Logger)+import Kenshou.Core.Phase (PhaseName (..), renderPhaseName)+import Kenshou.Core.Resource.Types (ResourceName)+import Kenshou.Core.RunSpec (withLegacyResource)++newtype RoleName = RoleName Text deriving stock (Eq, Ord, Show)++data WorkerRole = WorkerRole+ { name :: RoleName,+ summary :: Text,+ run :: RoleContext -> IO ()+ }++data PostgresConnInfo = PostgresConnInfo {connectionString :: Text, adminConnectionString :: Text} deriving stock (Eq, Show)++data WorkerInit = WorkerInit+ { runId :: RunId,+ scenario :: ScenarioId,+ role :: RoleName,+ instanceName :: Text,+ seed :: Seed,+ knobs :: ResolvedKnobs,+ dimensions :: Dimensions,+ postgres :: Maybe PostgresConnInfo,+ resources :: Map ResourceName Value,+ telemetry :: Maybe Value,+ outDir :: FilePath,+ args :: Value+ }+ deriving stock (Eq, Show)++data ControlMessage = CtlInit WorkerInit | CtlStart | CtlPhase PhaseName | CtlStop Int | CtlCustom Text Value deriving stock (Eq, Show)++data WorkerMessage = WrkReady | WrkProgress Int64 UTCTime | WrkFacts [Value] | WrkCustom Text Value | WrkDone (Maybe Text) | WrkError Text deriving stock (Eq, Show)++data RoleContext = RoleContext+ { init :: WorkerInit,+ receive :: IO (Maybe ControlMessage),+ send :: WorkerMessage -> IO (),+ logger :: Logger+ }++mkRoleName :: Text -> Either Text RoleName+mkRoleName value = case Text.splitOn "/" value of+ [layer, role] | not (Text.null layer) && not (Text.null role) -> Right (RoleName value)+ _ -> Left "role name must be <layer>/<name>"++renderRoleName :: RoleName -> Text+renderRoleName (RoleName value) = value++instance ToJSON RoleName where toJSON = toJSON . renderRoleName++instance FromJSON RoleName where parseJSON = withText "RoleName" (either (fail . Text.unpack) pure . mkRoleName)++instance ToJSON PostgresConnInfo where toJSON value = object ["connectionString" .= value.connectionString, "adminConnectionString" .= value.adminConnectionString]++instance FromJSON PostgresConnInfo where parseJSON = withObject "PostgresConnInfo" \value -> PostgresConnInfo <$> value .: "connectionString" <*> value .: "adminConnectionString"++instance ToJSON WorkerInit where+ toJSON value = object ["schema" .= ("kenshou.worker-init/v1" :: Text), "runId" .= value.runId, "scenario" .= value.scenario, "role" .= value.role, "instance" .= value.instanceName, "seed" .= value.seed, "knobs" .= value.knobs, "dimensions" .= value.dimensions, "postgres" .= value.postgres, "resources" .= value.resources, "telemetry" .= value.telemetry, "outDir" .= value.outDir, "args" .= value.args]++instance FromJSON WorkerInit where+ parseJSON = withObject "WorkerInit" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.worker-init/v1" :: Text) then fail "unsupported worker-init schema" else pure ()+ resources <- (value .:? "resources" .!= Map.empty) >>= withLegacyResource "worker-init" value+ WorkerInit <$> value .: "runId" <*> value .: "scenario" <*> value .: "role" <*> value .: "instance" <*> value .: "seed" <*> value .: "knobs" <*> value .: "dimensions" <*> value .:? "postgres" <*> pure resources <*> value .:? "telemetry" <*> value .: "outDir" <*> value .: "args"++instance ToJSON ControlMessage where+ toJSON (CtlInit value) = tagged "init" ["init" .= value]+ toJSON CtlStart = tagged "start" []+ toJSON (CtlPhase name) = tagged "phase" ["name" .= renderPhaseName name]+ toJSON (CtlStop grace) = tagged "stop" ["graceMillis" .= grace]+ toJSON (CtlCustom name payload) = tagged "custom" ["name" .= name, "payload" .= payload]++instance FromJSON ControlMessage where+ parseJSON = withObject "ControlMessage" \value -> do+ version <- value .: "v"+ if version /= (1 :: Int) then fail "unsupported worker protocol version" else pure ()+ messageType <- value .: "type" :: Parser Text+ case messageType of+ "init" -> CtlInit <$> value .: "init"+ "start" -> pure CtlStart+ "phase" -> CtlPhase <$> (value .: "name" >>= parsePhase)+ "stop" -> CtlStop <$> value .: "graceMillis"+ "custom" -> CtlCustom <$> value .: "name" <*> value .: "payload"+ _ -> fail "unknown control message"++instance ToJSON WorkerMessage where+ toJSON WrkReady = tagged "ready" []+ toJSON (WrkProgress count at) = tagged "progress" ["count" .= count, "at" .= at]+ toJSON (WrkFacts batch) = tagged "facts" ["batch" .= batch]+ toJSON (WrkCustom name payload) = tagged "custom" ["name" .= name, "payload" .= payload]+ toJSON (WrkDone detail) = tagged "done" ["detail" .= detail]+ toJSON (WrkError message) = tagged "error" ["message" .= message]++instance FromJSON WorkerMessage where+ parseJSON = withObject "WorkerMessage" \value -> do+ version <- value .: "v"+ if version /= (1 :: Int) then fail "unsupported worker protocol version" else pure ()+ messageType <- value .: "type" :: Parser Text+ case messageType of+ "ready" -> pure WrkReady+ "progress" -> WrkProgress <$> value .: "count" <*> value .: "at"+ "facts" -> WrkFacts <$> value .: "batch"+ "custom" -> WrkCustom <$> value .: "name" <*> value .: "payload"+ "done" -> WrkDone <$> value .:? "detail"+ "error" -> WrkError <$> value .: "message"+ _ -> fail "unknown worker message"++tagged :: Text -> [Pair] -> Value+tagged messageType fields = object (["v" .= (1 :: Int), "type" .= messageType] <> fields)++parsePhase :: Text -> Parser PhaseName+parsePhase "warm-up" = pure WarmUp+parsePhase "steady" = pure Steady+parsePhase "drain" = pure Drain+parsePhase _ = fail "unknown phase"
+ src/Kenshou/Core/Role/Dispatch.hs view
@@ -0,0 +1,45 @@+module Kenshou.Core.Role.Dispatch (runWorker) where++import Control.Exception (SomeException, displayException, try)+import Data.Aeson qualified as Aeson+import Data.ByteString.Char8 qualified as ByteString+import Data.ByteString.Lazy.Char8 qualified as LazyByteString+import Data.IORef+import Data.Text qualified as Text+import GHC.IO.Handle (hDuplicate, hDuplicateTo)+import Kenshou.Core.Bundle (Registry, lookupRole)+import Kenshou.Core.Log (nullLogger)+import Kenshou.Core.Role+import System.Exit (ExitCode (..))+import System.IO (Handle, hFlush, hIsEOF, stderr, stdin, stdout)++runWorker :: Registry -> RoleName -> IO ExitCode+runWorker registry roleName = case lookupRole registry roleName of+ Nothing -> pure (ExitFailure 2)+ Just role -> do+ channel <- hDuplicate stdout+ hDuplicateTo stderr stdout+ initial <- receiveControl stdin+ case initial of+ Just (CtlInit workerInit) | workerInit.role == roleName -> do+ parentGone <- newIORef False+ let receive = receiveControl stdin >>= \case Nothing -> writeIORef parentGone True >> pure Nothing; message -> pure message+ outcome <- try (role.run (RoleContext workerInit receive (sendWorker channel) nullLogger)) :: IO (Either SomeException ())+ case outcome of+ Left exception -> sendWorker channel (WrkError (Text.pack (displayException exception))) >> pure (ExitFailure 70)+ Right () -> do+ orphaned <- readIORef parentGone+ if orphaned then pure (ExitFailure 75) else sendWorker channel (WrkDone Nothing) >> pure ExitSuccess+ _ -> pure (ExitFailure 2)++receiveControl :: Handle -> IO (Maybe ControlMessage)+receiveControl handle = do+ atEnd <- hIsEOF handle+ if atEnd+ then pure Nothing+ else do+ line <- ByteString.hGetLine handle+ pure (either (const Nothing) Just (Aeson.eitherDecodeStrict' line))++sendWorker :: Handle -> WorkerMessage -> IO ()+sendWorker handle message = LazyByteString.hPutStrLn handle (Aeson.encode message) >> hFlush handle
+ src/Kenshou/Core/Role/Spawn.hs view
@@ -0,0 +1,66 @@+module Kenshou.Core.Role.Spawn (WorkerHandle (..), withWorker) where++import Control.Exception (bracket)+import Data.Aeson (Value, eitherDecodeStrict', encode)+import Data.ByteString.Char8 qualified as ByteString+import Data.ByteString.Lazy.Char8 qualified as LazyByteString+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (ArtifactDir (..), Environment (..), RunContext (..), artifactPath)+import Kenshou.Core.Env.Postgres (PostgresEnv (..))+import Kenshou.Core.Resource.Types (ProvisionedResource (..))+import Kenshou.Core.Role+import Kenshou.Core.RunSpec (EnvironmentSpec (..))+import System.Environment (getExecutablePath)+import System.Exit (ExitCode)+import System.IO (BufferMode (LineBuffering), Handle, IOMode (AppendMode), hClose, hFlush, hIsEOF, hSetBuffering, openFile)+import System.Posix.Types (CPid)+import System.Process+import System.Timeout (timeout)++data WorkerHandle = WorkerHandle+ { instanceName :: Text,+ pid :: CPid,+ send :: ControlMessage -> IO (),+ receive :: Int -> IO (Maybe WorkerMessage),+ waitExit :: IO ExitCode+ }++withWorker :: RunContext -> RoleName -> Text -> Value -> (WorkerHandle -> IO value) -> IO value+withWorker context roleName instanceName arguments action = do+ logPath <- artifactPath context LogsDir ("worker-" <> Text.unpack instanceName <> ".stderr.log")+ bracket (start logPath) stop (action . fst)+ where+ start logPath = do+ executable <- getExecutablePath+ errorHandle <- openFile logPath AppendMode+ (Just input, Just output, _, processHandle) <- createProcess (proc executable ["worker", "--role", Text.unpack (renderRoleName roleName), "--diagnosis-root", context.outDir]) {std_in = CreatePipe, std_out = CreatePipe, std_err = UseHandle errorHandle}+ hSetBuffering input LineBuffering+ processId <- getPid processHandle >>= maybe (ioError (userError "worker has no process id")) pure+ let sendMessage message = LazyByteString.hPutStrLn input (encode message) >> hFlush input+ receiveMessage timeoutMillis = timeout (timeoutMillis * 1000) (readWorker output) >>= pure . joinMaybe+ workerHandle = WorkerHandle instanceName processId sendMessage receiveMessage (waitForProcess processHandle)+ postgres = fmap (\environment -> PostgresConnInfo environment.connectionString environment.adminConnectionString) context.env.postgres+ workerInit = WorkerInit context.runId context.scenario roleName instanceName context.seed context.knobs context.dimensions postgres (fmap (.workerValue) context.env.resources) context.environmentSpec.telemetry context.outDir arguments+ sendMessage (CtlInit workerInit)+ pure (workerHandle, (input, output, errorHandle, processHandle))+ stop (_, (input, output, errorHandle, processHandle)) = do+ sendLine input (CtlStop 2000)+ hClose input+ finished <- timeout 2000000 (waitForProcess processHandle)+ case finished of+ Just _ -> pure ()+ Nothing -> terminateProcess processHandle >> waitForProcess processHandle >> pure ()+ hClose output+ hClose errorHandle++readWorker :: Handle -> IO (Maybe WorkerMessage)+readWorker handle = do+ atEnd <- hIsEOF handle+ if atEnd then pure Nothing else either (const Nothing) Just . eitherDecodeStrict' <$> ByteString.hGetLine handle++sendLine :: Handle -> ControlMessage -> IO ()+sendLine handle message = LazyByteString.hPutStrLn handle (encode message) >> hFlush handle++joinMaybe :: Maybe (Maybe value) -> Maybe value+joinMaybe = maybe Nothing id
+ src/Kenshou/Core/Run.hs view
@@ -0,0 +1,169 @@+module Kenshou.Core.Run+ ( RunnerConfig (..),+ RunOutput (..),+ executeRun,+ defectDisposition,+ )+where++import Control.Exception (SomeException, displayException, fromException, try)+import Control.Monad (when)+import Data.Aeson (ToJSON, Value, encode, object, (.=))+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (diffUTCTime, getCurrentTime)+import Kenshou.Core.Bundle (Registry, lookupResource, registryMigrationSets)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))+import Kenshou.Core.Compat (comparisonKey, compatInputs, seriesKey)+import Kenshou.Core.Context+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Env.Postgres (EnvError (..), PgSettingsSnapshot, PostgresEnv (..), withPostgresEnvKeeping, withPostgresEnvNamed)+import Kenshou.Core.Fingerprint (HostFingerprint (..), collectHostFingerprint, hostValue, kenshouValue, runtimeValue)+import Kenshou.Core.Id (renderRunId, unSeed)+import Kenshou.Core.Log (Severity (Warning), withLogger)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Core.Outcome (Outcome (..), outcomeExitCode)+import Kenshou.Core.Resource (ProvisionedResource (..), ResourceName, ResourceProvider (..), renderResourceName)+import Kenshou.Core.RunResult+import Kenshou.Core.RunSpec (CohortExpectation (..), EffectiveRunSpec (..), EnvironmentSpec (..), RunSpec)+import Kenshou.Core.RunSpec.Resolve (SpecError (..), resolveRunSpec)+import Kenshou.Core.Scenario (KnownDefect (..), Scenario (..), ScenarioReport (..), cohortScopeApplies, individualKnownDefects, infrastructureFailureBecause)+import System.Directory (createDirectory, createDirectoryIfMissing, doesDirectoryExist, getCurrentDirectory, renameFile)+import System.FilePath ((</>))+import System.Timeout (timeout)++data RunnerConfig = RunnerConfig+ { registry :: Registry,+ outRoot :: FilePath,+ cohort :: CohortIdentity,+ cellFingerprint :: Maybe Value,+ keepEnvironment :: Bool,+ strictKnownDefects :: Bool,+ argv :: [String]+ }++data RunOutput = RunOutput {directory :: FilePath, result :: RunResult, manifestSha256 :: Text}++executeRun :: RunnerConfig -> RunSpec -> IO (Either (NonEmpty SpecError) RunOutput)+executeRun config input = do+ resolution <- resolveRunSpec config.registry input+ case resolution of+ Left problems -> pure (Left problems)+ Right (scenario, spec) -> do+ let directory = config.outRoot </> Text.unpack (renderRunId spec.runId)+ exists <- doesDirectoryExist directory+ if exists+ then pure (Left (SpecError ("run directory already exists: " <> Text.pack directory) :| []))+ else do+ createDirectoryIfMissing True config.outRoot+ createDirectory directory+ createDirectory (directory </> "logs")+ atomicEncode (directory </> "run-spec.json") spec+ specBytes <- ByteString.readFile (directory </> "run-spec.json")+ (result, mediaTypes) <- withLogger (directory </> "logs" </> "harness.jsonl") spec.runId \logger -> do+ startedAt <- getCurrentTime+ state <- newRunState+ let context = RunContext spec.runId spec.scenario spec.knobs spec.dimensions spec.seed spec.phases (Environment Nothing Map.empty Map.empty) spec.environment spec.comparison directory logger state+ (report, postgresSnapshot, extraPostgresSnapshots, resourceFingerprints) <- runScenario config scenario spec context+ endedAt <- getCurrentTime+ host <- collectHostFingerprint+ when (host.revision == Nothing) (observe context "harness" Warning "harness revision is unavailable")+ (summaries, observations, phaseTimings, mediaTypes) <- readRunState state+ workingDirectory <- getCurrentDirectory+ let (defect, blocking, exitCode) = defectDisposition config.strictKnownDefects config.cohort scenario report+ schemas = maybe [] (.schemas) scenario.requires.postgres+ inputs = compatInputs spec postgresSnapshot schemas config.cohort+ compatibility = object ["algorithm" .= ("kenshou.compat-key/v1" :: Text), "comparisonKey" .= comparisonKey inputs, "seriesKey" .= seriesKey inputs, "inputs" .= inputs]+ fingerprint =+ object $+ ["placement" .= spec.environment.placement, "machineProfile" .= spec.environment.machineProfile, "host" .= hostValue host, "runtime" .= runtimeValue host, "kenshou" .= kenshouValue host, "postgres" .= postgresSnapshot, "extraPostgres" .= extraPostgresSnapshots, "cell" .= config.cellFingerprint]+ <> ["resources" .= resourceFingerprints | not (Map.null resourceFingerprints)]+ invocation = object ["argv" .= config.argv, "workingDirectory" .= workingDirectory]+ result = RunResult spec.runId spec.scenario spec.scenarioRevision scenario.tier report.outcome blocking exitCode report.reason report.failures defect (unSeed spec.seed) (sha256Hex specBytes) spec.comparison startedAt endedAt (realToFrac (diffUTCTime endedAt startedAt)) phaseTimings config.cohort fingerprint compatibility summaries observations invocation+ atomicEncode (directory </> "run-result.json") result+ pure (result, mediaTypes)+ manifest <- writeManifest directory spec.runId mediaTypes+ atomicEncode (directory </> "manifest.json") manifest+ manifestBytes <- ByteString.readFile (directory </> "manifest.json")+ pure (Right (RunOutput directory result (sha256Hex manifestBytes)))++runScenario :: RunnerConfig -> Scenario -> EffectiveRunSpec -> RunContext -> IO (ScenarioReport, Maybe PgSettingsSnapshot, Map.Map Text PgSettingsSnapshot, Map.Map ResourceName Value)+runScenario config scenario spec context+ | cohortMismatch config.cohort spec.cohortExpectation = pure (infrastructureFailureBecause "cohort-mismatch", Nothing, Map.empty, Map.empty)+ | otherwise = do+ nested <- withPrimary scenario.requires.postgres spec.environment.postgres \primary ->+ withExtras scenario.requires.extraPostgres spec.environment.extraPostgres Map.empty \extras ->+ withResources scenario.requires.resources Map.empty \resources -> do+ let environment = Environment primary extras resources+ report <- timed context {env = environment}+ pure (report, (.snapshot) <$> primary, fmap (.snapshot) extras, fmap (.fingerprint) resources)+ let provisioned = nested >>= id >>= id+ pure (either (\err -> (infrastructureFailureBecause (Text.pack (show err)), Nothing, Map.empty, Map.empty)) id provisioned)+ where+ -- Providers are bracketed in declaration order inside the PostgreSQL contexts; a+ -- provider failure is an infrastructure failure like a PostgreSQL one.+ withResources [] accumulated action = Right <$> action accumulated+ withResources (name : rest) accumulated action = case (lookupResource config.registry name, Map.lookup name spec.environment.resources) of+ (Nothing, _) -> pure (Left (EnvError ("no registered provider for resource " <> renderResourceName name)))+ (_, Nothing) -> pure (Left (EnvError ("resource specification is missing: " <> renderResourceName name)))+ (Just ResourceProvider {withLocal = acquire}, Just value) -> do+ attempted <- try (acquire context value \provisioned -> withResources rest (Map.insert name provisioned accumulated) action)+ pure case attempted of+ Left (exception :: SomeException) -> Left (EnvError ("resource " <> renderResourceName name <> " failed: " <> Text.pack (displayException exception)))+ Right inner -> inner++ withPrimary Nothing _ action = Right <$> action Nothing+ withPrimary (Just requirement) (Just postgresSpec) action = withPostgresEnvKeeping (registryMigrationSets config.registry) config.keepEnvironment context.logger context.outDir spec.runId requirement postgresSpec spec.dimensions (action . Just)+ withPrimary (Just _) Nothing _ = pure (Left (EnvError "PostgreSQL specification is missing"))++ withExtras [] _ accumulated action = Right <$> action accumulated+ withExtras ((name, requirement) : rest) postgresSpecs accumulated action = case Map.lookup name postgresSpecs of+ Nothing -> pure (Left (EnvError ("extra PostgreSQL specification is missing: " <> name)))+ Just postgresSpec -> do+ provisioned <- withPostgresEnvNamed (registryMigrationSets config.registry) config.keepEnvironment (Just name) context.logger context.outDir spec.runId requirement postgresSpec spec.dimensions \postgres ->+ withExtras rest postgresSpecs (Map.insert name postgres accumulated) action+ pure (provisioned >>= id)++ timed runContext = do+ execution <- try (timeout (spec.timeoutSeconds * 1000000) (scenario.run runContext)) :: IO (Either SomeException (Maybe ScenarioReport))+ pure case execution of+ Left exception -> case fromException exception of+ Just (InfrastructureError message) -> ScenarioReport InfrastructureFailure (Just message) []+ Nothing -> ScenarioReport Errored (Just (Text.pack (displayException exception))) []+ Right Nothing -> ScenarioReport Errored (Just ("timeout after " <> Text.pack (show spec.timeoutSeconds) <> " s")) []+ Right (Just report) -> report++cohortMismatch :: CohortIdentity -> Maybe CohortExpectation -> Bool+cohortMismatch _ Nothing = False+cohortMismatch cohort (Just expectation) = unPlanHash cohort.identityPlanHash /= expectation.planHash++defectDisposition :: Bool -> CohortIdentity -> Scenario -> ScenarioReport -> (Maybe (KnownDefect, KnownDefectStatus), Bool, Int)+defectDisposition strict cohort scenario report = case NonEmpty.nonEmpty applicable of+ Nothing -> ordinary+ Just defects ->+ let status+ | report.outcome == Failed && not (null report.failures) && all (`elem` coveredLabels) report.failures = DefectReproduced+ | report.outcome == Failed = DefectDifferentFailure+ | otherwise = DefectNotReproduced+ blocking = report.outcome /= Passed && status /= DefectReproduced+ exitCode = if status == DefectReproduced && not strict then 0 else outcomeExitCode report.outcome+ identified = case NonEmpty.toList defects of+ [single] -> single+ _ -> KnownDefectGroup defects+ in (Just (identified, status), blocking, exitCode)+ where+ applicable = filter (cohortScopeApplies cohort . (.appliesTo)) (maybe [] individualKnownDefects scenario.knownDefect)+ coveredLabels = concatMap (.expectedFailures) applicable+ ordinary = (Nothing, report.outcome /= Passed, outcomeExitCode report.outcome)++atomicEncode :: (ToJSON value) => FilePath -> value -> IO ()+atomicEncode path value = do+ let temporary = path <> ".tmp"+ LazyByteString.writeFile temporary (encode value <> "\n")+ renameFile temporary path
+ src/Kenshou/Core/RunResult.hs view
@@ -0,0 +1,114 @@+module Kenshou.Core.RunResult+ ( RunResult (..),+ KnownDefectStatus (..),+ )+where++import Data.Aeson hiding (Error)+import Data.Aeson.Key qualified as Key+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime)+import Data.Word (Word64)+import Kenshou.Core.Cohort (CohortIdentity)+import Kenshou.Core.Context (Observation (..), PhaseTiming, SummarySection (..))+import Kenshou.Core.Id (RunId, ScenarioId (..), renderKind, renderLayer, renderScenarioId, unSegment)+import Kenshou.Core.Log (Severity (..))+import Kenshou.Core.Outcome (Outcome)+import Kenshou.Core.RunSpec (ComparisonMembership)+import Kenshou.Core.Scenario (KnownDefect (..), Tier, individualKnownDefects, renderTier)++data KnownDefectStatus = DefectReproduced | DefectDifferentFailure | DefectNotReproduced deriving stock (Eq, Show)++data RunResult = RunResult+ { runId :: RunId,+ scenario :: ScenarioId,+ scenarioRevision :: Int,+ tier :: Tier,+ outcome :: Outcome,+ blocking :: Bool,+ exitCode :: Int,+ reason :: Maybe Text,+ failures :: [Text],+ knownDefect :: Maybe (KnownDefect, KnownDefectStatus),+ seed :: Word64,+ specSha256 :: Text,+ comparison :: Maybe ComparisonMembership,+ startedAt :: UTCTime,+ endedAt :: UTCTime,+ durationSeconds :: Double,+ phases :: [PhaseTiming],+ cohort :: CohortIdentity,+ fingerprint :: Value,+ compatibility :: Value,+ summaries :: Map SummarySection (Map Text Value),+ observations :: [Observation],+ invocation :: Value+ }+ deriving stock (Eq, Show)++instance ToJSON RunResult where+ toJSON result =+ object+ [ "schema" .= ("kenshou.run-result/v1" :: Text),+ "runId" .= result.runId,+ "scenario" .= renderScenarioId result.scenario,+ "scenarioRevision" .= result.scenarioRevision,+ "layer" .= renderLayer result.scenario.layer,+ "component" .= unSegment result.scenario.component,+ "kind" .= renderKind result.scenario.kind,+ "tier" .= renderTier result.tier,+ "outcome" .= result.outcome,+ "blocking" .= result.blocking,+ "exitCode" .= result.exitCode,+ "reason" .= result.reason,+ "failures" .= result.failures,+ "knownDefect" .= fmap (knownDefectValue result.failures) result.knownDefect,+ "seed" .= result.seed,+ "spec" .= object ["path" .= ("run-spec.json" :: Text), "sha256" .= result.specSha256],+ "comparison" .= result.comparison,+ "timings" .= object ["clock" .= ("GHC.Clock.getMonotonicTimeNSec" :: Text), "startedAt" .= result.startedAt, "endedAt" .= result.endedAt, "durationSeconds" .= result.durationSeconds, "phases" .= result.phases],+ "cohort" .= result.cohort,+ "fingerprint" .= result.fingerprint,+ "compatibility" .= result.compatibility,+ "summaries" .= summaryValue result.summaries,+ "observations" .= fmap observationValue result.observations,+ "invocation" .= result.invocation+ ]++knownDefectValue :: [Text] -> (KnownDefect, KnownDefectStatus) -> Value+knownDefectValue failures (defect, status) = case individualKnownDefects defect of+ [single] -> singleValue single status+ many ->+ object+ [ "status" .= statusText status,+ "defects" .= fmap (\item -> singleValue item (if any (`elem` item.expectedFailures) failures then DefectReproduced else DefectNotReproduced)) many+ ]+ where+ singleValue item itemStatus = object ["reference" .= item.reference, "summary" .= item.summary, "expectedFailures" .= item.expectedFailures, "status" .= statusText itemStatus]++statusText :: KnownDefectStatus -> Text+statusText DefectReproduced = "reproduced"+statusText DefectDifferentFailure = "different-failure"+statusText DefectNotReproduced = "not-reproduced"++summaryValue :: Map SummarySection (Map Text Value) -> Value+summaryValue summaries =+ object+ [ "measurements" .= section Measurements,+ "verdicts" .= section Verdicts,+ "diagnosis" .= section Diagnosis,+ "telemetry" .= section Telemetry+ ]+ where+ section name = object [Key.fromText key .= value | (key, value) <- Map.toAscList (Map.findWithDefault Map.empty name summaries)]++observationValue :: Observation -> Value+observationValue observation = object ["source" .= observation.source, "severity" .= severityText observation.severity, "message" .= observation.message, "at" .= observation.at]++severityText :: Severity -> Text+severityText Debug = "debug"+severityText Info = "info"+severityText Warning = "warning"+severityText Error = "error"
+ src/Kenshou/Core/RunSpec.hs view
@@ -0,0 +1,223 @@+module Kenshou.Core.RunSpec+ ( SpecPlacement (..),+ ConnectionSource (..),+ PostgresSpec (..),+ EnvironmentSpec (..),+ CohortExpectation (..),+ ComparisonMembership (..),+ RunSpec (..),+ EffectiveRunSpec (..),+ minimalRunSpec,+ redactConnectionString,+ withLegacyResource,+ )+where++import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.Types (Parser)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Dimension (Dimensions, renderDimensions)+import Kenshou.Core.Id (RunId, ScenarioId, Seed)+import Kenshou.Core.Knob (KnobName, RawKnob (..), ResolvedKnobs)+import Kenshou.Core.Knob qualified+import Kenshou.Core.Phase (PhasePlan)+import Kenshou.Core.Resource.Types (ResourceName, legacyKafkaName)++data SpecPlacement = RunLocal | RunOnCell deriving stock (Eq, Show)++data ConnectionSource = ConnLiteral Text | ConnFromEnv Text deriving stock (Eq, Show)++data PostgresSpec = PostgresEphemeral [(Text, Text)] | PostgresExternal ConnectionSource deriving stock (Eq, Show)++data EnvironmentSpec = EnvironmentSpec+ { placement :: SpecPlacement,+ machineProfile :: Maybe Text,+ postgres :: Maybe PostgresSpec,+ extraPostgres :: Map Text PostgresSpec,+ -- | Per-resource specification values, keyed by the resource name.+ resources :: Map ResourceName Value,+ telemetry :: Maybe Value+ }+ deriving stock (Eq, Show)++data CohortExpectation = CohortExpectation {name :: Maybe Text, planHash :: Text} deriving stock (Eq, Show)++data ComparisonMembership = ComparisonMembership {group :: Text, arm :: Text, trial :: Int, position :: Int} deriving stock (Eq, Show)++data RunSpec = RunSpec+ { runId :: Maybe RunId,+ scenario :: ScenarioId,+ scenarioRevision :: Maybe Int,+ knobs :: [(KnobName, RawKnob)],+ dimensions :: [(Text, Text)],+ seed :: Maybe Seed,+ phases :: Maybe PhasePlan,+ timeoutSeconds :: Maybe Int,+ environment :: EnvironmentSpec,+ cohortExpectation :: Maybe CohortExpectation,+ comparison :: Maybe ComparisonMembership,+ labels :: Map Text Text+ }+ deriving stock (Eq, Show)++data EffectiveRunSpec = EffectiveRunSpec+ { runId :: RunId,+ scenario :: ScenarioId,+ scenarioRevision :: Int,+ knobs :: ResolvedKnobs,+ dimensions :: Dimensions,+ seed :: Seed,+ phases :: PhasePlan,+ timeoutSeconds :: Int,+ environment :: EnvironmentSpec,+ cohortExpectation :: Maybe CohortExpectation,+ comparison :: Maybe ComparisonMembership,+ labels :: Map Text Text+ }+ deriving stock (Eq, Show)++minimalRunSpec :: ScenarioId -> RunSpec+minimalRunSpec scenario = RunSpec Nothing scenario Nothing [] [] Nothing Nothing Nothing (EnvironmentSpec RunLocal Nothing Nothing Map.empty Map.empty Nothing) Nothing Nothing Map.empty++redactConnectionString :: Text -> Text+redactConnectionString value = redactUri (Text.unwords (fmap redactWord (Text.words value)))+ where+ redactWord word+ | "password=" `Text.isPrefixOf` Text.toCaseFold word = "password=<redacted>"+ | otherwise = word+ redactUri input = case Text.breakOn "://" input of+ (scheme, rest)+ | not (Text.null rest) ->+ let afterScheme = Text.drop 3 rest+ (authority, suffix) = Text.breakOn "/" afterScheme+ in case Text.breakOnEnd "@" authority of+ (credentials, host)+ | not (Text.null credentials) ->+ let (user, password) = Text.breakOn ":" (Text.dropEnd 1 credentials)+ in if Text.null password then input else scheme <> "://" <> user <> ":<redacted>@" <> host <> suffix+ _ -> input+ _ -> input++instance FromJSON RunSpec where+ parseJSON = withObject "RunSpec" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.run-spec/v1" :: Text) then fail "unsupported run-spec schema" else pure ()+ runId <- value .:? "runId"+ scenario <- value .: "scenario"+ scenarioRevision <- value .:? "scenarioRevision"+ rawKnobs <- value .:? "knobs" .!= Map.empty+ knobs <- traverse parseKnob (Map.toList rawKnobs)+ dimensions <- Map.toList <$> value .:? "dimensions" .!= Map.empty+ seed <- value .:? "seed"+ phases <- value .:? "phases"+ timeoutSeconds <- value .:? "timeoutSeconds"+ environment <- value .:? "environment" .!= EnvironmentSpec RunLocal Nothing Nothing Map.empty Map.empty Nothing+ cohortExpectation <- value .:? "cohortExpectation"+ comparison <- value .:? "comparison"+ labels <- value .:? "labels" .!= Map.empty+ pure RunSpec {runId, scenario, scenarioRevision, knobs, dimensions, seed, phases, timeoutSeconds, environment, cohortExpectation, comparison, labels}+ where+ parseKnob (name, raw) = case Kenshou.Core.Knob.mkKnobName name of+ Left err -> fail (Text.unpack err)+ Right parsed -> pure (parsed, RawJson raw)++instance ToJSON RunSpec where+ toJSON spec =+ object $+ [ "schema" .= ("kenshou.run-spec/v1" :: Text),+ "scenario" .= spec.scenario,+ "knobs" .= Map.fromList [(Kenshou.Core.Knob.renderKnobName name, rawValue raw) | (name, raw) <- spec.knobs],+ "dimensions" .= Map.fromList spec.dimensions,+ "environment" .= spec.environment,+ "cohortExpectation" .= spec.cohortExpectation,+ "comparison" .= spec.comparison,+ "labels" .= spec.labels+ ]+ <> maybe [] (pure . ("runId" .=)) spec.runId+ <> maybe [] (pure . ("scenarioRevision" .=)) spec.scenarioRevision+ <> maybe [] (pure . ("seed" .=)) spec.seed+ <> maybe [] (pure . ("phases" .=)) spec.phases+ <> maybe [] (pure . ("timeoutSeconds" .=)) spec.timeoutSeconds+ where+ rawValue (RawText value) = String value+ rawValue (RawJson value) = value++instance ToJSON EffectiveRunSpec where+ toJSON spec =+ object+ [ "schema" .= ("kenshou.run-spec/v1" :: Text),+ "runId" .= spec.runId,+ "scenario" .= spec.scenario,+ "scenarioRevision" .= spec.scenarioRevision,+ "knobs" .= spec.knobs,+ "dimensions" .= object [Key.fromText name .= value | (name, value) <- renderDimensions spec.dimensions],+ "seed" .= spec.seed,+ "phases" .= spec.phases,+ "timeoutSeconds" .= spec.timeoutSeconds,+ "environment" .= spec.environment,+ "cohortExpectation" .= spec.cohortExpectation,+ "comparison" .= spec.comparison,+ "labels" .= spec.labels+ ]++instance ToJSON EnvironmentSpec where+ toJSON environment =+ object $+ ["placement" .= placementText environment.placement, "machineProfile" .= environment.machineProfile, "postgres" .= environment.postgres]+ <> ["extraPostgres" .= environment.extraPostgres | not (Map.null environment.extraPostgres)]+ <> ["resources" .= environment.resources | not (Map.null environment.resources)]+ <> ["telemetry" .= environment.telemetry]++instance FromJSON EnvironmentSpec where+ parseJSON = withObject "EnvironmentSpec" \value -> do+ resources <- (value .:? "resources" .!= Map.empty) >>= withLegacyResource "environment" value+ EnvironmentSpec <$> (value .:? "placement" .!= RunLocal) <*> value .:? "machineProfile" <*> value .:? "postgres" <*> (value .:? "extraPostgres" .!= Map.empty) <*> pure resources <*> value .:? "telemetry"++-- | Documents written before named resources carried the one resource they knew as a+-- top-level field; read a non-null one as that named resource.+withLegacyResource :: Text -> Object -> Map ResourceName Value -> Parser (Map ResourceName Value)+withLegacyResource label value resources = do+ legacy <- value .:? "kafka"+ case (legacy, Map.lookup legacyKafkaName resources) of+ (Nothing, _) -> pure resources+ (Just Null, _) -> pure resources+ (Just found, Nothing) -> pure (Map.insert legacyKafkaName found resources)+ (Just found, Just current)+ | found == current -> pure resources+ | otherwise -> fail (Text.unpack label <> ".kafka and " <> Text.unpack label <> ".resources.kafka disagree")++instance ToJSON SpecPlacement where toJSON = String . placementText++instance FromJSON SpecPlacement where parseJSON = withText "SpecPlacement" \case "local" -> pure RunLocal; "cell" -> pure RunOnCell; other -> fail ("unknown placement " <> Text.unpack other)++placementText :: SpecPlacement -> Text+placementText RunLocal = "local"+placementText RunOnCell = "cell"++instance ToJSON PostgresSpec where+ toJSON (PostgresEphemeral settings) = object ["mode" .= ("ephemeral" :: Text), "settings" .= Map.fromList settings]+ toJSON (PostgresExternal (ConnFromEnv name)) = object ["mode" .= ("external" :: Text), "connectionStringEnv" .= name]+ toJSON (PostgresExternal (ConnLiteral value)) = object ["mode" .= ("external" :: Text), "connectionString" .= redactConnectionString value]++instance FromJSON PostgresSpec where+ parseJSON = withObject "PostgresSpec" \value -> do+ mode <- value .: "mode" :: Parser Text+ case mode of+ "ephemeral" -> PostgresEphemeral . Map.toList <$> value .:? "settings" .!= Map.empty+ "external" -> do+ env <- value .:? "connectionStringEnv"+ literal <- value .:? "connectionString"+ case (env, literal) of (Just name, Nothing) -> pure (PostgresExternal (ConnFromEnv name)); (Nothing, Just connection) -> pure (PostgresExternal (ConnLiteral connection)); _ -> fail "external PostgreSQL requires exactly one connection source"+ _ -> fail "unknown PostgreSQL mode"++instance ToJSON CohortExpectation where toJSON value = object ["name" .= value.name, "planHash" .= value.planHash]++instance FromJSON CohortExpectation where parseJSON = withObject "CohortExpectation" \value -> CohortExpectation <$> value .:? "name" <*> value .: "planHash"++instance ToJSON ComparisonMembership where toJSON value = object ["group" .= value.group, "arm" .= value.arm, "trial" .= value.trial, "position" .= value.position]++instance FromJSON ComparisonMembership where parseJSON = withObject "ComparisonMembership" \value -> ComparisonMembership <$> value .: "group" <*> value .: "arm" <*> value .: "trial" <*> value .: "position"
+ src/Kenshou/Core/RunSpec/Resolve.hs view
@@ -0,0 +1,106 @@+module Kenshou.Core.RunSpec.Resolve (SpecError (..), resolveRunSpec) where++import Data.Aeson (Value)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (Registry, lookupResource, lookupScenario)+import Kenshou.Core.Dimension (resolveDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))+import Kenshou.Core.Id (mkSeed, newRunId)+import Kenshou.Core.Knob (resolveKnobs)+import Kenshou.Core.Phase (PhasePlan (..))+import Kenshou.Core.Resource (ResourceName, ResourceProvider (..), renderResourceName)+import Kenshou.Core.RunSpec+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Core.Scenario (Placement (..), Scenario (..), Tier (..))+import System.Info qualified as System+import System.Random (randomIO)++newtype SpecError = SpecError Text deriving stock (Eq, Show)++resolveRunSpec :: Registry -> RunSpec -> IO (Either (NonEmpty SpecError) (Scenario, EffectiveRunSpec))+resolveRunSpec registry spec = case lookupScenario registry spec.scenario of+ Nothing -> pure (Left (SpecError ("unknown scenario " <> Text.pack (show spec.scenario)) :| []))+ Just scenario -> do+ generatedRunId <- maybe newRunId pure spec.runId+ randomSeed <- randomIO+ let seedResult = maybe (mkSeed (randomSeed `mod` 9007199254740992)) Right spec.seed+ knobsResult = resolveKnobs scenario.knobs spec.knobs+ dimensionsResult = resolveDimensions scenario.dimensions spec.dimensions+ pure do+ checkedSeed <- firstOne seedResult+ checkedKnobs <- mapErrors knobsResult+ checkedDimensions <- mapErrors dimensionsResult+ if maybe True (== scenario.revision) spec.scenarioRevision then pure () else Left (SpecError "scenario revision does not match registry" :| [])+ validatePlacement scenario.placement spec.environment.placement+ validatePostgres scenario spec.environment.postgres+ validateExtraPostgres scenario spec.environment.extraPostgres+ resources <- resolveResources registry scenario spec.environment.resources+ let environment = (applyEnvironmentDefaults scenario spec.environment) {RunSpec.resources = resources}+ phases = maybe scenario.phases id spec.phases+ effective = EffectiveRunSpec generatedRunId scenario.id scenario.revision checkedKnobs checkedDimensions checkedSeed phases (maybe (tierTimeout scenario.tier phases) id spec.timeoutSeconds) environment spec.cohortExpectation spec.comparison spec.labels+ pure (scenario, effective)+ where+ firstOne :: Either Text value -> Either (NonEmpty SpecError) value+ firstOne = either (Left . (:| []) . SpecError) Right+ mapErrors :: (Show problem) => Either (NonEmpty problem) value -> Either (NonEmpty SpecError) value+ mapErrors = either (Left . fmap (SpecError . Text.pack . show)) Right++-- | Every resource the specification names must be required by the scenario, and+-- every required resource's value is normalized by its registered provider, so the+-- effective specification records the complete value.+resolveResources :: Registry -> Scenario -> Map ResourceName Value -> Either (NonEmpty SpecError) (Map ResourceName Value)+resolveResources registry scenario supplied = do+ case [name | name <- Map.keys supplied, name `notElem` scenario.requires.resources] of+ [] -> pure ()+ extra : _ -> Left (SpecError ("environment names a resource the scenario does not require: " <> renderResourceName extra) :| [])+ normalized <- traverse normalizeOne scenario.requires.resources+ pure (Map.fromList normalized)+ where+ normalizeOne name = case lookupResource registry name of+ Nothing -> Left (SpecError ("no registered provider for resource " <> renderResourceName name) :| [])+ Just provider -> either (\problem -> Left (SpecError ("resource " <> renderResourceName name <> ": " <> problem) :| [])) (Right . (name,)) (provider.normalize (Map.lookup name supplied))++validatePlacement :: Placement -> SpecPlacement -> Either (NonEmpty SpecError) ()+validatePlacement PlaceCell RunLocal = Left (SpecError "scenario requires cell placement" :| [])+validatePlacement PlaceLocal RunOnCell = Left (SpecError "scenario requires local placement" :| [])+validatePlacement _ _ = Right ()++validatePostgres :: Scenario -> Maybe PostgresSpec -> Either (NonEmpty SpecError) ()+validatePostgres scenario postgresSpec = case (scenario.requires.postgres, postgresSpec) of+ (Nothing, Just _) -> Left (SpecError "PostgreSQL is not applicable to this scenario" :| [])+ (Just requirement, Just (PostgresExternal _)) | requirement.needsServerControl -> Left (SpecError "scenario requires control of an ephemeral PostgreSQL server" :| [])+ _ -> Right ()++validateExtraPostgres :: Scenario -> Map.Map Text PostgresSpec -> Either (NonEmpty SpecError) ()+validateExtraPostgres scenario supplied =+ case errors of+ [] -> Right ()+ first : rest -> Left (first :| rest)+ where+ required = Map.fromList scenario.requires.extraPostgres+ unknown = Map.keys (supplied `Map.difference` required)+ controlErrors =+ [ SpecError ("extra PostgreSQL " <> name <> " requires an ephemeral server")+ | (name, requirement) <- Map.toList required,+ requirement.needsServerControl,+ Just (PostgresExternal _) <- [Map.lookup name supplied]+ ]+ errors = fmap (SpecError . ("unknown extra PostgreSQL environment " <>)) unknown <> controlErrors++applyEnvironmentDefaults :: Scenario -> EnvironmentSpec -> EnvironmentSpec+applyEnvironmentDefaults scenario environment =+ environment+ { machineProfile = Just (maybe ("local/" <> Text.pack System.os <> "-" <> Text.pack System.arch) id environment.machineProfile),+ postgres = case (scenario.requires.postgres, environment.postgres) of (Just _, Nothing) -> Just (PostgresEphemeral []); (_, existing) -> existing,+ extraPostgres = Map.union environment.extraPostgres (Map.fromList [(name, PostgresEphemeral []) | (name, _) <- scenario.requires.extraPostgres])+ }++tierTimeout :: Tier -> PhasePlan -> Int+tierTimeout TierSmoke _ = 120+tierTimeout TierStandard _ = 1200+tierTimeout TierExtended _ = 7200+tierTimeout TierSoak phases = ceiling (phases.warmUpSeconds + phases.steadySeconds + phases.drainSeconds) + 1800
+ src/Kenshou/Core/Scenario.hs view
@@ -0,0 +1,119 @@+module Kenshou.Core.Scenario+ ( Tier (..),+ Placement (..),+ KnownDefect (..),+ individualKnownDefects,+ CohortScope (..),+ PackageCondition (..),+ cohortScopeApplies,+ ScenarioReport (..),+ Scenario (..),+ renderTier,+ renderPlacement,+ passed,+ failedWith,+ inconclusiveBecause,+ infrastructureFailureBecause,+ )+where++import Data.List.NonEmpty (NonEmpty)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Version (parseVersion)+import Kenshou.Core.Cohort (CohortIdentity (..), PackageSource (..), ResolvedComponent (..), ResolvedPackage (..))+import Kenshou.Core.Context (RunContext)+import Kenshou.Core.Dimension (DimensionSupport)+import Kenshou.Core.Env (EnvRequirements)+import Kenshou.Core.Id (ScenarioId)+import Kenshou.Core.Knob (KnobSpec)+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.Phase (PhasePlan)+import Text.ParserCombinators.ReadP (readP_to_S)++data Tier = TierSmoke | TierStandard | TierExtended | TierSoak deriving stock (Eq, Ord, Show)++data Placement = PlaceLocal | PlaceCell | PlaceEither deriving stock (Eq, Ord, Show)++data KnownDefect+ = KnownDefect+ { reference :: Text,+ summary :: Text,+ expectedFailures :: [Text],+ appliesTo :: CohortScope+ }+ | KnownDefectGroup (NonEmpty KnownDefect)+ deriving stock (Eq, Show)++individualKnownDefects :: KnownDefect -> [KnownDefect]+individualKnownDefects (KnownDefectGroup defects) = concatMap individualKnownDefects defects+individualKnownDefects defect = [defect]++data CohortScope = AllCohorts | OnlyWhen (NonEmpty PackageCondition) deriving stock (Eq, Show)++data PackageCondition+ = ResolvedFromHackage Text+ | ResolvedFromGit Text+ | VersionBelow Text Text+ | RevisionIs Text Text+ deriving stock (Eq, Show)++cohortScopeApplies :: CohortIdentity -> CohortScope -> Bool+cohortScopeApplies _ AllCohorts = True+cohortScopeApplies cohort (OnlyWhen conditions) = all (conditionApplies packages) conditions+ where+ packages = concatMap (.resolvedComponentPackages) cohort.identityComponents++conditionApplies :: [ResolvedPackage] -> PackageCondition -> Bool+conditionApplies packages condition = any matches packages+ where+ matches package = case condition of+ ResolvedFromHackage name -> package.resolvedPackageName == name && case package.resolvedPackageSource of FromHackage _ -> True; _ -> False+ ResolvedFromGit name -> package.resolvedPackageName == name && case package.resolvedPackageSource of FromGit _ _ _ -> True; _ -> False+ VersionBelow name boundary -> package.resolvedPackageName == name && maybe False (uncurry (<)) ((,) <$> parse package.resolvedPackageVersion <*> parse boundary)+ RevisionIs name revision -> package.resolvedPackageName == name && case package.resolvedPackageSource of FromGit _ actual _ -> actual == revision; _ -> False+ parse value = case [version | (version, "") <- readP_to_S parseVersion (Text.unpack value)] of [] -> Nothing; versions -> Just (last versions)++data ScenarioReport = ScenarioReport+ { outcome :: Outcome,+ reason :: Maybe Text,+ failures :: [Text]+ }+ deriving stock (Eq, Show)++data Scenario = Scenario+ { id :: ScenarioId,+ revision :: Int,+ summary :: Text,+ tier :: Tier,+ placement :: Placement,+ knobs :: [KnobSpec],+ dimensions :: DimensionSupport,+ phases :: PhasePlan,+ requires :: EnvRequirements,+ knownDefect :: Maybe KnownDefect,+ run :: RunContext -> IO ScenarioReport+ }++renderTier :: Tier -> Text+renderTier TierSmoke = "smoke"+renderTier TierStandard = "standard"+renderTier TierExtended = "extended"+renderTier TierSoak = "soak"++renderPlacement :: Placement -> Text+renderPlacement PlaceLocal = "local"+renderPlacement PlaceCell = "cell"+renderPlacement PlaceEither = "either"++passed :: ScenarioReport+passed = ScenarioReport Passed Nothing []++failedWith :: [Text] -> Text -> ScenarioReport+failedWith labels message = ScenarioReport Failed (Just message) labels++inconclusiveBecause :: Text -> ScenarioReport+inconclusiveBecause message = ScenarioReport Inconclusive (Just message) []++infrastructureFailureBecause :: Text -> ScenarioReport+infrastructureFailureBecause message = ScenarioReport InfrastructureFailure (Just message) []
+ src/Kenshou/Core/Selector.hs view
@@ -0,0 +1,40 @@+module Kenshou.Core.Selector+ ( ScenarioSelector,+ parseSelector,+ renderSelector,+ matchesSelector,+ )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (ScenarioId, renderScenarioId)++newtype ScenarioSelector = ScenarioSelector [Text]+ deriving stock (Eq, Show)++parseSelector :: Text -> Either Text ScenarioSelector+parseSelector value+ | null pieces = Left "selector is empty"+ | "**" `elem` initSafe pieces = Left "** is permitted only as the final segment"+ | length pieces < 4 && last pieces /= "**" = Left "short selectors must end in **"+ | length pieces > 4 = Left "selector has more than four segments"+ | any invalid pieces = Left ("invalid selector \"" <> value <> "\"")+ | otherwise = Right (ScenarioSelector pieces)+ where+ pieces = Text.splitOn "/" value+ initSafe [] = []+ initSafe xs = init xs+ invalid piece = Text.null piece || (piece /= "*" && piece /= "**" && Text.any (== '*') piece)++renderSelector :: ScenarioSelector -> Text+renderSelector (ScenarioSelector pieces) = Text.intercalate "/" pieces++matchesSelector :: ScenarioSelector -> ScenarioId -> Bool+matchesSelector (ScenarioSelector patterns) scenario = go patterns (Text.splitOn "/" (renderScenarioId scenario))+ where+ go ["**"] _ = True+ go [] [] = True+ go (patternPart : restPatterns) (actual : restActual) =+ (patternPart == "*" || patternPart == actual) && go restPatterns restActual+ go _ _ = False
+ src/Kenshou/Core/Selftest.hs view
@@ -0,0 +1,151 @@+module Kenshou.Core.Selftest (bundle) where++import Control.Concurrent (threadDelay)+import Data.Aeson (object, (.=))+import Data.Either (isLeft)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Context (RunContext (..), SummarySection (..), putSummary, requirePostgres)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), noDimensions, postgresDimensions)+import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..), noEnvironment)+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot (..), PostgresEnv (..))+import Kenshou.Core.Id (parseScenarioId, selftestLayer)+import Kenshou.Core.Knob+import Kenshou.Core.Outcome (parseOutcome)+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.Role+import Kenshou.Core.Role.Spawn (WorkerHandle (..), withWorker)+import Kenshou.Core.Scenario+import Kenshou.Core.Selftest.Schema (selftestMigrationSet, selftestSchema)+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)++bundle :: LayerBundle+bundle = (mkBundle selftestLayer [alwaysFail, alwaysPass, errors, outcomeScenario, knownDefectScenario, postgresRoundtripScenario, workerEchoScenario] [echoRole]) {migrations = [selftestMigrationSet]}++alwaysPass, alwaysFail, errors :: Scenario+alwaysPass = scenario "selftest/kernel/correctness/always-pass" "Always reports passed." (const (pure passed))+alwaysFail = scenario "selftest/kernel/correctness/always-fail" "Always reports failed." (const (pure (failedWith ["seeded-failure"] "this scenario always fails")))+errors = scenario "selftest/kernel/correctness/errors" "Always throws." (const (ioError (userError "seeded self-test error")))++outcomeScenario :: Scenario+outcomeScenario =+ (scenario "selftest/kernel/correctness/outcome" "Returns an operator-selected outcome." runOutcome)+ { knobs =+ [ KnobSpec outcomeKnob "Outcome to report" KnobText (VText "passed") (OneOf (VText "passed" :| fmap VText ["failed", "errored", "inconclusive", "infrastructure-failure"])) [],+ KnobSpec sleepKnob "Delay before reporting" KnobDouble (VDouble 0) (DoubleRange 0 3600) []+ ]+ }++knownDefectScenario :: Scenario+knownDefectScenario =+ (scenario "selftest/kernel/correctness/known-defect" "Reproduces a declared known defect." (const (pure (failedWith ["seeded-defect"] "seeded known defect"))))+ { knownDefect = Just (KnownDefect "mori://shinzui/kenshou/plans/2-open-the-kernel-layer-identity-and-schema-composition-to-registered-suites" "Seeded harness defect" ["seeded-defect"] AllCohorts)+ }++workerEchoScenario :: Scenario+workerEchoScenario =+ (scenario "selftest/kernel/concurrency/worker-echo" "Exchanges messages with a child worker process." runWorkerEcho)+ { knobs = [KnobSpec messagesKnob "Messages to echo" KnobInt (VInt 1) (IntRange 1 1000) []]+ }++postgresRoundtripScenario :: Scenario+postgresRoundtripScenario =+ (scenario "selftest/kernel/correctness/postgres-roundtrip" "Verifies a composed migration ledger and PostgreSQL durability." runPostgresRoundtrip)+ { revision = 2,+ knobs = [KnobSpec eventsKnob "Rows to round-trip" KnobInt (VInt 3) (IntRange 1 1000) [VInt 1, VInt 100]],+ dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| []) noDimensions,+ requires = noEnvironment {postgres = Just (PostgresRequirement [selftestSchema] [] False)}+ }++echoRole :: WorkerRole+echoRole = WorkerRole (roleName "selftest/echo") "Echoes custom worker messages." \context -> do+ context.send WrkReady+ let loop =+ context.receive >>= \case+ Just (CtlCustom "echo" payload) -> context.send (WrkCustom "echo" payload) >> loop+ Just (CtlStop _) -> pure ()+ Just _ -> loop+ Nothing -> pure ()+ loop++runWorkerEcho :: RunContext -> IO ScenarioReport+runWorkerEcho context = withWorker context (roleName "selftest/echo") "echo-1" (object []) \worker -> do+ ready <- worker.receive 10000+ case ready of+ Just WrkReady -> echoMessages worker [1 .. fromIntegral (knobInt context.knobs messagesKnob)]+ _ -> pure (failedWith ["echo-timeout"] "worker did not become ready")+ where+ echoMessages worker [] = do+ putSummary context Verdicts "worker-echo" (object ["childPid" .= show worker.pid])+ pure passed+ echoMessages worker (number : rest) = do+ let payload = object ["n" .= (number :: Int)]+ worker.send (CtlCustom "echo" payload)+ reply <- worker.receive 10000+ case reply of+ Just (WrkCustom "echo" echoed) | echoed == payload -> echoMessages worker rest+ Nothing -> pure (failedWith ["echo-timeout"] "worker reply timed out")+ _ -> pure (failedWith ["echo-mismatch"] "worker reply differed")++runPostgresRoundtrip :: RunContext -> IO ScenarioReport+runPostgresRoundtrip context = do+ let environment = requirePostgres context+ count = knobInt context.knobs eventsKnob+ schemas <- psql environment.connectionString "select string_agg(schema_name, ',' order by schema_name) from information_schema.schemata where schema_name not like 'pg\\_%' and schema_name not in ('information_schema','public')"+ ledger <- psql environment.connectionString "select string_agg(distinct component, ',' order by component) from pgmigrate.migrations"+ inserted <- psql environment.connectionString ("insert into kenshou_selftest.roundtrip (id, payload) select n, 'row-' || n from generate_series(1," <> Text.pack (show count) <> ") as n")+ roundtrip <- psql environment.connectionString "select count(*) from kenshou_selftest.roundtrip where payload = 'row-' || id"+ let actualFsync = Map.lookup "fsync" environment.snapshot.settings+ expectedFsync = case context.dimensions.pgDurability of Just PgDurable -> "on"; _ -> "off"+ failures =+ ["schemas" | schemas /= Right "kenshou_selftest,pgmigrate"]+ <> ["one-ledger" | ledger /= Right "kenshou-selftest"]+ <> ["roundtrip" | isLeft inserted || roundtrip /= Right (Text.pack (show count))]+ <> ["durability-honoured" | actualFsync /= Just expectedFsync]+ putSummary context Verdicts "postgres-roundtrip" (object ["rows" .= count, "ledgerComponents" .= (["kenshou-selftest"] :: [Text])])+ pure (if null failures then passed else failedWith failures "PostgreSQL round-trip checks failed")++psql :: Text -> Text -> IO (Either Text Text)+psql connection query = do+ (code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+ pure case code of ExitSuccess -> Right (Text.strip (Text.pack output)); _ -> Left (Text.strip (Text.pack err))++runOutcome :: RunContext -> IO ScenarioReport+runOutcome context = do+ let delay = knobDouble context.knobs sleepKnob+ threadDelay (round (delay * 1000000))+ pure case parseOutcome (knobText context.knobs outcomeKnob) of+ Right result -> ScenarioReport result Nothing []+ Left message -> failedWith ["invalid-outcome"] message++outcomeKnob, sleepKnob, messagesKnob, eventsKnob :: KnobName+outcomeKnob = knobName "selftest.outcome"+sleepKnob = knobName "selftest.sleep-seconds"+messagesKnob = knobName "selftest.messages"+eventsKnob = knobName "selftest.events"++knobName :: Text -> KnobName+knobName = either (error . show) id . mkKnobName++roleName :: Text -> RoleName+roleName = either (error . show) id . mkRoleName++scenario :: Text -> Text -> (RunContext -> IO ScenarioReport) -> Scenario+scenario identifier summary action =+ Scenario+ { id = either (error . show) id (parseScenarioId identifier),+ revision = 1,+ summary,+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = [],+ dimensions = noDimensions,+ phases = zeroPhases,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = action+ }
+ src/Kenshou/Core/Selftest/Schema.hs view
@@ -0,0 +1,26 @@+-- | The kernel's own migration set: one schema and one table that the self-test+-- scenarios of every toolkit may use without depending on a suite's schema.+module Kenshou.Core.Selftest.Schema+ ( selftestSchema,+ selftestMigrationSet,+ )+where++import Data.List.NonEmpty (NonEmpty (..))+import Data.Set qualified as Set+import Data.Text qualified as Text+import Database.PostgreSQL.Migrate (migrationComponent, sqlMigration)+import Kenshou.Core.Env (MigrationSet (..), MigrationSetName (..))++selftestSchema :: MigrationSetName+selftestSchema = MigrationSetName "kenshou-selftest"++selftestMigrationSet :: MigrationSet+selftestMigrationSet =+ MigrationSet+ { name = selftestSchema,+ requires = [],+ component = either (Left . Text.pack . show) Right do+ roundtrip <- sqlMigration "create-roundtrip" "create schema kenshou_selftest;\ncreate table kenshou_selftest.roundtrip (id bigint primary key, payload text not null);\n"+ migrationComponent "kenshou-selftest" Set.empty (roundtrip :| [])+ }
+ src/Kenshou/Core/Version.hs view
@@ -0,0 +1,9 @@+module Kenshou.Core.Version (suiteVersion) where++import Data.Text (Text)+import Data.Text qualified as Text+import Data.Version (showVersion)+import Paths_kenshou_core (version)++suiteVersion :: Text+suiteVersion = Text.pack (showVersion version)
+ src/Kenshou/Perf/Artifacts.hs view
@@ -0,0 +1,64 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Artifacts (verifyPerfManifest, sealPerfDirectory, safePath) where++import Control.Exception (IOException, displayException, try)+import Control.Monad (unless)+import Data.Aeson (eitherDecodeFileStrict', encodeFile)+import Data.ByteString qualified as ByteString+import Data.List (nub, sort)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id (newRunId)+import Kenshou.Core.Manifest (Manifest (..), ManifestFile (..), writeManifest)+import System.Directory (doesDirectoryExist, listDirectory, pathIsSymbolicLink)+import System.FilePath (isAbsolute, makeRelative, splitDirectories, (</>))++-- Performance documents are replayed after retrieval. Reject links, escaping+-- paths, omitted files and duplicate entries before opening any manifest path.+verifyPerfManifest :: FilePath -> IO (Either Text ())+verifyPerfManifest root = ioResult do+ decoded <- eitherDecodeFileStrict' (root </> "manifest.json")+ case decoded of+ Left problem -> pure (Left (Text.pack problem))+ Right manifest -> do+ paths <- regularFiles root+ let listed = map (.path) (manifest :: Manifest).files+ if any (not . safePath) listed || nub listed /= listed || sort listed /= filter (/= "manifest.json") paths+ then pure (Left "performance manifest file inventory is unsafe or incomplete")+ else do+ problems <- traverse verifyFile manifest.files+ pure (sequence_ problems)+ where+ verifyFile file = do+ bytes <- ByteString.readFile (root </> file.path)+ pure do+ unless (sha256Hex bytes == file.sha256) (Left (Text.pack file.path <> ": digest mismatch"))+ unless (fromIntegral (ByteString.length bytes) == file.bytes) (Left (Text.pack file.path <> ": byte count mismatch"))++sealPerfDirectory :: FilePath -> IO (Either Text ())+sealPerfDirectory root = ioResult do+ _ <- regularFiles root+ identifier <- newRunId+ manifest <- writeManifest root identifier Map.empty+ encodeFile (root </> "manifest.json") manifest+ pure (Right ())++regularFiles :: FilePath -> IO [FilePath]+regularFiles root = sort <$> visit root+ where+ visit path = do+ link <- pathIsSymbolicLink path+ if link then ioError (userError "performance evidence may not contain symbolic links") else pure ()+ directory <- doesDirectoryExist path+ if directory then listDirectory path >>= fmap concat . traverse (visit . (path </>)) else pure [makeRelative root path]++safePath :: FilePath -> Bool+safePath path = not (null path) && not (isAbsolute path) && all (`notElem` ["", ".", ".."]) (splitDirectories path) && '\\' `notElem` path++ioResult :: IO (Either Text a) -> IO (Either Text a)+ioResult action = do+ result <- try @IOException action+ pure (either (Left . Text.pack . displayException) id result)
+ src/Kenshou/Perf/Calibration.hs view
@@ -0,0 +1,185 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Calibration+ ( CalibrationReference (..),+ CalibrationDefinition (..),+ CalibrationSlot (..),+ CalibrationAcceptance (..),+ CalibrationReceipt (..),+ VerifiedCalibrationTrial (..),+ validateCalibration,+ validateCalibrationDefinition,+ toolchainDigest,+ )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List (nub, sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Id (RunId)+import Kenshou.Perf.Prepared (BuildInventory (..))+import Kenshou.Perf.Request (canonicalProject, immutableRevision, strictObject)++data CalibrationReference = CalibrationReference {definitionUri :: Text, definitionSha256 :: Text, acceptanceUri :: Text, acceptanceSha256 :: Text}+ deriving stock (Eq, Show)++data CalibrationDefinition = CalibrationDefinition+ { batchId :: RunId,+ project :: Text,+ profile :: Text,+ machineProfile :: Text,+ healthPolicySha256 :: Text,+ workloadSha256 :: Text,+ toolchainSha256 :: Text,+ harnessSemanticsSha256 :: Text,+ baselineRevision :: Text,+ slowdownRevision :: Text,+ injectionMechanism :: Text,+ targetMetric :: Text,+ targetRelativeEffect :: Double,+ targetAbsoluteEffect :: Double,+ seeds :: [Integer],+ infrastructureRetryCap :: Int,+ expiresAt :: UTCTime+ }+ deriving stock (Eq, Show)++data CalibrationReceipt = CalibrationReceipt {uri :: Text, sha256 :: Text}+ deriving stock (Eq, Show)++data CalibrationSlot = CalibrationSlot {seed :: Integer, receipts :: [CalibrationReceipt]}+ deriving stock (Eq, Show)++data CalibrationAcceptance = CalibrationAcceptance {batchId :: RunId, aa :: [CalibrationSlot], slowdown :: [CalibrationSlot]}+ deriving stock (Eq, Show)++-- These observations are constructed only after the ordinary full report replay.+-- They are not accepted as a calibration input document or catalog assertion.+data VerifiedCalibrationTrial = VerifiedCalibrationTrial+ { sessionId :: RunId,+ seed :: Integer,+ control :: Text,+ project :: Text,+ baselineRevision :: Text,+ candidateRevision :: Text,+ profile :: Text,+ trialMachineProfile :: Text,+ trialHealthPolicySha256 :: Text,+ workloadSha256 :: Text,+ trialToolchainSha256 :: Text,+ trialHarnessSemanticsSha256 :: Text,+ verdict :: Text,+ infrastructureState :: Text,+ runtimeBlockers :: [Text],+ createdAt :: UTCTime,+ elapsedSeconds :: Double+ }+ deriving stock (Eq, Show)++-- Commit identity belongs to preparation, not this toolchain identity: attaching+-- a calibration link in a new catalog commit must not invalidate itself.+toolchainDigest :: BuildInventory -> Text+toolchainDigest inventory = sha256Hex (canonicalEncode (object ["compiler" .= inventory.compiler, "platform" .= inventory.platform, "toolchain" .= stable]))+ where+ stable = case inventory.toolchain of Object value -> Object (KeyMap.delete "harnessRevision" value); value -> value++validateCalibration :: UTCTime -> UTCTime -> CalibrationDefinition -> CalibrationAcceptance -> [[VerifiedCalibrationTrial]] -> [[VerifiedCalibrationTrial]] -> Either Text Double+validateCalibration now declaredAt definition acceptance aaTrials slowdownTrials = do+ validateCalibrationDefinition now declaredAt definition+ require (acceptance.batchId == definition.batchId) "calibration batch identity differs"+ mapM_ (\slots -> require (map (.seed) slots == definition.seeds && all (not . null . (.receipts)) slots) "acceptance omitted or reordered a declared slot") [acceptance.aa, acceptance.slowdown]+ let references = concatMap (.receipts) (acceptance.aa <> acceptance.slowdown)+ trials = concat (aaTrials <> slowdownTrials)+ require (nub (map (.uri) references) == map (.uri) references && nub (map (.sessionId) trials) == map (.sessionId) trials) "duplicate calibration evidence or session"+ aaAccepted <- check "aa" acceptance.aa aaTrials+ slowdownAccepted <- check "none" acceptance.slowdown slowdownTrials+ require (length (filter ((== "failed") . (.infrastructureState)) trials) <= definition.infrastructureRetryCap) "calibration infrastructure retry cap exceeded"+ require (counts "pass" aaAccepted >= 19 && counts "regression" aaAccepted == 0 && counts "inconclusive" aaAccepted <= 1) "A/A statistical acceptance failed"+ require (counts "regression" slowdownAccepted >= 19 && counts "pass" slowdownAccepted == 0 && counts "inconclusive" slowdownAccepted <= 1) "slowdown sensitivity acceptance failed"+ let elapsed = sort (map (sum . map (.elapsedSeconds)) aaTrials)+ p95 = elapsed !! 18 -- nearest rank ceil(0.95 * 20), including every retained retry+ require (p95 <= 900) "A/A warm turnaround p95 exceeds fifteen minutes"+ pure p95+ where+ require condition message = unless condition (Left message)+ counts outcome = length . filter ((== outcome) . (.verdict))+ check control slots groups = do+ require (length groups == 20 && length groups == length slots) "missing calibration trial group"+ traverse (checkSlot control) (zip slots groups)+ checkSlot control (slot, trials) = do+ require (not (null trials) && length trials == length slot.receipts) "calibration receipt/trial count differs"+ mapM_ (checkTrial slot.seed control) trials+ require (all ((== "failed") . (.infrastructureState)) (init trials) && (.infrastructureState) (last trials) == "healthy") "statistical outcomes were discarded or slot never completed"+ pure (last trials)+ checkTrial seed control trial = do+ require (trial.createdAt >= declaredAt && trial.createdAt <= definition.expiresAt) "trial is outside the frozen batch lifetime"+ require ((trial.seed, trial.control, trial.project, trial.profile, trial.trialMachineProfile, trial.trialHealthPolicySha256, trial.workloadSha256, trial.trialToolchainSha256, trial.trialHarnessSemanticsSha256) == (seed, control, definition.project, definition.profile, definition.machineProfile, definition.healthPolicySha256, definition.workloadSha256, definition.toolchainSha256, definition.harnessSemanticsSha256)) "calibration workload, seed, toolchain or harness drift"+ require (trial.baselineRevision == definition.baselineRevision && trial.candidateRevision == if control == "aa" then definition.baselineRevision else definition.slowdownRevision) "calibration source revision differs"+ require (null trial.runtimeBlockers) "runtime blocker invalidates calibration"+ require (trial.infrastructureState `elem` ["healthy", "failed"] && trial.verdict `elem` ["pass", "regression", "inconclusive", "not-run"]) "unsupported calibration outcome"+ require (trial.infrastructureState /= "healthy" || trial.verdict /= "not-run") "unexecuted slot is not a statistical outcome"+ require (not (isNaN trial.elapsedSeconds || isInfinite trial.elapsedSeconds) && trial.elapsedSeconds >= 0) "invalid retained calibration timing"++finitePositive :: Double -> Bool+finitePositive value = not (isNaN value || isInfinite value) && value > 0++validateCalibrationDefinition :: UTCTime -> UTCTime -> CalibrationDefinition -> Either Text ()+validateCalibrationDefinition now declaredAt definition = do+ require (canonicalProject definition.project && all immutableRevision [definition.baselineRevision, definition.slowdownRevision] && definition.baselineRevision /= definition.slowdownRevision) "invalid calibration source identity"+ require (length definition.seeds == 20 && nub definition.seeds == definition.seeds && all (\seed -> seed >= 0 && seed <= 9007199254740991) definition.seeds) "calibration needs twenty unique declared seeds"+ require (definition.infrastructureRetryCap >= 0 && definition.infrastructureRetryCap <= 20) "invalid calibration infrastructure retry cap"+ require (declaredAt <= now && declaredAt < definition.expiresAt && now <= definition.expiresAt) "calibration expired or predates its declaration"+ require (not (Text.null definition.injectionMechanism) && not (Text.null definition.targetMetric) && finitePositive definition.targetRelativeEffect && finitePositive definition.targetAbsoluteEffect) "missing or invalid injected effect"+ require (not (Text.null definition.profile) && not (Text.null definition.machineProfile) && not ("pending" `Text.isInfixOf` definition.machineProfile) && all digest [definition.healthPolicySha256, definition.workloadSha256, definition.toolchainSha256, definition.harnessSemanticsSha256]) "calibration needs immutable workload, toolchain and harness digests"+ where+ require condition message = unless condition (Left message)+ digest value = Text.length value == 71 && "sha256:" `Text.isPrefixOf` value && Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) (Text.drop 7 value)++instance ToJSON CalibrationReference where+ toJSON value = object ["definitionUri" .= value.definitionUri, "definitionSha256" .= value.definitionSha256, "acceptanceUri" .= value.acceptanceUri, "acceptanceSha256" .= value.acceptanceSha256]++instance FromJSON CalibrationReference where+ parseJSON = withObject "CalibrationReference" \v -> do+ strictObject ["definitionUri", "definitionSha256", "acceptanceUri", "acceptanceSha256"] v+ CalibrationReference <$> v .: "definitionUri" <*> v .: "definitionSha256" <*> v .: "acceptanceUri" <*> v .: "acceptanceSha256"++instance ToJSON CalibrationDefinition where+ toJSON value = object ["schema" .= ("kenshou.perf-calibration-definition/v1" :: Text), "batchId" .= value.batchId, "project" .= value.project, "profile" .= value.profile, "machineProfile" .= value.machineProfile, "healthPolicySha256" .= value.healthPolicySha256, "workloadSha256" .= value.workloadSha256, "toolchainSha256" .= value.toolchainSha256, "harnessSemanticsSha256" .= value.harnessSemanticsSha256, "baselineRevision" .= value.baselineRevision, "slowdownRevision" .= value.slowdownRevision, "injectionMechanism" .= value.injectionMechanism, "targetMetric" .= value.targetMetric, "targetRelativeEffect" .= value.targetRelativeEffect, "targetAbsoluteEffect" .= value.targetAbsoluteEffect, "seeds" .= value.seeds, "infrastructureRetryCap" .= value.infrastructureRetryCap, "expiresAt" .= value.expiresAt]++instance FromJSON CalibrationDefinition where+ parseJSON = withObject "CalibrationDefinition" \v -> do+ strictObject ["schema", "batchId", "project", "profile", "machineProfile", "healthPolicySha256", "workloadSha256", "toolchainSha256", "harnessSemanticsSha256", "baselineRevision", "slowdownRevision", "injectionMechanism", "targetMetric", "targetRelativeEffect", "targetAbsoluteEffect", "seeds", "infrastructureRetryCap", "expiresAt"] v+ schema <- v .: "schema"+ unless (schema == ("kenshou.perf-calibration-definition/v1" :: Text)) (fail "unsupported calibration definition")+ CalibrationDefinition <$> v .: "batchId" <*> v .: "project" <*> v .: "profile" <*> v .: "machineProfile" <*> v .: "healthPolicySha256" <*> v .: "workloadSha256" <*> v .: "toolchainSha256" <*> v .: "harnessSemanticsSha256" <*> v .: "baselineRevision" <*> v .: "slowdownRevision" <*> v .: "injectionMechanism" <*> v .: "targetMetric" <*> v .: "targetRelativeEffect" <*> v .: "targetAbsoluteEffect" <*> v .: "seeds" <*> v .: "infrastructureRetryCap" <*> v .: "expiresAt"++instance ToJSON CalibrationReceipt where+ toJSON value = object ["uri" .= value.uri, "sha256" .= value.sha256]++instance FromJSON CalibrationReceipt where+ parseJSON = withObject "CalibrationReceipt" \v -> do+ strictObject ["uri", "sha256"] v+ CalibrationReceipt <$> v .: "uri" <*> v .: "sha256"++instance ToJSON CalibrationSlot where+ toJSON value = object ["seed" .= value.seed, "receipts" .= value.receipts]++instance FromJSON CalibrationSlot where+ parseJSON = withObject "CalibrationSlot" \v -> do+ strictObject ["seed", "receipts"] v+ CalibrationSlot <$> v .: "seed" <*> v .: "receipts"++instance ToJSON CalibrationAcceptance where+ toJSON value = object ["schema" .= ("kenshou.perf-calibration-acceptance/v1" :: Text), "batchId" .= value.batchId, "aa" .= value.aa, "slowdown" .= value.slowdown]++instance FromJSON CalibrationAcceptance where+ parseJSON = withObject "CalibrationAcceptance" \v -> do+ strictObject ["schema", "batchId", "aa", "slowdown"] v+ schema <- v .: "schema"+ unless (schema == ("kenshou.perf-calibration-acceptance/v1" :: Text)) (fail "unsupported calibration acceptance")+ CalibrationAcceptance <$> v .: "batchId" <*> v .: "aa" <*> v .: "slowdown"
+ src/Kenshou/Perf/Contracts.hs view
@@ -0,0 +1,40 @@+-- | What a suite supplies to performance planning and preparation. The kernel's+-- validators are generic; which scenario revisions honour count-bound steady work,+-- which knob carries that count, and how an isolated cohort lane is built are+-- facts about the suite's scenarios and packages, so the executable passes them in+-- through 'Kenshou.Core.Cli.Program'.+module Kenshou.Perf.Contracts+ ( PerfContracts (..),+ noPerfContracts,+ )+where++import Data.Text (Text)+import Kenshou.Core.Cohort (CohortDescriptor)+import Kenshou.Core.Knob (KnobName)++data PerfContracts = PerfContracts+ { -- | Benchmark scenario identifiers and revisions whose phase contract has been+ -- reviewed: they implement count-bound steady work rather than timed phases.+ -- A profile naming any other scenario revision is rejected.+ reviewedPhaseContracts :: [(Text, Int)],+ -- | The knob whose effective value must equal a profile's steady count, when+ -- the suite has one.+ steadyCountKnob :: Maybe KnobName,+ -- | Extra Cabal project stanzas for a cohort lane, such as package flags that+ -- leave out parts of the suite the lane cannot build.+ laneStanzas :: CohortDescriptor -> [Text],+ -- | The harness packages, by directory name, that a cohort lane builds.+ lanePackages :: CohortDescriptor -> [String] -> [String]+ }++-- | No reviewed contracts: every performance profile is rejected, and cohort+-- lanes build every harness package without extra stanzas.+noPerfContracts :: PerfContracts+noPerfContracts =+ PerfContracts+ { reviewedPhaseContracts = [],+ steadyCountKnob = Nothing,+ laneStanzas = const [],+ lanePackages = const id+ }
+ src/Kenshou/Perf/Plan.hs view
@@ -0,0 +1,345 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Plan+ ( RequiredMetric (..),+ PerfProfile (..),+ ProfileCatalog (..),+ PlannedProfile (..),+ PerfPlan (..),+ validateProfile,+ workloadDigest,+ planPerformance,+ conservativeChanges,+ )+where++import Control.Monad (unless)+import Data.Aeson+import Data.List (nub, sort, sortOn)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), renderDimensions, resolveDimensions)+import Kenshou.Core.Id (Kind (..), ScenarioId (..), renderScenarioId)+import Kenshou.Core.Knob+import Kenshou.Perf.Contracts (PerfContracts (..))+import Kenshou.Perf.Request+import Kenshou.Plan.Catalog qualified as Catalog+import Kenshou.Plan.Change+import Kenshou.Plan.Components++data RequiredMetric = RequiredMetric {name :: Text, unit :: Text, minimumSteadySamples :: Int}+ deriving stock (Eq, Show)++data PerfProfile = PerfProfile+ { name :: Text,+ revision :: Int,+ scenario :: Text,+ scenarioRevision :: Int,+ knobs :: Map Text Value,+ dimensions :: Map Text Text,+ workloadPurpose :: Text,+ warmupSeconds :: Int,+ steadyCount :: Int,+ cooldownSeconds :: Int,+ timeoutSeconds :: Int,+ cachePolicy :: Text,+ policy :: FilePath,+ requiredMetrics :: [RequiredMetric],+ capabilities :: [Text],+ machineProfile :: Text,+ trialSeconds :: Int,+ resetSeconds :: Int,+ sealSeconds :: Int,+ minimumPairs :: Int,+ replacementPairs :: Int,+ calibration :: Maybe Value+ }+ deriving stock (Eq, Show)++newtype ProfileCatalog = ProfileCatalog {profiles :: [PerfProfile]}+ deriving stock (Eq, Show)++data PlannedProfile = PlannedProfile+ { profile :: PerfProfile,+ workloadSha256 :: Text,+ effectiveKnobs :: ResolvedKnobs,+ reasons :: [Value],+ estimateSeconds :: Integer+ }+ deriving stock (Eq, Show)++data PerfPlan = PerfPlan+ { request :: PerfRequest,+ requestSha256 :: Text,+ graphSha256 :: Text,+ catalogSha256 :: Text,+ profileCatalogSha256 :: Text,+ selected :: [PlannedProfile],+ excluded :: [(Text, Text)],+ affectedScenarios :: [Text],+ uncoveredScenarios :: [Text],+ unmappedPaths :: [FilePath],+ documentationProof :: [FilePath],+ coverage :: Text,+ changeCoverage :: Text,+ estimateSeconds :: Integer,+ gatingEligible :: Bool+ }+ deriving stock (Eq, Show)++-- Calibration references are deliberately excluded from the workload identity.+-- Attaching evidence to an unchanged workload must not create a digest cycle.+workloadDigest :: PerfProfile -> Text -> Text+workloadDigest profile policyDigest = sha256Hex (canonicalEncode (object ["workload" .= profile {calibration = Nothing}, "policySha256" .= policyDigest]))++validateProfile :: PerfContracts -> [Catalog.ScenarioInfo] -> PerfProfile -> Either Text ResolvedKnobs+validateProfile contracts catalog profile = do+ scenario <- case filter ((== profile.scenario) . renderScenarioId . (.id)) catalog of+ [value] -> Right value+ _ -> Left ("profile names an unknown or duplicate scenario: " <> profile.scenario)+ require (scenario.id.kind == Benchmark && scenario.revision == profile.scenarioRevision) "profile scenario revision/kind mismatch"+ require (profile.revision > 0 && not (Text.null profile.name) && Text.all (`elem` (['a' .. 'z'] <> ['0' .. '9'] <> ['-'])) profile.name) "invalid profile identity"+ require (profile.workloadPurpose == "fixed-load-latency") "unsupported workload purpose; paced throughput is not capacity"+ -- Only reviewed revisions implement count-bound steady work, not timed phases.+ require ((profile.scenario, profile.scenarioRevision) `elem` contracts.reviewedPhaseContracts) "scenario phase contract has not been reviewed"+ require (profile.warmupSeconds == 0 && profile.cooldownSeconds == 0 && profile.timeoutSeconds == 180) "scenario revision does not honor requested phases/timeout"+ require (profile.cachePolicy == "cold-reset") "unsupported cache policy"+ require (profile.minimumPairs >= 5 && profile.replacementPairs >= 0 && profile.replacementPairs <= 2) "invalid fixed pair/replacement allowance"+ require (profile.trialSeconds > 0 && profile.resetSeconds >= 0 && profile.sealSeconds >= 0) "cost estimate needs positive trial time and nonnegative overheads"+ require (not (null profile.requiredMetrics) && nub (map (.name) profile.requiredMetrics) == map (.name) profile.requiredMetrics) "required metrics must be nonempty and unique"+ require (all (\metric -> not (Text.null metric.name) && not (Text.null metric.unit) && metric.minimumSteadySamples > 0) profile.requiredMetrics) "invalid required metric"+ require (all ((<= profile.steadyCount) . (.minimumSteadySamples)) profile.requiredMetrics) "required sample count exceeds planned work"+ require (not (Text.null profile.machineProfile) && not (null profile.capabilities)) "profile needs a machine profile and cell capabilities"+ assignments <- traverse (\(key, value) -> (,RawJson value) <$> mkKnobName key) (Map.toAscList profile.knobs)+ resolved <- either (Left . Text.pack . show) Right (resolveKnobs scenario.knobs assignments)+ dimensionValues <- either (Left . Text.pack . show) Right (resolveDimensions scenario.dimensions (Map.toAscList profile.dimensions))+ require (dimensionValues.pgDurability /= Just PgFsyncOff) "performance profiles require durable PostgreSQL"+ rawKnob <- mkKnobName "measure.raw-samples"+ case contracts.steadyCountKnob of+ Just countKnob -> require (Map.lookup countKnob (resolvedKnobsMap resolved) == Just (VInt (fromIntegral profile.steadyCount))) "profile steady count disagrees with effective workload"+ Nothing -> pure ()+ require (Map.lookup rawKnob (resolvedKnobsMap resolved) == Just (VText "full")) "performance profiles require full raw samples"+ rateKnob <- mkKnobName "bench.rate"+ let offeredRate = case Map.lookup rateKnob (resolvedKnobsMap resolved) of Just (VInt rate) -> rate; _ -> 0+ require (offeredRate > 0 && fromIntegral profile.trialSeconds * offeredRate >= fromIntegral profile.steadyCount) "trial estimate is shorter than the declared offered workload"+ pure resolved+ where+ require condition message = unless condition (Left (profile.name <> ": " <> message))++-- Paths absent from the graph are not a documentation proof. Only explicit+-- documentation names outside all source roots can be excluded safely.+conservativeChanges :: ComponentGraph -> Text -> [FilePath] -> [Change] -> ([Change], [FilePath], [FilePath])+conservativeChanges graph repository paths mapped = (changes, unknown, documentation)+ where+ components = filter ((== repository) . (.repository)) graph.components+ inRoot path = any (\component -> any (\root -> Text.pack root `Text.isPrefixOf` Text.pack path) component.sourceRoots) components+ isDocumentation path = not (inRoot path) && ("docs/" `Text.isPrefixOf` Text.pack path || path `elem` ["README.md", "CHANGELOG.md", "LICENSE", "LICENSE.md"])+ documentation = filter isDocumentation paths+ unknown = filter (\path -> not (inRoot path) && not (isDocumentation path)) paths+ changes =+ if null components || not (null unknown)+ then mapped <> [Change (ComponentRef (ComponentId "all") Nothing) Everything "unmapped upstream input; conservative selection"]+ else mapped++-- The admission map is populated only by an evidence verifier, never by a+-- catalog's self-asserted calibrated flag. It maps workload digest to verified+-- calibration evidence digest. A catalog link cannot populate this map without+-- full receipt retrieval, raw report replay and acceptance-batch verification.+planPerformance :: PerfContracts -> PerfRequest -> ComponentGraph -> [Catalog.ScenarioInfo] -> ProfileCatalog -> [Change] -> [FilePath] -> [FilePath] -> Map Text Text -> Map Text Text -> Either Text PerfPlan+planPerformance contracts request graph catalog profiles changes unknown documentation policyDigests admitted = do+ validateRequest request+ unless (nub profileNames == profileNames) (Left "duplicate performance profile")+ unless (all (`elem` profileNames) request.profiles) (Left "unknown requested performance profile")+ checked <- traverse check (sortOn (.name) profiles.profiles)+ let affected = selectScenarios graph catalog changes+ affectedNames = sort (map (renderScenarioId . (.id) . (.scenario)) affected)+ candidates = filter (wanted affectedNames) checked+ (selected, excluded, _) = foldl admit ([], [], 0) candidates+ covered = map (.scenario) (map (.profile) selected)+ telemetryMissing = [renderScenarioId item.scenario.id | item <- affected, item.minPolicy == Just "telemetry-corners", not (any (coversTelemetry item) selected)]+ uncovered = nub (filter (`notElem` covered) affectedNames <> telemetryMissing)+ notApplicable = request.control == "none" && null changes && null unknown && not (null documentation) && null request.profiles+ scopeComplete = if request.coverageMode == "selected-profiles" then sort (map (.name) (map (.profile) selected)) == sort request.profiles else null uncovered && not (null selected)+ scope = if notApplicable then "not-applicable" else if scopeComplete && null excluded then "complete" else "incomplete"+ pure+ PerfPlan+ { request,+ requestSha256 = requestDigest request,+ graphSha256 = graphDigest graph,+ catalogSha256 = sha256Hex (canonicalEncode (toJSON catalog)),+ profileCatalogSha256 = sha256Hex (canonicalEncode (toJSON profiles)),+ selected,+ excluded,+ affectedScenarios = affectedNames,+ uncoveredScenarios = uncovered,+ unmappedPaths = unknown,+ documentationProof = documentation,+ coverage = scope,+ changeCoverage = if notApplicable then "not-applicable" else if null uncovered && not (null affectedNames) then "complete" else "incomplete",+ estimateSeconds = sum (map (.estimateSeconds) selected),+ gatingEligible = request.purpose == "gating" && scope == "complete" && all (\item -> Map.member item.workloadSha256 admitted) selected+ }+ where+ profileNames = map (.name) profiles.profiles+ check profile = do+ effective <- validateProfile contracts catalog profile+ scenario <- case filter ((== profile.scenario) . renderScenarioId . (.id)) catalog of+ [value] -> Right value+ _ -> Left "profile scenario is missing"+ effectiveDimensions <- either (Left . Text.pack . show) Right (resolveDimensions scenario.dimensions (Map.toAscList profile.dimensions))+ let normalized =+ (profile :: PerfProfile)+ { knobs = Map.fromList [(renderKnobName name, toJSON value) | (name, value) <- Map.toAscList (resolvedKnobsMap effective)],+ dimensions = Map.fromList (renderDimensions effectiveDimensions)+ }+ policyDigest <- maybe (Left ("missing policy digest: " <> Text.pack profile.policy)) Right (Map.lookup (Text.pack profile.policy) policyDigests)+ pure (normalized, (effective, workloadDigest normalized policyDigest))+ wanted affected (profile, _) = (null request.profiles || profile.name `elem` request.profiles) && (request.control == "aa" || request.coverageMode == "selected-profiles" || profile.scenario `elem` affected)+ admit (selected, excluded, spent) (profile, (effective, digest))+ | request.purpose /= "calibration" && Map.notMember digest admitted = (selected, excluded <> [(profile.name, "uncalibrated")], spent)+ | spent + cost > fromIntegral request.budgetMinutes * 60 = (selected, excluded <> [(profile.name, "over-budget")], spent)+ | otherwise = (selected <> [PlannedProfile profile digest effective reasons cost], excluded, spent + cost)+ where+ cost = 2 * (fromIntegral profile.minimumPairs + fromIntegral profile.replacementPairs) * (fromIntegral profile.trialSeconds + fromIntegral profile.resetSeconds + fromIntegral profile.sealSeconds)+ reasons =+ [toJSON reason | item <- selectScenarios graph catalog changes, renderScenarioId item.scenario.id == profile.scenario, reason <- NonEmpty.toList item.reasons]+ <> [ object+ [ "component" .= ("request/profile/" <> profile.name),+ "source" .= ("named" :: Text),+ "via" .= [if request.control == "aa" then "aa-control" else "selected-profiles" :: Text],+ "selector" .= profile.scenario,+ "distance" .= (0 :: Int)+ ]+ | profile.name `elem` request.profiles+ ]+ coversTelemetry item selected =+ selected.profile.scenario == renderScenarioId item.scenario.id+ && Map.lookup "telemetry.tracing" selected.profile.dimensions `elem` [Just "sdk-inmemory", Just "sdk-otlp"]+ && Map.lookup "telemetry.metrics" selected.profile.dimensions `elem` [Just "collect", Just "serve", Just "serve-scraped"]++instance ToJSON RequiredMetric where+ toJSON metric = object ["name" .= metric.name, "unit" .= metric.unit, "minimumSteadySamples" .= metric.minimumSteadySamples]++instance FromJSON RequiredMetric where+ parseJSON = withObject "RequiredMetric" \value -> do+ strictObject ["name", "unit", "minimumSteadySamples"] value+ RequiredMetric <$> value .: "name" <*> value .: "unit" <*> value .: "minimumSteadySamples"++instance ToJSON PerfProfile where+ toJSON profile =+ object+ [ "name" .= profile.name,+ "revision" .= profile.revision,+ "scenario" .= profile.scenario,+ "scenarioRevision" .= profile.scenarioRevision,+ "knobs" .= profile.knobs,+ "dimensions" .= profile.dimensions,+ "workloadPurpose" .= profile.workloadPurpose,+ "warmupSeconds" .= profile.warmupSeconds,+ "steadyCount" .= profile.steadyCount,+ "cooldownSeconds" .= profile.cooldownSeconds,+ "timeoutSeconds" .= profile.timeoutSeconds,+ "cachePolicy" .= profile.cachePolicy,+ "policy" .= profile.policy,+ "requiredMetrics" .= profile.requiredMetrics,+ "capabilities" .= profile.capabilities,+ "machineProfile" .= profile.machineProfile,+ "trialSeconds" .= profile.trialSeconds,+ "resetSeconds" .= profile.resetSeconds,+ "sealSeconds" .= profile.sealSeconds,+ "minimumPairs" .= profile.minimumPairs,+ "replacementPairs" .= profile.replacementPairs,+ "calibration" .= profile.calibration+ ]++instance FromJSON PerfProfile where+ parseJSON = withObject "PerfProfile" \value -> do+ strictObject ["name", "revision", "scenario", "scenarioRevision", "knobs", "dimensions", "workloadPurpose", "warmupSeconds", "steadyCount", "cooldownSeconds", "timeoutSeconds", "cachePolicy", "policy", "requiredMetrics", "capabilities", "machineProfile", "trialSeconds", "resetSeconds", "sealSeconds", "minimumPairs", "replacementPairs", "calibration"] value+ PerfProfile+ <$> value .: "name"+ <*> value .: "revision"+ <*> value .: "scenario"+ <*> value .: "scenarioRevision"+ <*> value .: "knobs"+ <*> value .: "dimensions"+ <*> value .: "workloadPurpose"+ <*> value .: "warmupSeconds"+ <*> value .: "steadyCount"+ <*> value .: "cooldownSeconds"+ <*> value .: "timeoutSeconds"+ <*> value .: "cachePolicy"+ <*> value .: "policy"+ <*> value .: "requiredMetrics"+ <*> value .: "capabilities"+ <*> value .: "machineProfile"+ <*> value .: "trialSeconds"+ <*> value .: "resetSeconds"+ <*> value .: "sealSeconds"+ <*> value .: "minimumPairs"+ <*> value .: "replacementPairs"+ <*> value .:? "calibration"++instance ToJSON ProfileCatalog where+ toJSON catalog = object ["schema" .= ("kenshou.perf-catalog/v1" :: Text), "profiles" .= catalog.profiles]++instance FromJSON ProfileCatalog where+ parseJSON = withObject "ProfileCatalog" \value -> do+ strictObject ["schema", "profiles"] value+ schema <- value .: "schema"+ unless (schema == ("kenshou.perf-catalog/v1" :: Text)) (fail "unsupported performance catalog schema")+ ProfileCatalog <$> value .: "profiles"++instance ToJSON PlannedProfile where+ toJSON planned = object ["profile" .= planned.profile, "workloadSha256" .= planned.workloadSha256, "effectiveKnobs" .= planned.effectiveKnobs, "reasons" .= planned.reasons, "estimateSeconds" .= planned.estimateSeconds]++instance ToJSON PerfPlan where+ toJSON plan =+ object+ [ "schema" .= ("kenshou.perf-plan/v1" :: Text),+ "request" .= plan.request,+ "requestSha256" .= plan.requestSha256,+ "graphSha256" .= plan.graphSha256,+ "catalogSha256" .= plan.catalogSha256,+ "profileCatalogSha256" .= plan.profileCatalogSha256,+ "selected" .= plan.selected,+ "excluded" .= [object ["profile" .= name, "reason" .= reason] | (name, reason) <- plan.excluded],+ "affectedScenarios" .= plan.affectedScenarios,+ "uncoveredScenarios" .= plan.uncoveredScenarios,+ "unmappedPaths" .= plan.unmappedPaths,+ "documentationProof" .= plan.documentationProof,+ "coverage" .= plan.coverage,+ "changeCoverage" .= plan.changeCoverage,+ "estimateSeconds" .= plan.estimateSeconds,+ "gatingEligible" .= plan.gatingEligible+ ]++instance FromJSON PlannedProfile where+ parseJSON = withObject "PlannedProfile" \value -> do+ strictObject ["profile", "workloadSha256", "effectiveKnobs", "reasons", "estimateSeconds"] value+ planned <- PlannedProfile <$> value .: "profile" <*> value .: "workloadSha256" <*> value .: "effectiveKnobs" <*> value .: "reasons" <*> value .: "estimateSeconds"+ unless (planned.estimateSeconds > 0) (fail "invalid planned configuration estimate")+ pure planned++instance FromJSON PerfPlan where+ parseJSON = withObject "PerfPlan" \value -> do+ strictObject ["schema", "request", "requestSha256", "graphSha256", "catalogSha256", "profileCatalogSha256", "selected", "excluded", "affectedScenarios", "uncoveredScenarios", "unmappedPaths", "documentationProof", "coverage", "changeCoverage", "estimateSeconds", "gatingEligible"] value+ schema <- value .: "schema"+ unless (schema == ("kenshou.perf-plan/v1" :: Text)) (fail "unsupported performance plan schema")+ omissions <-+ value .: "excluded"+ >>= traverse+ ( withObject "ExcludedProfile" \item -> do+ strictObject ["profile", "reason"] item+ (,) <$> item .: "profile" <*> item .: "reason"+ )+ plan <- PerfPlan <$> value .: "request" <*> value .: "requestSha256" <*> value .: "graphSha256" <*> value .: "catalogSha256" <*> value .: "profileCatalogSha256" <*> value .: "selected" <*> pure omissions <*> value .: "affectedScenarios" <*> value .: "uncoveredScenarios" <*> value .: "unmappedPaths" <*> value .: "documentationProof" <*> value .: "coverage" <*> value .: "changeCoverage" <*> value .: "estimateSeconds" <*> value .: "gatingEligible"+ unless (plan.requestSha256 == requestDigest plan.request) (fail "performance plan request digest mismatch")+ unless (plan.estimateSeconds == sum (map (.estimateSeconds) plan.selected) && plan.estimateSeconds <= fromIntegral plan.request.budgetMinutes * 60) (fail "performance plan estimate mismatch or budget exceeded")+ unless (all (`elem` ["complete", "incomplete", "not-applicable"]) [plan.coverage, plan.changeCoverage]) (fail "unknown performance coverage")+ unless (nub (map (.name) (map (.profile) plan.selected)) == map (.name) (map (.profile) plan.selected)) (fail "duplicate selected performance profile")+ unless (not plan.gatingEligible || plan.request.purpose == "gating" && plan.coverage == "complete" && not (null plan.selected)) (fail "invalid gating eligibility")+ pure plan
+ src/Kenshou/Perf/Prepared.hs view
@@ -0,0 +1,352 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Prepared+ ( InventoryPackage (..),+ BuildInventory (..),+ PerfPrepared (..),+ PreparedArm (..),+ cabalInventory,+ compareInventories,+ inventoryDigest,+ preparedDigest,+ buildKeyFor,+ validatePrepared,+ validateArmInventories,+ renderBoundsProject,+ )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser, parseEither)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (nub, sortOn)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Cohort+import Kenshou.Perf.Contracts (PerfContracts (..))+import Kenshou.Perf.Plan (PerfPlan (..), PerfProfile (..), PlannedProfile (..), validateProfile, workloadDigest)+import Kenshou.Perf.Request+import Kenshou.Perf.Source (SourceSnapshot, descriptorForSnapshot, legacyDescriptorForSnapshot, targetPackages)+import Kenshou.Plan.Catalog (ScenarioInfo)+import System.FilePath (makeRelative, normalise, splitDirectories)++data InventoryPackage = InventoryPackage+ {name :: Text, version :: Text, source :: Value, flags :: Map Text Bool, settings :: Value}+ deriving stock (Eq, Show)++data BuildInventory = BuildInventory+ { resolver :: Text,+ compiler :: Text,+ platform :: Text,+ toolchain :: Value,+ packages :: [InventoryPackage]+ }+ deriving stock (Eq, Show)++data PreparedArm = PreparedArm+ { descriptor :: CohortDescriptor,+ sourceLock :: Value,+ boundsInventory :: BuildInventory,+ buildInventory :: BuildInventory,+ payload :: Value,+ buildKey :: Text+ }+ deriving stock (Eq, Show)++data PerfPrepared = PerfPrepared+ { plan :: PerfPlan,+ inputs :: Value,+ harnessRevision :: Text,+ baseline :: PreparedArm,+ candidate :: PreparedArm+ }+ deriving stock (Eq, Show)++inventoryDigest :: BuildInventory -> Text+inventoryDigest = sha256Hex . canonicalEncode . toJSON++preparedDigest :: PerfPrepared -> Text+preparedDigest = sha256Hex . canonicalEncode . toJSON++buildKeyFor :: Text -> CohortDescriptor -> Value -> BuildInventory -> BuildInventory -> Text+buildKeyFor revision descriptor lock inventory bounds =+ sha256Hex+ ( canonicalEncode+ ( object+ ["harness" .= revision, "descriptorSha256" .= sha256Hex (LazyByteString.toStrict (encode descriptor)), "sourceLock" .= lock, "inventory" .= inventory, "boundsInventory" .= bounds]+ )+ )++-- Recheck the relationships inside a sealed preparation before any cell action.+-- The original files and their manifest are verified separately by the loader.+validatePrepared :: PerfContracts -> PerfPrepared -> Either Text ()+validatePrepared contracts prepared = do+ (request, template, before, after, catalog, policies) <-+ parse+ "pinned inputs"+ ( withObject "PinnedInputs" \v -> do+ request <- v .: "request"+ harness <- v .: "harness"+ withObject+ "PinnedHarness"+ ( \h -> do+ revision <- h .: "revision"+ dirty <- h .: "dirty"+ unless (revision == prepared.harnessRevision && not dirty) (fail "pinned harness must be the clean prepared commit")+ )+ harness+ graph <- v .: "graph"+ scenarios <- v .: "scenarioCatalog"+ profiles <- v .: "profileCatalog"+ unless (sha256Hex (canonicalEncode graph) == prepared.plan.graphSha256 && sha256Hex (canonicalEncode scenarios) == prepared.plan.catalogSha256 && sha256Hex (canonicalEncode profiles) == prepared.plan.profileCatalogSha256) (fail "pinned graph/catalog digest mismatch")+ (,,,,,) request <$> v .: "template" <*> v .: "baselineSource" <*> v .: "candidateSource" <*> parseJSON scenarios <*> v .: "policies"+ )+ prepared.inputs+ unless (request == prepared.plan.request) (Left "prepared request differs from the plan")+ targets <- targetPackages request template+ baselineDescriptor <- descriptorForSnapshot request template "baseline" (before :: SourceSnapshot)+ candidateDescriptor <- descriptorForSnapshot request template "candidate" after+ legacyBaseline <- legacyDescriptorForSnapshot request template "baseline" before+ legacyCandidate <- legacyDescriptorForSnapshot request template "candidate" after+ unless ((baselineDescriptor == prepared.baseline.descriptor && candidateDescriptor == prepared.candidate.descriptor) || (legacyBaseline == prepared.baseline.descriptor && legacyCandidate == prepared.candidate.descriptor)) (Left "prepared arm descriptor differs from pinned source manifests")+ compareInventories targets prepared.baseline.boundsInventory prepared.candidate.boundsInventory+ compareInventories targets prepared.baseline.buildInventory prepared.candidate.buildInventory+ mapM_ (validateSelected catalog policies) prepared.plan.selected+ mapM_ validateArm [prepared.baseline, prepared.candidate]+ where+ parse label parser value = either (Left . ((label <> ": ") <>) . Text.pack) Right (parseEither parser value)+ validateSelected :: [ScenarioInfo] -> Map FilePath Value -> PlannedProfile -> Either Text ()+ validateSelected catalog policies selected = do+ effective <- validateProfile contracts catalog selected.profile+ unless (toJSON effective == toJSON selected.effectiveKnobs && toJSON effective == toJSON selected.profile.knobs) (Left "prepared effective workload differs from its profile")+ policy <- maybe (Left "pinned profile policy missing") Right (Map.lookup selected.profile.policy policies)+ unless (workloadDigest selected.profile (sha256Hex (canonicalEncode policy)) == selected.workloadSha256) (Left "prepared workload/policy digest mismatch")+ validateArm arm = do+ unless (arm.buildInventory.resolver == "nix" && arm.buildInventory.platform == "x86_64-linux" && arm.buildInventory.compiler == arm.descriptor.descriptorCompiler && arm.boundsInventory.resolver == "cabal-bounds" && arm.boundsInventory.compiler == arm.descriptor.descriptorCompiler) (Left "prepared inventory compiler/platform/resolver mismatch")+ _ <-+ parse+ "prepared toolchain"+ ( withObject "Toolchain" \t -> do+ revision <- t .: "harnessRevision"+ strict <- t .: "strictBounds"+ nixpkgs <- t .: "nixpkgsRevision"+ haskellNix <- t .: "haskellNixRevision"+ unless (revision == prepared.harnessRevision && strict && immutableRevision nixpkgs && immutableRevision haskellNix) (fail "unverified or mutable prepared toolchain")+ )+ arm.buildInventory.toolchain+ expected <- parse "pinned build toolchain" (withObject "PinnedInputs" (.:? "buildToolchain")) prepared.inputs+ case expected of+ Nothing -> unless (not prepared.plan.gatingEligible) (Left "gating preparation requires a pinned build toolchain")+ Just value -> case arm.buildInventory.toolchain of+ Object actual -> unless (Object (KeyMap.delete "harnessRevision" actual) == value) (Left "actual build toolchain differs from frozen planning inputs")+ _ -> Left "actual build toolchain is not an object"+ let descriptorDigest = sha256Hex (LazyByteString.toStrict (encode arm.descriptor))+ _ <-+ parse+ "source lock"+ ( withObject "SourceLock" \v -> do+ schema <- v .: "schema"+ name <- v .: "cohort"+ digest <- v .: "descriptorSha256"+ unless (schema == ("kenshou.cohort-nix-lock/v1" :: Text) && name == arm.descriptor.descriptorName && digest == Text.drop 7 descriptorDigest) (fail "source lock does not bind its descriptor")+ )+ arm.sourceLock+ unless (arm.buildKey == buildKeyFor prepared.harnessRevision arm.descriptor arm.sourceLock arm.buildInventory arm.boundsInventory) (Left "prepared build key mismatch")+ validateArmInventories arm.descriptor arm.buildInventory arm.boundsInventory+ _ <-+ parse+ "payload identity"+ ( withObject "Payload" \v -> do+ identity <- v .: "cohortIdentity"+ harness <- v .: "harness"+ withObject+ "PayloadHarness"+ ( \h -> do+ revision <- h .: "revision"+ dirty <- h .: "dirty"+ unless (revision == prepared.harnessRevision && not dirty) (fail "payload harness mismatch")+ )+ harness+ unless (null (checkCohort arm.descriptor identity) && identity.identityDescriptorSha256 == Text.drop 7 descriptorDigest && identity.identityResolver == Just "nix") (fail "payload cohort identity mismatch")+ )+ arm.payload+ pure ()++-- Run this before publication, and repeat it during offline preparation loading.+validateArmInventories :: CohortDescriptor -> BuildInventory -> BuildInventory -> Either Text ()+validateArmInventories descriptor nix bounds = do+ compareInventories [] nix nix+ compareInventories [] bounds bounds+ mapM_ strictPackage nix.packages+ let nixPackages = Map.fromList [(entry.name, entry) | entry <- nix.packages]+ cabalPackages = Map.fromList [(entry.name, entry) | entry <- bounds.packages]+ mapM_+ ( \pin -> do+ actual <- maybe (Left "cohort pin missing from Nix inventory") Right (Map.lookup pin.pinName nixPackages)+ bound <- maybe (Left "cohort pin missing from bounds inventory") Right (Map.lookup pin.pinName cabalPackages)+ unless (actual.version == pin.pinVersion && bound.version == pin.pinVersion) (Left "resolved inventory version differs from cohort pin")+ )+ (concatMap (.componentPackages) descriptor.descriptorComponents)+ mapM_+ ( \entry -> case Map.lookup entry.name cabalPackages of+ Nothing -> Right ()+ Just bound -> unless (entry.version == bound.version && entry.flags == bound.flags) (Left ("Nix package version or flags differs from bounds plan: " <> entry.name))+ )+ nix.packages+ where+ strictPackage entry =+ either+ (Left . Text.pack)+ Right+ ( parseEither+ ( withObject "PackageSettings" \s -> do+ jailbreak <- s .: "jailbreak"+ flags <- s .: "configureFlags" :: Parser [Text]+ let allowed = map ("--allow-newer=" <>) descriptor.descriptorAllowNewer+ unless (not jailbreak && all (\flag -> not ("--allow-newer" `Text.isPrefixOf` flag) || flag `elem` allowed) flags) (fail "package has unreviewed bound lifting")+ )+ entry.settings+ )++-- Cabal's plan source and revision fields are documented by the upstream+-- ProjectPlanOutput implementation. Include description revisions and all+-- resolved flags, and identify local harness packages by the clean commit.+-- Build-product IDs are excluded: relinking a dependent is expected.+cabalInventory :: FilePath -> Text -> Value -> Either Text BuildInventory+cabalInventory harnessRoot harnessRevision document = do+ unless (immutableRevision harnessRevision) (Left "bounds inventory requires a clean harness commit")+ either (Left . Text.pack) Right (parseEither parsePlan document)+ where+ parsePlan = withObject "Cabal plan" \root -> do+ compiler <- root .: "compiler-id"+ cabal <- root .: "cabal-version" :: Parser Text+ os <- root .: "os" :: Parser Text+ arch <- root .: "arch" :: Parser Text+ units <- root .: "install-plan" >>= traverse parseUnit+ let grouped = Map.fromListWith (<>) [(unit.name, [unit]) | unit <- units]+ unless (all (\values -> nub values == take 1 values) (Map.elems grouped)) (fail "ambiguous package source/version/flags in Cabal inventory")+ pure (BuildInventory "cabal-bounds" compiler (arch <> "-" <> os) (object ["cabal" .= cabal, "harnessRevision" .= harnessRevision]) (sortOn (.name) (nub units)))+ parseUnit = withObject "Cabal unit" \unit -> do+ name <- unit .: "pkg-name"+ version <- unit .: "pkg-version"+ kind <- unit .: "type" :: Parser Text+ flags <- unit .:? "flags" .!= Map.empty+ source <-+ if kind == "pre-existing"+ then pure (object ["type" .= ("boot" :: Text)])+ else do+ descriptionHash <- unit .:? "pkg-cabal-sha256" :: Parser (Maybe Text)+ sourceHash <- unit .:? "pkg-src-sha256" :: Parser (Maybe Text)+ revision <- unit .:? "pkg-revision" :: Parser (Maybe Int)+ location <- unit .: "pkg-src" >>= normalizeSource name+ pure (object ["location" .= location, "sourceSha256" .= sourceHash, "descriptionSha256" .= descriptionHash, "descriptionRevision" .= revision])+ pure (InventoryPackage name version source flags (object []))+ normalizeSource name = withObject "Cabal source" \source -> do+ kind <- source .: "type" :: Parser Text+ case kind of+ "local" -> do+ path <- source .: "path"+ let relative = makeRelative (normalise harnessRoot) (normalise path)+ unless ("kenshou-" `Text.isPrefixOf` name && not (null relative) && all (/= "..") (splitDirectories relative) && relative /= normalise path) (fail "Cabal inventory contains a local source outside the pinned harness")+ pure (object ["type" .= ("harness" :: Text), "revision" .= harnessRevision, "path" .= relative])+ "source-repo" -> do+ repo <- source .: "source-repo"+ withObject+ "Cabal Git source"+ ( \fields -> do+ location <- fields .: "location" :: Parser Text+ revision <- fields .: "tag"+ branch <- fields .:? "branch" :: Parser (Maybe Text)+ unless ("https://" `Text.isPrefixOf` location && immutableRevision revision && branch == Nothing) (fail "Cabal inventory contains a mutable Git source")+ pure (Object source)+ )+ repo+ "repo-tar" -> pure (Object source)+ _ -> fail "Cabal inventory contains an unsupported source"++compareInventories :: [Text] -> BuildInventory -> BuildInventory -> Either Text ()+compareInventories targets baseline candidate = do+ unless (not (null baseline.packages) && not (null candidate.packages)) (Left "empty resolved inventory")+ unless ((baseline.resolver, baseline.compiler, baseline.platform, baseline.toolchain) == (candidate.resolver, candidate.compiler, candidate.platform, candidate.toolchain)) (Left "compiler, platform or toolchain drift")+ let before = Map.fromList [(entry.name, entry) | entry <- baseline.packages]+ after = Map.fromList [(entry.name, entry) | entry <- candidate.packages]+ unless (Map.size before == length baseline.packages && Map.size after == length candidate.packages && Map.keys before == Map.keys after) (Left "resolved package inventory changed or contains duplicate names")+ unless (all (`Map.member` before) targets) (Left "target package missing from resolved inventory")+ mapM_ (check after) (Map.toAscList before)+ where+ check after (name, previous) = case Map.lookup name after of+ Nothing -> Left ("missing resolved package: " <> name)+ Just current+ | previous.flags /= current.flags || previous.settings /= current.settings -> Left ("resolved flags/settings changed: " <> name)+ | name `elem` targets -> Right ()+ | previous /= current -> Left ("unrelated resolved dependency changed: " <> name)+ | otherwise -> Right ()++-- Standalone projects never import active.project or project.local. They pin+-- every cohort package and use only already reviewed, package-scoped exceptions.+-- The suite's lane stanzas follow the pins.+renderBoundsProject :: PerfContracts -> [FilePath] -> CohortDescriptor -> Either Text Text+renderBoundsProject contracts harnessPackages descriptor = do+ unless (not (null harnessPackages) && all safeAtom harnessPackages) (Left "unsafe or empty harness package paths")+ unless (all scopedException descriptor.descriptorAllowNewer) (Left "blanket bound lifting is forbidden in performance preparation")+ unless (all (safeAtom . Text.unpack) (descriptor.descriptorCompiler : descriptor.descriptorIndexState : map (.pinName) pins <> map (.pinVersion) pins)) (Left "unsafe cohort project field")+ gitStanzas <- traverse gitComponent descriptor.descriptorComponents+ pure+ ( Text.unlines+ (["packages: " <> Text.intercalate ", " (map Text.pack harnessPackages), "with-compiler: " <> descriptor.descriptorCompiler, "index-state: " <> descriptor.descriptorIndexState, "tests: False", "benchmarks: False", "extra-packages: " <> Text.intercalate ", " (map (.pinName) pins), "constraints: " <> Text.intercalate ", " [pin.pinName <> " ==" <> pin.pinVersion | pin <- pins], "allow-newer: " <> (if null descriptor.descriptorAllowNewer then "none" else Text.intercalate ", " descriptor.descriptorAllowNewer)] <> laneFlags <> concat gitStanzas)+ )+ where+ laneFlags = contracts.laneStanzas descriptor+ pins = concatMap (.componentPackages) descriptor.descriptorComponents+ safeAtom value = not (null value) && all (`notElem` ['\n', '\r', '\t', ' ', ',', '"']) value+ scopedException value = case Text.splitOn ":" value of+ [owner, dependency] -> all (\part -> not (Text.null part) && Text.all (\c -> c `elem` ['a' .. 'z'] || c `elem` ['0' .. '9'] || c == '-') part) [owner, dependency]+ _ -> False+ gitComponent component = case component.componentSource of+ HackageSource -> Right []+ GitSource location revision -> do+ unless ("https://" `Text.isPrefixOf` location && immutableRevision revision && safeAtom (Text.unpack location)) (Left "source project must use immutable HTTPS Git")+ let dirs = nub [maybe "." id pin.pinSubdir | pin <- component.componentPackages]+ unless (all (\path -> safeAtom (Text.unpack path) && not ("/" `Text.isPrefixOf` path) && all (/= "..") (Text.splitOn "/" path)) dirs) (Left "unsafe Git package subdirectory")+ pure ["", "source-repository-package", " type: git", " location: " <> location, " tag: " <> revision, " subdir: " <> Text.intercalate " " dirs]++instance ToJSON InventoryPackage where+ toJSON entry = object ["name" .= entry.name, "version" .= entry.version, "source" .= entry.source, "flags" .= entry.flags, "settings" .= entry.settings]++instance FromJSON InventoryPackage where+ parseJSON = withObject "InventoryPackage" \v -> do+ strictObject ["name", "version", "source", "flags", "settings"] v+ InventoryPackage <$> v .: "name" <*> v .: "version" <*> v .: "source" <*> v .: "flags" <*> v .: "settings"++instance ToJSON BuildInventory where+ toJSON inventory = object ["resolver" .= inventory.resolver, "compiler" .= inventory.compiler, "platform" .= inventory.platform, "toolchain" .= inventory.toolchain, "packages" .= sortOn (.name) inventory.packages]++instance FromJSON BuildInventory where+ parseJSON = withObject "BuildInventory" \v -> do+ strictObject ["resolver", "compiler", "platform", "toolchain", "packages"] v+ BuildInventory <$> v .: "resolver" <*> v .: "compiler" <*> v .: "platform" <*> v .: "toolchain" <*> v .: "packages"++instance ToJSON PreparedArm where+ toJSON arm = object ["descriptor" .= arm.descriptor, "sourceLock" .= arm.sourceLock, "boundsInventory" .= arm.boundsInventory, "buildInventory" .= arm.buildInventory, "payload" .= arm.payload, "buildKey" .= arm.buildKey]++instance FromJSON PreparedArm where+ parseJSON = withObject "PreparedArm" \v -> do+ strictObject ["descriptor", "sourceLock", "boundsInventory", "buildInventory", "payload", "buildKey"] v+ PreparedArm <$> v .: "descriptor" <*> v .: "sourceLock" <*> v .: "boundsInventory" <*> v .: "buildInventory" <*> v .: "payload" <*> v .: "buildKey"++instance ToJSON PerfPrepared where+ toJSON prepared = object ["schema" .= ("kenshou.perf-prepared/v1" :: Text), "plan" .= prepared.plan, "inputs" .= prepared.inputs, "harnessRevision" .= prepared.harnessRevision, "baseline" .= prepared.baseline, "candidate" .= prepared.candidate]++instance FromJSON PerfPrepared where+ parseJSON = withObject "PerfPrepared" \v -> do+ strictObject ["schema", "plan", "inputs", "harnessRevision", "baseline", "candidate"] v+ schema <- v .: "schema"+ unless (schema == ("kenshou.perf-prepared/v1" :: Text)) (fail "unsupported performance preparation schema")+ prepared <- PerfPrepared <$> v .: "plan" <*> v .: "inputs" <*> v .: "harnessRevision" <*> v .: "baseline" <*> v .: "candidate"+ unless (immutableRevision prepared.harnessRevision) (fail "invalid prepared harness revision")+ pure prepared
+ src/Kenshou/Perf/Request.hs view
@@ -0,0 +1,126 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Request+ ( PerfRequest (..),+ validateRequest,+ requestDigest,+ immutableRevision,+ canonicalProject,+ strictObject,+ )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser)+import Data.Char (isAsciiLower, isDigit)+import Data.List (nub, sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)++-- Lists are sets on the wire; encoding sorts them to give equivalent requests+-- the same identity. Duplicates are rejected rather than silently discarded.+data PerfRequest = PerfRequest+ { project :: Text,+ baselineRevision :: Text,+ candidateRevision :: Text,+ cohortTemplate :: FilePath,+ targetComponents :: [Text],+ suite :: Text,+ coverageMode :: Text,+ purpose :: Text,+ control :: Text,+ profiles :: [Text],+ seed :: Integer,+ cells :: [Text],+ waitSeconds :: Int,+ budgetMinutes :: Int+ }+ deriving stock (Eq, Show)++immutableRevision :: Text -> Bool+immutableRevision revision = Text.length revision == 40 && Text.all (\c -> c `elem` ['0' .. '9'] || c `elem` ['a' .. 'f']) revision++canonicalProject :: Text -> Bool+canonicalProject uri = case Text.splitOn "/" <$> Text.stripPrefix "mori://" uri of+ Just [namespace, projectName] -> valid namespace && valid projectName+ _ -> False+ where+ valid part = not (Text.null part) && Text.all (\c -> isAsciiLower c || isDigit c || c `elem` ("-_" :: String)) part++validateRequest :: PerfRequest -> Either Text ()+validateRequest request = do+ require (canonicalProject request.project) "project must be a canonical mori://namespace/project URI"+ require (all immutableRevision [request.baselineRevision, request.candidateRevision]) "revisions must be full lowercase 40-character Git commits"+ require (not (null request.cohortTemplate)) "cohortTemplate must be nonempty"+ require (request.suite == "change-performance") "unsupported performance suite"+ require (request.coverageMode `elem` ["affected", "selected-profiles"]) "unknown coverageMode"+ require (request.purpose `elem` ["gating", "calibration"]) "unknown purpose"+ require (request.control `elem` ["none", "aa"]) "unknown control"+ require (request.control /= "aa" || request.baselineRevision == request.candidateRevision) "A/A control requires identical revisions"+ require (request.control /= "aa" || not (null request.profiles)) "A/A control requires explicit profiles"+ require (request.coverageMode /= "selected-profiles" || not (null request.profiles)) "selected-profiles requires explicit profiles"+ require (request.seed >= 0 && request.seed <= 9007199254740991) "seed must be an exactly representable nonnegative JSON integer"+ require (request.waitSeconds >= 0 && request.budgetMinutes > 0) "waitSeconds must be nonnegative and budgetMinutes positive"+ require (not (null request.targetComponents) && not (null request.cells)) "targetComponents and cells must be nonempty"+ mapM_ validateSet [request.targetComponents, request.profiles, request.cells]+ where+ require condition message = unless condition (Left message)+ validateSet values = do+ require (all (\v -> not (Text.null v) && Text.strip v == v) values) "set entries must be nonempty and have no surrounding whitespace"+ require (nub values == values) "duplicate set entry"++requestDigest :: PerfRequest -> Text+requestDigest = sha256Hex . canonicalEncode . toJSON++strictObject :: [Text] -> Object -> Parser ()+strictObject allowed value = unless (null unknown) (fail ("unknown fields: " <> show unknown))+ where+ unknown = filter (`notElem` allowed) (map Key.toText (KeyMap.keys value))++instance ToJSON PerfRequest where+ toJSON request =+ object+ [ "schema" .= ("kenshou.perf-request/v1" :: Text),+ "project" .= request.project,+ "baselineRevision" .= request.baselineRevision,+ "candidateRevision" .= request.candidateRevision,+ "cohortTemplate" .= request.cohortTemplate,+ "targetComponents" .= sort request.targetComponents,+ "suite" .= request.suite,+ "coverageMode" .= request.coverageMode,+ "purpose" .= request.purpose,+ "control" .= request.control,+ "profiles" .= sort request.profiles,+ "seed" .= request.seed,+ "cells" .= sort request.cells,+ "waitSeconds" .= request.waitSeconds,+ "budgetMinutes" .= request.budgetMinutes+ ]++instance FromJSON PerfRequest where+ parseJSON = withObject "PerfRequest" \value -> do+ strictObject ["schema", "project", "baselineRevision", "candidateRevision", "cohortTemplate", "targetComponents", "suite", "coverageMode", "purpose", "control", "profiles", "seed", "cells", "waitSeconds", "budgetMinutes"] value+ schema <- value .: "schema"+ unless (schema == ("kenshou.perf-request/v1" :: Text)) (fail "unsupported performance request schema")+ request <-+ PerfRequest+ <$> value .: "project"+ <*> value .: "baselineRevision"+ <*> value .: "candidateRevision"+ <*> value .: "cohortTemplate"+ <*> value .: "targetComponents"+ <*> value .:? "suite" .!= "change-performance"+ <*> value .:? "coverageMode" .!= "affected"+ <*> value .:? "purpose" .!= "gating"+ <*> value .:? "control" .!= "none"+ <*> value .:? "profiles" .!= []+ <*> value .: "seed"+ <*> value .: "cells"+ <*> value .:? "waitSeconds" .!= 90+ <*> value .:? "budgetMinutes" .!= 30+ either (fail . Text.unpack) pure (validateRequest request)+ pure request {targetComponents = sort request.targetComponents, profiles = sort request.profiles, cells = sort request.cells}
+ src/Kenshou/Perf/Result.hs view
@@ -0,0 +1,192 @@+module Kenshou.Perf.Result+ ( ConfigurationResult (..),+ PerfResult (..),+ ArtifactReference (..),+ aggregateResult,+ validateResult,+ semanticComparison,+ )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.Types (Parser, parseEither)+import Data.List (nub, sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Perf.Plan+import Kenshou.Perf.Request++-- Attempts include unsuccessful infrastructure attempts. Only the complete,+-- healthy comparison represented by comparison supplies validPairs.+data ConfigurationResult = ConfigurationResult+ { profile :: Text,+ workloadSha256 :: Text,+ performanceVerdict :: Text,+ infrastructureState :: Text,+ runtimeFailures :: [Text],+ validPairs :: Int,+ requiredPairs :: Int,+ attempts :: [Value],+ comparison :: Maybe Value+ }+ deriving stock (Eq, Show)++data PerfResult = PerfResult+ { requestSha256 :: Text,+ preparedSha256 :: Text,+ purpose :: Text,+ coverage :: Text,+ changeCoverage :: Text,+ performanceVerdict :: Text,+ infrastructureState :: Text,+ blockingRuntimeFailures :: [Text],+ gatingEligible :: Bool,+ configurations :: [ConfigurationResult],+ exclusions :: [(Text, Text)],+ uncoveredScenarios :: [Text],+ evidenceVerified :: Bool,+ exitCode :: Int,+ stageSeconds :: Map Text Double,+ artifactReferences :: [ArtifactReference]+ }+ deriving stock (Eq, Show)++-- The root manifest and retrieval receipt anchor their own digests externally,+-- avoiding a self-referential report. Other links carry an artifact digest.+data ArtifactReference = ArtifactReference {kind :: Text, uri :: Text, sha256 :: Maybe Text}+ deriving stock (Eq, Show)++instance ToJSON ArtifactReference where+ toJSON reference = object ["kind" .= reference.kind, "uri" .= reference.uri, "sha256" .= reference.sha256]++instance FromJSON ArtifactReference where+ parseJSON = withObject "PerformanceArtifactReference" \v -> do+ strictObject ["kind", "uri", "sha256"] v+ reference <- ArtifactReference <$> v .: "kind" <*> v .: "uri" <*> v .:? "sha256"+ unless (not (Text.null reference.kind) && "gs://" `Text.isPrefixOf` reference.uri && not (Text.any (`elem` ['\n', '\r', ' ', '?', '#']) reference.uri) && maybe True digest reference.sha256) (fail "invalid performance artifact reference")+ pure reference++-- An aggregate's ordinary exit mapping also applies to calibration exercises;+-- their purpose and gatingEligible fields keep them out of production gates.+aggregateResult :: PerfPlan -> Text -> Bool -> [ConfigurationResult] -> Either Text PerfResult+aggregateResult plan preparedDigest verified configurations = do+ mapM_ validateConfiguration configurations+ let expected = sort [entry.profile.name | entry <- plan.selected]+ supplied = map (.profile) configurations+ unless (nub supplied == supplied && all (`elem` expected) supplied) (Left "unexpected or duplicate configuration result")+ mapM_ checkBinding configurations+ let complete = sort supplied == expected && all ((/= "pending") . (.infrastructureState)) configurations+ scope = if plan.coverage == "not-applicable" then "not-applicable" else if plan.coverage == "complete" && complete then "complete" else "incomplete"+ blockers = concatMap (.runtimeFailures) configurations+ infrastructure = if not verified || any ((== "failed") . (.infrastructureState)) configurations then "failed" else if any ((== "pending") . (.infrastructureState)) configurations || not complete then "pending" else "healthy"+ regression = any ((== "regression") . (.performanceVerdict)) configurations+ accepted = all ((== "pass") . (.performanceVerdict)) configurations && (not (null configurations) || scope == "not-applicable")+ verdict = if scope == "not-applicable" then "not-applicable" else if regression then "regression" else if accepted && scope == "complete" && infrastructure == "healthy" then "pass" else "inconclusive"+ code = outcomeExit infrastructure blockers scope verdict+ eligible = plan.gatingEligible && code == 0 && verdict == "pass"+ result = PerfResult plan.requestSha256 preparedDigest plan.request.purpose scope plan.changeCoverage verdict infrastructure blockers eligible configurations plan.excluded plan.uncoveredScenarios verified code Map.empty []+ validateResult result+ pure result+ where+ checkBinding result = case filter ((== result.profile) . (.name) . (.profile)) plan.selected of+ [entry] -> unless (result.workloadSha256 == entry.workloadSha256 && result.requiredPairs == entry.profile.minimumPairs) (Left "configuration workload or fixed pair count differs from pinned plan")+ _ -> Left "configuration has no unique selected profile"++outcomeExit :: Text -> [Text] -> Text -> Text -> Int+outcomeExit infrastructure blockers scope verdict+ | infrastructure == "failed" = 4+ | not (null blockers) || verdict == "regression" = 1+ | infrastructure /= "healthy" || scope == "incomplete" || verdict == "inconclusive" = 3+ | otherwise = 0++validateResult :: PerfResult -> Either Text ()+validateResult result = do+ unless (digest result.requestSha256 && digest result.preparedSha256) (Left "invalid aggregate input digest")+ unless (result.purpose `elem` ["gating", "calibration"] && result.coverage `elem` ["complete", "incomplete", "not-applicable"] && result.changeCoverage `elem` ["complete", "incomplete", "not-applicable"] && result.performanceVerdict `elem` ["pass", "regression", "inconclusive", "not-applicable"]) (Left "invalid aggregate purpose, scope or verdict")+ unless (result.infrastructureState `elem` ["healthy", "pending", "failed"]) (Left "invalid aggregate infrastructure state")+ mapM_ validateConfiguration result.configurations+ unless (all (\seconds -> not (isNaN seconds || isInfinite seconds) && seconds >= 0) (Map.elems result.stageSeconds)) (Left "invalid performance stage timing")+ mapM_ (\reference -> case fromJSON (toJSON reference) :: Result ArtifactReference of Error problem -> Left (Text.pack problem); Success _ -> Right ()) result.artifactReferences+ let names = map (.profile) result.configurations+ blockers = concatMap (.runtimeFailures) result.configurations+ regression = any ((== "regression") . (.performanceVerdict)) result.configurations+ unless (nub names == names && result.blockingRuntimeFailures == blockers) (Left "aggregate duplicates configurations or omits runtime blockers")+ unless (result.exitCode == outcomeExit result.infrastructureState blockers result.coverage result.performanceVerdict) (Left "aggregate exit mapping mismatch")+ unless (result.evidenceVerified || result.infrastructureState == "failed") (Left "unverified publication must be an infrastructure failure")+ unless (not (any ((== "failed") . (.infrastructureState)) result.configurations) || result.infrastructureState == "failed") (Left "aggregate hides unresolved infrastructure failure")+ unless (not regression || result.performanceVerdict == "regression") (Left "aggregate hides a regression")+ unless (not result.gatingEligible || (result.purpose == "gating" && result.exitCode == 0 && result.performanceVerdict == "pass")) (Left "ineligible aggregate claims a production gate")+ unless (result.performanceVerdict /= "pass" || (result.coverage == "complete" && result.infrastructureState == "healthy" && not (null result.configurations) && all ((== "pass") . (.performanceVerdict)) result.configurations)) (Left "aggregate performance pass lacks complete healthy evidence")+ unless (result.performanceVerdict /= "not-applicable" || (result.coverage == "not-applicable" && null result.configurations)) (Left "not-applicable aggregate contains executed configurations")++validateConfiguration :: ConfigurationResult -> Either Text ()+validateConfiguration result = do+ unless (not (Text.null result.profile) && digest result.workloadSha256 && result.requiredPairs >= 5 && result.validPairs >= 0 && result.validPairs <= result.requiredPairs) (Left "invalid configuration identity or pair count")+ unless (result.performanceVerdict `elem` ["pass", "regression", "inconclusive", "not-run"] && result.infrastructureState `elem` ["healthy", "pending", "failed"]) (Left "invalid configuration outcome")+ unless (result.performanceVerdict `notElem` ["pass", "regression"] || (result.validPairs == result.requiredPairs && result.infrastructureState == "healthy")) (Left "statistical verdict lacks complete healthy pairs")+ case result.comparison of+ Nothing -> unless (result.performanceVerdict `elem` ["inconclusive", "not-run"]) (Left "statistical verdict lacks its retained comparison")+ Just document ->+ either+ (Left . Text.pack)+ Right+ ( parseEither+ ( withObject "RetainedComparison" \v -> do+ schema <- v .: "schema"+ verdict <- v .: "verdict"+ count <- v .: "pairCount"+ let sameVerdict = verdict == result.performanceVerdict || (verdict == "infrastructure-failure" && result.infrastructureState == "failed" && result.performanceVerdict == "inconclusive")+ unless (schema == ("kenshou.comparison/v1" :: Text) && sameVerdict && count == result.validPairs) (fail "retained comparison differs from configuration result")+ )+ document+ )++digest :: Text -> Bool+digest value = "sha256:" `Text.isPrefixOf` value && Text.length value == 71 && Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) (Text.drop 7 value)++-- Reanalysis creates a new analysis ID, clock and host fingerprint. Compare the+-- evidence identities and calculation instead; preserve original sealed bytes.+semanticComparison :: Value -> Either Text Value+semanticComparison document = either (Left . Text.pack) Right (parseEither parser document)+ where+ parser = withObject "Comparison" \v -> do+ schema <- v .: "schema"+ unless (schema == ("kenshou.comparison/v1" :: Text)) (fail "unsupported retained comparison")+ fields <-+ traverse+ (\name -> (Key.fromText name,) <$> (v .: Key.fromText name :: Parser Value))+ ["schema", "design", "baselineRuns", "candidateRuns", "algorithm", "policy", "variedFactors", "pairCount", "metrics", "reasons", "verdict", "exitCode"]+ pure (object fields)++instance ToJSON ConfigurationResult where+ toJSON result = object ["profile" .= result.profile, "workloadSha256" .= result.workloadSha256, "performanceVerdict" .= result.performanceVerdict, "infrastructureState" .= result.infrastructureState, "runtimeFailures" .= result.runtimeFailures, "validPairs" .= result.validPairs, "requiredPairs" .= result.requiredPairs, "attempts" .= result.attempts, "comparison" .= result.comparison]++instance FromJSON ConfigurationResult where+ parseJSON = withObject "ConfigurationResult" \v -> do+ strictObject ["profile", "workloadSha256", "performanceVerdict", "infrastructureState", "runtimeFailures", "validPairs", "requiredPairs", "attempts", "comparison"] v+ result <- ConfigurationResult <$> v .: "profile" <*> v .: "workloadSha256" <*> v .: "performanceVerdict" <*> v .: "infrastructureState" <*> v .: "runtimeFailures" <*> v .: "validPairs" <*> v .: "requiredPairs" <*> v .: "attempts" <*> v .:? "comparison"+ either (fail . Text.unpack) pure (validateConfiguration result)+ pure result++instance ToJSON PerfResult where+ toJSON result = object ["schema" .= ("kenshou.perf-result/v1" :: Text), "requestSha256" .= result.requestSha256, "preparedSha256" .= result.preparedSha256, "purpose" .= result.purpose, "coverage" .= result.coverage, "changeCoverage" .= result.changeCoverage, "performanceVerdict" .= result.performanceVerdict, "infrastructureState" .= result.infrastructureState, "blockingRuntimeFailures" .= result.blockingRuntimeFailures, "gatingEligible" .= result.gatingEligible, "configurations" .= result.configurations, "exclusions" .= [object ["profile" .= name, "reason" .= reason] | (name, reason) <- result.exclusions], "uncoveredScenarios" .= result.uncoveredScenarios, "evidenceVerified" .= result.evidenceVerified, "exitCode" .= result.exitCode, "stageSeconds" .= result.stageSeconds, "artifactReferences" .= result.artifactReferences]++instance FromJSON PerfResult where+ parseJSON = withObject "PerfResult" \v -> do+ strictObject ["schema", "requestSha256", "preparedSha256", "purpose", "coverage", "changeCoverage", "performanceVerdict", "infrastructureState", "blockingRuntimeFailures", "gatingEligible", "configurations", "exclusions", "uncoveredScenarios", "evidenceVerified", "exitCode", "stageSeconds", "artifactReferences"] v+ schema <- v .: "schema"+ unless (schema == ("kenshou.perf-result/v1" :: Text)) (fail "unsupported performance result schema")+ exclusions <-+ v .: "exclusions"+ >>= traverse+ ( withObject "Exclusion" \e -> do+ strictObject ["profile", "reason"] e+ (,) <$> e .: "profile" <*> e .: "reason"+ )+ result <- PerfResult <$> v .: "requestSha256" <*> v .: "preparedSha256" <*> v .: "purpose" <*> v .: "coverage" <*> v .: "changeCoverage" <*> v .: "performanceVerdict" <*> v .: "infrastructureState" <*> v .: "blockingRuntimeFailures" <*> v .: "gatingEligible" <*> v .: "configurations" <*> pure exclusions <*> v .: "uncoveredScenarios" <*> v .: "evidenceVerified" <*> v .: "exitCode" <*> (v .:? "stageSeconds" .!= Map.empty) <*> (v .:? "artifactReferences" .!= [])+ either (fail . Text.unpack) pure (validateResult result)+ pure result
+ src/Kenshou/Perf/Source.hs view
@@ -0,0 +1,173 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Perf.Source+ ( PackageManifest (..),+ SourceSnapshot (..),+ parsePackageManifest,+ targetPackages,+ sourceSnapshot,+ descriptorForSnapshot,+ legacyDescriptorForSnapshot,+ )+where++import Control.Monad (unless)+import Data.Aeson+import Data.Char (isAsciiLower, isDigit)+import Data.List (nub, sort)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Cohort+import Kenshou.Perf.Request+import System.FilePath (takeDirectory)++data PackageManifest = PackageManifest+ { name :: Text,+ version :: Text,+ path :: FilePath,+ sha256 :: Text+ }+ deriving stock (Eq, Show)++data SourceSnapshot = SourceSnapshot+ { project :: Text,+ location :: Text,+ revision :: Text,+ packages :: [PackageManifest]+ }+ deriving stock (Eq, Show)++-- Source manifests are inspected without executing candidate code. Cabal will+-- perform the full syntax/bounds validation during preparation. Reject an+-- ambiguous identity here rather than guessing which declaration Cabal means.+parsePackageManifest :: FilePath -> Text -> Either Text PackageManifest+parsePackageManifest path contents = do+ name <- field "name"+ version <- field "version"+ unless (validName name && validVersion version) (Left ("invalid manifest identity: " <> Text.pack path))+ unless (safePath path) (Left "unsafe manifest path")+ pure (PackageManifest name version path (sha256Hex (Text.encodeUtf8 contents)))+ where+ field key = case [Text.strip (fst (Text.breakOn "--" rest)) | line <- Text.lines contents, let (label, value) = Text.breakOn ":" line, Text.toLower (Text.strip label) == key, let rest = Text.drop 1 value] of+ [value] | not (Text.null value) -> Right value+ _ -> Left ("missing or ambiguous " <> key <> " in " <> Text.pack path)+ validName value = not (Text.null value) && Text.all (\c -> isAsciiLower c || isDigit c || c == '-') value+ validVersion value = all (\part -> not (Text.null part) && Text.all isDigit part) (Text.splitOn "." value)++targetPackages :: PerfRequest -> CohortDescriptor -> Either Text [Text]+targetPackages request descriptor = do+ validateRequest request+ unless (nub ids == ids && nub names == names) (Left "cohort component/package IDs are not unique")+ unless (all (`elem` ids) request.targetComponents) (Left "unknown target component in cohort template")+ unless (all owns targets) (Left "all target components must belong to the request's source project")+ unless (not (null targets) && all (not . null . (.componentPackages)) targets) (Left "target components must declare packages")+ mapM_ checkSource descriptor.descriptorComponents+ pure (sort (concatMap (map (.pinName) . (.componentPackages)) targets))+ where+ ids = map (unComponentId . (.componentId)) descriptor.descriptorComponents+ names = concatMap (map (.pinName) . (.componentPackages)) descriptor.descriptorComponents+ targets = filter ((`elem` request.targetComponents) . unComponentId . (.componentId)) descriptor.descriptorComponents+ owns component = component.componentMoriUri == request.project || (request.project <> "/packages/") `Text.isPrefixOf` component.componentMoriUri+ checkSource component = case component.componentSource of+ HackageSource -> Right ()+ GitSource location revision -> unless (validLocation location && immutableRevision revision) (Left "template Git sources must be immutable HTTPS sources")++sourceSnapshot :: PerfRequest -> CohortDescriptor -> Text -> Text -> [PackageManifest] -> Either Text SourceSnapshot+sourceSnapshot request descriptor location revision manifests = do+ expected <- targetPackages request descriptor+ unless (validLocation location && immutableRevision revision) (Left "source snapshot requires immutable HTTPS revision")+ unless (revision `elem` [request.baselineRevision, request.candidateRevision]) (Left "source snapshot is not one of the requested revisions")+ unless (all sameRepository targets) (Left "target template Git source belongs to another repository")+ let selected = filter ((`elem` expected) . (.name)) manifests+ unless (sort (map (.name) selected) == expected) (Left "source revision has a missing or duplicate target package")+ mapM_ validateManifest selected+ pure (SourceSnapshot request.project location revision (sortOnName selected))+ where+ targets = filter ((`elem` request.targetComponents) . unComponentId . (.componentId)) descriptor.descriptorComponents+ sameRepository component = case component.componentSource of+ HackageSource -> True+ GitSource upstream _ -> Text.dropWhileEnd (== '/') (maybe upstream id (Text.stripSuffix ".git" upstream)) == Text.dropWhileEnd (== '/') (maybe location id (Text.stripSuffix ".git" location))+ sortOnName values = map snd (Map.toAscList (Map.fromList [(value.name, value) | value <- values]))++descriptorForSnapshot :: PerfRequest -> CohortDescriptor -> Text -> SourceSnapshot -> Either Text CohortDescriptor+descriptorForSnapshot = descriptorWithIdentity False++-- Read compatibility for previously sealed v1 preparations. New preparation+-- always uses the resolved-cohort identity; legacy names remain bound to the+-- exact retained request and independent manifests, never accepted arbitrarily.+legacyDescriptorForSnapshot :: PerfRequest -> CohortDescriptor -> Text -> SourceSnapshot -> Either Text CohortDescriptor+legacyDescriptorForSnapshot = descriptorWithIdentity True++descriptorWithIdentity :: Bool -> PerfRequest -> CohortDescriptor -> Text -> SourceSnapshot -> Either Text CohortDescriptor+descriptorWithIdentity legacy request descriptor arm snapshot = do+ expected <- targetPackages request descriptor+ unless (snapshot.project == request.project && sort (map (.name) snapshot.packages) == expected) (Left "snapshot project/package inventory differs from the request")+ unless (validLocation snapshot.location && immutableRevision snapshot.revision) (Left "snapshot source must be immutable HTTPS Git")+ unless (arm `elem` ["baseline", "candidate"]) (Left "unknown comparison arm")+ unless (snapshot.revision == if arm == "baseline" then request.baselineRevision else request.candidateRevision) (Left "snapshot revision differs from its arm")+ mapM_ validateManifest snapshot.packages+ components <- traverse replace descriptor.descriptorComponents+ let effectiveArm = if request.baselineRevision == request.candidateRevision then "aa" else arm+ resolved =+ descriptor+ { descriptorName = CohortName "perf",+ descriptorDescription = "Exact-revision performance cohort for " <> request.project <> "; template scope: " <> descriptor.descriptorDescription,+ descriptorProjectFile = "perf.project",+ descriptorComponents = components+ }+ generatedName =+ if legacy+ then "perf-" <> Text.take 16 (Text.drop 7 (requestDigest request)) <> "-" <> effectiveArm+ else "perf-" <> Text.drop 7 (sha256Hex (canonicalEncode (toJSON resolved)))+ pure+ resolved+ { descriptorName = CohortName generatedName,+ descriptorDescription = if legacy then "Exact-revision " <> effectiveArm <> " for " <> request.project <> "; template scope: " <> descriptor.descriptorDescription else resolved.descriptorDescription,+ descriptorProjectFile = Text.unpack generatedName <> ".project"+ }+ where+ replace component+ | unComponentId component.componentId `notElem` request.targetComponents = Right component+ | otherwise = do+ pins <- traverse pin component.componentPackages+ pure component {componentSource = GitSource snapshot.location snapshot.revision, componentPackages = pins}+ pin previous = case filter ((== previous.pinName) . (.name)) snapshot.packages of+ [manifest] -> pure previous {pinVersion = manifest.version, pinSubdir = let directory = takeDirectory manifest.path in if directory == "." then Nothing else Just (Text.pack directory)}+ _ -> Left "snapshot package identity mismatch"++instance ToJSON PackageManifest where+ toJSON manifest = object ["name" .= manifest.name, "version" .= manifest.version, "path" .= manifest.path, "sha256" .= manifest.sha256]++instance FromJSON PackageManifest where+ parseJSON = withObject "PackageManifest" \value -> do+ strictObject ["name", "version", "path", "sha256"] value+ manifest <- PackageManifest <$> value .: "name" <*> value .: "version" <*> value .: "path" <*> value .: "sha256"+ either (fail . Text.unpack) pure (validateManifest manifest)+ pure manifest++instance ToJSON SourceSnapshot where+ toJSON snapshot = object ["project" .= snapshot.project, "location" .= snapshot.location, "revision" .= snapshot.revision, "packages" .= snapshot.packages]++instance FromJSON SourceSnapshot where+ parseJSON = withObject "SourceSnapshot" \value -> do+ strictObject ["project", "location", "revision", "packages"] value+ snapshot <- SourceSnapshot <$> value .: "project" <*> value .: "location" <*> value .: "revision" <*> value .: "packages"+ unless (canonicalProject snapshot.project && validLocation snapshot.location && immutableRevision snapshot.revision) (fail "invalid immutable source identity")+ unless (not (null snapshot.packages) && nub (map (.name) snapshot.packages) == map (.name) snapshot.packages) (fail "invalid source package inventory")+ pure snapshot++validLocation :: Text -> Bool+validLocation location = "https://" `Text.isPrefixOf` location && Text.length location > 8 && not (Text.any (`elem` ['@', '#', '?', '\n', '\r', ' ']) location)++safePath :: FilePath -> Bool+safePath value = not (null value) && not ("/" `Text.isPrefixOf` Text.pack value) && all (`notElem` ["", ".", ".."]) (Text.splitOn "/" (Text.pack value)) && not (Text.any (`elem` ['\n', '\r', '\t', ' ', '\\']) (Text.pack value))++validateManifest :: PackageManifest -> Either Text ()+validateManifest manifest = do+ unless (not (Text.null manifest.name) && Text.all (\c -> isAsciiLower c || isDigit c || c == '-') manifest.name) (Left "invalid manifest package name")+ unless (all (\part -> not (Text.null part) && Text.all isDigit part) (Text.splitOn "." manifest.version)) (Left "invalid manifest package version")+ unless (safePath manifest.path && ".cabal" `Text.isSuffixOf` Text.pack manifest.path) (Left "unsafe manifest path")+ unless (maybe False (\digest -> Text.length digest == 64 && Text.all (\c -> c `elem` ['0' .. '9'] || c `elem` ['a' .. 'f']) digest) (Text.stripPrefix "sha256:" manifest.sha256)) (Left "invalid manifest digest")
+ src/Kenshou/Plan/Catalog.hs view
@@ -0,0 +1,208 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Catalog+ ( ScenarioInfo (..),+ CatalogDocument (..),+ fromScenario,+ fromBundles,+ readCatalogFile,+ decodeCatalog,+ )+where++import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (Parser)+import Data.Bifunctor (first)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Bundle (LayerBundle, allScenarios, mkRegistry)+import Kenshou.Core.Dimension+import Kenshou.Core.Id (ScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Scenario++data ScenarioInfo = ScenarioInfo+ { id :: ScenarioId,+ revision :: Int,+ summary :: Text,+ tier :: Tier,+ placement :: Placement,+ knobs :: [KnobSpec],+ dimensions :: DimensionSupport,+ knownDefect :: Maybe KnownDefect+ }+ deriving stock (Eq, Show)++newtype CatalogDocument = CatalogDocument {scenarios :: [ScenarioInfo]}+ deriving stock (Eq, Show)++fromScenario :: Scenario -> ScenarioInfo+fromScenario scenario =+ ScenarioInfo+ { id = scenario.id,+ revision = scenario.revision,+ summary = scenario.summary,+ tier = scenario.tier,+ placement = scenario.placement,+ knobs = scenario.knobs,+ dimensions = scenario.dimensions,+ knownDefect = scenario.knownDefect+ }++fromBundles :: [LayerBundle] -> [ScenarioInfo]+fromBundles bundles = case mkRegistry bundles of+ Left _ -> []+ Right registry -> fmap fromScenario (allScenarios registry)++readCatalogFile :: FilePath -> IO (Either Text [ScenarioInfo])+readCatalogFile path = decodeCatalog <$> ByteString.readFile path++decodeCatalog :: ByteString -> Either Text [ScenarioInfo]+decodeCatalog bytes = first Text.pack do+ document <- eitherDecodeStrict' bytes :: Either String CatalogDocument+ pure document.scenarios++instance FromJSON CatalogDocument where+ parseJSON = withObject "CatalogDocument" \value -> do+ schema <- value .:? "schema" .!= ("kenshou.scenario-list/v1" :: Text)+ if schema /= "kenshou.scenario-list/v1" then fail "unsupported catalog schema" else pure ()+ CatalogDocument <$> value .: "scenarios"++instance ToJSON CatalogDocument where+ toJSON document = object ["schema" .= ("kenshou.scenario-list/v1" :: Text), "scenarios" .= document.scenarios, "roles" .= ([] :: [Value])]++instance FromJSON ScenarioInfo where+ parseJSON = withObject "ScenarioInfo" \value ->+ ScenarioInfo+ <$> value .: "id"+ <*> value .:? "revision" .!= 1+ <*> value .:? "summary" .!= ""+ <*> (value .:? "tier" .!= "smoke" >>= parseTier)+ <*> (value .:? "placement" .!= "either" >>= parsePlacement)+ <*> value .:? "knobs" .!= []+ <*> value .:? "dimensions" .!= noDimensions+ <*> pure Nothing++instance ToJSON ScenarioInfo where+ toJSON scenario =+ object+ [ "id" .= scenario.id,+ "revision" .= scenario.revision,+ "summary" .= scenario.summary,+ "tier" .= renderTier scenario.tier,+ "placement" .= renderPlacement scenario.placement,+ "knobs" .= scenario.knobs,+ "dimensions" .= scenario.dimensions,+ "knownDefect" .= fmap defectReferences scenario.knownDefect+ ]+ where+ defectReferences defect = case individualKnownDefects defect of+ [single] -> toJSON single.reference+ many -> toJSON (fmap (.reference) many)++parseTier :: Text -> Parser Tier+parseTier = \case+ "smoke" -> pure TierSmoke+ "standard" -> pure TierStandard+ "extended" -> pure TierExtended+ "soak" -> pure TierSoak+ value -> fail ("unknown tier " <> Text.unpack value)++parsePlacement :: Text -> Parser Placement+parsePlacement = \case+ "local" -> pure PlaceLocal+ "cell" -> pure PlaceCell+ "either" -> pure PlaceEither+ value -> fail ("unknown placement " <> Text.unpack value)++instance ToJSON KnobSpec where+ toJSON spec =+ object+ [ "name" .= spec.name,+ "summary" .= spec.summary,+ "type" .= knobTypeText spec.knobType,+ "default" .= spec.def,+ "allowed" .= allowedValue spec.allowed,+ "variants" .= spec.variants+ ]++instance FromJSON KnobSpec where+ parseJSON = withObject "KnobSpec" \value -> do+ name <- value .: "name"+ summary <- value .:? "summary" .!= ""+ knobType <- value .: "type" >>= parseKnobType+ def <- value .: "default"+ allowed <- value .:? "allowed" >>= maybe (pure AnyValue) parseAllowed+ variants <- value .:? "variants" .!= []+ pure KnobSpec {name, summary, knobType, def, allowed, variants}++knobTypeText :: KnobType -> Text+knobTypeText KnobBool = "bool"+knobTypeText KnobInt = "int"+knobTypeText KnobDouble = "double"+knobTypeText KnobText = "text"++parseKnobType :: Text -> Parser KnobType+parseKnobType = \case+ "bool" -> pure KnobBool+ "int" -> pure KnobInt+ "double" -> pure KnobDouble+ "text" -> pure KnobText+ value -> fail ("unknown knob type " <> Text.unpack value)++allowedValue :: Allowed -> Value+allowedValue AnyValue = object ["kind" .= ("any" :: Text)]+allowedValue (OneOf values) = object ["kind" .= ("one-of" :: Text), "values" .= values]+allowedValue (IntRange low high) = object ["kind" .= ("int-range" :: Text), "min" .= low, "max" .= high]+allowedValue (DoubleRange low high) = object ["kind" .= ("double-range" :: Text), "min" .= low, "max" .= high]++parseAllowed :: Value -> Parser Allowed+parseAllowed = withObject "Allowed" \value ->+ value .: "kind" >>= \case+ ("any" :: Text) -> pure AnyValue+ "one-of" -> do+ values <- value .: "values"+ case values of [] -> fail "one-of must not be empty"; first : rest -> pure (OneOf (first :| rest))+ "int-range" -> IntRange <$> value .: "min" <*> value .: "max"+ "double-range" -> DoubleRange <$> value .: "min" <*> value .: "max"+ other -> fail ("unknown allowed kind " <> Text.unpack other)++instance ToJSON DimensionSupport where+ toJSON support =+ object+ [ "telemetry.tracing" .= supported renderTracing support.tracing,+ "telemetry.metrics" .= supported renderMetrics support.metrics,+ "pg.durability" .= supported renderDurability support.pgDurability,+ "pg.version" .= supported renderVersion support.pgVersion+ ]+ where+ supported _ NotApplicable = Null+ supported render (Supported value) = object ["values" .= fmap render value.values, "default" .= render value.def]++instance FromJSON DimensionSupport where+ parseJSON = withObject "DimensionSupport" \value ->+ DimensionSupport+ <$> parseSupported value "telemetry.tracing" parseTracing+ <*> parseSupported value "telemetry.metrics" parseMetrics+ <*> parseSupported value "pg.durability" parseDurability+ <*> parseSupported value "pg.version" parseVersion+ where+ parseSupported objectValue name parse = case KeyMap.lookup (Key.fromText name) objectValue of+ Nothing -> pure NotApplicable+ Just Null -> pure NotApplicable+ Just raw ->+ withObject+ "Support"+ ( \supportValue -> do+ rawValues <- supportValue .: "values"+ rawDefault <- supportValue .: "default"+ values <- traverse (maybe (fail ("unknown value in " <> Text.unpack name)) pure . parse) rawValues+ def <- maybe (fail ("unknown default in " <> Text.unpack name)) pure (parse rawDefault)+ case values of [] -> fail "dimension support must not be empty"; first : rest -> pure (Supported (Support (first :| rest) def))+ )+ raw
+ src/Kenshou/Plan/Change.hs view
@@ -0,0 +1,131 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Change+ ( ChangeSource (..),+ Change (..),+ Warning (..),+ Reason (..),+ Selected (..),+ selectScenarios,+ selectAll,+ selectBySelectors,+ applySelectors,+ )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.List (sortOn)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (mapMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Components+import Kenshou.Plan.Selector++data ChangeSource = Named | CohortDiff | Since | UpstreamDiff | Everything+ deriving stock (Eq, Ord, Show)++data Change = Change+ { ref :: ComponentRef,+ source :: ChangeSource,+ detail :: Text+ }+ deriving stock (Eq, Show)++data Warning = Warning {code :: Text, message :: Text}+ deriving stock (Eq, Show)++data Reason = Reason+ { change :: Change,+ via :: [ComponentRef],+ selector :: Selector,+ distance :: Int+ }+ deriving stock (Eq, Show)++data Selected = Selected+ { scenario :: ScenarioInfo,+ reasons :: NonEmpty Reason,+ minPolicy :: Maybe Text,+ minKnobPolicy :: Maybe Text+ }+ deriving stock (Eq, Show)++selectScenarios :: ComponentGraph -> [ScenarioInfo] -> [Change] -> [Selected]+selectScenarios graph catalog changes+ | any ((== Everything) . (.source)) changes = selectAll catalog+ | otherwise = sortOn (scenarioKey . (.scenario)) (mapMaybe selected catalog)+ where+ affectedByChange changeValue = dependents graph (Set.singleton changeValue.ref)+ closure = [(changeValue, affectedByChange changeValue) | changeValue <- changes]+ selected scenario = case sortOn reasonKey (concatMap (reasonsFor scenario) closure) of+ [] -> Nothing+ first : rest ->+ Just+ Selected+ { scenario,+ reasons = first :| rest,+ minPolicy = maximumPolicy [minPolicyForRef graph changeValue.ref | changeValue <- changes, any ((== changeValue) . (.change)) (first : rest)],+ minKnobPolicy = Nothing+ }+ reasonsFor scenario (changeValue, affected) = do+ affectedValue <- Map.elems affected+ selector <- selectorsForRef graph affectedValue.ref+ if matches selector scenario.id+ then pure (Reason changeValue affectedValue.path selector affectedValue.distance)+ else []+ reasonKey reason = (reason.distance, renderRef reason.change.ref, renderSelector reason.selector)+ scenarioKey = show . (.id)++selectAll :: [ScenarioInfo] -> [Selected]+selectAll catalog = case parseSelector "**" of+ Left _ -> []+ Right selector ->+ [ Selected scenario (Reason change [change.ref] selector 0 :| []) Nothing Nothing+ | scenario <- catalog+ ]+ where+ change = Change (ComponentRef (ComponentId "all") Nothing) Everything "all scenarios"++selectBySelectors :: ChangeSource -> Text -> [Selector] -> [ScenarioInfo] -> [Selected]+selectBySelectors source detail selectors catalog =+ [ Selected scenario (Reason change [change.ref] selector 0 :| []) Nothing Nothing+ | scenario <- catalog,+ selector <- take 1 (filter (`matches` scenario.id) selectors)+ ]+ where+ change = Change (ComponentRef (ComponentId "repository-path") Nothing) source detail++applySelectors :: [Selector] -> [Selector] -> [Selected] -> [Selected]+applySelectors includes excludes = filter keep+ where+ keep selectedValue =+ (null includes || any (`matches` selectedValue.scenario.id) includes)+ && not (any (`matches` selectedValue.scenario.id) excludes)++maximumPolicy :: [Maybe Text] -> Maybe Text+maximumPolicy policies = case mapMaybe (\value -> value) policies of+ [] -> Nothing+ values -> Just (maximumByRank values)+ where+ maximumByRank = foldl1 (\left right -> if rank left >= rank right then left else right)+ rank "default-only" = (0 :: Int)+ rank "telemetry-corners" = 1+ rank "pairwise" = 2+ rank "full" = 3+ rank _ = 0++instance ToJSON Change where+ toJSON change = object ["component" .= renderRef change.ref, "source" .= renderChangeSource change.source, "detail" .= change.detail]++instance ToJSON Reason where+ toJSON reason = object ["component" .= renderRef reason.change.ref, "source" .= renderChangeSource reason.change.source, "via" .= fmap renderRef reason.via, "selector" .= renderSelector reason.selector, "distance" .= reason.distance]++renderChangeSource :: ChangeSource -> Text+renderChangeSource Named = "named"+renderChangeSource CohortDiff = "cohort-diff"+renderChangeSource Since = "since"+renderChangeSource UpstreamDiff = "upstream-diff"+renderChangeSource Everything = "all"
+ src/Kenshou/Plan/Change/Cohort.hs view
@@ -0,0 +1,87 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Change.Cohort+ ( CohortInput (..),+ PackageSource (..),+ parseCohortInput,+ readCohortPackages,+ packagesFromDescriptor,+ diffCohorts,+ )+where++import Data.Aeson qualified as Aeson+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Kenshou.Core.Cohort qualified as Cohort+import Kenshou.Plan.Change+import Kenshou.Plan.Components+import System.Exit (ExitCode (..))+import System.Process (readProcessWithExitCode)++data CohortInput = CohortName Text | CohortFile FilePath | CohortGitObject Text+ deriving stock (Eq, Show)++data PackageSource = FromHackage Text | FromGit Text Text+ deriving stock (Eq, Ord, Show)++parseCohortInput :: Text -> CohortInput+parseCohortInput value+ | ":" `Text.isInfixOf` value = CohortGitObject value+ | "/" `Text.isInfixOf` value || ".json" `Text.isSuffixOf` value = CohortFile (Text.unpack value)+ | otherwise = CohortName value++readCohortPackages :: FilePath -> CohortInput -> IO (Either Text (Map Text PackageSource))+readCohortPackages root = \case+ CohortName name -> loadFile (root <> "/cohort/" <> Text.unpack name <> ".json")+ CohortFile path -> loadFile path+ CohortGitObject objectName -> do+ (code, output, err) <- readProcessWithExitCode "git" ["-C", root, "show", Text.unpack objectName] ""+ pure case code of+ ExitSuccess -> decodeBytes (Text.Encoding.encodeUtf8 (Text.pack output))+ _ -> Left (Text.strip (Text.pack err))+ where+ loadFile path = do+ descriptor <- Cohort.loadCohortDescriptor path+ pure (either (Left . Text.pack . show) (Right . packagesFromDescriptor) descriptor)+ decodeBytes bytes = case Aeson.eitherDecodeStrict' bytes of+ Left err -> Left (Text.pack err)+ Right descriptor -> Right (packagesFromDescriptor descriptor)++packagesFromDescriptor :: Cohort.CohortDescriptor -> Map Text PackageSource+packagesFromDescriptor descriptor = Map.fromList do+ component <- descriptor.descriptorComponents+ package <- component.componentPackages+ pure+ ( package.pinName,+ case component.componentSource of+ Cohort.HackageSource -> FromHackage package.pinVersion+ Cohort.GitSource gitLocation gitRevision -> FromGit gitLocation gitRevision+ )++diffCohorts :: ComponentGraph -> Map Text PackageSource -> Map Text PackageSource -> ([Change], [Warning])+diffCohorts graph old new = (changes, warnings)+ where+ packageOwners = Map.fromList [(package, componentValue.id) | componentValue <- graph.components, package <- componentValue.packages]+ names = Map.keysSet old <> Map.keysSet new+ changed = [name | name <- Set.toAscList names, Map.lookup name old /= Map.lookup name new]+ mapped = [(name, owner) | name <- changed, Just owner <- [Map.lookup name packageOwners]]+ changes =+ [ Change (ComponentRef owner Nothing) CohortDiff (name <> " " <> renderSource (Map.lookup name old) <> " -> " <> renderSource (Map.lookup name new))+ | (name, owner) <- mapped+ ]+ <> [Change (ComponentRef (ComponentId "all") Nothing) Everything ("unmapped package " <> name) | name <- changed, Map.notMember name packageOwners, name `notElem` graph.ignoredPackages]+ warnings =+ [ Warning "unmapped-package" ("unmapped package " <> name <> "; selecting everything")+ | name <- changed,+ Map.notMember name packageOwners,+ name `notElem` graph.ignoredPackages+ ]+ <> [Warning "ignored-package" ("ignored package changed: " <> name) | name <- changed, name `elem` graph.ignoredPackages]+ renderSource Nothing = "absent"+ renderSource (Just (FromHackage version)) = version+ renderSource (Just (FromGit _ revision)) = "git " <> revision
+ src/Kenshou/Plan/Change/Git.hs view
@@ -0,0 +1,128 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Change.Git+ ( UpstreamDiff (..),+ parseUpstreamDiff,+ changesSince,+ changesFromUpstream,+ )+where++import Data.Aeson qualified as Aeson+import Data.ByteString qualified as ByteString+import Data.List (isPrefixOf, maximumBy, nub)+import Data.Ord (comparing)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Kenshou.Core.Cohort (CohortDescriptor)+import Kenshou.Plan.Change hiding (UpstreamDiff)+import Kenshou.Plan.Change qualified as Change+import Kenshou.Plan.Change.Cohort (diffCohorts, packagesFromDescriptor)+import Kenshou.Plan.Components+import Kenshou.Plan.Selector (Selector)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Process (readProcessWithExitCode)++data UpstreamDiff = UpstreamDiff+ { repository :: Text,+ path :: FilePath,+ revisionA :: Text,+ revisionB :: Text+ }+ deriving stock (Eq, Show)++parseUpstreamDiff :: Text -> Either Text UpstreamDiff+parseUpstreamDiff raw = do+ let (repository, afterEquals) = Text.breakOn "=" raw+ withoutEquals = Text.drop 1 afterEquals+ (pathAndAt, revisions) = Text.breakOnEnd "@" withoutEquals+ path = Text.dropEnd 1 pathAndAt+ (revisionA, dotsAndB) = Text.breakOn ".." revisions+ revisionB = Text.drop 2 dotsAndB+ if any Text.null [repository, afterEquals, path, revisionA, dotsAndB, revisionB]+ then Left "expected REPO=PATH@REVA..REVB"+ else Right (UpstreamDiff repository (Text.unpack path) revisionA revisionB)++changesSince :: ComponentGraph -> FilePath -> Text -> IO (Either Text ([Change], [Selector], [Warning]))+changesSince graph root revision = do+ tracked <- git root ["diff", "--name-only", Text.unpack revision]+ untracked <- git root ["ls-files", "--others", "--exclude-standard"]+ case (tracked, untracked) of+ (Right trackedPaths, Right untrackedPaths) -> classifyPaths graph root revision (nub (trackedPaths <> untrackedPaths))+ (Left err, _) -> pure (Left err)+ (_, Left err) -> pure (Left err)++classifyPaths :: ComponentGraph -> FilePath -> Text -> [FilePath] -> IO (Either Text ([Change], [Selector], [Warning]))+classifyPaths graph root revision paths = do+ classified <- traverse classify paths+ pure do+ values <- sequence classified+ let changes = concatMap (\(items, _, _) -> items) values+ selectors = concatMap (\(_, items, _) -> items) values+ warnings = concatMap (\(_, _, items) -> items) values+ Right (changes, selectors, warnings)+ where+ classify path = case longestRule graph path of+ Nothing -> pure (Right ([everything ("unmapped repository path " <> Text.pack path)], [], [Warning "unmapped-path" ("unmapped repository path " <> Text.pack path <> "; selecting everything")]))+ Just rule -> case rule.effect of+ PathNothing -> pure (Right ([], [], []))+ PathAll -> pure (Right ([everything (Text.pack path)], [], []))+ PathSelectors selectors -> pure (Right ([], selectors, []))+ PathCohort -> classifyCohort path+ classifyCohort path+ | path == "cohort/active.project" || ".project" `Text.isSuffixOf` Text.pack path =+ pure (Right ([everything (Text.pack path)], [], [Warning "cohort-project-change" ("cohort project changed without a safe descriptor mapping: " <> Text.pack path)]))+ | ".json" `Text.isSuffixOf` Text.pack path = do+ oldBytes <- gitShow root (revision <> ":" <> Text.pack path)+ newBytes <- ByteString.readFile (root </> path)+ pure do+ old <- oldBytes >>= decodeDescriptor+ new <- decodeDescriptor newBytes+ let (changes, warnings) = diffCohorts graph (packagesFromDescriptor old) (packagesFromDescriptor new)+ Right (changes, [], warnings)+ | otherwise = pure (Right ([everything (Text.pack path)], [], [Warning "unknown-cohort-change" ("cannot map cohort change " <> Text.pack path)]))+ everything detail = Change (ComponentRef (ComponentId "all") Nothing) Everything detail++changesFromUpstream :: ComponentGraph -> UpstreamDiff -> IO (Either Text ([Change], Int))+changesFromUpstream graph input = do+ result <- git input.path ["diff", "--name-only", Text.unpack input.revisionA, Text.unpack input.revisionB]+ pure do+ paths <- result+ let mapped = fmap (mapUpstreamPath graph input.repository) paths+ changes = nub [change | Just change <- mapped]+ ignored = length [() | Nothing <- mapped]+ Right (changes, ignored)++mapUpstreamPath :: ComponentGraph -> Text -> FilePath -> Maybe Change+mapUpstreamPath graph repository path = do+ componentValue <- bestComponent+ let subMatches = [(prefix, subcomponentValue.id) | subcomponentValue <- componentValue.subcomponents, prefix <- subcomponentValue.pathPrefixes, prefix `isPrefixOf` path]+ reference = case subMatches of+ [] -> ComponentRef componentValue.id Nothing+ _ -> ComponentRef componentValue.id (Just (snd (maximumBy (comparing (length . fst)) subMatches)))+ pure (Change reference Change.UpstreamDiff (repository <> ":" <> Text.pack path))+ where+ candidates = [componentValue | componentValue <- graph.components, componentValue.repository == repository, any (`isPrefixOf` path) componentValue.sourceRoots]+ bestComponent = case candidates of [] -> Nothing; _ -> Just (maximumBy (comparing (maximum . (0 :) . fmap length . (.sourceRoots))) candidates)++longestRule :: ComponentGraph -> FilePath -> Maybe RepositoryPathRule+longestRule graph path = case filter ((`isPrefixOf` path) . (.prefix)) graph.repositoryPaths of+ [] -> Nothing+ rules -> Just (maximumBy (comparing (length . (.prefix))) rules)++git :: FilePath -> [String] -> IO (Either Text [FilePath])+git root arguments = do+ (code, output, err) <- readProcessWithExitCode "git" (["-C", root] <> arguments) ""+ pure case code of+ ExitSuccess -> Right (filter (not . null) (lines output))+ _ -> Left (Text.strip (Text.pack err))++gitShow :: FilePath -> Text -> IO (Either Text ByteString.ByteString)+gitShow root objectName = do+ (code, output, err) <- readProcessWithExitCode "git" ["-C", root, "show", Text.unpack objectName] ""+ pure case code of ExitSuccess -> Right (Text.Encoding.encodeUtf8 (Text.pack output)); _ -> Left (Text.strip (Text.pack err))++decodeDescriptor :: ByteString.ByteString -> Either Text CohortDescriptor+decodeDescriptor = either (Left . Text.pack) Right . Aeson.eitherDecodeStrict'
+ src/Kenshou/Plan/Components.hs view
@@ -0,0 +1,347 @@+{-# LANGUAGE FieldSelectors #-}+{-# LANGUAGE TemplateHaskell #-}++module Kenshou.Plan.Components+ ( ComponentId (..),+ SubId (..),+ ComponentRef (..),+ EdgeKind (..),+ ComponentKind (..),+ ChangeImplies (..),+ Subcomponent (..),+ DependencyEdge (..),+ Component (..),+ PathEffect (..),+ RepositoryPathRule (..),+ ComponentGraph (..),+ Affected (..),+ GraphProblem (..),+ GraphError (..),+ selftestGraph,+ selftestGraphBytes,+ loadGraphFile,+ decodeGraph,+ validateGraph,+ parseRef,+ renderRef,+ dependents,+ graphDigest,+ componentById,+ selectorsForRef,+ minPolicyForRef,+ )+where++import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.FileEmbed (embedFile)+import Data.Graph (SCC (..), stronglyConnComp)+import Data.List (group, nub, sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Sequence (Seq (..), (|>))+import Data.Sequence qualified as Seq+import Data.Set (Set)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Plan.Selector (Selector, parseSelector, renderSelector)++newtype ComponentId = ComponentId {unComponentId :: Text}+ deriving stock (Eq, Ord, Show)++newtype SubId = SubId {unSubId :: Text}+ deriving stock (Eq, Ord, Show)++data ComponentRef = ComponentRef {component :: ComponentId, sub :: Maybe SubId}+ deriving stock (Eq, Ord, Show)++data EdgeKind = Build | Runtime deriving stock (Eq, Ord, Show)++data ComponentKind = Library | Migrations | Service | Harness | Assembly+ deriving stock (Eq, Ord, Show)++newtype ChangeImplies = ChangeImplies {minDimensionPolicy :: Text}+ deriving stock (Eq, Show)++data Subcomponent = Subcomponent+ { id :: SubId,+ pathPrefixes :: [FilePath],+ selectors :: [Selector],+ uses :: [SubId],+ changeImplies :: Maybe ChangeImplies+ }+ deriving stock (Eq, Show)++data DependencyEdge = DependencyEdge+ { to :: ComponentId,+ kind :: EdgeKind,+ toSubs :: [SubId],+ fromSubs :: [SubId]+ }+ deriving stock (Eq, Show)++data Component = Component+ { id :: ComponentId,+ kind :: ComponentKind,+ uri :: Maybe Text,+ repository :: Text,+ sourceRoots :: [FilePath],+ packages :: [Text],+ selectors :: [Selector],+ changeImplies :: Maybe ChangeImplies,+ verifiedAgainst :: Text,+ subcomponents :: [Subcomponent],+ dependsOn :: [DependencyEdge]+ }+ deriving stock (Eq, Show)++data PathEffect = PathAll | PathNothing | PathCohort | PathSelectors [Selector]+ deriving stock (Eq, Show)++data RepositoryPathRule = RepositoryPathRule {prefix :: FilePath, effect :: PathEffect}+ deriving stock (Eq, Show)++data ComponentGraph = ComponentGraph+ { schema :: Text,+ ignoredPackages :: [Text],+ components :: [Component],+ repositoryPaths :: [RepositoryPathRule]+ }+ deriving stock (Eq, Show)++data Affected = Affected+ { ref :: ComponentRef,+ origin :: ComponentRef,+ path :: [ComponentRef],+ distance :: Int+ }+ deriving stock (Eq, Show)++newtype GraphProblem = GraphProblem {problemText :: Text}+ deriving stock (Eq, Ord, Show)++newtype GraphError = GraphError {errorText :: Text}+ deriving stock (Eq, Show)++selftestGraphBytes :: ByteString+selftestGraphBytes = $(embedFile "data/components.json")++selftestGraph :: Either GraphError ComponentGraph+selftestGraph = decodeGraph selftestGraphBytes++loadGraphFile :: FilePath -> IO (Either GraphError ComponentGraph)+loadGraphFile path = decodeGraph <$> ByteString.readFile path++decodeGraph :: ByteString -> Either GraphError ComponentGraph+decodeGraph bytes = case eitherDecodeStrict' bytes of+ Left err -> Left (GraphError (Text.pack err))+ Right graph -> case validateGraph graph of+ [] -> Right graph+ problems -> Left (GraphError (Text.intercalate "; " (fmap (.problemText) problems)))++validateGraph :: ComponentGraph -> [GraphProblem]+validateGraph graph =+ schemaProblems+ <> duplicateComponentProblems+ <> duplicatePackageProblems+ <> concatMap validateComponent graph.components+ <> cycleProblems+ where+ componentIds = Set.fromList (fmap (.id) graph.components)+ schemaProblems = [GraphProblem ("unsupported component graph schema " <> graph.schema) | graph.schema /= "kenshou.component-graph/v1"]+ duplicateComponentProblems = duplicateProblems "component" (fmap (unComponentId . (.id)) graph.components)+ duplicatePackageProblems = duplicateProblems "package" (concatMap (.packages) graph.components)+ validateComponent componentValue =+ duplicateProblems ("subcomponent of " <> unComponentId componentValue.id) (fmap (unSubId . (.id)) componentValue.subcomponents)+ <> concatMap (validateEdge componentValue) componentValue.dependsOn+ <> concatMap (validateSub componentValue) componentValue.subcomponents+ validateEdge componentValue edge =+ [GraphProblem (unComponentId componentValue.id <> ": unknown dependency " <> unComponentId edge.to) | edge.to `Set.notMember` componentIds]+ <> case componentById graph edge.to of+ Nothing -> []+ Just target ->+ [ GraphProblem (unComponentId componentValue.id <> ": unknown target subcomponent " <> unSubId subId)+ | subId <- edge.toSubs,+ subId `notElem` fmap (.id) target.subcomponents+ ]+ <> [ GraphProblem (unComponentId componentValue.id <> ": unknown source subcomponent " <> unSubId subId)+ | subId <- edge.fromSubs,+ subId `notElem` fmap (.id) componentValue.subcomponents+ ]+ validateSub componentValue subcomponentValue =+ [ GraphProblem (unComponentId componentValue.id <> ": " <> unSubId subcomponentValue.id <> " uses unknown subcomponent " <> unSubId used)+ | used <- subcomponentValue.uses,+ used `notElem` fmap (.id) componentValue.subcomponents+ ]+ cycleProblems =+ [ GraphProblem ("component dependency cycle: " <> Text.intercalate " -> " (sort (fmap (unComponentId . (.id)) members)))+ | CyclicSCC members <- stronglyConnComp [(componentValue, componentValue.id, fmap (.to) componentValue.dependsOn) | componentValue <- graph.components]+ ]++duplicateProblems :: Text -> [Text] -> [GraphProblem]+duplicateProblems label values =+ [GraphProblem ("duplicate " <> label <> " " <> value) | entries@(value : _) <- group (sort values), length entries > 1]++componentById :: ComponentGraph -> ComponentId -> Maybe Component+componentById graph wanted = Map.lookup wanted (Map.fromList [(componentValue.id, componentValue) | componentValue <- graph.components])++parseRef :: ComponentGraph -> Text -> Either Text ComponentRef+parseRef graph raw = case Text.splitOn ":" raw of+ [componentText] -> do+ let componentId = ComponentId componentText+ case componentById graph componentId of+ Nothing -> Left ("unknown component " <> raw)+ Just _ -> Right (ComponentRef componentId Nothing)+ [componentText, subText] -> do+ let componentId = ComponentId componentText+ subId = SubId subText+ componentValue <- maybe (Left ("unknown component " <> componentText)) Right (componentById graph componentId)+ if subId `elem` fmap (.id) componentValue.subcomponents+ then Right (ComponentRef componentId (Just subId))+ else Left ("unknown subcomponent " <> raw)+ _ -> Left ("invalid component reference " <> raw)++renderRef :: ComponentRef -> Text+renderRef reference = unComponentId reference.component <> maybe "" ((":" <>) . unSubId) reference.sub++dependents :: ComponentGraph -> Set ComponentRef -> Map ComponentRef Affected+dependents graph origins = go initialQueue initialMap+ where+ seeded = concatMap expandOrigin (Set.toAscList origins)+ expandOrigin originRef@(ComponentRef componentId Nothing) =+ (originRef, Affected originRef originRef [originRef] 0)+ : case componentById graph componentId of+ Nothing -> []+ Just componentValue ->+ [ let subRef = ComponentRef componentId (Just subcomponentValue.id)+ in (subRef, Affected subRef originRef [originRef, subRef] 0)+ | subcomponentValue <- componentValue.subcomponents+ ]+ expandOrigin originRef = [(originRef, Affected originRef originRef [originRef] 0)]+ initialMap = Map.fromListWith chooseShorter seeded+ initialQueue = Seq.fromList (Map.keys initialMap)+ go Empty results = results+ go (current :<| rest) results =+ let currentAffected = results Map.! current+ candidates = fmap (mkAffected currentAffected) (neighbors current)+ (results', queue') = foldl insertCandidate (results, rest) candidates+ in go queue' results'+ mkAffected previous next =+ Affected next previous.origin (previous.path <> [next]) (previous.distance + if next.component == previous.ref.component then 0 else 1)+ insertCandidate (results, queue) candidate = case Map.lookup candidate.ref results of+ Just existing | existing.distance <= candidate.distance -> (results, queue)+ _ -> (Map.insert candidate.ref candidate results, queue |> candidate.ref)+ chooseShorter left right = if left.distance <= right.distance then left else right+ neighbors target = nub (siblingUsers target <> edgeUsers target)+ siblingUsers (ComponentRef componentId (Just subId)) = case componentById graph componentId of+ Nothing -> []+ Just componentValue -> [ComponentRef componentId (Just sibling.id) | sibling <- componentValue.subcomponents, subId `elem` sibling.uses]+ siblingUsers (ComponentRef componentId Nothing) = case componentById graph componentId of+ Nothing -> []+ Just componentValue+ | null componentValue.selectors -> fmap (ComponentRef componentId . Just . (.id)) componentValue.subcomponents+ | otherwise -> []+ edgeUsers target = concatMap (sourceFor target) graph.components+ sourceFor target source = concatMap (edgeSource target source) source.dependsOn+ edgeSource target source edge+ | edge.to /= target.component = []+ | not (targetMatches target edge) = []+ | null edge.fromSubs = [ComponentRef source.id Nothing]+ | otherwise = fmap (ComponentRef source.id . Just) edge.fromSubs+ targetMatches (ComponentRef _ Nothing) _ = True+ targetMatches (ComponentRef _ (Just subId)) edge = null edge.toSubs || subId `elem` edge.toSubs++selectorsForRef :: ComponentGraph -> ComponentRef -> [Selector]+selectorsForRef graph (ComponentRef componentId Nothing) = maybe [] (.selectors) (componentById graph componentId)+selectorsForRef graph (ComponentRef componentId (Just subId)) = do+ componentValue <- maybe [] pure (componentById graph componentId)+ subcomponentValue <- filter ((== subId) . (.id)) componentValue.subcomponents+ subcomponentValue.selectors++minPolicyForRef :: ComponentGraph -> ComponentRef -> Maybe Text+minPolicyForRef graph (ComponentRef componentId Nothing) = do+ componentValue <- componentById graph componentId+ implies <- componentValue.changeImplies+ pure implies.minDimensionPolicy+minPolicyForRef graph (ComponentRef componentId (Just subId)) = do+ componentValue <- componentById graph componentId+ subcomponentValue <- case filter ((== subId) . (.id)) componentValue.subcomponents of [] -> Nothing; value : _ -> Just value+ implies <- subcomponentValue.changeImplies+ pure implies.minDimensionPolicy++graphDigest :: ComponentGraph -> Text+graphDigest = sha256Hex . canonicalEncode . toJSON++instance FromJSON ComponentGraph where+ parseJSON = withObject "ComponentGraph" \value -> ComponentGraph <$> value .: "schema" <*> value .:? "ignoredPackages" .!= [] <*> value .: "components" <*> value .: "repositoryPaths"++instance ToJSON ComponentGraph where+ toJSON graph = object ["schema" .= graph.schema, "ignoredPackages" .= graph.ignoredPackages, "components" .= graph.components, "repositoryPaths" .= graph.repositoryPaths]++instance FromJSON Component where+ parseJSON = withObject "Component" \value ->+ Component <$> value .: "id" <*> value .: "kind" <*> value .:? "uri" <*> value .: "repository" <*> value .:? "sourceRoots" .!= [] <*> value .:? "packages" .!= [] <*> parseSelectors value <*> value .:? "changeImplies" <*> value .:? "verifiedAgainst" .!= "" <*> value .:? "subcomponents" .!= [] <*> value .:? "dependsOn" .!= []++instance ToJSON Component where+ toJSON componentValue = object ["id" .= componentValue.id, "kind" .= componentValue.kind, "uri" .= componentValue.uri, "repository" .= componentValue.repository, "sourceRoots" .= componentValue.sourceRoots, "packages" .= componentValue.packages, "selectors" .= fmap renderSelector componentValue.selectors, "changeImplies" .= componentValue.changeImplies, "verifiedAgainst" .= componentValue.verifiedAgainst, "subcomponents" .= componentValue.subcomponents, "dependsOn" .= componentValue.dependsOn]++instance FromJSON Subcomponent where+ parseJSON = withObject "Subcomponent" \value -> Subcomponent <$> value .: "id" <*> value .:? "pathPrefixes" .!= [] <*> parseSelectors value <*> value .:? "uses" .!= [] <*> value .:? "changeImplies"++instance ToJSON Subcomponent where+ toJSON subcomponentValue = object ["id" .= subcomponentValue.id, "pathPrefixes" .= subcomponentValue.pathPrefixes, "selectors" .= fmap renderSelector subcomponentValue.selectors, "uses" .= subcomponentValue.uses, "changeImplies" .= subcomponentValue.changeImplies]++parseSelectors :: Object -> Parser [Selector]+parseSelectors value = do+ raw <- value .:? "selectors" .!= []+ traverse (either (fail . Text.unpack) pure . parseSelector) raw++instance FromJSON DependencyEdge where+ parseJSON = withObject "DependencyEdge" \value -> DependencyEdge <$> value .: "to" <*> value .: "kind" <*> value .:? "toSubs" .!= [] <*> value .:? "fromSubs" .!= []++instance ToJSON DependencyEdge where+ toJSON edge = object ["to" .= edge.to, "kind" .= edge.kind, "toSubs" .= edge.toSubs, "fromSubs" .= edge.fromSubs]++instance FromJSON RepositoryPathRule where+ parseJSON = withObject "RepositoryPathRule" \value -> RepositoryPathRule <$> value .: "prefix" <*> value .: "effect"++instance ToJSON RepositoryPathRule where toJSON rule = object ["prefix" .= rule.prefix, "effect" .= rule.effect]++instance FromJSON PathEffect where+ parseJSON = withObject "PathEffect" \value ->+ value .: "kind" >>= \case+ ("all" :: Text) -> pure PathAll+ "nothing" -> pure PathNothing+ "cohort" -> pure PathCohort+ "selectors" -> PathSelectors <$> parseSelectors value+ other -> fail ("unknown path effect " <> Text.unpack other)++instance ToJSON PathEffect where+ toJSON PathAll = object ["kind" .= ("all" :: Text)]+ toJSON PathNothing = object ["kind" .= ("nothing" :: Text)]+ toJSON PathCohort = object ["kind" .= ("cohort" :: Text)]+ toJSON (PathSelectors selectors) = object ["kind" .= ("selectors" :: Text), "selectors" .= fmap renderSelector selectors]++instance FromJSON ChangeImplies where parseJSON = withObject "ChangeImplies" \value -> ChangeImplies <$> value .: "minDimensionPolicy"++instance ToJSON ChangeImplies where toJSON value = object ["minDimensionPolicy" .= value.minDimensionPolicy]++instance FromJSON ComponentId where parseJSON = withText "ComponentId" (pure . ComponentId)++instance ToJSON ComponentId where toJSON = toJSON . unComponentId++instance FromJSON SubId where parseJSON = withText "SubId" (pure . SubId)++instance ToJSON SubId where toJSON = toJSON . unSubId++instance FromJSON EdgeKind where parseJSON = withText "EdgeKind" \case "build" -> pure Build; "runtime" -> pure Runtime; other -> fail ("unknown edge kind " <> Text.unpack other)++instance ToJSON EdgeKind where toJSON Build = String "build"; toJSON Runtime = String "runtime"++instance FromJSON ComponentKind where parseJSON = withText "ComponentKind" \case "library" -> pure Library; "migrations" -> pure Migrations; "service" -> pure Service; "harness" -> pure Harness; "assembly" -> pure Assembly; other -> fail ("unknown component kind " <> Text.unpack other)++instance ToJSON ComponentKind where toJSON Library = String "library"; toJSON Migrations = String "migrations"; toJSON Service = String "service"; toJSON Harness = String "harness"; toJSON Assembly = String "assembly"
+ src/Kenshou/Plan/Components/Check.hs view
@@ -0,0 +1,121 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Components.Check+ ( PackageName (..),+ EdgeDrift (..),+ LintFinding (..),+ checkAgainstPlanJson,+ lintAgainstCatalog,+ isDriftError,+ isLintError,+ )+where++import Data.Aeson+import Data.ByteString qualified as ByteString+import Data.List (nub, sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set (Set)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (ScenarioId (..), renderScenarioId, selftestLayer)+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Components+import Kenshou.Plan.Selector (Selector, matches, renderSelector)++newtype PackageName = PackageName Text+ deriving stock (Eq, Ord, Show)++data EdgeDrift+ = MissingEdge ComponentId ComponentId [PackageName]+ | ExtraEdge ComponentId ComponentId+ | NotInBuildPlan PackageName+ deriving stock (Eq, Ord, Show)++data LintFinding+ = OrphanScenario ScenarioId+ | DeadSelector ComponentRef Selector+ deriving stock (Eq, Show)++data PlanUnit = PlanUnit {unitId :: Text, package :: PackageName, componentName :: Maybe Text, dependencies :: [Text]}++newtype CabalPlan = CabalPlan {units :: [PlanUnit]}++instance FromJSON CabalPlan where parseJSON = withObject "CabalPlan" \value -> CabalPlan <$> value .: "install-plan"++instance FromJSON PlanUnit where+ parseJSON = withObject "PlanUnit" \value ->+ PlanUnit <$> value .: "id" <*> (PackageName <$> value .: "pkg-name") <*> value .:? "component-name" <*> value .:? "depends" .!= []++checkAgainstPlanJson :: ComponentGraph -> FilePath -> IO (Either Text [EdgeDrift])+checkAgainstPlanJson graph path = do+ bytes <- ByteString.readFile path+ pure case eitherDecodeStrict' bytes of+ Left err -> Left (Text.pack err)+ Right plan -> Right (compareEdges graph plan)++compareEdges :: ComponentGraph -> CabalPlan -> [EdgeDrift]+compareEdges graph plan =+ fmap (uncurry missingEdgeWithPackages) missing+ <> fmap (uncurry ExtraEdge) extra+ <> fmap (NotInBuildPlan . PackageName) absentPackages+ where+ libraryUnits = filter ((== Just "lib") . (.componentName)) plan.units+ idToPackage = Map.fromList [(unit.unitId, unit.package) | unit <- plan.units]+ packageToComponent = Map.fromList [(PackageName package, componentValue.id) | componentValue <- graph.components, package <- componentValue.packages]+ presentPackages = Set.fromList (fmap (.package) libraryUnits)+ presentComponents = Set.fromList (Map.elems (Map.restrictKeys packageToComponent presentPackages))+ observed = Set.fromList do+ unit <- libraryUnits+ source <- maybe [] pure (Map.lookup unit.package packageToComponent)+ dependencyId <- unit.dependencies+ dependencyPackage <- maybe [] pure (Map.lookup dependencyId idToPackage)+ target <- maybe [] pure (Map.lookup dependencyPackage packageToComponent)+ if source == target then [] else pure (source, target)+ declared = Set.fromList [(componentValue.id, edge.to) | componentValue <- graph.components, edge <- componentValue.dependsOn, edge.kind == Build, componentValue.id `Set.member` presentComponents, edge.to `Set.member` presentComponents]+ missing = Set.toAscList (observed `Set.difference` declared)+ extra = Set.toAscList (declared `Set.difference` observed)+ absentPackages = sort [package | componentValue <- graph.components, package <- componentValue.packages, PackageName package `Set.notMember` presentPackages]+ directPackages source target =+ nub+ [ dependencyPackage+ | unit <- libraryUnits,+ Map.lookup unit.package packageToComponent == Just source,+ dependencyId <- unit.dependencies,+ Just dependencyPackage <- [Map.lookup dependencyId idToPackage],+ Map.lookup dependencyPackage packageToComponent == Just target+ ]+ missingEdgeWithPackages source target = MissingEdge source target (directPackages source target)++lintAgainstCatalog :: ComponentGraph -> [ScenarioInfo] -> [LintFinding]+lintAgainstCatalog graph catalog = orphaned <> dead+ where+ references =+ [ComponentRef componentValue.id Nothing | componentValue <- graph.components]+ <> [ComponentRef componentValue.id (Just subcomponentValue.id) | componentValue <- graph.components, subcomponentValue <- componentValue.subcomponents]+ selectorPairs = [(reference, selector) | reference <- references, selector <- selectorsForRef graph reference]+ orphaned =+ [ OrphanScenario scenario.id+ | scenario <- catalog,+ scenario.id.layer /= selftestLayer,+ not (any ((`matches` scenario.id) . snd) selectorPairs)+ ]+ dead =+ [ DeadSelector reference selector+ | (reference, selector) <- selectorPairs,+ selectorHasRegisteredLayer selector,+ not (any (matches selector . (.id)) catalog)+ ]+ selectorHasRegisteredLayer selector =+ let prefix = Text.takeWhile (/= '/') (renderSelector selector)+ in prefix == "*" || any ((== prefix) . Text.takeWhile (/= '/') . renderScenarioId . (.id)) catalog++isDriftError :: EdgeDrift -> Bool+isDriftError NotInBuildPlan {} = False+isDriftError _ = True++isLintError :: LintFinding -> Bool+isLintError OrphanScenario {} = True+isLintError DeadSelector {} = False
+ src/Kenshou/Plan/Execute.hs view
@@ -0,0 +1,237 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Execute+ ( ExecuteOptions (..),+ executePlan,+ )+where++import Control.Exception (finally)+import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (traverse_)+import Data.Maybe (isJust)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Id+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.RunSpec (EnvironmentSpec, RunSpec)+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Plan.Summary+import System.Directory+import System.Environment (getExecutablePath, lookupEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Posix.Process (getProcessID)+import System.Posix.Signals (sigKILL, signalProcess)+import System.Process (createProcess, getPid, proc, readProcessWithExitCode, terminateProcess, waitForProcess)+import System.Timeout (timeout)++data ExecuteOptions = ExecuteOptions+ { planFile :: FilePath,+ outDir :: FilePath,+ resume :: Bool,+ failFast :: Bool,+ environment :: Maybe FilePath,+ timeoutFactor :: Maybe Double+ }+ deriving stock (Eq, Show)++data ExecutablePlan = ExecutablePlan {planId :: RunId, runs :: [ExecutableRun]}+ deriving stock (Eq, Show)++data ExecutableRun = ExecutableRun+ { ordinal :: Int,+ runId :: RunId,+ estimateMinutes :: Int,+ scenario :: ScenarioId,+ spec :: RunSpec+ }+ deriving stock (Eq, Show)++data ResultSummary = ResultSummary {outcome :: Outcome, blocking :: Bool, knownDefect :: Bool}+ deriving stock (Eq, Show)++executePlan :: ExecuteOptions -> IO PlanSummary+executePlan options = do+ planBytes <- ByteString.readFile options.planFile+ plan <- either (ioError . userError . Text.unpack . ("invalid run plan: " <>) . Text.pack) pure (eitherDecodeStrict' planBytes)+ environmentOverride <- traverse loadEnvironment options.environment+ createDirectoryIfMissing True options.outDir+ preparePlanCopy options planBytes+ acquireLock options.outDir+ runWithLock plan planBytes environmentOverride `finally` removeLock options.outDir+ where+ runWithLock plan _planBytes environmentOverride = do+ createDirectoryIfMissing True (options.outDir </> "specs")+ summary <- loadOrCreateSummary options plan+ executeEntries options plan environmentOverride summary++loadEnvironment :: FilePath -> IO EnvironmentSpec+loadEnvironment path = do+ bytes <- ByteString.readFile path+ either (ioError . userError . ("invalid environment document: " <>)) pure (eitherDecodeStrict' bytes)++preparePlanCopy :: ExecuteOptions -> ByteString -> IO ()+preparePlanCopy options bytes = do+ let destination = options.outDir </> "run-plan.json"+ exists <- doesFileExist destination+ if exists+ then do+ existing <- ByteString.readFile destination+ if not options.resume+ then ioError (userError "execution output already contains a plan; use --resume")+ else if sha256Hex existing /= sha256Hex bytes then ioError (userError "resume plan digest does not match run-plan.json") else pure ()+ else ByteString.writeFile destination bytes++acquireLock :: FilePath -> IO ()+acquireLock outDir = do+ let path = outDir </> ".execute.lock"+ exists <- doesFileExist path+ if exists+ then do+ contents <- readFile path+ live <- processAlive contents+ if live then ioError (userError ("executor lock is held by pid " <> contents)) else writePid path+ else writePid path+ where+ writePid path = getProcessID >>= writeFile path . show+ processAlive raw = case reads raw of+ [(pid, _)] -> do+ (code, _, _) <- readProcessWithExitCode "kill" ["-0", show (pid :: Int)] ""+ pure (code == ExitSuccess)+ _ -> pure False++removeLock :: FilePath -> IO ()+removeLock outDir = do+ let path = outDir </> ".execute.lock"+ exists <- doesFileExist path+ if exists then removeFile path else pure ()++loadOrCreateSummary :: ExecuteOptions -> ExecutablePlan -> IO PlanSummary+loadOrCreateSummary options plan = do+ let path = options.outDir </> "plan-summary.json"+ exists <- doesFileExist path+ if exists && options.resume+ then do+ bytes <- ByteString.readFile path+ either (ioError . userError . ("invalid plan summary: " <>)) pure (eitherDecodeStrict' bytes)+ else pure (emptySummary plan.planId [(run.ordinal, run.scenario) | run <- plan.runs])++executeEntries :: ExecuteOptions -> ExecutablePlan -> Maybe EnvironmentSpec -> PlanSummary -> IO PlanSummary+executeEntries options plan environmentOverride = go plan.runs+ where+ summaryPath = options.outDir </> "plan-summary.json"+ go [] summary = writeSummary summaryPath (finalizeSummary summary) >> pure (finalizeSummary summary)+ go (run : rest) summary = case findEntry run.ordinal summary.entries of+ Just entry | entry.status == Completed -> go rest summary+ _ -> do+ let priorAttempts = maybe [] (.attempts) (findEntry run.ordinal summary.entries)+ attemptId <- if null priorAttempts then pure run.runId else newRunId+ startedAt <- getCurrentTime+ let attempt = Attempt attemptId startedAt Nothing Nothing False Nothing+ running = updateEntry run.ordinal (\entry -> entry {status = Running, attempts = entry.attempts <> [attempt]}) summary+ writeSummary summaryPath running+ writeSpec options environmentOverride run+ childCode <- spawnRun options run attemptId+ finishedAt <- getCurrentTime+ result <- readResult options attemptId childCode+ let completedAttempt = attempt {finishedAt = Just finishedAt, outcome = Just result.outcome, knownDefect = result.knownDefect, childExitCode = Just (exitCodeInt childCode)}+ completed = finalizeSummary (updateEntry run.ordinal (\entry -> entry {status = Completed, attempts = priorAttempts <> [completedAttempt]}) running)+ writeSummary summaryPath completed+ if options.failFast && result.blocking && result.outcome == Failed then pure completed else go rest completed++writeSpec :: ExecuteOptions -> Maybe EnvironmentSpec -> ExecutableRun -> IO ()+writeSpec options environmentOverride run = LazyByteString.writeFile path (encode effective)+ where+ path = options.outDir </> "specs" </> pad run.ordinal <> ".json"+ effective = maybe run.spec (replaceEnvironment run.spec) environmentOverride+ pad value = let rendered = show value in replicate (4 - min 4 (length rendered)) '0' <> rendered++replaceEnvironment :: RunSpec -> EnvironmentSpec -> RunSpec+replaceEnvironment spec environment =+ RunSpec.RunSpec+ spec.runId+ spec.scenario+ spec.scenarioRevision+ spec.knobs+ spec.dimensions+ spec.seed+ spec.phases+ spec.timeoutSeconds+ environment+ spec.cohortExpectation+ spec.comparison+ spec.labels++spawnRun :: ExecuteOptions -> ExecutableRun -> RunId -> IO ExitCode+spawnRun options run runId = do+ executable <- getExecutablePath+ cellFingerprint <- lookupEnv "KENSHOU_CELL_FINGERPRINT"+ let specPath = options.outDir </> "specs" </> pad run.ordinal <> ".json"+ arguments = ["run", "--spec", specPath, "--out", options.outDir, "--run-id", Text.unpack (renderRunId runId)] <> maybe [] (\path -> ["--cell-fingerprint", path]) cellFingerprint+ (_, _, _, processHandle) <- createProcess (proc executable arguments)+ case options.timeoutFactor of+ Nothing -> waitForProcess processHandle+ Just factor -> do+ result <- timeout (floor (factor * fromIntegral run.estimateMinutes * 60 * 1000000)) (waitForProcess processHandle)+ case result of+ Just code -> pure code+ Nothing -> do+ terminateProcess processHandle+ stopped <- timeout 30000000 (waitForProcess processHandle)+ case stopped of+ Just _ -> pure (ExitFailure 124)+ Nothing -> do+ traverse_ (signalProcess sigKILL) =<< getPid processHandle+ _ <- waitForProcess processHandle+ pure (ExitFailure 124)+ where+ pad value = let rendered = show value in replicate (4 - min 4 (length rendered)) '0' <> rendered++readResult :: ExecuteOptions -> RunId -> ExitCode -> IO ResultSummary+readResult options runId _childCode = do+ let path = options.outDir </> Text.unpack (renderRunId runId) </> "run-result.json"+ exists <- doesFileExist path+ if not exists+ then pure (ResultSummary Errored True False)+ else do+ bytes <- ByteString.readFile path+ pure (either (const (ResultSummary Errored True False)) (\value -> value) (eitherDecodeStrict' bytes))++findEntry :: Int -> [SummaryEntry] -> Maybe SummaryEntry+findEntry ordinal = foldr (\entry found -> if entry.ordinal == ordinal then Just entry else found) Nothing++updateEntry :: Int -> (SummaryEntry -> SummaryEntry) -> PlanSummary -> PlanSummary+updateEntry ordinal update summary = summary {entries = fmap (\entry -> if entry.ordinal == ordinal then update entry else entry) summary.entries}++exitCodeInt :: ExitCode -> Int+exitCodeInt ExitSuccess = 0+exitCodeInt (ExitFailure value) = value++instance FromJSON ExecutablePlan where+ parseJSON = withObject "ExecutablePlan" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.run-plan/v1" :: Text) then fail "unsupported run-plan schema" else pure ()+ ExecutablePlan <$> value .: "planId" <*> value .: "runs"++instance FromJSON ExecutableRun where+ parseJSON = withObject "ExecutableRun" \value -> do+ ordinal <- value .: "ordinal"+ runId <- value .: "runId"+ estimateMinutes <- value .: "estimateMinutes"+ spec <- value .: "spec"+ let scenario = spec.scenario+ pure ExecutableRun {ordinal, runId, estimateMinutes, scenario, spec}++instance FromJSON ResultSummary where+ parseJSON = withObject "ResultSummary" \value -> do+ outcome <- value .: "outcome"+ blocking <- value .:? "blocking" .!= True+ defect <- value .:? "knownDefect" :: Parser (Maybe Value)+ pure (ResultSummary outcome blocking (isJust defect))
+ src/Kenshou/Plan/Matrix.hs view
@@ -0,0 +1,167 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Matrix+ ( DimensionPolicy (..),+ KnobPolicy (..),+ Config (..),+ MatrixSkip (..),+ expandScenario,+ pairwiseCover,+ )+where++import Data.List (maximumBy, nub, sortOn)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Ord (Down (..), comparing)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Dimension+import Kenshou.Core.Id (Kind (..), ScenarioId (..))+import Kenshou.Core.Knob+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Change (Selected (..))+import Kenshou.Plan.Policy++data Config = Config+ { knobs :: Map KnobName KnobValue,+ dimensions :: Map Text Text,+ isDefault :: Bool+ }+ deriving stock (Eq, Show)++data MatrixSkip = MatrixSkip {reason :: Text, detail :: Text}+ deriving stock (Eq, Show)++expandScenario :: PlanPolicy -> Selected -> ([Config], [MatrixSkip])+expandScenario policy selected = case dimensionRows of+ Left problem -> ([], [problem])+ Right rows -> case knobRows of+ Left problem -> ([], [problem])+ Right (defaultKnobs, variants) ->+ ( [Config defaultKnobs row (index == 0) | (index, row) <- zip [0 :: Int ..] rows]+ <> [Config variant (headOrEmpty rows) False | variant <- variants],+ []+ )+ where+ scenario = selected.scenario+ effectivePolicy = max policy.dimensionPolicy (maybe DefaultOnly parseMinimum selected.minPolicy)+ dimensionRows = expandDimensions effectivePolicy policy.pinnedDimensions scenario+ effectiveKnobPolicy = max policy.knobPolicy (maybe KnobDefaults parseMinimumKnob selected.minKnobPolicy)+ knobRows = expandKnobs (policy {knobPolicy = effectiveKnobPolicy}) scenario+ parseMinimum value = maybe DefaultOnly (\policyValue -> policyValue) (parseDimensionPolicy value)+ parseMinimumKnob value = maybe KnobDefaults (\policyValue -> policyValue) (parseKnobPolicy value)+ headOrEmpty [] = Map.empty+ headOrEmpty (first : _) = first++expandDimensions :: DimensionPolicy -> [(Text, Text)] -> ScenarioInfo -> Either MatrixSkip [Map Text Text]+expandDimensions policy pins scenario+ | scenario.id.kind == Benchmark && not durableSupported = Left (MatrixSkip "benchmark-without-durable" "benchmark does not support pg.durability=durable")+ | otherwise = do+ pinned <- traverse validatePin pins+ let valueSets = fmap (applyPin pinned) supported+ effectiveSets = if scenario.id.kind == Benchmark then fmap forceDurable valueSets else valueSets+ defaults = Map.fromList [(name, def) | (name, _, def) <- effectiveSets]+ rows = case policy of+ DefaultOnly -> [defaults]+ TelemetryCorners -> telemetryCorners effectiveSets defaults+ Pairwise -> pairwiseCover [(name, values) | (name, values, _) <- effectiveSets]+ Full -> productRows [(name, values) | (name, values, _) <- effectiveSets]+ pure rows+ where+ supported = supportRows scenario.dimensions+ durableSupported = case scenario.dimensions.pgDurability of+ Supported support -> PgDurable `elem` support.values+ NotApplicable -> True+ validatePin (name, value) = case lookup name [(rowName, values) | (rowName, values, _) <- supported] of+ Nothing -> Left (MatrixSkip "unsupported-dimension" (name <> " is not applicable"))+ Just values | value `elem` values -> Right (name, value)+ Just _ -> Left (MatrixSkip "unsupported-dimension" (name <> "=" <> value <> " is not supported"))+ applyPin pinnedValues row@(name, _, _) = case lookup name pinnedValues of+ Nothing -> row+ Just value -> (name, [value], value)+ forceDurable ("pg.durability", _, _) = ("pg.durability", ["durable"], "durable")+ forceDurable row = row++supportRows :: DimensionSupport -> [(Text, [Text], Text)]+supportRows support =+ concat+ [ supportRow "telemetry.tracing" renderTracing support.tracing,+ supportRow "telemetry.metrics" renderMetrics support.metrics,+ supportRow "pg.durability" renderDurability support.pgDurability,+ supportRow "pg.version" renderVersion support.pgVersion+ ]+ where+ supportRow _ _ NotApplicable = []+ supportRow name render (Supported value) = [(name, fmap render (NonEmpty.toList value.values), render value.def)]++telemetryCorners :: [(Text, [Text], Text)] -> Map Text Text -> [Map Text Text]+telemetryCorners rows defaults = nub [set tracing metrics defaults | tracing <- tracingValues, metrics <- metricsValues]+ where+ values name fallback = maybe fallback (\(items, def) -> nub [def, preferred name items def]) (lookupRow name)+ lookupRow name = case [(items, def) | (rowName, items, def) <- rows, rowName == name] of [] -> Nothing; first : _ -> Just first+ tracingValues = values "telemetry.tracing" [""]+ metricsValues = values "telemetry.metrics" [""]+ preferred "telemetry.tracing" items def = firstSupported ["sdk-inmemory", "sdk-otlp", "noop"] items def+ preferred "telemetry.metrics" items def = firstSupported ["serve-scraped", "serve", "collect"] items def+ preferred _ _ def = def+ firstSupported preferences items def = case filter (`elem` items) preferences of [] -> def; first : _ -> first+ set tracing metrics = setIfPresent "telemetry.metrics" metrics . setIfPresent "telemetry.tracing" tracing+ setIfPresent _ "" valuesMap = valuesMap+ setIfPresent name value valuesMap = Map.insert name value valuesMap++productRows :: [(Text, [Text])] -> [Map Text Text]+productRows = foldr step [Map.empty]+ where+ step (name, values) rows = [Map.insert name value row | value <- values, row <- rows]++pairwiseCover :: [(Text, [Text])] -> [Map Text Text]+pairwiseCover dimensions+ | length dimensions < 2 = take 1 (productRows dimensions)+ | otherwise = go uncovered [] candidates+ where+ candidates = sortOn renderRow (productRows dimensions)+ uncovered = Set.unions (fmap rowPairs candidates)+ go remaining chosen available+ | Set.null remaining = reverse chosen+ | otherwise = case available of+ [] -> reverse chosen+ _ ->+ let best = maximumBy (comparing (\row -> (Set.size (Set.intersection remaining (rowPairs row)), Down (renderRow row)))) available+ in go (remaining `Set.difference` rowPairs best) (best : chosen) (filter (/= best) available)+ rowPairs row =+ Set.fromList+ [ ((leftName, leftValue), (rightName, rightValue))+ | (leftIndex, (leftName, leftValue)) <- zip [0 :: Int ..] (Map.toAscList row),+ (rightIndex, (rightName, rightValue)) <- zip [0 :: Int ..] (Map.toAscList row),+ leftIndex < rightIndex+ ]+ renderRow = Text.pack . show . Map.toAscList++expandKnobs :: PlanPolicy -> ScenarioInfo -> Either MatrixSkip (Map KnobName KnobValue, [Map KnobName KnobValue])+expandKnobs policy scenario = case resolveKnobs scenario.knobs supplied of+ Left problems -> Left (MatrixSkip "unsupported-knob" (Text.intercalate "; " (fmap renderProblem (NonEmpty.toList problems))))+ Right resolved -> Right (base, variants)+ where+ base = resolvedKnobsMap resolved+ variants+ | policy.knobPolicy == KnobDefaults = []+ | otherwise =+ [ Map.insert spec.name value base+ | spec <- scenario.knobs,+ length spec.variants <= 8,+ value <- spec.variants,+ value /= Map.findWithDefault spec.def spec.name base+ ]+ where+ declaredNames = fmap (renderKnobName . (.name)) scenario.knobs+ supplied =+ [ (spec.name, RawText value)+ | (name, value) <- policy.pinnedKnobs,+ name `elem` declaredNames,+ spec <- scenario.knobs,+ renderKnobName spec.name == name+ ]+ renderProblem (KnobError message) = message
+ src/Kenshou/Plan/Policy.hs view
@@ -0,0 +1,100 @@+module Kenshou.Plan.Policy+ ( DimensionPolicy (..),+ KnobPolicy (..),+ PlanPolicy (..),+ defaultPlanPolicy,+ mergePolicy,+ renderDimensionPolicy,+ parseDimensionPolicy,+ renderKnobPolicy,+ parseKnobPolicy,+ )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set (Set)+import Data.Set qualified as Set+import Data.Text (Text)+import Kenshou.Core.Id (Kind (..), Seed, renderKind)+import Kenshou.Core.RunSpec (SpecPlacement (..))+import Kenshou.Core.Scenario (Tier (..), renderTier)++data DimensionPolicy = DefaultOnly | TelemetryCorners | Pairwise | Full+ deriving stock (Eq, Ord, Show, Enum, Bounded)++data KnobPolicy = KnobDefaults | KnobDeclaredVariants+ deriving stock (Eq, Ord, Show)++data PlanPolicy = PlanPolicy+ { maxTier :: Tier,+ kinds :: Set Kind,+ placement :: SpecPlacement,+ dimensionPolicy :: DimensionPolicy,+ knobPolicy :: KnobPolicy,+ trials :: Int,+ budgetMinutes :: Maybe Int,+ tierMinutes :: Map Tier Int,+ seed :: Seed,+ pinnedKnobs :: [(Text, Text)],+ pinnedDimensions :: [(Text, Text)]+ }+ deriving stock (Eq, Show)++defaultPlanPolicy :: Seed -> PlanPolicy+defaultPlanPolicy seed =+ PlanPolicy+ { maxTier = TierStandard,+ kinds = Set.fromList [Correctness, Concurrency, Benchmark, Soak],+ placement = RunLocal,+ dimensionPolicy = DefaultOnly,+ knobPolicy = KnobDefaults,+ trials = 3,+ budgetMinutes = Nothing,+ tierMinutes = Map.fromList [(TierSmoke, 1), (TierStandard, 10), (TierExtended, 60), (TierSoak, 240)],+ seed,+ pinnedKnobs = [],+ pinnedDimensions = []+ }++-- | Policies are concrete after parsing. A later policy therefore wins, while+-- pins are accumulated so suite defaults can be refined on the command line.+mergePolicy :: PlanPolicy -> PlanPolicy -> PlanPolicy+mergePolicy _base override = override++renderDimensionPolicy :: DimensionPolicy -> Text+renderDimensionPolicy DefaultOnly = "default-only"+renderDimensionPolicy TelemetryCorners = "telemetry-corners"+renderDimensionPolicy Pairwise = "pairwise"+renderDimensionPolicy Full = "full"++parseDimensionPolicy :: Text -> Maybe DimensionPolicy+parseDimensionPolicy value = lookup value [(renderDimensionPolicy policy, policy) | policy <- [minBound .. maxBound]]++renderKnobPolicy :: KnobPolicy -> Text+renderKnobPolicy KnobDefaults = "defaults"+renderKnobPolicy KnobDeclaredVariants = "declared-variants"++parseKnobPolicy :: Text -> Maybe KnobPolicy+parseKnobPolicy "defaults" = Just KnobDefaults+parseKnobPolicy "declared-variants" = Just KnobDeclaredVariants+parseKnobPolicy _ = Nothing++instance ToJSON PlanPolicy where+ toJSON policy =+ object+ [ "maxTier" .= renderTier policy.maxTier,+ "kinds" .= fmap renderKind (Set.toAscList policy.kinds),+ "placement" .= policy.placement,+ "dimensionPolicy" .= renderDimensionPolicy policy.dimensionPolicy,+ "knobPolicy" .= renderKnobPolicy policy.knobPolicy,+ "trials" .= policy.trials,+ "trialOrdering" .= ("alternating/v1" :: Text),+ "budgetMinutes" .= policy.budgetMinutes,+ "tierMinutes" .= object [Key.fromText (renderTier tier) .= minutes | (tier, minutes) <- Map.toAscList policy.tierMinutes],+ "seed" .= policy.seed,+ "set" .= policy.pinnedKnobs,+ "dimensions" .= policy.pinnedDimensions+ ]
+ src/Kenshou/Plan/RunPlan.hs view
@@ -0,0 +1,270 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.RunPlan+ ( PlanInputs (..),+ PlanContext (..),+ TrialInfo (..),+ PlannedRun (..),+ SkipReason (..),+ Skipped (..),+ SkeletonRun (..),+ PlanSkeleton (..),+ RunPlan (..),+ buildPlan,+ stampPlan,+ )+where++import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.List (sortOn)+import Data.List.NonEmpty (NonEmpty)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import Kenshou.Core.Id+import Kenshou.Core.Knob (RawKnob (..))+import Kenshou.Core.RunSpec+import Kenshou.Core.RunSpec qualified as RunSpec+import Kenshou.Core.Scenario (Placement (..), renderPlacement, renderTier)+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Change+import Kenshou.Plan.Matrix+import Kenshou.Plan.Policy+import System.Random.SplitMix (nextWord64)++newtype PlanInputs = PlanInputs {value :: Value}+ deriving stock (Eq, Show)++data PlanContext = PlanContext+ { suite :: Maybe Text,+ graphDigest :: Text,+ cohortName :: Text,+ cohortPlanHash :: Text,+ inputs :: PlanInputs,+ changes :: [Change],+ warnings :: [Text]+ }+ deriving stock (Eq, Show)++data TrialInfo = TrialInfo {group :: Text, arm :: Text, index :: Int, of_ :: Int}+ deriving stock (Eq, Show)++data PlannedRun = PlannedRun+ { ordinal :: Int,+ runId :: RunId,+ estimateMinutes :: Int,+ reasons :: NonEmpty Reason,+ trial :: Maybe TrialInfo,+ spec :: RunSpec+ }+ deriving stock (Eq, Show)++data SkipReason+ = SkipTier+ | SkipKind+ | SkipPlacement+ | SkipExcluded+ | SkipUnsupportedDimension+ | SkipBenchmarkWithoutDurable+ | SkipUnsupportedKnob+ | SkipOverBudget+ deriving stock (Eq, Ord, Show)++data Skipped = Skipped {scenario :: ScenarioId, reason :: SkipReason, detail :: Text}+ deriving stock (Eq, Show)++data SkeletonRun = SkeletonRun+ { estimateMinutes :: Int,+ reasons :: NonEmpty Reason,+ trial :: Maybe TrialInfo,+ spec :: RunSpec+ }+ deriving stock (Eq, Show)++data PlanSkeleton = PlanSkeleton+ { context :: PlanContext,+ policy :: PlanPolicy,+ runs :: [SkeletonRun],+ skipped :: [Skipped],+ estimateMinutes :: Int+ }+ deriving stock (Eq, Show)++data RunPlan = RunPlan+ { planId :: RunId,+ createdAt :: UTCTime,+ context :: PlanContext,+ policy :: PlanPolicy,+ runs :: [PlannedRun],+ skipped :: [Skipped],+ estimateMinutes :: Int+ }+ deriving stock (Eq, Show)++buildPlan :: PlanContext -> PlanPolicy -> [Selected] -> PlanSkeleton+buildPlan context policy selected =+ PlanSkeleton+ { context,+ policy,+ runs = kept,+ skipped = initialSkipped <> budgetSkipped,+ estimateMinutes = sum (fmap (.estimateMinutes) kept)+ }+ where+ ordered = sortOn selectedKey selected+ expanded = fmap expandOne ordered+ candidates = concatMap fst expanded+ initialSkipped = concatMap snd expanded+ (kept, budgetSkipped) = applyBudget policy.budgetMinutes candidates+ selectedKey value =+ ( minimum (fmap (.distance) (NonEmpty.toList value.reasons)),+ value.scenario.tier,+ kindRank value.scenario.id.kind,+ renderScenarioId value.scenario.id+ )+ kindRank Correctness = (0 :: Int)+ kindRank Concurrency = 1+ kindRank Benchmark = 2+ kindRank Soak = 3+ expandOne selectedValue+ | selectedValue.scenario.tier > policy.maxTier = ([], [skip SkipTier ("tier " <> renderTier selectedValue.scenario.tier <> " exceeds " <> renderTier policy.maxTier)])+ | selectedValue.scenario.id.kind `Set.notMember` policy.kinds = ([], [skip SkipKind "scenario kind is not enabled"])+ | not (placementMatches policy.placement selectedValue.scenario.placement) = ([], [skip SkipPlacement ("scenario placement is " <> renderPlacement selectedValue.scenario.placement)])+ | otherwise = case expandScenario policy selectedValue of+ ([], matrixSkips) -> ([], fmap (matrixSkip selectedValue.scenario.id) matrixSkips)+ (configs, _) -> (makeRuns selectedValue configs, [])+ where+ skip reason detail = Skipped selectedValue.scenario.id reason detail+ matrixSkip scenarioId problem = Skipped scenarioId (matrixReason problem.reason) problem.detail+ matrixReason "benchmark-without-durable" = SkipBenchmarkWithoutDurable+ matrixReason "unsupported-dimension" = SkipUnsupportedDimension+ matrixReason _ = SkipUnsupportedKnob+ makeRuns selectedValue configs+ | selectedValue.scenario.id.kind == Benchmark = concatMap makeTrial [0 .. policy.trials - 1]+ | otherwise = fmap (makeRun selectedValue Nothing 0) configs+ where+ makeTrial trialIndex =+ [ makeRun selectedValue (Just (TrialInfo (renderScenarioId selectedValue.scenario.id) ("arm-" <> Text.pack (show armIndex)) trialIndex policy.trials)) trialIndex config+ | (armIndex, config) <- zip [0 :: Int ..] (if even trialIndex then configs else reverse configs)+ ]+ makeRun selectedValue trial trialIndex config =+ SkeletonRun+ { estimateMinutes = Map.findWithDefault 10 selectedValue.scenario.tier policy.tierMinutes,+ reasons = selectedValue.reasons,+ trial,+ spec =+ (minimalRunSpec selectedValue.scenario.id)+ { RunSpec.scenarioRevision = Just selectedValue.scenario.revision,+ RunSpec.knobs = [(name, RawJson (toJSON value)) | (name, value) <- Map.toAscList config.knobs],+ RunSpec.dimensions = Map.toAscList config.dimensions,+ RunSpec.seed = Just (derivedSeed policy.seed selectedValue.scenario.id trialIndex),+ RunSpec.environment = defaultEnvironment policy.placement,+ RunSpec.cohortExpectation = Just (CohortExpectation (Just context.cohortName) context.cohortPlanHash),+ RunSpec.comparison = fmap toComparison trial+ }+ }+ toComparison trial = ComparisonMembership trial.group trial.arm trial.index (trial.index + 1)++applyBudget :: Maybe Int -> [SkeletonRun] -> ([SkeletonRun], [Skipped])+applyBudget Nothing runs = (runs, [])+applyBudget (Just budget) runs = go 0 [] [] (groupRuns runs)+ where+ go _ kept skipped [] = (reverse kept, reverse skipped)+ go used kept skipped ([] : rest) = go used kept skipped rest+ go used kept skipped (group@(first : _) : rest)+ | used + groupCost <= budget = go (used + groupCost) (reverse group <> kept) skipped rest+ | otherwise =+ let scenarioId = first.spec.scenario+ item = Skipped scenarioId SkipOverBudget ("group costs " <> Text.pack (show groupCost) <> " minutes with " <> Text.pack (show (budget - used)) <> " remaining")+ in go used kept (item : skipped) rest+ where+ groupCost = sum (fmap (.estimateMinutes) group)+ groupRuns [] = []+ groupRuns (first : rest) =+ let (same, later) = span ((== first.spec.scenario) . (.scenario) . (.spec)) rest+ in (first : same) : groupRuns later++stampPlan :: PlanSkeleton -> IO RunPlan+stampPlan skeleton = do+ planId <- newRunId+ createdAt <- getCurrentTime+ runs <- traverse stamp (zip [1 :: Int ..] skeleton.runs)+ pure+ RunPlan+ { planId,+ createdAt,+ context = skeleton.context,+ policy = skeleton.policy,+ runs,+ skipped = skeleton.skipped,+ estimateMinutes = skeleton.estimateMinutes+ }+ where+ stamp (ordinal, candidate) = do+ runId <- newRunId+ pure+ PlannedRun+ { ordinal,+ runId,+ estimateMinutes = candidate.estimateMinutes,+ reasons = candidate.reasons,+ trial = candidate.trial,+ spec = candidate.spec+ }++derivedSeed :: Seed -> ScenarioId -> Int -> Seed+derivedSeed seed scenario trialIndex = case mkSeed (word `mod` 9007199254740992) of+ Right value -> value+ Left _ -> seed+ where+ (word, _) = nextWord64 (deriveGen seed (renderScenarioId scenario <> ":" <> Text.pack (show trialIndex)))++placementMatches :: SpecPlacement -> Placement -> Bool+placementMatches RunLocal PlaceLocal = True+placementMatches RunLocal PlaceEither = True+placementMatches RunOnCell PlaceCell = True+placementMatches RunOnCell PlaceEither = True+placementMatches _ _ = False++defaultEnvironment :: SpecPlacement -> EnvironmentSpec+defaultEnvironment placement = EnvironmentSpec placement Nothing Nothing Map.empty Map.empty Nothing++instance ToJSON RunPlan where+ toJSON plan =+ object+ [ "schema" .= ("kenshou.run-plan/v1" :: Text),+ "planId" .= plan.planId,+ "createdAt" .= plan.createdAt,+ "suite" .= plan.context.suite,+ "graphDigest" .= plan.context.graphDigest,+ "cohort" .= object ["name" .= plan.context.cohortName, "planHash" .= plan.context.cohortPlanHash],+ "inputs" .= plan.context.inputs.value,+ "policy" .= plan.policy,+ "changes" .= plan.context.changes,+ "runs" .= plan.runs,+ "skipped" .= plan.skipped,+ "warnings" .= plan.context.warnings,+ "estimateMinutes" .= plan.estimateMinutes+ ]++instance ToJSON PlannedRun where+ toJSON run = object ["ordinal" .= run.ordinal, "runId" .= run.runId, "estimateMinutes" .= run.estimateMinutes, "reasons" .= run.reasons, "trial" .= run.trial, "spec" .= run.spec]++instance ToJSON TrialInfo where+ toJSON trial = object ["group" .= trial.group, "arm" .= trial.arm, "index" .= trial.index, "of" .= trial.of_]++instance ToJSON Skipped where+ toJSON skipped = object ["scenario" .= skipped.scenario, "reason" .= renderSkipReason skipped.reason, "detail" .= skipped.detail]++renderSkipReason :: SkipReason -> Text+renderSkipReason SkipTier = "tier"+renderSkipReason SkipKind = "kind"+renderSkipReason SkipPlacement = "placement"+renderSkipReason SkipExcluded = "excluded"+renderSkipReason SkipUnsupportedDimension = "unsupported-dimension"+renderSkipReason SkipBenchmarkWithoutDurable = "benchmark-without-durable"+renderSkipReason SkipUnsupportedKnob = "unsupported-knob"+renderSkipReason SkipOverBudget = "over-budget"
+ src/Kenshou/Plan/Selector.hs view
@@ -0,0 +1,23 @@+module Kenshou.Plan.Selector+ ( Selector,+ parseSelector,+ renderSelector,+ matches,+ )+where++import Data.Text (Text)+import Kenshou.Core.Id (ScenarioId)+import Kenshou.Core.Selector (ScenarioSelector)+import Kenshou.Core.Selector qualified as Core++type Selector = ScenarioSelector++parseSelector :: Text -> Either Text Selector+parseSelector = Core.parseSelector++renderSelector :: Selector -> Text+renderSelector = Core.renderSelector++matches :: Selector -> ScenarioId -> Bool+matches = Core.matchesSelector
+ src/Kenshou/Plan/Suite.hs view
@@ -0,0 +1,148 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Suite+ ( SuiteMode (..),+ SuiteExclusion (..),+ SuiteSelection (..),+ PolicyPatch (..),+ Suite (..),+ readSuite,+ decodeSuite,+ suitePolicy,+ applySuite,+ )+where++import Control.Applicative ((<|>))+import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.Bifunctor (first)+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.List.NonEmpty qualified as NonEmpty+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Id (Seed, parseKind)+import Kenshou.Core.RunSpec (SpecPlacement (..))+import Kenshou.Core.Scenario (Tier (..))+import Kenshou.Plan.Catalog (ScenarioInfo)+import Kenshou.Plan.Change+import Kenshou.Plan.Policy+import Kenshou.Plan.Policy qualified as Policy+import Kenshou.Plan.Selector++data SuiteMode = SuiteAll | SuiteChanged deriving stock (Eq, Show)++data SuiteExclusion = SuiteExclusion {select :: Selector, why :: Text}+ deriving stock (Eq, Show)++data SuiteSelection = SuiteSelection+ { mode :: SuiteMode,+ always :: [Selector],+ exclude :: [SuiteExclusion]+ }+ deriving stock (Eq, Show)++data PolicyPatch = PolicyPatch+ { maxTier :: Maybe Tier,+ kinds :: Maybe [Text],+ placement :: Maybe SpecPlacement,+ dimensionPolicy :: Maybe DimensionPolicy,+ knobPolicy :: Maybe KnobPolicy,+ trials :: Maybe Int,+ budgetMinutes :: Maybe Int+ }+ deriving stock (Eq, Show)++data Suite = Suite+ { name :: Text,+ description :: Text,+ selection :: SuiteSelection,+ policy :: PolicyPatch,+ directlyChanged :: Maybe PolicyPatch+ }+ deriving stock (Eq, Show)++readSuite :: FilePath -> IO (Either Text Suite)+readSuite path = decodeSuite <$> ByteString.readFile path++decodeSuite :: ByteString -> Either Text Suite+decodeSuite = first Text.pack . eitherDecodeStrict'++suitePolicy :: Seed -> Suite -> Either Text PlanPolicy+suitePolicy seed suite = applyPatch suite.policy (defaultPlanPolicy seed)++applySuite :: Suite -> [ScenarioInfo] -> [Selected] -> [Selected]+applySuite suite catalog changed = applyDirectlyChanged (applySelectors [] exclusions (base <> alwaysSelected))+ where+ base = case suite.selection.mode of SuiteAll -> selectAll catalog; SuiteChanged -> changed+ alwaysSelected = selectBySelectors Everything ("suite " <> suite.name <> " always") suite.selection.always catalog+ exclusions = fmap (.select) suite.selection.exclude+ applyDirectlyChanged = case suite.directlyChanged of+ Nothing -> id+ Just patch -> fmap (raiseDirect patch)+ raiseDirect patch selected+ | minimum (fmap (.distance) (NonEmpty.toList selected.reasons)) == 0 =+ selected+ { minPolicy = fmap (renderDimensionPolicy . max (maybe DefaultOnly parseMinimum selected.minPolicy)) patch.dimensionPolicy <|> selected.minPolicy,+ minKnobPolicy = fmap (renderKnobPolicy . max (maybe KnobDefaults parseMinimumKnob selected.minKnobPolicy)) patch.knobPolicy <|> selected.minKnobPolicy+ }+ | otherwise = selected+ parseMinimum value = maybe DefaultOnly (\parsed -> parsed) (parseDimensionPolicy value)+ parseMinimumKnob value = maybe KnobDefaults (\parsed -> parsed) (parseKnobPolicy value)++applyPatch :: PolicyPatch -> PlanPolicy -> Either Text PlanPolicy+applyPatch patch base = do+ kinds <- maybe (Right base.kinds) (fmap Set.fromList . traverse parseKind) patch.kinds+ pure+ base+ { Policy.maxTier = maybe base.maxTier (\value -> value) patch.maxTier,+ Policy.kinds = kinds,+ Policy.placement = maybe base.placement (\value -> value) patch.placement,+ Policy.dimensionPolicy = maybe base.dimensionPolicy (\value -> value) patch.dimensionPolicy,+ Policy.knobPolicy = maybe base.knobPolicy (\value -> value) patch.knobPolicy,+ Policy.trials = maybe base.trials (\value -> value) patch.trials,+ Policy.budgetMinutes = patch.budgetMinutes <|> base.budgetMinutes+ }++instance FromJSON Suite where+ parseJSON = withObject "Suite" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.suite/v1" :: Text) then fail "unsupported suite schema" else pure ()+ Suite <$> value .: "name" <*> value .: "description" <*> value .: "selection" <*> value .: "policy" <*> value .:? "directlyChanged"++instance FromJSON SuiteSelection where+ parseJSON = withObject "SuiteSelection" \value -> SuiteSelection <$> (value .: "mode" >>= parseMode) <*> (value .:? "always" .!= [] >>= traverse parseSelectorValue) <*> value .:? "exclude" .!= []+ where+ parseMode ("all" :: Text) = pure SuiteAll+ parseMode "changed" = pure SuiteChanged+ parseMode other = fail ("unknown suite mode " <> Text.unpack other)++instance FromJSON SuiteExclusion where+ parseJSON = withObject "SuiteExclusion" \value -> SuiteExclusion <$> (value .: "select" >>= parseSelectorValue) <*> value .: "why"++instance FromJSON PolicyPatch where+ parseJSON = withObject "PolicyPatch" \value ->+ PolicyPatch+ <$> (value .:? "maxTier" >>= traverse parseTier)+ <*> value .:? "kinds"+ <*> (value .:? "placement" >>= traverse parsePlacement)+ <*> (value .:? "dimensionPolicy" >>= traverse parseDimension)+ <*> (value .:? "knobPolicy" >>= traverse parseKnob)+ <*> value .:? "trials"+ <*> value .:? "budgetMinutes"+ where+ parseTier "smoke" = pure TierSmoke+ parseTier "standard" = pure TierStandard+ parseTier "extended" = pure TierExtended+ parseTier "soak" = pure TierSoak+ parseTier other = fail ("unknown tier " <> Text.unpack other)+ parsePlacement "local" = pure RunLocal+ parsePlacement "cell" = pure RunOnCell+ parsePlacement other = fail ("unknown placement " <> Text.unpack other)+ parseDimension raw = maybe (fail ("unknown dimension policy " <> Text.unpack raw)) pure (parseDimensionPolicy raw)+ parseKnob raw = maybe (fail ("unknown knob policy " <> Text.unpack raw)) pure (parseKnobPolicy raw)++parseSelectorValue :: Text -> Parser Selector+parseSelectorValue = either (fail . Text.unpack) pure . parseSelector
+ src/Kenshou/Plan/Summary.hs view
@@ -0,0 +1,127 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.Plan.Summary+ ( EntryStatus (..),+ Attempt (..),+ SummaryEntry (..),+ PlanSummary (..),+ emptySummary,+ worstOutcome,+ summaryExitCode,+ finalizeSummary,+ writeSummary,+ )+where++import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime)+import Kenshou.Core.Id (RunId, ScenarioId)+import Kenshou.Core.Outcome (Outcome (..), outcomeExitCode, renderOutcome)+import System.Directory (renameFile)+import System.Exit (ExitCode (..))++data EntryStatus = Pending | Running | Completed deriving stock (Eq, Show)++data Attempt = Attempt+ { runId :: RunId,+ startedAt :: UTCTime,+ finishedAt :: Maybe UTCTime,+ outcome :: Maybe Outcome,+ knownDefect :: Bool,+ childExitCode :: Maybe Int+ }+ deriving stock (Eq, Show)++data SummaryEntry = SummaryEntry+ { ordinal :: Int,+ scenario :: ScenarioId,+ status :: EntryStatus,+ attempts :: [Attempt]+ }+ deriving stock (Eq, Show)++data PlanSummary = PlanSummary+ { planId :: RunId,+ entries :: [SummaryEntry],+ counts :: Map Text Int,+ worst :: Outcome,+ exitCode :: Int+ }+ deriving stock (Eq, Show)++emptySummary :: RunId -> [(Int, ScenarioId)] -> PlanSummary+emptySummary planId entries = finalizeSummary (PlanSummary planId [SummaryEntry ordinal scenario Pending [] | (ordinal, scenario) <- entries] Map.empty Passed 0)++worstOutcome :: PlanSummary -> Outcome+worstOutcome summary = case blocking of+ [] -> Passed+ values -> foldl1 worse values+ where+ blocking = concatMap entryOutcome summary.entries+ entryOutcome entry+ | entry.status /= Completed = [Errored]+ | otherwise = case reverse entry.attempts of+ attempt : _ | attempt.knownDefect -> []+ attempt : _ -> maybe [Errored] pure attempt.outcome+ [] -> [Errored]+ worse left right = if rank left >= rank right then left else right+ rank Passed = (0 :: Int)+ rank Inconclusive = 1+ rank InfrastructureFailure = 2+ rank Errored = 3+ rank Failed = 4++summaryExitCode :: PlanSummary -> ExitCode+summaryExitCode summary = case outcomeExitCode (worstOutcome summary) of 0 -> ExitSuccess; value -> ExitFailure value++finalizeSummary :: PlanSummary -> PlanSummary+finalizeSummary summary = summary {counts = outcomeCounts, worst, exitCode = outcomeExitCode worst}+ where+ worst = worstOutcome summary+ latest = [attempt | entry <- summary.entries, attempt <- take 1 (reverse entry.attempts)]+ outcomes = [outcome | attempt <- latest, not attempt.knownDefect, Just outcome <- [attempt.outcome]]+ defectCount = length (filter (.knownDefect) latest)+ outcomeCounts = Map.fromListWith (+) ([(renderOutcome outcome, 1) | outcome <- outcomes] <> [("known-defect", defectCount) | defectCount > 0])++writeSummary :: FilePath -> PlanSummary -> IO ()+writeSummary path summary = do+ let temporary = path <> ".tmp"+ LazyByteString.writeFile temporary (encode (finalizeSummary summary))+ renameFile temporary path++instance ToJSON PlanSummary where+ toJSON summary = object ["schema" .= ("kenshou.plan-summary/v1" :: Text), "planId" .= summary.planId, "entries" .= summary.entries, "counts" .= summary.counts, "worstOutcome" .= summary.worst, "exitCode" .= summary.exitCode]++instance FromJSON PlanSummary where+ parseJSON = withObject "PlanSummary" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.plan-summary/v1" :: Text) then fail "unsupported plan summary schema" else pure ()+ PlanSummary <$> value .: "planId" <*> value .: "entries" <*> value .:? "counts" .!= Map.empty <*> value .:? "worstOutcome" .!= Passed <*> value .:? "exitCode" .!= 0++instance ToJSON SummaryEntry where+ toJSON entry = object ["ordinal" .= entry.ordinal, "scenario" .= entry.scenario, "status" .= renderStatus entry.status, "attempts" .= entry.attempts]++instance FromJSON SummaryEntry where+ parseJSON = withObject "SummaryEntry" \value -> SummaryEntry <$> value .: "ordinal" <*> value .: "scenario" <*> (value .: "status" >>= parseStatus) <*> value .:? "attempts" .!= []++instance ToJSON Attempt where+ toJSON attempt = object ["runId" .= attempt.runId, "startedAt" .= attempt.startedAt, "finishedAt" .= attempt.finishedAt, "outcome" .= attempt.outcome, "knownDefect" .= attempt.knownDefect, "childExitCode" .= attempt.childExitCode]++instance FromJSON Attempt where+ parseJSON = withObject "Attempt" \value -> Attempt <$> value .: "runId" <*> value .: "startedAt" <*> value .:? "finishedAt" <*> value .:? "outcome" <*> value .:? "knownDefect" .!= False <*> value .:? "childExitCode"++renderStatus :: EntryStatus -> Text+renderStatus Pending = "pending"+renderStatus Running = "running"+renderStatus Completed = "completed"++parseStatus :: Text -> Parser EntryStatus+parseStatus "pending" = pure Pending+parseStatus "running" = pure Running+parseStatus "completed" = pure Completed+parseStatus other = fail ("unknown entry status " <> show other)
+ test/Kenshou/ConsumerBaseline.hs view
@@ -0,0 +1,25 @@+-- | The first consumer's performance contracts, read from+-- ../fixtures/consumer-baseline/perf-contracts.json. The captured preparation in+-- that directory pins the consumer's reviewed benchmark revisions, so validating it+-- needs the same contracts the consumer supplies.+module Kenshou.ConsumerBaseline (consumerContracts) where++import Data.Aeson (eitherDecodeFileStrict', withObject, (.:))+import Data.Aeson.Types (Parser, Value, parseEither)+import Data.Text (Text)+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)++consumerContracts :: IO PerfContracts+consumerContracts = do+ document <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/perf-contracts.json" >>= either fail pure+ (reviewed, knob) <- either fail pure (parseEither contractsParser (document :: Value))+ countKnob <- either (fail . show) pure (mkKnobName knob)+ pure noPerfContracts {reviewedPhaseContracts = reviewed, steadyCountKnob = Just countKnob}+ where+ contractsParser :: Value -> Parser ([(Text, Int)], Text)+ contractsParser = withObject "PerfContracts" \value -> do+ entries <- value .: "reviewedPhaseContracts"+ reviewed <- traverse (withObject "ReviewedPhaseContract" \entry -> (,) <$> entry .: "scenario" <*> entry .: "revision") entries+ knob <- value .: "steadyCountKnob"+ pure (reviewed, knob)
+ test/Kenshou/PerfCalibrationSpec.hs view
@@ -0,0 +1,80 @@+module Kenshou.PerfCalibrationSpec (spec) where++import Data.Aeson+import Data.Either (isLeft)+import Data.Text qualified as Text+import Data.Time (UTCTime (..), addUTCTime, fromGregorian, secondsToDiffTime)+import Kenshou.Core.Id (newRunId)+import Kenshou.Perf.Calibration+import Test.Hspec++spec :: Spec+spec = describe "performance calibration admission" do+ it "accepts twenty independently replayed slots and preserves frozen protocol roundtrips" do+ (definition, acceptance, aa, slowdown) <- fixture+ validateCalibration now now definition acceptance aa slowdown `shouldBe` Right 90+ eitherDecode (encode definition) `shouldBe` Right definition+ eitherDecode (encode acceptance) `shouldBe` Right acceptance+ it "never drops a completed statistical outcome to replace it with a better result" do+ (definition, acceptance, aa, slowdown) <- fixture+ replacementId <- newRunId+ let slot = first acceptance.aa+ replaced = slot {receipts = slot.receipts <> [CalibrationReceipt "gs://fixture/replacement/receipt.json" "sha256:replacement"]}+ statisticalRetry = first aa <> [(first (first aa)) {sessionId = replacementId}]+ validateCalibration now now definition acceptance {aa = replaced : drop 1 acceptance.aa} (statisticalRetry : drop 1 aa) slowdown `shouldSatisfy` isLeft+ it "requires at least nineteen expected outcomes and zero opposite outcomes" do+ (definition, acceptance, aa, slowdown) <- fixture+ let change verdict = map (\trial -> trial {verdict})+ validateCalibration now now definition acceptance (change "inconclusive" (first aa) : drop 1 aa) slowdown `shouldBe` Right 90+ validateCalibration now now definition acceptance (change "regression" (first aa) : drop 1 aa) slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance aa (change "pass" (first slowdown) : drop 1 slowdown) `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (map (change "inconclusive") (take 2 aa) <> drop 2 aa) slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (change "not-run" (first aa) : drop 1 aa) slowdown `shouldSatisfy` isLeft+ it "invalidates expired, drifted, duplicate, missing and runtime-blocked batches" do+ (definition, acceptance, aa, slowdown) <- fixture+ validateCalibration (addUTCTime 3601 now) now definition acceptance aa slowdown `shouldSatisfy` isLeft+ validateCalibration now (addUTCTime 1 now) definition acceptance aa slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (drop 1 aa) slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (first aa : first aa : drop 2 aa) slowdown `shouldSatisfy` isLeft+ let corrupt :: (VerifiedCalibrationTrial -> VerifiedCalibrationTrial) -> [[VerifiedCalibrationTrial]]+ corrupt change = map change (first aa) : drop 1 aa+ validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialHealthPolicySha256 = "relaxed-health-limits"})) slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialMachineProfile = "different-cell-image"})) slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialToolchainSha256 = "changed"})) slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (corrupt (\trial -> trial {trialHarnessSemanticsSha256 = "changed"})) slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition acceptance (corrupt (\trial -> trial {runtimeBlockers = ["known exit-zero defect"]})) slowdown `shouldSatisfy` isLeft+ it "retains infrastructure retries under the frozen cap and includes their time" do+ (definition, acceptance, aa, slowdown) <- fixture+ extra <- newRunId+ let failed = (first (first aa)) {sessionId = extra, infrastructureState = "failed", verdict = "not-run", elapsedSeconds = 850}+ original = first acceptance.aa+ slot = original {receipts = CalibrationReceipt "gs://fixture/failed/receipt.json" "sha256:failed" : original.receipts}+ revised = acceptance {aa = slot : drop 1 acceptance.aa}+ trials = (failed : first aa) : drop 1 aa+ validateCalibration now now definition {infrastructureRetryCap = 0} revised trials slowdown `shouldSatisfy` isLeft+ validateCalibration now now definition revised trials slowdown `shouldBe` Right 90+ let late = map (map (\trial -> trial {elapsedSeconds = 901})) aa+ validateCalibration now now definition acceptance late slowdown `shouldSatisfy` isLeft++fixture :: IO (CalibrationDefinition, CalibrationAcceptance, [[VerifiedCalibrationTrial]], [[VerifiedCalibrationTrial]])+fixture = do+ identifier <- newRunId+ let seeds = [1 .. 20]+ definition = CalibrationDefinition identifier "mori://fixture/runtime" "core" "synthetic-machine" (digest "d") (digest "a") (digest "b") (digest "c") (Text.replicate 40 "a") (Text.replicate 40 "b") "delay at operation boundary" "op.latency.p99" 0.5 1000000 seeds 1 (addUTCTime 3600 now)+ slot control seed = CalibrationSlot seed [CalibrationReceipt ("gs://fixture/" <> control <> "/" <> Text.pack (show seed) <> "/receipt.json") "sha256:receipt"]+ acceptance = CalibrationAcceptance identifier (map (slot "aa") seeds) (map (slot "slowdown") seeds)+ trial control verdict seed = do+ session <- newRunId+ pure [VerifiedCalibrationTrial session seed control definition.project definition.baselineRevision (if control == "aa" then definition.baselineRevision else definition.slowdownRevision) definition.profile definition.machineProfile definition.healthPolicySha256 definition.workloadSha256 definition.toolchainSha256 definition.harnessSemanticsSha256 verdict "healthy" [] now 90]+ aa <- traverse (trial "aa" "pass") seeds+ slowdown <- traverse (trial "none" "regression") seeds+ pure (definition, acceptance, aa, slowdown)+ where+ digest letter = "sha256:" <> Text.replicate 64 letter++now :: UTCTime+now = UTCTime (fromGregorian 2026 10 11) (secondsToDiffTime 0)++first :: [a] -> a+first (value : _) = value+first [] = error "empty synthetic calibration fixture"
+ test/Kenshou/PerfPreparedSpec.hs view
@@ -0,0 +1,171 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.PerfPreparedSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseEither)+import Data.ByteString qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified+import Kenshou.ConsumerBaseline qualified as ConsumerBaseline+import Kenshou.Core.Canonical (sha256Hex)+import Kenshou.Core.Cohort qualified as Cohort+import Kenshou.Core.Manifest qualified as Manifest+import Kenshou.Perf.Artifacts+import Kenshou.Perf.Calibration qualified as Calibration+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)+import Kenshou.Perf.Prepared hiding (plan)+import Kenshou.Perf.Source qualified as Source+import System.Directory (createFileLink)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance preparation integrity" do+ it "validates the pinned synthetic protocol fixture and rejects changed build keys" do+ consumerContracts <- loadConsumerContracts+ prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+ validatePrepared consumerContracts prepared `shouldBe` Right ()+ let baselineArm = (prepared :: PerfPrepared).baseline+ validatePrepared consumerContracts prepared {baseline = baselineArm {buildKey = "sha256:changed"}} `shouldSatisfy` isLeft+ decoded <- right (eitherDecode (encode prepared) :: Either String PerfPrepared)+ toJSON decoded `shouldBe` toJSON prepared+ it "admits resolved-cohort identities, replays legacy identities and rejects a rehashed alias" do+ consumerContracts <- loadConsumerContracts+ prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+ (request, template, baselineSource, candidateSource) <- right (parseEither (withObject "Inputs" \fields -> (,,,) <$> fields .: "request" <*> fields .: "template" <*> fields .: "baselineSource" <*> fields .: "candidateSource") (prepared :: PerfPrepared).inputs)+ baselineDescriptor <- right (Source.descriptorForSnapshot request template "baseline" baselineSource)+ candidateDescriptor <- right (Source.descriptorForSnapshot request template "candidate" candidateSource)+ let bind descriptor arm = do+ let digest = Data.Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor)))+ lock <- case arm.sourceLock of+ Object fields -> pure (Object (KeyMap.insert "cohort" (toJSON descriptor.descriptorName) (KeyMap.insert "descriptorSha256" (String digest) fields)))+ _ -> fail "fixture source lock is not an object"+ payload <- case arm.payload of+ Object fields -> do+ original <- maybe (fail "fixture payload has no identity") pure (KeyMap.lookup "cohortIdentity" fields)+ identity <- case fromJSON original of Error problem -> fail problem; Success value -> pure value+ let changed = (identity :: Cohort.CohortIdentity) {Cohort.identityCohort = descriptor.descriptorName, Cohort.identityDescriptorSha256 = digest}+ pure (Object (KeyMap.insert "cohortIdentity" (toJSON changed) (KeyMap.insert "flakeAttr" (String ("packages.x86_64-linux.kenshou-" <> Cohort.unCohortName descriptor.descriptorName)) fields)))+ _ -> fail "fixture payload is not an object"+ pure arm {descriptor = descriptor, sourceLock = lock, payload = payload, buildKey = buildKeyFor prepared.harnessRevision descriptor lock arm.buildInventory arm.boundsInventory}+ baseline <- bind baselineDescriptor prepared.baseline+ candidate <- bind candidateDescriptor prepared.candidate+ let modern = prepared {baseline = baseline, candidate = candidate}+ validatePrepared consumerContracts modern `shouldBe` Right ()+ validatePrepared consumerContracts prepared `shouldBe` Right ()+ validatePrepared consumerContracts modern {candidate = prepared.candidate} `shouldSatisfy` isLeft+ alias <- bind baselineDescriptor {Cohort.descriptorName = Cohort.CohortName "perf-unrelated-alias", Cohort.descriptorProjectFile = "perf-unrelated-alias.project"} baseline+ validatePrepared consumerContracts modern {baseline = alias, candidate = alias} `shouldSatisfy` isLeft+ it "binds actual inventories to the planning toolchain while allowing catalog-only promotion" do+ consumerContracts <- loadConsumerContracts+ prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+ let inventory = (prepared :: PerfPrepared).baseline.buildInventory+ case (prepared.inputs, inventory.toolchain) of+ (Object inputs, Object toolchain) -> do+ let expected = Object (KeyMap.delete "harnessRevision" toolchain)+ pinned = prepared {inputs = Object (KeyMap.insert "buildToolchain" expected inputs)}+ changed = inventory {toolchain = Object (KeyMap.insert "harnessRevision" (String "another catalog commit") toolchain)}+ validatePrepared consumerContracts pinned `shouldBe` Right ()+ Calibration.toolchainDigest inventory `shouldBe` Calibration.toolchainDigest changed+ validatePrepared consumerContracts prepared {inputs = Object (KeyMap.insert "buildToolchain" (object ["strictBounds" .= False]) inputs)} `shouldSatisfy` isLeft+ _ -> expectationFailure "invalid synthetic inventory fixture"+ it "rejects bound lifting inside a resealed inventory even when its toolchain claims strict bounds" do+ consumerContracts <- loadConsumerContracts+ prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right+ let arm = (prepared :: PerfPrepared).baseline+ corrupt = arm.buildInventory {packages = [entry {settings = object ["jailbreak" .= True, "configureFlags" .= ([] :: [Text])]} | entry <- arm.buildInventory.packages]}+ changed = arm {buildInventory = corrupt, buildKey = buildKeyFor prepared.harnessRevision arm.descriptor arm.sourceLock corrupt arm.boundsInventory}+ validatePrepared consumerContracts prepared {baseline = changed, candidate = changed} `shouldSatisfy` isLeft+ it "allows only target source/version changes; target flags still must match" do+ compareInventories ["runtime"] inventory inventory {packages = [runtime {version = "2", source = String "candidate"}, dependency]} `shouldBe` Right ()+ compareInventories ["runtime"] inventory inventory {packages = [runtime {flags = Map.singleton "feature" True}, dependency]} `shouldSatisfy` isLeft+ compareInventories ["runtime"] inventory inventory {packages = [runtime {settings = String "different"}, dependency]} `shouldSatisfy` isLeft+ it "rejects unrelated source, version, toolchain and package-set drift" do+ mapM_+ (\changed -> compareInventories ["runtime"] inventory changed `shouldSatisfy` isLeft)+ [inventory {packages = [runtime, dependency {source = String "other"}]}, inventory {packages = [runtime, dependency {version = "2"}]}, inventory {toolchain = String "other"}, inventory {packages = [runtime]}]+ it "rejects missing targets and ambiguous package identities" do+ compareInventories ["missing"] inventory inventory `shouldSatisfy` isLeft+ compareInventories ["runtime"] inventory inventory {packages = [runtime, dependency, dependency]} `shouldSatisfy` isLeft+ it "identifies harness sources by commit and includes transitive flags/revisions" do+ resolved <- right (cabalInventory "/workspace/harness" revision plan)+ length resolved.packages `shouldBe` 2+ map (.flags) resolved.packages `shouldContain` [Map.singleton "feature" True]+ cabalInventory "/workspace/other" revision plan `shouldSatisfy` isLeft+ cabalInventory "/workspace/harness" "main" plan `shouldSatisfy` isLeft+ eitherDecode (encode resolved) `shouldBe` Right resolved+ it "renders a standalone bounds project and rejects blanket exceptions" do+ descriptor <- syntheticCohort+ project <- right (renderBoundsProject noPerfContracts ["/harness/kenshou-core/kenshou-core.cabal"] descriptor)+ project `shouldNotContainText` "active.project"+ renderBoundsProject noPerfContracts ["/harness/package.cabal"] descriptor {Cohort.descriptorAllowNewer = ["all"]} `shouldSatisfy` isLeft+ renderBoundsProject noPerfContracts ["/unsafe path/package.cabal"] descriptor `shouldSatisfy` isLeft+ it "appends the suite's lane stanzas for the cohorts it selects" do+ descriptor <- syntheticCohort+ let laneName = Cohort.CohortName "acme-lane"+ contracts = noPerfContracts {laneStanzas = \selected -> if selected.descriptorName == laneName then ["package acme-suite", " flags: -extras"] else []}+ lane <- right (renderBoundsProject contracts ["/harness/acme-suite/acme-suite.cabal"] descriptor {Cohort.descriptorName = laneName})+ lane `shouldSatisfy` Data.Text.isInfixOf "package acme-suite\n flags: -extras"+ full <- right (renderBoundsProject contracts ["/harness/acme-suite/acme-suite.cabal"] descriptor)+ full `shouldNotContainText` "flags: -extras"+ it "verifies the complete sealed file set and refuses an omitted file" $ withSystemTempDirectory "perf-seal" \root -> do+ writeFile (root </> "input.json") "{}"+ sealPerfDirectory root `shouldReturn` Right ()+ verifyPerfManifest root `shouldReturn` Right ()+ writeFile (root </> "unlisted.json") "{}"+ verifyPerfManifest root >>= (`shouldSatisfy` isLeft)+ it "refuses duplicate and escaping paths before opening a listed file" $ withSystemTempDirectory "perf-seal" \root -> do+ writeFile (root </> "input.json") "{}"+ sealPerfDirectory root `shouldReturn` Right ()+ original <- eitherDecodeFileStrict' (root </> "manifest.json") >>= right+ encodeFile (root </> "manifest.json") (original :: Manifest.Manifest) {Manifest.files = original.files <> original.files}+ verifyPerfManifest root >>= (`shouldSatisfy` isLeft)+ encodeFile (root </> "manifest.json") original {Manifest.files = map (\file -> file {Manifest.path = "../outside.json"}) original.files}+ verifyPerfManifest root >>= (`shouldSatisfy` isLeft)+ it "refuses symbolic links when sealing or verifying evidence" $ withSystemTempDirectory "perf-seal" \root -> do+ writeFile (root </> "input.json") "{}"+ sealPerfDirectory root `shouldReturn` Right ()+ createFileLink (root </> "input.json") (root </> "link.json")+ sealPerfDirectory root >>= (`shouldSatisfy` isLeft)+ verifyPerfManifest root >>= (`shouldSatisfy` isLeft)++runtime, dependency :: InventoryPackage+runtime = InventoryPackage "runtime" "1" (String "baseline") Map.empty (object [])+dependency = InventoryPackage "dependency" "1" (String "fixed") Map.empty (object [])++inventory :: BuildInventory+inventory = BuildInventory "nix" "ghc-9.12.4" "x86_64-linux" (String "pins") [dependency, runtime]++revision :: Text+revision = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"++plan :: Value+plan =+ object+ [ "compiler-id" .= ("ghc-9.12.4" :: Text),+ "cabal-version" .= ("3.16" :: Text),+ "os" .= ("linux" :: Text),+ "arch" .= ("x86_64" :: Text),+ "install-plan"+ .= [ object ["type" .= ("configured" :: Text), "pkg-name" .= ("kenshou-core" :: Text), "pkg-version" .= ("0.1" :: Text), "pkg-src" .= object ["type" .= ("local" :: Text), "path" .= ("/workspace/harness/kenshou-core" :: Text)]],+ object ["type" .= ("configured" :: Text), "pkg-name" .= ("dependency" :: Text), "pkg-version" .= ("1" :: Text), "pkg-src" .= object ["type" .= ("repo-tar" :: Text)], "flags" .= Map.singleton ("feature" :: Text) True, "pkg-revision" .= (1 :: Int), "pkg-cabal-sha256" .= ("description" :: Text), "pkg-src-sha256" .= ("archive" :: Text)]+ ]+ ]++right :: (Show e) => Either e a -> IO a+right = either (\problem -> expectationFailure (show problem) >> fail "expected Right") pure++shouldNotContainText :: Text -> Text -> Expectation+shouldNotContainText actual needle = actual `shouldSatisfy` (not . Data.Text.isInfixOf needle)++loadConsumerContracts :: IO PerfContracts+loadConsumerContracts = ConsumerBaseline.consumerContracts++syntheticCohort :: IO Cohort.CohortDescriptor+syntheticCohort = ByteString.readFile "test/fixtures/perf/cohort.json" >>= right . eitherDecodeStrict'
+ test/Kenshou/PerfResultSpec.hs view
@@ -0,0 +1,116 @@+module Kenshou.PerfResultSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Perf.Plan+import Kenshou.Perf.Plan qualified as Plan+import Kenshou.Perf.Result+import Kenshou.Perf.Result qualified as Result+import Test.Hspec++spec :: Spec+spec = describe "performance aggregate results" do+ it "decodes the synthetic outcome fixtures with their distinct exit meanings" do+ let cases = [("pass", 0), ("incomplete", 3), ("runtime-failure", 1), ("regression", 1), ("inconclusive", 3), ("infrastructure-failure", 4), ("not-applicable", 0)]+ mapM_+ ( \(name, code) -> do+ result <- eitherDecodeFileStrict' ((if name == "not-applicable" then "test/golden/" else "../fixtures/consumer-baseline/kenshou-core/test/golden/") <> "perf-result." <> name <> ".fixture.json") >>= either fail pure+ (result :: PerfResult).exitCode `shouldBe` code+ )+ cases+ it "keeps calibration passes non-gating and distinguishes requested from change scope" do+ plan <- fixture+ result <- accepted (aggregateResult plan digest True (map (configuration "pass") plan.selected))+ result.exitCode `shouldBe` 0+ result.gatingEligible `shouldBe` False+ result.purpose `shouldBe` "calibration"+ result.changeCoverage `shouldBe` plan.changeCoverage+ (eitherDecode (encode result) :: Either String PerfResult) `shouldBe` Right result+ it "retains report references and measured stages while rejecting invalid metadata" do+ plan <- fixture+ result <- accepted (aggregateResult plan digest True (map (configuration "pass") plan.selected))+ let reported = result {stageSeconds = Map.singleton "totalAtReportAssembly" 17.5, artifactReferences = [ArtifactReference "aggregate-manifest" "gs://fixture-evidence/performance/manifest.json" Nothing]}+ (eitherDecode (encode reported) :: Either String PerfResult) `shouldBe` Right reported+ validateResult reported {stageSeconds = Map.singleton "negative" (-1)} `shouldSatisfy` isLeft+ validateResult reported {artifactReferences = [ArtifactReference "prepared" "gs://fixture-evidence/performance/prepared.json" (Just "mutable")]} `shouldSatisfy` isLeft+ it "uses 4 before 1 before 3 while preserving blockers and omissions" do+ plan <- fixture+ let (first, second) = twoProfiles plan+ blocker = (configuration "inconclusive" first) {runtimeFailures = ["known defect reproduced with exit zero"], comparison = Nothing}+ failure = (configuration "inconclusive" second) {infrastructureState = "failed", validPairs = 0, comparison = Nothing}+ result <- accepted (aggregateResult plan digest True [blocker, failure])+ result.exitCode `shouldBe` 4+ result.blockingRuntimeFailures `shouldBe` ["known defect reproduced with exit zero"]+ regression <- accepted (aggregateResult plan {Plan.coverage = "incomplete"} digest True [configuration "regression" first])+ regression.exitCode `shouldBe` 1+ regression.coverage `shouldBe` "incomplete"+ incomplete <- accepted (aggregateResult plan digest True [configuration "pass" first])+ incomplete.exitCode `shouldBe` 3+ it "does not turn an unresolved publication failure into success" do+ plan <- fixture+ result <- accepted (aggregateResult plan digest False (map (configuration "pass") plan.selected))+ result.exitCode `shouldBe` 4+ result.performanceVerdict `shouldBe` "inconclusive"+ it "keeps a statistical pass separate from a blocking runtime defect" do+ plan <- fixture+ let (first, second) = twoProfiles plan+ blocked = (configuration "pass" first) {runtimeFailures = ["known-defect-reproduced"]}+ result <- accepted (aggregateResult plan digest True [blocked, configuration "pass" second])+ result.performanceVerdict `shouldBe` "pass"+ result.exitCode `shouldBe` 1+ result.gatingEligible `shouldBe` False+ it "retains recovered invalid attempts without making healthy evidence fail" do+ plan <- fixture+ let results = [(configuration "pass" entry) {attempts = [object ["state" .= ("infrastructure-invalid" :: Text), "reason" .= ("reset failed" :: Text)]]} | entry <- plan.selected]+ result <- accepted (aggregateResult plan digest True results)+ result.exitCode `shouldBe` 0+ it "rejects a forged gate, missing pairs and inconsistent retained verdict" do+ plan <- fixture+ result <- accepted (aggregateResult plan digest True (map (configuration "pass") plan.selected))+ let first = fst (twoProfiles plan)+ validateResult result {Result.gatingEligible = True} `shouldSatisfy` isLeft+ aggregateResult plan digest True [(configuration "pass" first) {validPairs = 4}] `shouldSatisfy` isLeft+ aggregateResult plan digest True [(configuration "pass" first) {comparison = Just (comparisonValue "regression" 5)}] `shouldSatisfy` isLeft+ it "requires workload and pair counts from the pinned selection" do+ plan <- fixture+ let first = fst (twoProfiles plan)+ aggregateResult plan digest True [(configuration "pass" first) {Result.workloadSha256 = "sha256:" <> "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}] `shouldSatisfy` isLeft+ aggregateResult plan digest True [(configuration "pass" first) {requiredPairs = 6, validPairs = 6, comparison = Just (comparisonValue "pass" 6)}] `shouldSatisfy` isLeft+ it "uses not-applicable only for a proved unexecuted scope" do+ plan <- fixture+ let noWork = plan {selected = [], coverage = "not-applicable", changeCoverage = "not-applicable", gatingEligible = False}+ result <- accepted (aggregateResult noWork digest True [])+ result.performanceVerdict `shouldBe` "not-applicable"+ result.exitCode `shouldBe` 0+ result.gatingEligible `shouldBe` False+ it "replays across analysis IDs, clocks and hosts but retains metric and evidence identity" do+ let core = object ["schema" .= ("kenshou.comparison/v1" :: Text), "design" .= ("abba" :: Text), "baselineRuns" .= ["run-a" :: Text], "candidateRuns" .= ["run-b" :: Text], "algorithm" .= object [], "policy" .= object [], "variedFactors" .= ["control" :: Text], "pairCount" .= (5 :: Int), "metrics" .= object ["latency" .= (10 :: Int)], "reasons" .= ([] :: [Text]), "verdict" .= ("pass" :: Text), "exitCode" .= (0 :: Int)]+ change key value (Object fields) = Object (KeyMap.insert key value fields)+ change _ _ value = value+ original = change "comparisonId" (String "original") (change "harnessRevision" (String "host-a") core)+ replay = change "comparisonId" (String "replay") (change "finishedAt" (String "later") (change "harnessRevision" (String "host-b") core))+ semanticComparison original `shouldBe` semanticComparison replay+ semanticComparison (change "metrics" (object ["latency" .= (11 :: Int)]) replay) `shouldNotBe` semanticComparison original++twoProfiles :: PerfPlan -> (PlannedProfile, PlannedProfile)+twoProfiles plan = case plan.selected of+ [first, second] -> (first, second)+ _ -> error "synthetic plan fixture must contain exactly two profiles"++fixture :: IO PerfPlan+fixture = eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-plan.json" >>= either (fail . show) pure++digest :: Text+digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"++configuration :: Text -> PlannedProfile -> ConfigurationResult+configuration verdict entry = ConfigurationResult entry.profile.name entry.workloadSha256 verdict "healthy" [] entry.profile.minimumPairs entry.profile.minimumPairs [] (Just (comparisonValue verdict entry.profile.minimumPairs))++comparisonValue :: Text -> Int -> Value+comparisonValue verdict count = object ["schema" .= ("kenshou.comparison/v1" :: Text), "verdict" .= verdict, "pairCount" .= count]++accepted :: Either Text a -> IO a+accepted = either (fail . show) pure
+ test/Kenshou/PerfSpec.hs view
@@ -0,0 +1,230 @@+{-# LANGUAGE FieldSelectors #-}++module Kenshou.PerfSpec (spec) where++import Control.Monad (forM_)+import Data.Aeson+import Data.ByteString qualified as ByteString+import Data.Either (isLeft)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Cohort qualified as Cohort+import Kenshou.Core.Dimension+import Kenshou.Core.Id (parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Knob (mkKnobName)+import Kenshou.Core.Scenario (Placement (..), Tier (..))+import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)+import Kenshou.Perf.Plan qualified as Perf+import Kenshou.Perf.Request qualified as Request+import Kenshou.Perf.Source qualified as Source+import Kenshou.Plan.Catalog (ScenarioInfo (..))+import Kenshou.Plan.Change+import Kenshou.Plan.Components qualified as Graph+import Test.Hspec hiding (context)++spec :: Spec+spec = do+ describe "performance requests" do+ it "normalizes defaults and round-trips explicit request fields" do+ eitherDecode (encode request) `shouldBe` Right request+ it "binds scope, purpose, control and budgets to request identity" do+ map Request.requestDigest [request {Request.coverageMode = "affected"}, request {Request.purpose = "gating"}, request {Request.control = "none"}, request {Request.budgetMinutes = 31}]+ `shouldSatisfy` all (/= Request.requestDigest request)+ it "sorts set fields without changing the digest" do+ Request.requestDigest request {Request.profiles = reverse request.profiles} `shouldBe` Request.requestDigest request+ it "rejects abbreviated, mutable, uppercase and placeholder revisions" do+ mapM_ (\revision -> Request.validateRequest request {Request.baselineRevision = revision} `shouldSatisfy` isLeft) ["main", "HEAD", "BASE_SHA", "abcd", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"]+ it "rejects cross-project artifact URIs and malformed canonical project URIs" do+ mapM_ (\project -> Request.validateRequest request {Request.project = project} `shouldSatisfy` isLeft) ["acme", "mori://shinzui/acme/packages/acme-core", "mori://shinzui/../acme"]+ it "rejects A/A with distinct revisions and A/A without explicit profiles" do+ Request.validateRequest request {Request.candidateRevision = shaB} `shouldSatisfy` isLeft+ Request.validateRequest request {Request.profiles = []} `shouldSatisfy` isLeft+ it "rejects duplicate target IDs, empty scope and invalid budgets" do+ mapM_+ (\value -> Request.validateRequest value `shouldSatisfy` isLeft)+ [request {Request.targetComponents = ["acme", "acme"]}, request {Request.profiles = []}, request {Request.budgetMinutes = 0}, request {Request.waitSeconds = -1}]+ it "rejects unknown wire fields instead of silently ignoring a typo" do+ let malformed = object ["schema" .= ("kenshou.perf-request/v1" :: Text), "purpos" .= ("calibration" :: Text)]+ (eitherDecode (encode malformed) :: Either String Request.PerfRequest) `shouldSatisfy` isLeft+ describe "performance source identities for generated cohorts" do+ it "reuses an unchanged cohort across candidate revisions, calibration seeds and measurement scope" do+ descriptor <- template+ manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")+ metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.3\n")+ snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])+ baseline <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)+ let otherCandidate = request {Request.control = "none", Request.candidateRevision = shaB}+ revised = [request {Request.seed = request.seed + 1}, request {Request.cells = ["beta"]}, request {Request.budgetMinutes = 31}, request {Request.waitSeconds = 0}, request {Request.profiles = take 1 request.profiles}, request {Request.purpose = "gating"}, otherCandidate, otherCandidate {Request.candidateRevision = Text.replicate 40 "c"}]+ forM_ revised \selection -> Source.descriptorForSnapshot selection descriptor "baseline" snapshot `shouldBe` Right baseline+ Source.descriptorForSnapshot otherCandidate {Request.baselineRevision = shaB, Request.candidateRevision = shaA} descriptor "candidate" snapshot `shouldBe` Right baseline+ different <- right (Source.sourceSnapshot otherCandidate descriptor upstream shaB [manifest, metrics])+ Source.descriptorForSnapshot otherCandidate descriptor "candidate" different `shouldNotBe` Right baseline+ it "invalidates cohort identity when compiler, index state or fixed dependency inputs change" do+ descriptor <- template+ manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")+ metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.3\n")+ snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])+ original <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)+ let altered = descriptor {Cohort.descriptorComponents = map (\component -> if component.componentId == Cohort.ComponentId "acme" then component else component {Cohort.componentPackages = map (\pin -> pin {Cohort.pinVersion = "99.0.0"}) component.componentPackages}) descriptor.descriptorComponents}+ forM_ [descriptor {Cohort.descriptorCompiler = "ghc-9.14.1"}, descriptor {Cohort.descriptorIndexState = "2026-10-11T00:00:00Z"}, altered] \changed -> Source.descriptorForSnapshot request changed "baseline" snapshot `shouldNotBe` Right original+ it "reads manifest version and package directory independently in each revision" do+ descriptor <- template+ manifest <- right (Source.parsePackageManifest "moved/acme-core.cabal" "name: acme-core\nversion: 0.9.0.4\n")+ metrics <- right (Source.parsePackageManifest "acme-metrics/acme-metrics.cabal" "name: acme-metrics\nversion: 0.9.0.4\n")+ snapshot <- right (Source.sourceSnapshot request descriptor upstream shaA [manifest, metrics])+ generated <- right (Source.descriptorForSnapshot request descriptor "baseline" snapshot)+ otherArm <- right (Source.descriptorForSnapshot request descriptor "candidate" snapshot)+ generated `shouldBe` otherArm+ target <- first (filter ((== Cohort.ComponentId "acme") . (.componentId)) generated.descriptorComponents)+ target.componentSource `shouldBe` Cohort.GitSource upstream shaA+ map (.pinVersion) target.componentPackages `shouldBe` ["0.9.0.4", "0.9.0.4"]+ map (.pinSubdir) target.componentPackages `shouldBe` [Just "moved", Just "acme-metrics"]+ filter ((/= Cohort.ComponentId "acme") . (.componentId)) generated.descriptorComponents+ `shouldBe` filter ((/= Cohort.ComponentId "acme") . (.componentId)) descriptor.descriptorComponents+ it "rejects unknown descriptor IDs and targets owned by a different project" do+ descriptor <- template+ Source.targetPackages request {Request.targetComponents = ["unknown"]} descriptor `shouldSatisfy` isLeft+ Source.targetPackages request {Request.targetComponents = ["acme-quill-adapter"]} descriptor `shouldSatisfy` isLeft+ it "rejects missing and duplicate packages at a requested revision" do+ descriptor <- template+ manifest <- right (Source.parsePackageManifest "acme-core/acme-core.cabal" "name: acme-core\nversion: 0.9.0.3\n")+ Source.sourceSnapshot request descriptor upstream shaA [manifest] `shouldSatisfy` isLeft+ Source.sourceSnapshot request descriptor upstream shaA [manifest, manifest] `shouldSatisfy` isLeft+ it "rejects mutable template source pins" do+ descriptor <- template+ let components = map (\component -> component {Cohort.componentSource = Cohort.GitSource upstream "main"}) descriptor.descriptorComponents+ Source.targetPackages request descriptor {Cohort.descriptorComponents = components} `shouldSatisfy` isLeft+ it "rejects forged wire paths and manifest digests" do+ manifest <- right (Source.parsePackageManifest "safe/package.cabal" "name: package\nversion: 1.0\n")+ (eitherDecode (encode manifest {Source.path = "../outside.cabal"}) :: Either String Source.PackageManifest) `shouldSatisfy` isLeft+ (eitherDecode (encode manifest {Source.sha256 = "sha256:fake"}) :: Either String Source.PackageManifest) `shouldSatisfy` isLeft+ it "rejects ambiguous identity and paths that escape generated inputs" do+ Source.parsePackageManifest "../bad.cabal" "name: bad\nversion: 1.0\n" `shouldSatisfy` isLeft+ Source.parsePackageManifest "bad.cabal" "name: bad\nname: other\nversion: 1.0\n" `shouldSatisfy` isLeft+ describe "bounded performance planning" do+ it "retains explicit A/A profiles in calibration mode with an empty diff" do+ (graph, profiles, catalog) <- context+ plan <- right (Perf.planPerformance syntheticContracts request graph catalog profiles [] [] [] policyDigests Map.empty)+ length plan.selected `shouldBe` 2+ plan.coverage `shouldBe` "complete"+ plan.gatingEligible `shouldBe` False+ plan.estimateSeconds `shouldBe` 1540+ eitherDecode (encode plan) `shouldBe` Right plan+ map (.reasons) plan.selected `shouldSatisfy` all (not . null)+ it "cannot admit a catalog's unverified calibration claim to a gating plan" do+ (graph, profiles, catalog) <- context+ let claimed = profiles {Perf.profiles = map (\profile -> profile {Perf.calibration = Just (object ["calibrated" .= True])}) profiles.profiles}+ plan <- right (Perf.planPerformance syntheticContracts request {Request.purpose = "gating"} graph catalog claimed [] [] [] policyDigests Map.empty)+ plan.coverage `shouldBe` "incomplete"+ map snd plan.excluded `shouldBe` ["uncalibrated", "uncalibrated"]+ plan.gatingEligible `shouldBe` False+ it "binds workload identity without making calibration promotion self-referential" do+ (_, profiles, _) <- context+ profile <- first profiles.profiles+ Perf.workloadDigest profile "sha256:policy" `shouldBe` Perf.workloadDigest profile {Perf.calibration = Just (String "evidence")} "sha256:policy"+ Perf.workloadDigest profile "sha256:policy" `shouldNotBe` Perf.workloadDigest profile {Perf.scenarioRevision = 3} "sha256:policy"+ it "defers an entire configuration when the paired estimate exceeds budget" do+ (graph, profiles, catalog) <- context+ plan <- right (Perf.planPerformance syntheticContracts request {Request.budgetMinutes = 13} graph catalog profiles [] [] [] policyDigests Map.empty)+ length plan.selected `shouldBe` 1+ map snd plan.excluded `shouldBe` ["over-budget"]+ plan.coverage `shouldBe` "incomplete"+ it "reports transitive dependent and uncovered correctness paths" do+ (graph, profiles, catalog) <- context+ reference <- right (Graph.parseRef graph "acme-core:contract")+ let changes = [Change reference Named "core contract changed"]+ plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none", Request.coverageMode = "affected", Request.profiles = []} graph catalog profiles changes [] [] policyDigests Map.empty)+ plan.affectedScenarios `shouldContain` ["acme/quill-adapter/benchmark/end-to-end-throughput-latency"]+ plan.uncoveredScenarios `shouldContain` ["acme/metrics/correctness/test"]+ plan.coverage `shouldBe` "incomplete"+ it "separates selected-profile completion from whole-change coverage" do+ (graph, profiles, catalog) <- context+ reference <- right (Graph.parseRef graph "acme-core:contract")+ plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none"} graph catalog profiles [Change reference Named "core changed"] [] [] policyDigests Map.empty)+ plan.coverage `shouldBe` "complete"+ plan.changeCoverage `shouldBe` "incomplete"+ it "does not certify a telemetry change with disabled telemetry" do+ (graph, profiles, catalog) <- context+ reference <- right (Graph.parseRef graph "acme-core:telemetry")+ plan <- right (Perf.planPerformance syntheticContracts request {Request.control = "none", Request.coverageMode = "affected", Request.profiles = []} graph catalog profiles [Change reference Named "telemetry changed"] [] [] policyDigests Map.empty)+ plan.uncoveredScenarios `shouldContain` ["acme/core-ordering/benchmark/concurrency-sweep"]+ plan.coverage `shouldBe` "incomplete"+ it "selects conservatively for unmapped build inputs" do+ (graph, _, _) <- context+ let (changes, unknown, documentation) = Perf.conservativeChanges graph "acme" ["cabal.project", "docs/guide.md"] []+ map (.source) changes `shouldBe` [Everything]+ unknown `shouldBe` ["cabal.project"]+ documentation `shouldBe` ["docs/guide.md"]+ it "requires a repository-scoped documentation proof for not-applicable" do+ (graph, profiles, catalog) <- context+ let normal = request {Request.control = "none", Request.profiles = [], Request.coverageMode = "affected"}+ empty <- right (Perf.planPerformance syntheticContracts normal graph catalog profiles [] [] [] policyDigests Map.empty)+ empty.coverage `shouldBe` "incomplete"+ proved <- right (Perf.planPerformance syntheticContracts normal graph catalog profiles [] [] ["docs/guide.md"] policyDigests Map.empty)+ proved.coverage `shouldBe` "not-applicable"+ proved.gatingEligible `shouldBe` False+ it "rejects phases the pinned scenario revision cannot honor" do+ (_, profiles, catalog) <- context+ profile <- first profiles.profiles+ Perf.validateProfile syntheticContracts catalog profile {Perf.warmupSeconds = 1} `shouldSatisfy` isLeft+ Perf.validateProfile syntheticContracts catalog profile {Perf.steadyCount = 2000} `shouldSatisfy` isLeft+ Perf.validateProfile syntheticContracts catalog profile {Perf.timeoutSeconds = 181} `shouldSatisfy` isLeft+ it "rejects unknown profile names and raw sample suppression" do+ (graph, profiles, catalog) <- context+ Perf.planPerformance syntheticContracts request {Request.profiles = ["missing"]} graph catalog profiles [] [] [] policyDigests Map.empty `shouldSatisfy` isLeft+ profile <- first profiles.profiles+ Perf.validateProfile syntheticContracts catalog profile {Perf.knobs = Map.insert "measure.raw-samples" (String "off") profile.knobs} `shouldSatisfy` isLeft++shaA, shaB, upstream :: Text+shaA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+shaB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+upstream = "https://github.com/shinzui/acme.git"++request :: Request.PerfRequest+request = Request.PerfRequest "mori://shinzui/acme" shaA shaA "cohort/released.json" ["acme"] "change-performance" "selected-profiles" "calibration" "aa" ["acme-core-quick", "acme-quill-quick"] 20261010 ["alpha"] 90 30++policyDigests :: Map.Map Text Text+policyDigests = Map.singleton "policies/default.json" "sha256:policy"++context :: IO (Graph.ComponentGraph, Perf.ProfileCatalog, [ScenarioInfo])+context = do+ graph <- Graph.loadGraphFile "test/fixtures/plan/components.synthetic.json" >>= right+ profiles <- load "catalog.v1.json"+ catalog <- traverse makeScenario [(profile.scenario, profile.scenarioRevision, True) | profile <- profiles.profiles]+ correctness <- makeScenario ("acme/metrics/correctness/test", 1, False)+ pure (graph, profiles, catalog <> [correctness])+ where+ makeScenario (name, revision, benchmark) = do+ identifier <- right (parseScenarioId name)+ let dimensions = if name == "acme/core-ordering/benchmark/concurrency-sweep" then allTelemetryArms noDimensions else postgresDimensions (PgDurable :| []) (Pg18 :| [Pg17]) noDimensions+ knobs <- traverse makeKnob [("bench.arm", KnobText, VText "configured"), ("bench.rate", KnobInt, VInt 200), ("acme.messages", KnobInt, VInt 1000), ("measure.raw-samples", KnobText, VText "full")]+ pure (ScenarioInfo identifier revision "fixture" TierStandard PlaceEither (if benchmark then knobs else []) dimensions Nothing)+ makeKnob (name, kind, def) = do+ key <- right (mkKnobName name)+ pure (KnobSpec key "fixture" kind def AnyValue [])++template :: IO Cohort.CohortDescriptor+template = load "cohort.json"++load :: (FromJSON a) => FilePath -> IO a+load path = ByteString.readFile ("test/fixtures/perf/" <> path) >>= right . eitherDecodeStrict'++right :: (Show e) => Either e a -> IO a+right (Right value) = pure value+right (Left problem) = expectationFailure (show problem) >> fail "expected Right"++first :: [a] -> IO a+first (value : _) = pure value+first [] = expectationFailure "expected a nonempty fixture" >> fail "empty fixture"++-- The two synthetic benchmark revisions the catalog's profiles pin implement+-- count-bound steady work; their steady count is the acme.messages knob.+syntheticContracts :: PerfContracts+syntheticContracts =+ noPerfContracts+ { reviewedPhaseContracts = [("acme/core-ordering/benchmark/concurrency-sweep", 2), ("acme/quill-adapter/benchmark/end-to-end-throughput-latency", 1)],+ steadyCountKnob = either (const Nothing) Just (mkKnobName "acme.messages")+ }
+ test/Kenshou/PlanSpec.hs view
@@ -0,0 +1,336 @@+module Kenshou.PlanSpec (spec) where++import Data.Aeson qualified as Aeson+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Dimension+import Kenshou.Core.Id (Kind (..), mkSeed, parseRunId, parseScenarioId, renderScenarioId)+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.RunSpec (SpecPlacement (..))+import Kenshou.Core.Scenario (Placement (..), Tier (..))+import Kenshou.Plan.Catalog (ScenarioInfo (..), readCatalogFile)+import Kenshou.Plan.Change+import Kenshou.Plan.Change.Cohort+import Kenshou.Plan.Change.Git qualified as Git+import Kenshou.Plan.Components+import Kenshou.Plan.Components.Check+import Kenshou.Plan.Matrix+import Kenshou.Plan.Policy+import Kenshou.Plan.Policy qualified as Policy+import Kenshou.Plan.RunPlan (PlanContext (..), PlanInputs (..), PlanSkeleton (..), SkipReason (..), Skipped (..), buildPlan)+import Kenshou.Plan.Selector+import Kenshou.Plan.Suite+import Kenshou.Plan.Summary qualified as Summary+import System.Directory (createDirectoryIfMissing)+import System.FilePath (takeDirectory, (</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (callProcess)+import Test.Hspec++spec :: Spec+spec = do+ describe "Kenshou.Plan.Selector" do+ it "round-trips and matches recursive selectors" do+ selector <- expectRight (parseSelector "acme/quill-adapter/**")+ scenario <- expectRight (parseScenarioId "acme/quill-adapter/correctness/finalize")+ renderSelector selector `shouldBe` "acme/quill-adapter/**"+ matches selector scenario `shouldBe` True+ it "rejects recursive wildcards before the final position" do+ parseSelector "acme/**/correctness/*" `shouldBe` Left "** is permitted only as the final segment"++ describe "Kenshou.Plan.Catalog" do+ it "decodes the planned scenario catalog" do+ result <- readCatalogFile "test/fixtures/plan/catalog-planned.json"+ catalog <- expectRight result+ length catalog `shouldSatisfy` (> 40)++ describe "Kenshou.Plan.Components" do+ it "validates the embedded graph" do+ graph <- syntheticGraph >>= expectRight+ validateGraph graph `shouldBe` []+ it "computes the quill dependent closure" do+ graph <- syntheticGraph >>= expectRight+ origin <- expectRight (parseRef graph "quill-hs")+ let affected = dependents graph (Set.singleton origin)+ present text = either (const False) (`Map.member` affected) (parseRef graph text)+ present "acme-quill-adapter" `shouldBe` True+ present "flux-quill" `shouldBe` True+ present "kenshou-harness" `shouldBe` True+ present "runtime-assembly" `shouldBe` True+ it "keeps acme runner changes away from flux contract consumers" do+ graph <- syntheticGraph >>= expectRight+ origin <- expectRight (parseRef graph "acme-core:runner")+ let affected = dependents graph (Set.singleton origin)+ present text = either (const False) (`Map.member` affected) (parseRef graph text)+ present "flux-quill" `shouldBe` True+ present "acme-metrics" `shouldBe` True+ present "acme-tally-adapter" `shouldBe` True+ present "runtime-assembly" `shouldBe` True+ present "flux" `shouldBe` False+ present "flux:router" `shouldBe` False+ it "routes acme contract changes to the three flux consumers" do+ graph <- syntheticGraph >>= expectRight+ origin <- expectRight (parseRef graph "acme-core:contract")+ let affected = dependents graph (Set.singleton origin)+ present text = either (const False) (`Map.member` affected) (parseRef graph text)+ present "flux:process-manager" `shouldBe` True+ present "flux:router" `shouldBe` True+ present "flux:inbox" `shouldBe` True+ present "flux:command" `shouldBe` False+ present "flux:outbox" `shouldBe` False+ present "flux:workflow" `shouldBe` False++ describe "Kenshou.Plan.Components.Check" do+ it "resolves shortened unit ids and finds a planted missing edge" do+ graph <- syntheticGraph >>= expectRight+ result <- checkAgainstPlanJson graph "test/fixtures/plan/plan-missing-edge.json"+ findings <- expectRight result+ findings `shouldSatisfy` any isMissing++ describe "Kenshou.Plan.Change" do+ it "selects only the QUILL family and its dependents" do+ (graph, catalog) <- fixtureInputs+ changed <- named graph "quill-hs"+ selectedIds (selectScenarios graph catalog [changed])+ `shouldBe` expectedIds catalog ["quill/", "acme/quill-adapter/", "flux/queue/", "runtime/", "selftest/kernel/correctness/postgres-roundtrip"]+ it "selects all Tally, Flux, runtime, and kernel evidence for tally-store" do+ (graph, catalog) <- fixtureInputs+ changed <- named graph "tally-store"+ selectedIds (selectScenarios graph catalog [changed])+ `shouldBe` expectedIds catalog ["tally/", "acme/tally-adapter/", "flux/", "runtime/", "selftest/kernel/"]+ it "distinguishes acme runner changes from contract changes" do+ (graph, catalog) <- fixtureInputs+ runner <- named graph "acme-core:runner"+ contract <- named graph "acme-core:contract"+ selectedIds (selectScenarios graph catalog [runner])+ `shouldBe` expectedIds catalog ["acme/", "brook/", "flux/queue/", "runtime/"]+ let contractIds = selectedIds (selectScenarios graph catalog [contract])+ contractIds `shouldSatisfy` Set.member "flux/process-manager/correctness/react"+ contractIds `shouldSatisfy` Set.member "flux/router/correctness/route"+ contractIds `shouldSatisfy` Set.member "flux/inbox/correctness/deduplicate"+ contractIds `shouldSatisfy` Set.notMember "flux/command/correctness/decide"+ contractIds `shouldSatisfy` Set.notMember "flux/outbox/concurrency/publish"+ contractIds `shouldSatisfy` Set.notMember "flux/workflow/correctness/resume"+ it "maps an outbox-only upstream diff and ignores upstream documentation" do+ (graph, catalog) <- fixtureInputs+ withSystemTempDirectory "kenshou-upstream" \repository -> do+ initialiseRepository repository+ writeRepoFile repository "flux/src/Flux/Outbox/Schema.hs" "module Schema where\n"+ writeRepoFile repository "docs/readme.md" "initial\n"+ commitAll repository "initial"+ writeRepoFile repository "flux/src/Flux/Outbox/Schema.hs" "module Schema where\nvalue = 1\n"+ commitAll repository "outbox"+ result <- Git.changesFromUpstream graph (Git.UpstreamDiff "flux" repository "HEAD^" "HEAD")+ (changes, ignored) <- expectRight result+ ignored `shouldBe` 0+ selectedIds (selectScenarios graph catalog changes)+ `shouldBe` expectedIds catalog ["flux/outbox/", "runtime/"]+ writeRepoFile repository "docs/readme.md" "changed\n"+ commitAll repository "docs"+ docsResult <- Git.changesFromUpstream graph (Git.UpstreamDiff "flux" repository "HEAD^" "HEAD")+ (docsChanges, docsIgnored) <- expectRight docsResult+ docsChanges `shouldBe` []+ docsIgnored `shouldBe` 1+ it "maps --since repository paths to selectors and ignores docs" do+ (graph, catalog) <- fixtureInputs+ withSystemTempDirectory "kenshou-since" \repository -> do+ initialiseRepository repository+ writeRepoFile repository "kenshou-tally/x" "initial\n"+ writeRepoFile repository "docs/z" "initial\n"+ commitAll repository "initial"+ writeRepoFile repository "kenshou-tally/x" "changed\n"+ result <- Git.changesSince graph repository "HEAD"+ (changes, selectors, warnings) <- expectRight result+ changes `shouldBe` []+ warnings `shouldBe` []+ selectedIds (selectBySelectors Since "repository path" selectors catalog)+ `shouldBe` expectedIds catalog ["tally/"]+ it "maps a cohort package difference and fails safe for unknown packages" do+ (graph, _catalog) <- fixtureInputs+ let old = Map.fromList [("tally-store", FromHackage "0.8.0.1")]+ new = Map.fromList [("tally-store", FromHackage "0.8.0.2"), ("unmapped", FromHackage "1")]+ (changes, warnings) = diffCohorts graph old new+ fmap (.ref) changes `shouldContain` [ComponentRef (ComponentId "tally-store") Nothing]+ fmap (.source) changes `shouldContain` [Everything]+ fmap (.code) warnings `shouldContain` ["unmapped-package"]++ describe "Kenshou.Plan.Matrix" do+ it "covers every pair in at most twenty rows" do+ let dimensions = [("a", ["0", "1", "2", "3"]), ("b", ["0", "1", "2", "3"]), ("c", ["0", "1"]), ("d", ["0", "1"])]+ rows = pairwiseCover dimensions+ length rows `shouldSatisfy` (<= 20)+ pairSet rows `shouldBe` pairSet (fullRows dimensions)+ it "raises an observability change to telemetry corners" do+ selected <- matrixSelected Correctness (Just "telemetry-corners")+ seed <- expectRight (mkSeed 42)+ let (configs, skipped) = expandScenario (defaultPlanPolicy seed) selected+ skipped `shouldBe` []+ length configs `shouldBe` 4+ it "never emits fsync-off for a benchmark" do+ selected <- matrixSelected Benchmark Nothing+ seed <- expectRight (mkSeed 42)+ let policy = (defaultPlanPolicy seed :: PlanPolicy) {Policy.placement = RunOnCell}+ (configs, skipped) = expandScenario policy selected+ skipped `shouldBe` []+ fmap (Map.lookup "pg.durability" . (.dimensions)) configs `shouldSatisfy` all (== Just "durable")+ it "admits a benchmark without PostgreSQL without inventing a durability dimension" do+ selected <- matrixSelected Benchmark Nothing+ seed <- expectRight (mkSeed 42)+ let scenario = (selected.scenario :: ScenarioInfo) {dimensions = noDimensions}+ policy = (defaultPlanPolicy seed :: PlanPolicy) {Policy.placement = RunOnCell}+ (configs, skipped) = expandScenario policy (selected {scenario = scenario})+ skipped `shouldBe` []+ configs `shouldSatisfy` (not . null)+ fmap (Map.lookup "pg.durability" . (.dimensions)) configs `shouldSatisfy` all (== Nothing)++ describe "Kenshou.Plan.RunPlan" do+ it "keeps benchmark trial groups whole under a budget" do+ selected <- matrixSelected Benchmark Nothing+ seed <- expectRight (mkSeed 42)+ let policy = (defaultPlanPolicy seed :: PlanPolicy) {Policy.placement = RunOnCell, Policy.budgetMinutes = Just 2}+ skeleton = buildPlan testPlanContext policy [selected]+ skeleton.runs `shouldBe` []+ fmap (.reason) skeleton.skipped `shouldContain` [SkipOverBudget]++ describe "Kenshou.Plan.Suite" do+ it "decodes every checked-in suite" do+ let names = ["smoke", "change", "nightly", "weekly-soak", "release"] :: [Text]+ results <- traverse (readSuite . ("../suites/" <>) . (<> ".json") . Text.unpack) names+ suites <- traverse expectRight results+ fmap (.name) suites `shouldBe` names+ it "raises knob policy only for directly selected scenarios" do+ suite <- readSuite "../suites/change.json" >>= expectRight+ (_, catalog) <- fixtureInputs+ scenario <- maybe (expectationFailure "fixture has no scenario with knobs" >> fail "unreachable") pure (find (not . null . (.knobs)) catalog)+ selector <- expectRight (parseSelector "**")+ let change = Change (ComponentRef (ComponentId "fixture") Nothing) Named "fixture"+ selected distance = Selected scenario (Reason change [change.ref] selector distance :| []) Nothing Nothing+ direct = applySuite suite [scenario] [selected 0]+ dependent = applySuite suite [scenario] [selected 1]+ fmap (.minKnobPolicy) direct `shouldBe` [Just "declared-variants"]+ fmap (.minKnobPolicy) dependent `shouldBe` [Nothing]++ describe "Kenshou.Plan.Summary" do+ it "treats reproduced known defects as non-blocking" do+ runId <- expectRight (parseRunId "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011")+ scenarioId <- expectRight (parseScenarioId "selftest/kernel/correctness/known-defect")+ let attempt = Summary.Attempt runId (read "2026-09-21 00:00:00 UTC") (Just (read "2026-09-21 00:00:01 UTC")) (Just Failed) True (Just 0)+ summary = Summary.PlanSummary runId [Summary.SummaryEntry 1 scenarioId Summary.Completed [attempt]] Map.empty Passed 0+ Summary.worstOutcome summary `shouldBe` Passed+ it "counts an unfinished entry as errored" do+ runId <- expectRight (parseRunId "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011")+ scenarioId <- expectRight (parseScenarioId "selftest/kernel/correctness/always-pass")+ let summary = Summary.PlanSummary runId [Summary.SummaryEntry 1 scenarioId Summary.Pending []] Map.empty Passed 0+ Summary.worstOutcome summary `shouldBe` Errored+ where+ isMissing (MissingEdge (ComponentId "quill-hs") (ComponentId "tally-store") _) = True+ isMissing _ = False++fixtureInputs :: IO (ComponentGraph, [ScenarioInfo])+fixtureInputs = do+ graph <- syntheticGraph >>= expectRight+ catalog <- readCatalogFile "test/fixtures/plan/catalog-planned.json" >>= expectRight+ pure (graph, catalog)++named :: ComponentGraph -> Text -> IO Change+named graph reference = do+ parsed <- expectRight (parseRef graph reference)+ pure (Change parsed Named ("--changed " <> reference))++selectedIds :: [Selected] -> Set.Set Text+selectedIds = Set.fromList . fmap (renderScenarioId . (.id) . (.scenario))++expectedIds :: [ScenarioInfo] -> [Text] -> Set.Set Text+expectedIds catalog prefixes =+ Set.fromList+ [ identifier+ | scenario <- catalog,+ let identifier = renderScenarioId scenario.id,+ any (`Text.isPrefixOf` identifier) prefixes+ ]++initialiseRepository :: FilePath -> IO ()+initialiseRepository repository = do+ callProcess "git" ["-C", repository, "init", "-q"]+ callProcess "git" ["-C", repository, "config", "user.email", "kenshou@example.invalid"]+ callProcess "git" ["-C", repository, "config", "user.name", "Kenshou Test"]++writeRepoFile :: FilePath -> FilePath -> String -> IO ()+writeRepoFile repository relative contents = do+ createDirectoryIfMissing True (repository </> takeDirectory relative)+ writeFile (repository </> relative) contents++commitAll :: FilePath -> String -> IO ()+commitAll repository message = do+ callProcess "git" ["-C", repository, "add", "."]+ callProcess "git" ["-C", repository, "commit", "-q", "-m", message]++matrixSelected :: Kind -> Maybe Text -> IO Selected+matrixSelected kind minimumPolicy = do+ scenarioId <- expectRight (parseScenarioId ("tally/append/" <> kindText kind <> "/matrix"))+ selector <- expectRight (parseSelector "tally/**")+ let scenario =+ ScenarioInfo+ { id = scenarioId,+ revision = 1,+ summary = "matrix fixture",+ tier = TierSmoke,+ placement = if kind == Benchmark then PlaceCell else PlaceEither,+ knobs = [],+ dimensions = allMatrixDimensions,+ knownDefect = Nothing+ }+ change = Change (ComponentRef (ComponentId "tally-store") Nothing) Named "fixture"+ pure (Selected scenario (Reason change [change.ref] selector 0 :| []) minimumPolicy Nothing)+ where+ kindText Correctness = "correctness"+ kindText Concurrency = "concurrency"+ kindText Benchmark = "benchmark"+ kindText Soak = "soak"++allMatrixDimensions :: DimensionSupport+allMatrixDimensions =+ DimensionSupport+ (Supported (Support (TracingOff :| [TracingNoop, TracingSdkInMemory, TracingSdkOtlp]) TracingOff))+ (Supported (Support (MetricsOff :| [MetricsCollect, MetricsServe, MetricsServeScraped]) MetricsOff))+ (Supported (Support (PgFsyncOff :| [PgDurable]) PgFsyncOff))+ (Supported (Support (Pg17 :| [Pg18]) Pg18))++pairSet :: [Map.Map Text Text] -> Set.Set ((Text, Text), (Text, Text))+pairSet rows =+ Set.fromList+ [ (left, right)+ | row <- rows,+ (leftIndex, left) <- zip [0 :: Int ..] (Map.toAscList row),+ (rightIndex, right) <- zip [0 :: Int ..] (Map.toAscList row),+ leftIndex < rightIndex+ ]++fullRows :: [(Text, [Text])] -> [Map.Map Text Text]+fullRows = foldr (\(name, values) rows -> [Map.insert name value row | value <- values, row <- rows]) [Map.empty]++testPlanContext :: PlanContext+testPlanContext =+ PlanContext+ { suite = Nothing,+ graphDigest = "sha256:test",+ cohortName = "released",+ cohortPlanHash = "sha256:test",+ inputs = PlanInputs (Aeson.object []),+ changes = [],+ warnings = []+ }++expectRight :: (Show problem) => Either problem value -> IO value+expectRight (Right value) = pure value+expectRight (Left problem) = expectationFailure (show problem) >> fail "unreachable"++-- A component graph with the shapes the planner must handle (libraries with+-- subcomponents and uses edges, dependsOn edges with fromSubs and toSubs, a+-- changeImplies policy, and repository paths) under synthetic names.+syntheticGraph :: IO (Either GraphError ComponentGraph)+syntheticGraph = loadGraphFile "test/fixtures/plan/components.synthetic.json"
+ test/Kenshou/ResourceSpec.hs view
@@ -0,0 +1,51 @@+module Kenshou.ResourceSpec (spec) where++import Data.Aeson (FromJSON, Value (..), eitherDecodeFileStrict', object, toJSON, (.=))+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Either (isLeft)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Core.Canonical (canonicalEncode)+import Kenshou.Core.Compat (renderResourceInputs)+import Kenshou.Core.Resource.Types (ResourceName, mkResourceName, renderResourceName)+import Kenshou.Core.Role (WorkerInit (..))+import Kenshou.Core.RunSpec (RunSpec (..))+import Kenshou.Core.RunSpec qualified as RunSpec+import Test.Hspec++spec :: Spec+spec = describe "named resources" do+ it "validates resource names" do+ renderResourceName <$> mkResourceName "object-store" `shouldBe` Right "object-store"+ mapM_ (\bad -> isLeft (mkResourceName bad) `shouldBe` True) ["", "Object", "9lives", "has space", "x_y"]+ it "round-trips a run specification with a resource map to identical canonical JSON" do+ raw <- decodeFile "test/golden/resources/run-spec.resources.json" :: IO Value+ decoded <- decodeFile "test/golden/resources/run-spec.resources.json" :: IO RunSpec+ Map.keys decoded.environment.resources `shouldBe` [name "broker", name "object-store"]+ canonicalEncode (toJSON decoded) `shouldBe` canonicalEncode raw+ it "round-trips a worker initialization with a resource map to identical canonical JSON" do+ raw <- decodeFile "test/golden/resources/worker-init.resources.json" :: IO Value+ decoded <- decodeFile "test/golden/resources/worker-init.resources.json" :: IO WorkerInit+ Map.keys decoded.resources `shouldBe` [name "broker", name "object-store"]+ canonicalEncode (toJSON decoded) `shouldBe` canonicalEncode raw+ it "omits an empty resource map from the run specification but always writes it for workers" do+ decoded <- decodeFile "test/golden/resources/run-spec.resources.json" :: IO RunSpec+ let emptied = decoded {environment = decoded.environment {RunSpec.resources = Map.empty}}+ environmentKeys (toJSON emptied) `shouldSatisfy` notElem "resources"+ worker <- decodeFile "test/golden/resources/worker-init.resources.json" :: IO WorkerInit+ case toJSON worker {resources = Map.empty} of+ Object fields -> KeyMap.lookup "resources" fields `shouldBe` Just (object [])+ _ -> expectationFailure "worker-init is not an object"+ it "renders resources under one compatibility member, omitted when empty" do+ let value = object ["bucket" .= ("fixture" :: Text)]+ object (renderResourceInputs (Map.singleton (name "object-store") value)) `shouldBe` object ["resources" .= object ["object-store" .= value]]+ object (renderResourceInputs Map.empty) `shouldBe` object []+ where+ name text = either (error . show) id (mkResourceName text) :: ResourceName+ environmentKeys (Object fields) = case KeyMap.lookup "environment" fields of+ Just (Object environment) -> KeyMap.keys environment+ _ -> []+ environmentKeys _ = []++decodeFile :: (FromJSON a) => FilePath -> IO a+decodeFile path = eitherDecodeFileStrict' path >>= either fail pure
+ test/Main.hs view
@@ -0,0 +1,582 @@+{-# LANGUAGE FieldSelectors #-}++module Main (main) where++import Control.Exception (bracket_)+import Data.Aeson (FromJSON, Value (..), eitherDecode, eitherDecodeFileStrict', encode, encodeFile, object, toJSON, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.Either (isLeft, isRight)+import Data.Foldable (toList)+import Data.IORef (modifyIORef', newIORef, readIORef)+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text.Encoding+import Database.PostgreSQL.Migrate (migrationComponent, sqlMigration)+import Kenshou.Core.Bundle (LayerBundle (..), ListFilter (..), RegistryError (..), ScenarioRule (..), allScenarios, emptyListFilter, mkBundle, mkRegistry, registryMigrationSets, requirePgVersionsFor, selectScenarios)+import Kenshou.Core.Bundle qualified as Bundle+import Kenshou.Core.Canonical (canonicalEncode)+import Kenshou.Core.Cohort+import Kenshou.Core.Compat (comparisonKey, compatInputs, seriesKey)+import Kenshou.Core.Context (Environment (..), RunContext (..))+import Kenshou.Core.Dimension+import Kenshou.Core.Env (EnvRequirements (..), MigrationSet (..), MigrationSetName (..), PostgresRequirement (..), noEnvironment, requiresResource)+import Kenshou.Core.Env.Migration (composePlan, orderMigrationSets)+import Kenshou.Core.Env.Postgres (EnvError (..), PgSettingsSnapshot (..), PostgresEnv (..), ServerControl (..), StopMode (..), withPostgresEnv, withPostgresEnvNamed)+import Kenshou.Core.Id+import Kenshou.Core.Knob+import Kenshou.Core.Log (nullLogger)+import Kenshou.Core.Manifest (verifyManifest, writeManifest)+import Kenshou.Core.Outcome (Outcome (Passed))+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.Resource (ProvisionedResource (..), ResourceProvider (..), mkResourceName)+import Kenshou.Core.Run (RunOutput (..), RunnerConfig (..), defectDisposition, executeRun)+import Kenshou.Core.RunResult (KnownDefectStatus (..))+import Kenshou.Core.RunSpec+import Kenshou.Core.RunSpec.Resolve (resolveRunSpec)+import Kenshou.Core.Scenario (CohortScope (..), KnownDefect (..), PackageCondition (..), Placement (..), Scenario (..), ScenarioReport (..), Tier (..), failedWith)+import Kenshou.Core.Selector (matchesSelector, parseSelector)+import Kenshou.Core.Selftest qualified as Selftest+import Kenshou.Core.Selftest.Schema (selftestMigrationSet, selftestSchema)+import Kenshou.PerfCalibrationSpec qualified+import Kenshou.PerfPreparedSpec qualified+import Kenshou.PerfResultSpec qualified+import Kenshou.PerfSpec qualified+import Kenshou.PlanSpec qualified+import Kenshou.ResourceSpec qualified+import System.Environment (lookupEnv)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import System.Process (readProcessWithExitCode)+import Test.Hspec+ ( Spec,+ describe,+ expectationFailure,+ hspec,+ it,+ pendingWith,+ shouldBe,+ shouldContain,+ shouldNotBe,+ shouldReturn,+ )++main :: IO ()+main = hspec do+ Kenshou.PerfCalibrationSpec.spec+ Kenshou.PerfResultSpec.spec+ descriptorSpec+ hashSpec+ mismatchSpec+ activeCohortSpec+ identifierSpec+ layerIdentifierSpec+ selectorSpec+ registrySpec+ openRegistrySpec+ resourceProvisioningSpec+ knownDefectGroupSpec+ knobSpec+ dimensionSpec+ runSpecSpec+ compatibilitySpec+ manifestSpec+ goldenSpec+ postgresEnvironmentSpec+ migrationPlanSpec+ Kenshou.PlanSpec.spec+ Kenshou.ResourceSpec.spec+ Kenshou.PerfPreparedSpec.spec+ Kenshou.PerfSpec.spec++-- The consumer's own layer strings are checked against its sealed records in the+-- kenshou-evidence consumer compatibility tests; these cases use synthetic names.+layerIdentifierSpec :: Spec+layerIdentifierSpec = describe "layer identifiers" do+ it "renders the self-test layer" do+ renderLayer selftestLayer `shouldBe` "selftest"+ it "accepts any valid segment and renders scenario identifiers in any layer" do+ renderLayer <$> parseLayer "acme" `shouldBe` Right "acme"+ renderScenarioId <$> parseScenarioId "acme/widget/correctness/roundtrip" `shouldBe` Right "acme/widget/correctness/roundtrip"+ it "rejects invalid segments" do+ isLeft (parseLayer "Acme") `shouldBe` True+ isLeft (parseLayer "") `shouldBe` True++descriptorSpec :: Spec+descriptorSpec = describe "cohort identity" do+ it "matches the stable JSON golden" do+ descriptor <- decodeFixture "descriptor-released.json"+ plan <- decodeFixture "plan-released.json"+ golden <- decodeFixture "cohort-identity.golden.json"+ case identityFromPlan descriptor "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" plan of+ Left err -> expectationFailure (show err)+ Right identity -> toJSON identity `shouldBe` (golden :: Value)+ it "round-trips a Nix resolver while old Cabal records omit it" do+ let cabalIdentity = identityWith []+ nixIdentity = cabalIdentity {identityResolver = Just "nix"}+ eitherDecode (encode cabalIdentity) `shouldBe` Right cabalIdentity+ eitherDecode (encode nixIdentity) `shouldBe` Right nixIdentity++hashSpec :: Spec+hashSpec = describe "planHash" do+ it "ignores install-plan order and local checkout paths" do+ baseline <- hashFixture "plan-released.json"+ reordered <- hashFixture "plan-reordered.json"+ moved <- hashFixture "plan-local-change.json"+ reordered `shouldBe` baseline+ moved `shouldBe` baseline++ it "changes with a package version or git revision" do+ baseline <- hashFixture "plan-released.json"+ versionChanged <- hashFixture "plan-version-change.json"+ commitChanged <- hashFixture "plan-commit-change.json"+ versionChanged `shouldNotBe` baseline+ commitChanged `shouldNotBe` baseline++mismatchSpec :: Spec+mismatchSpec = describe "checkCohort" do+ it "reports every mismatch constructor" do+ let descriptor = onePackageDescriptor+ missing = identityWith []+ wrongVersion = identityWith [ResolvedPackage "foo" "2.0.0" (FromHackage Nothing)]+ wrongSource = identityWith [ResolvedPackage "foo" "1.0.0" (FromGit "https://example.com/foo.git" "abc" Nothing)]+ localSource = identityWith [ResolvedPackage "foo" "1.0.0" (FromLocalPath "/tmp/foo")]+ wrongCohort = (identityWith [ResolvedPackage "foo" "1.0.0" (FromHackage Nothing)]) {identityCohort = CohortName "head"}+ wrongCompiler = (identityWith [ResolvedPackage "foo" "1.0.0" (FromHackage Nothing)]) {identityCompiler = "ghc-9.10.3"}+ checkCohort descriptor missing `shouldContain` [MissingPackage "foo"]+ checkCohort descriptor wrongVersion `shouldContain` [VersionMismatch "foo" "1.0.0" "2.0.0"]+ checkCohort descriptor wrongSource `shouldContain` [SourceMismatch "foo" HackageSource (FromGit "https://example.com/foo.git" "abc" Nothing)]+ checkCohort descriptor localSource `shouldContain` [LocalPathSource "foo" "/tmp/foo"]+ checkCohort descriptor wrongCohort `shouldContain` [CohortNameMismatch (CohortName "released") (CohortName "head")]+ checkCohort descriptor wrongCompiler `shouldContain` [CompilerMismatch "ghc-9.12.4" "ghc-9.10.3"]++activeCohortSpec :: Spec+activeCohortSpec = describe "activeCohortName" do+ it "accepts exactly one cohort import" do+ result <- activeCohortName (fixtures <> "/valid-project")+ result `shouldBe` Right (CohortName "released")++ it "rejects extra lines" do+ result <- activeCohortName (fixtures <> "/invalid-project")+ case result of+ Left (CohortInvalidActive _) -> pure ()+ other -> expectationFailure ("expected CohortInvalidActive, got " <> show other)++identifierSpec :: Spec+identifierSpec = describe "scenario identifiers" do+ it "round-trips canonical identifiers" do+ let rendered = "acme/append/concurrency/expected-version-race"+ fmap renderScenarioId (parseScenarioId rendered) `shouldBe` Right rendered++ it "rejects malformed identifiers" do+ parseScenarioId "acme/append/correctness" `shouldBe` Left "scenario identifier must have four segments: \"acme/append/correctness\""+ parseScenarioId "ACME/append/correctness/example" `shouldBe` Left "invalid layer \"ACME\": invalid segment \"ACME\""++ it "generates canonical UUIDv7 run ids" do+ generated <- newRunId+ parseRunId (renderRunId generated) `shouldBe` Right generated++selectorSpec :: Spec+selectorSpec = describe "scenario selectors" do+ it "supports one-segment and recursive wildcards" do+ scenario <- expectRight (parseScenarioId "selftest/kernel/correctness/always-pass")+ one <- expectRight (parseSelector "*/*/correctness/*")+ recursive <- expectRight (parseSelector "selftest/**")+ matchesSelector one scenario `shouldBe` True+ matchesSelector recursive scenario `shouldBe` True++-- A suite that is not the kernel's: its own layer, migration set, and rule.+openRegistrySpec :: Spec+openRegistrySpec = describe "open registry" do+ let layer = either (error . Text.unpack) id (mkLayerId "acme")+ acmeSet = syntheticSet "acme" []+ roundtrip pgVersions =+ Scenario+ { id = either (error . show) id (parseScenarioId "acme/widget/correctness/roundtrip"),+ revision = 1,+ summary = "Round-trips a widget.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = [],+ dimensions = postgresDimensions (PgDurable :| []) pgVersions noDimensions,+ phases = zeroPhases,+ requires = noEnvironment {postgres = Just (PostgresRequirement [MigrationSetName "acme"] [] False)},+ knownDefect = Nothing,+ run = const (pure (ScenarioReport Passed Nothing []))+ }+ bundleWith scenario = (mkBundle layer [scenario] []) {migrations = [acmeSet], rules = [requirePgVersionsFor (MigrationSetName "acme") (Pg18 :| [])]}+ registryErrors bundles = either (fmap (\(RegistryError message) -> message) . toList) (const []) (mkRegistry bundles)+ it "accepts a bundle for a new layer with its own migration set and rule" do+ registry <- expectRight (mkRegistry [Selftest.bundle, bundleWith (roundtrip (Pg18 :| []))])+ Map.keys (registryMigrationSets registry) `shouldBe` [MigrationSetName "acme", MigrationSetName "kenshou-selftest"]+ fmap (renderScenarioId . (.id)) (selectScenarios registry emptyListFilter {layers = [layer]}) `shouldBe` ["acme/widget/correctness/roundtrip"]+ it "rejects a scenario that violates a contributed rule with the rule's message" do+ registryErrors [bundleWith (roundtrip (Pg17 :| [Pg18]))] `shouldBe` ["acme/widget/correctness/roundtrip: acme scenarios require PostgreSQL 18"]+ it "applies a rule to scenarios of every bundle" do+ let rule = ScenarioRule "no-selftest" (\scenario -> ["rejected" | renderLayer scenario.id.layer == "selftest"])+ length (registryErrors [Selftest.bundle, (bundleWith (roundtrip (Pg18 :| []))) {rules = [rule]}]) `shouldBe` length Selftest.bundle.scenarios+ it "rejects duplicate and undefined migration sets" do+ registryErrors [bundleWith (roundtrip (Pg18 :| [])), (mkBundle layer [] []) {migrations = [acmeSet]}] `shouldBe` ["duplicate migration set: acme"]+ registryErrors [(bundleWith (roundtrip (Pg18 :| []))) {migrations = []}] `shouldBe` ["acme/widget/correctness/roundtrip: no registered bundle defines migration set acme"]++-- A suite resource with a stub provider that records when it is acquired and released.+resourceProvisioningSpec :: Spec+resourceProvisioningSpec = describe "resource provisioning" do+ let layer = either (error . Text.unpack) id (mkLayerId "acme")+ stub = either (error . Text.unpack) id (mkResourceName "stub")+ provider :: (Text -> IO ()) -> ResourceProvider+ provider record =+ ResourceProvider+ { name = stub,+ summary = "Stub resource",+ normalize = Right . fromMaybe (object ["mode" .= ("default" :: Text)]),+ withLocal = \_ value action ->+ bracket_ (record "acquire") (record "release") (action (ProvisionedResource (object ["spec" .= value]) (object ["kind" .= ("stub" :: Text)]))),+ cell = Nothing+ }+ roundtrip :: (Text -> IO ()) -> Scenario+ roundtrip record =+ Scenario+ { id = either (error . show) id (parseScenarioId "acme/widget/correctness/uses-stub"),+ revision = 1,+ summary = "Uses the stub resource.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = [],+ dimensions = noDimensions,+ phases = zeroPhases,+ requires = requiresResource stub,+ knownDefect = Nothing,+ run = \context -> do+ record "run"+ pure (if fmap (.workerValue) (Map.lookup stub context.env.resources) == Just (object ["spec" .= object ["mode" .= ("default" :: Text)]]) then ScenarioReport Passed Nothing [] else failedWith ["no-stub"] "the stub resource was not provisioned")+ }+ bundleWith record = (mkBundle layer [roundtrip record] []) {Bundle.resources = [provider record]}+ registryErrors bundles = either (fmap (\(RegistryError message) -> message) . toList) (const []) (mkRegistry bundles)+ ignore = const (pure ())+ it "rejects a duplicate provider and an unprovided resource" do+ registryErrors [bundleWith ignore, (mkBundle layer [] []) {Bundle.resources = [provider ignore]}] `shouldBe` ["duplicate resource provider: stub"]+ registryErrors [(bundleWith ignore) {Bundle.resources = []}] `shouldBe` ["scenario requires unprovided resource stub: acme/widget/correctness/uses-stub"]+ it "normalizes the value, provisions around the run, and records the fingerprint" $ withSystemTempDirectory "kenshou-resource" \root -> do+ events <- newIORef ([] :: [Text])+ let record event = modifyIORef' events (<> [event])+ registry <- expectRight (mkRegistry [bundleWith record])+ scenarioId <- expectRight (parseScenarioId "acme/widget/correctness/uses-stub")+ output <- executeRun (RunnerConfig registry root (identityWith []) Nothing False False []) (minimalRunSpec scenarioId) >>= expectRight+ readIORef events `shouldReturn` ["acquire", "run", "release"]+ runSpec <- decodeFile (output.directory </> "run-spec.json")+ runResult <- decodeFile (output.directory </> "run-result.json")+ path ["environment", "resources", "stub"] runSpec `shouldBe` Just (object ["mode" .= ("default" :: Text)])+ path ["fingerprint", "resources", "stub"] runResult `shouldBe` Just (object ["kind" .= ("stub" :: Text)])+ path ["outcome"] runResult `shouldBe` Just (String "passed")+ where+ decodeFile file = eitherDecodeFileStrict' file >>= either fail pure :: IO Value+ path [] value = Just value+ path (key : rest) (Object fields) = KeyMap.lookup (Key.fromText key) fields >>= path rest+ path _ _ = Nothing++registrySpec :: Spec+registrySpec = describe "scenario registry" do+ it "sorts the kernel self-tests by identifier" do+ case mkRegistry [Selftest.bundle] of+ Left errors -> expectationFailure (show errors)+ Right registry ->+ fmap (renderScenarioId . (.id)) (allScenarios registry)+ `shouldBe` [ "selftest/kernel/concurrency/worker-echo",+ "selftest/kernel/correctness/always-fail",+ "selftest/kernel/correctness/always-pass",+ "selftest/kernel/correctness/errors",+ "selftest/kernel/correctness/known-defect",+ "selftest/kernel/correctness/outcome",+ "selftest/kernel/correctness/postgres-roundtrip"+ ]++knownDefectGroupSpec :: Spec+knownDefectGroupSpec = describe "grouped known defects" do+ let base = case filter (\item -> renderScenarioId item.id == "selftest/kernel/correctness/known-defect") Selftest.bundle.scenarios of+ [item] -> item+ _ -> error "missing known-defect self-test"+ first = KnownDefect "mori://example/bugs/1" "first" ["first-failure"] AllCohorts+ second = KnownDefect "mori://example/bugs/2" "second" ["second-failure"] AllCohorts+ scenario = base {knownDefect = Just (KnownDefectGroup (first :| [second]))}+ cohort = identityWith []+ disposition strict labels = defectDisposition strict cohort scenario (failedWith labels "grouped defect probe")+ it "accepts either or both independently reproduced defects" do+ let check labels = case disposition False labels of+ (Just (KnownDefectGroup _, DefectReproduced), False, 0) -> pure ()+ other -> expectationFailure (show other)+ check ["first-failure"]+ check ["second-failure"]+ check ["first-failure", "second-failure"]+ it "blocks unrelated failures and strict known-defect runs" do+ case disposition False ["second-failure", "new-failure"] of+ (Just (_, DefectDifferentFailure), True, 1) -> pure ()+ other -> expectationFailure (show other)+ case disposition True ["second-failure"] of+ (Just (_, DefectReproduced), False, 1) -> pure ()+ other -> expectationFailure (show other)+ it "does not excuse a failed result without a failure label" do+ case disposition False [] of+ (Just (_, DefectDifferentFailure), True, 1) -> pure ()+ other -> expectationFailure (show other)+ it "ignores labels owned only by a defect outside the resolved cohort" do+ let scoped = KnownDefect "mori://example/bugs/2" "second" ["second-failure"] (OnlyWhen (ResolvedFromGit "foo" :| []))+ scopedScenario = base {knownDefect = Just (KnownDefectGroup (first :| [scoped]))}+ result = defectDisposition False (identityWith [ResolvedPackage "foo" "1.0.0" (FromHackage Nothing)]) scopedScenario (failedWith ["second-failure"] "scoped defect probe")+ case result of+ (Just (_, DefectDifferentFailure), True, 1) -> pure ()+ other -> expectationFailure (show other)++knobSpec :: Spec+knobSpec = describe "knob resolution" do+ let workers = knobName "selftest.workers"+ enabled = knobName "selftest.enabled"+ declarations =+ [ KnobSpec workers "Worker count" KnobInt (VInt 2) (IntRange 1 8) [VInt 1, VInt 8],+ KnobSpec enabled "Enable work" KnobBool (VBool False) AnyValue []+ ]+ it "fills defaults and parses typed command-line values" do+ resolved <- expectRight (resolveKnobs declarations [(workers, RawText "4"), (enabled, RawText "true")])+ knobInt resolved workers `shouldBe` (4 :: Int64)+ knobBool resolved enabled `shouldBe` True+ it "reports duplicates, unknown names and range violations together" do+ let unknown = knobName "selftest.unknown"+ result = resolveKnobs declarations [(workers, RawText "9"), (workers, RawText "1"), (unknown, RawText "x")]+ case result of+ Left problems -> length problems `shouldBe` 3+ Right _ -> expectationFailure "expected knob errors"++dimensionSpec :: Spec+dimensionSpec = describe "dimension resolution" do+ let support = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| []) noDimensions+ it "fills every applicable default" do+ resolveDimensions support [] `shouldBe` Right (Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18))+ it "rejects unsupported and inapplicable values" do+ case resolveDimensions support [("pg.version", "17"), ("telemetry.tracing", "off")] of+ Left problems -> length problems `shouldBe` 2+ Right _ -> expectationFailure "expected dimension errors"++runSpecSpec :: Spec+runSpecSpec = describe "run specification" do+ it "decodes a minimal document and emits a round-trippable effective document" do+ input <- case eitherDecode "{\"schema\":\"kenshou.run-spec/v1\",\"scenario\":\"selftest/kernel/correctness/always-pass\",\"seed\":7}" of+ Left err -> expectationFailure err >> fail "unreachable"+ Right value -> pure value+ registry <- expectRight (mkRegistry [Selftest.bundle])+ resolution <- resolveRunSpec registry input+ (_, effective) <- expectRight resolution+ eitherDecode (encode effective) `shouldBe` Right (toJSON effective :: Value)+ it "rejects a non-v7 run id while decoding" do+ let document = "{\"schema\":\"kenshou.run-spec/v1\",\"scenario\":\"selftest/kernel/correctness/always-pass\",\"runId\":\"550e8400-e29b-41d4-a716-446655440000\"}"+ case (eitherDecode document :: Either String RunSpec) of+ Left err -> err `shouldContain` "UUIDv7"+ Right _ -> expectationFailure "expected UUIDv7 rejection"++compatibilitySpec :: Spec+compatibilitySpec = describe "compatibility keys" do+ it "exclude run identity and cohort from comparison keys, but include the cohort in series keys" do+ registry <- expectRight (mkRegistry [Selftest.bundle])+ scenarioId <- expectRight (parseScenarioId "selftest/kernel/correctness/always-pass")+ firstResolution <- resolveRunSpec registry (minimalRunSpec scenarioId)+ secondResolution <- resolveRunSpec registry (minimalRunSpec scenarioId)+ (_, firstSpec) <- expectRight firstResolution+ (_, secondSpec) <- expectRight secondResolution+ let firstCohort = identityWith []+ secondCohort = firstCohort {identityPlanHash = PlanHash "sha256:other"}+ firstInputs = compatInputs firstSpec Nothing [] firstCohort+ repeatedInputs = compatInputs secondSpec Nothing [] firstCohort+ otherCohortInputs = compatInputs firstSpec Nothing [] secondCohort+ comparisonKey firstInputs `shouldBe` comparisonKey repeatedInputs+ comparisonKey firstInputs `shouldBe` comparisonKey otherCohortInputs+ seriesKey firstInputs `shouldNotBe` seriesKey otherCohortInputs++knobName :: Text -> KnobName+knobName = either (error . show) id . mkKnobName++manifestSpec :: Spec+manifestSpec = describe "artifact manifests" do+ it "detects a changed file" $ withSystemTempDirectory "kenshou-manifest" \directory -> do+ runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+ ByteString.writeFile (directory <> "/run-spec.json") "{}\n"+ manifest <- writeManifest directory runId Map.empty+ encodeFile (directory <> "/manifest.json") manifest+ verifyManifest directory `shouldReturn` Right ()+ ByteString.appendFile (directory <> "/run-spec.json") "x"+ verification <- verifyManifest directory+ case verification of+ Left _ -> pure ()+ Right () -> expectationFailure "expected a digest mismatch"++goldenSpec :: Spec+goldenSpec = describe "JSON goldens" do+ mapM_ roundTrips ["run-spec.minimal.json", "run-spec.effective.json", "run-spec.external.json", "run-result.passed.json", "run-result.known-defect.json", "manifest.json", "scenario-list.json"]+ where+ roundTrips name = it ("round-trips " <> name <> " canonically") do+ original <- decodeGolden name+ redecoded <- case eitherDecode (encode original) of+ Left err -> expectationFailure err >> fail "unreachable"+ Right value -> pure value+ canonicalEncode original `shouldBe` canonicalEncode (redecoded :: Value)+ decodeGolden name = do+ result <- eitherDecodeFileStrict' ("test/golden/" <> name)+ case result of+ Left err -> expectationFailure err >> fail "unreachable"+ Right value -> pure (value :: Value)++postgresEnvironmentSpec :: Spec+postgresEnvironmentSpec = describe "PostgreSQL environments" do+ it "migrates, clones, restarts, and crash-restarts PostgreSQL 18" $ withSystemTempDirectory "kenshou-pg18" \directory -> do+ runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+ let requirement = PostgresRequirement [selftestSchema] [] True+ expectedSchemas = "kenshou_selftest,pgmigrate"+ dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18)+ result <- withPostgresEnv selftestSets nullLogger directory runId requirement (PostgresEphemeral []) dimensions \environment -> do+ schemas <- psqlTest environment.connectionString "select string_agg(schema_name, ',' order by schema_name) from information_schema.schemata where schema_name in ('kenshou_selftest','pgmigrate')"+ clone <- environment.newDatabase "clone"+ clonedSchemas <- psqlTest clone "select string_agg(schema_name, ',' order by schema_name) from information_schema.schemata where schema_name in ('kenshou_selftest','pgmigrate')"+ case environment.control of+ Nothing -> expectationFailure "expected server control"+ Just control -> do+ control.restartServer+ psqlTest environment.connectionString "select 1" `shouldReturn` Right "1"+ control.stopServer StopImmediate+ control.startServer+ psqlTest environment.connectionString "select 1" `shouldReturn` Right "1"+ pure (schemas, clonedSchemas, Map.lookup "fsync" environment.snapshot.settings)+ result `shouldBe` Right (Right expectedSchemas, Right expectedSchemas, Just "off")++ it "creates and drops fresh databases in external mode" $ withSystemTempDirectory "kenshou-pg-external" \directory -> do+ runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+ let dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18)+ outerRequirement = PostgresRequirement [] [] False+ externalRequirement = PostgresRequirement [selftestSchema] [("shared_preload_libraries", "'pg_stat_statements'")] False+ outer <- withPostgresEnv selftestSets nullLogger directory runId outerRequirement (PostgresEphemeral [("shared_preload_libraries", "'pg_stat_statements'")]) dimensions \server -> do+ inner <- withPostgresEnv selftestSets nullLogger directory runId externalRequirement (PostgresExternal (ConnLiteral server.adminConnectionString)) dimensions (\environment -> psqlTest environment.connectionString "select to_regnamespace('kenshou_selftest') is not null")+ missing <- withPostgresEnv selftestSets nullLogger directory runId (PostgresRequirement [] [("shared_preload_libraries", "'missing_extension'")] False) (PostgresExternal (ConnLiteral server.adminConnectionString)) dimensions (const (pure ()))+ missing `shouldBe` Left (EnvError "external PostgreSQL is missing required preloaded libraries: missing_extension")+ remaining <- psqlTest server.adminConnectionString "select count(*) from pg_database where datname like 'kenshou_01997f3a5b7c7%'"+ pure (inner, remaining)+ outer `shouldBe` Right (Right (Right "t"), Right "0")++ it "keeps two named contexts of one run apart on a shared external server" $ withSystemTempDirectory "kenshou-pg-shared" \directory -> do+ runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e56")+ let dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg18)+ requirement = PostgresRequirement [] [] False+ outer <- withPostgresEnv selftestSets nullLogger directory runId requirement (PostgresEphemeral []) dimensions \server -> do+ let external = PostgresExternal (ConnLiteral server.adminConnectionString)+ nested <- withPostgresEnvNamed selftestSets False (Just "shop") nullLogger directory runId requirement external dimensions \shop ->+ withPostgresEnvNamed selftestSets False (Just "warehouse") nullLogger directory runId requirement external dimensions \warehouse ->+ pure (shop.databaseName /= warehouse.databaseName)+ remaining <- psqlTest server.adminConnectionString "select count(*) from pg_database where datname like 'kenshou_01997f3a5b7c7%'"+ pure (nested, remaining)+ outer `shouldBe` Right (Right (Right True), Right "0")++ it "selects PostgreSQL 17 and applies a composed plan" do+ available <- lookupEnv "KENSHOU_PG17_BIN"+ case available of+ Nothing -> pendingWith "KENSHOU_PG17_BIN is not set"+ Just _ -> withSystemTempDirectory "kenshou-pg17" \directory -> do+ runId <- expectRight (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+ let requirement = PostgresRequirement [selftestSchema] [] False+ dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just Pg17)+ result <- withPostgresEnv selftestSets nullLogger directory runId requirement (PostgresEphemeral []) dimensions (pure . (.snapshot.serverVersionNum))+ case result of+ Left err -> expectationFailure (show err)+ Right version -> (version >= 170000 && version < 180000) `shouldBe` True++psqlTest :: Text -> Text -> IO (Either Text Text)+psqlTest connection query = do+ (code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", Text.unpack query] ""+ pure case code of ExitSuccess -> Right (Text.strip (Text.pack output)); _ -> Left (Text.strip (Text.pack err))++onePackageDescriptor :: CohortDescriptor+onePackageDescriptor =+ CohortDescriptor+ "kenshou.cohort/v1"+ (CohortName "released")+ "fixture"+ "2026-09-20"+ "ghc-9.12.4"+ "2026-09-20T13:44:47Z"+ "cohort/released.project"+ [ComponentSpec (ComponentId "foo") "mori://example/foo" HackageSource [PackagePin "foo" "1.0.0" Nothing]]+ []+ []++identityWith :: [ResolvedPackage] -> CohortIdentity+identityWith packages =+ CohortIdentity+ (CohortName "released")+ "ghc-9.12.4"+ "3.16.1.0"+ "linux"+ "x86_64"+ (Just "2026-09-20T13:44:47Z")+ (PlanHash "sha256:fixture")+ "fixture"+ [ResolvedComponent (ComponentId "foo") "mori://example/foo" packages]+ Nothing++hashFixture :: FilePath -> IO PlanHash+hashFixture name = do+ value <- decodeFixture name+ case planHash value of+ Left err -> expectationFailure (show err) >> fail "unreachable"+ Right result -> pure result++decodeFixture :: (FromJSON value) => FilePath -> IO value+decodeFixture name = do+ result <- eitherDecodeFileStrict' (fixtures <> "/" <> name)+ case result of+ Left err -> expectationFailure err >> fail "unreachable"+ Right value -> pure value++fixtures :: FilePath+fixtures = "test/fixtures"++expectRight :: (Show problem) => Either problem value -> IO value+expectRight (Right value) = pure value+expectRight (Left problem) = expectationFailure (show problem) >> fail "unreachable"++migrationPlanSpec :: Spec+migrationPlanSpec = describe "migration plan composition" do+ let alpha = syntheticSet "alpha" []+ beta = syntheticSet "beta" [MigrationSetName "alpha"]+ gamma = syntheticSet "gamma" [MigrationSetName "beta"]+ unavailable = MigrationSet (MigrationSetName "delta") [] (Left "not built")+ cyclic = syntheticSet "epsilon" [MigrationSetName "epsilon"]+ sets = Map.fromList [(set.name, set) | set <- [alpha, beta, gamma, unavailable, cyclic]]+ order names = fmap (fmap (.name)) (orderMigrationSets sets (fmap MigrationSetName names))+ it "composes the self-test migration set" do+ isRight (composePlan selftestSets [selftestSchema]) `shouldBe` True+ it "keeps the declared order and inserts required sets first, transitively" do+ order ["gamma"] `shouldBe` Right (fmap MigrationSetName ["alpha", "beta", "gamma"])+ order ["beta", "alpha"] `shouldBe` Right (fmap MigrationSetName ["alpha", "beta"])+ order ["alpha", "gamma", "beta"] `shouldBe` Right (fmap MigrationSetName ["alpha", "beta", "gamma"])+ isRight (composePlan sets [MigrationSetName "gamma"]) `shouldBe` True+ it "rejects an empty selection, an undefined set, an unavailable set, and a cycle" do+ isLeft (composePlan sets []) `shouldBe` True+ isLeft (composePlan sets [MigrationSetName "missing"]) `shouldBe` True+ isLeft (composePlan sets [MigrationSetName "delta"]) `shouldBe` True+ isLeft (order ["epsilon"]) `shouldBe` True++-- A set whose one migration creates a schema of the same name.+syntheticSet :: Text -> [MigrationSetName] -> MigrationSet+syntheticSet name requires =+ MigrationSet+ { name = MigrationSetName name,+ requires,+ component = either (Left . Text.pack . show) Right do+ migration <- sqlMigration "create-schema" (Text.Encoding.encodeUtf8 ("create schema " <> name <> ";\n"))+ migrationComponent name (Set.fromList [dependency | MigrationSetName dependency <- requires]) (migration :| [])+ }++selftestSets :: Map.Map MigrationSetName MigrationSet+selftestSets = Map.singleton selftestSchema selftestMigrationSet
+ test/fixtures/cohort-identity.golden.json view
@@ -0,0 +1,19 @@+{+ "schema": "kenshou.cohort-identity/v1",+ "cohort": "released",+ "compiler": "ghc-9.12.4",+ "cabalVersion": "3.16.1.0",+ "os": "linux",+ "arch": "x86_64",+ "indexState": "2026-09-20T13:44:47Z",+ "planHash": "sha256:857e6ff76844ccdf1903dc62d6518fe59b64e34d4bc626873efe2534934f1cc7",+ "descriptorSha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",+ "components": [+ { "id": "foo", "moriUri": "mori://example/foo", "packages": [+ { "name": "foo", "version": "1.0.0", "source": { "type": "hackage", "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } }+ ] },+ { "id": "bar", "moriUri": "mori://example/bar", "packages": [+ { "name": "bar", "version": "2.0.0", "source": { "type": "git", "location": "https://example.com/bar.git", "rev": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }+ ] }+ ]+}
+ test/fixtures/descriptor-released.json view
@@ -0,0 +1,29 @@+{+ "schema": "kenshou.cohort/v1",+ "name": "released",+ "description": "Trimmed unit-test cohort.",+ "verifiedAt": "2026-09-20",+ "compiler": "ghc-9.12.4",+ "indexState": "2026-09-20T13:44:47Z",+ "projectFile": "cohort/released.project",+ "components": [+ {+ "id": "foo",+ "moriUri": "mori://example/foo",+ "source": { "type": "hackage" },+ "packages": [{ "name": "foo", "version": "1.0.0" }]+ },+ {+ "id": "bar",+ "moriUri": "mori://example/bar",+ "source": {+ "type": "git",+ "location": "https://example.com/bar.git",+ "rev": "0123456789abcdef0123456789abcdef01234567"+ },+ "packages": [{ "name": "bar", "version": "2.0.0", "subdir": "bar" }]+ }+ ],+ "allowNewer": [],+ "exceptions": []+}
+ test/fixtures/invalid-project/cohort/active.project view
@@ -0,0 +1,2 @@+import: released.project+import: head.project
+ test/fixtures/perf/catalog.v1.json view
@@ -0,0 +1,109 @@+{+ "schema": "kenshou.perf-catalog/v1",+ "profiles": [+ {+ "name": "acme-core-quick",+ "revision": 1,+ "scenario": "acme/core-ordering/benchmark/concurrency-sweep",+ "scenarioRevision": 2,+ "knobs": {+ "bench.arm": "configured",+ "bench.rate": 200,+ "acme.messages": 1000,+ "measure.raw-samples": "full"+ },+ "dimensions": {+ "telemetry.tracing": "off",+ "telemetry.metrics": "off"+ },+ "workloadPurpose": "fixed-load-latency",+ "warmupSeconds": 0,+ "steadyCount": 1000,+ "cooldownSeconds": 0,+ "timeoutSeconds": 180,+ "cachePolicy": "cold-reset",+ "policy": "policies/default.json",+ "requiredMetrics": [+ {+ "name": "op.publish-to-finalize.throughput",+ "unit": "ops/s",+ "minimumSteadySamples": 1000+ },+ {+ "name": "op.publish-to-finalize.latency.p50",+ "unit": "ns",+ "minimumSteadySamples": 1000+ },+ {+ "name": "op.publish-to-finalize.latency.p99",+ "unit": "ns",+ "minimumSteadySamples": 1000+ }+ ],+ "capabilities": [+ "cold-reset",+ "full-samples"+ ],+ "machineProfile": "quick-cell-pending-calibration",+ "trialSeconds": 15,+ "resetSeconds": 30,+ "sealSeconds": 10,+ "minimumPairs": 5,+ "replacementPairs": 2,+ "calibration": null+ },+ {+ "name": "acme-quill-quick",+ "revision": 1,+ "scenario": "acme/quill-adapter/benchmark/end-to-end-throughput-latency",+ "scenarioRevision": 1,+ "knobs": {+ "bench.arm": "adapter",+ "bench.rate": 200,+ "acme.messages": 1000,+ "measure.raw-samples": "full"+ },+ "dimensions": {+ "pg.durability": "durable",+ "pg.version": "18"+ },+ "workloadPurpose": "fixed-load-latency",+ "warmupSeconds": 0,+ "steadyCount": 1000,+ "cooldownSeconds": 0,+ "timeoutSeconds": 180,+ "cachePolicy": "cold-reset",+ "policy": "policies/default.json",+ "requiredMetrics": [+ {+ "name": "op.publish-to-finalize.throughput",+ "unit": "ops/s",+ "minimumSteadySamples": 1000+ },+ {+ "name": "op.publish-to-finalize.latency.p50",+ "unit": "ns",+ "minimumSteadySamples": 1000+ },+ {+ "name": "op.publish-to-finalize.latency.p99",+ "unit": "ns",+ "minimumSteadySamples": 1000+ }+ ],+ "capabilities": [+ "cold-reset",+ "full-samples",+ "postgresql-18",+ "quill"+ ],+ "machineProfile": "quick-cell-pending-calibration",+ "trialSeconds": 15,+ "resetSeconds": 30,+ "sealSeconds": 10,+ "minimumPairs": 5,+ "replacementPairs": 2,+ "calibration": null+ }+ ]+}
+ test/fixtures/perf/cohort.json view
@@ -0,0 +1,89 @@+{+ "schema": "kenshou.cohort/v1",+ "name": "released",+ "description": "Historical Hackage runtime baseline verified on 2026-09-20.",+ "verifiedAt": "2026-09-20",+ "compiler": "ghc-9.12.4",+ "indexState": "2026-09-20T13:44:47Z",+ "projectFile": "cohort/released.project",+ "components": [+ { "id": "keiki", "moriUri": "mori://shinzui/keiki", "source": { "type": "hackage" }, "packages": [+ { "name": "keiki", "version": "0.9.1.0" },+ { "name": "keiki-codec-json", "version": "0.9.1.0" }+ ] },+ { "id": "tally", "moriUri": "mori://shinzui/tally", "source": { "type": "hackage" }, "packages": [+ { "name": "tally-store", "version": "0.8.0.1" },+ { "name": "tally-store-migrations", "version": "0.4.0.0" },+ { "name": "tally-otel", "version": "0.2.0.8" },+ { "name": "tally-metrics", "version": "0.1.0.8" },+ { "name": "tally-cli", "version": "0.2.0.6" },+ { "name": "acme-tally-adapter", "version": "0.5.1.2" }+ ] },+ { "id": "flux", "moriUri": "mori://shinzui/flux", "source": { "type": "hackage" }, "packages": [+ { "name": "flux", "version": "0.17.0.0" },+ { "name": "flux-core", "version": "0.17.0.0" },+ { "name": "flux-quill", "version": "0.17.0.0" },+ { "name": "flux-migrations", "version": "0.17.0.0" },+ { "name": "flux-ops", "version": "0.17.0.0" },+ { "name": "flux-test-support", "version": "0.17.0.0" }+ ] },+ { "id": "acme", "moriUri": "mori://shinzui/acme", "source": { "type": "hackage" }, "packages": [+ { "name": "acme-core", "version": "0.9.0.3" },+ { "name": "acme-metrics", "version": "0.9.0.3" }+ ] },+ { "id": "acme-quill-adapter", "moriUri": "mori://shinzui/acme-quill-adapter", "source": { "type": "hackage" }, "packages": [+ { "name": "acme-quill-adapter", "version": "0.16.0.0" }+ ] },+ { "id": "acme-brook-adapter", "moriUri": "mori://shinzui/acme-brook-adapter", "source": { "type": "hackage" }, "packages": [+ { "name": "acme-brook-adapter", "version": "0.9.0.1" }+ ] },+ { "id": "brook-effectful", "moriUri": "mori://shinzui/brook-effectful", "source": { "type": "hackage" }, "packages": [+ { "name": "brook-effectful", "version": "0.3.1.0" }+ ] },+ { "id": "hw-brook-streamly", "moriUri": "mori://shinzui/hw-brook-streamly", "source": { "type": "hackage" }, "packages": [+ { "name": "hw-brook-streamly", "version": "0.2.0.0" }+ ] },+ { "id": "hw-brook-client", "moriUri": "mori://haskell-works/hw-brook-client", "source": { "type": "hackage" }, "packages": [+ { "name": "hw-brook-client", "version": "5.3.0" }+ ] },+ { "id": "quill-hs", "moriUri": "mori://shinzui/quill-hs", "source": { "type": "hackage" }, "packages": [+ { "name": "quill-core", "version": "0.6.1.0" },+ { "name": "quill-hasql", "version": "0.6.1.0" },+ { "name": "quill-effectful", "version": "0.6.1.0" },+ { "name": "quill-config", "version": "0.6.1.0" },+ { "name": "quill-migration", "version": "0.6.1.0" }+ ] },+ { "id": "pg-migrate", "moriUri": "mori://shinzui/pg-migrate", "source": { "type": "hackage" }, "packages": [+ { "name": "pg-migrate", "version": "1.1.0.0" },+ { "name": "pg-migrate-embed", "version": "1.1.0.0" },+ { "name": "pg-migrate-cli", "version": "1.1.0.0" },+ { "name": "pg-migrate-import-codd", "version": "1.1.0.0" },+ { "name": "pg-migrate-import-hasql-migration", "version": "1.1.0.0" }+ ] },+ { "id": "ephemeral-pg", "moriUri": "mori://shinzui/ephemeral-pg", "source": { "type": "hackage" }, "packages": [+ { "name": "ephemeral-pg", "version": "0.3.1.0" }+ ] },+ { "id": "effectful", "moriUri": "mori://effectful/effectful", "source": { "type": "hackage" }, "packages": [+ { "name": "effectful", "version": "2.6.1.0" },+ { "name": "effectful-core", "version": "2.6.1.0" }+ ] },+ { "id": "hs-opentelemetry", "moriUri": "mori://iand675/hs-opentelemetry", "source": { "type": "hackage" }, "packages": [+ { "name": "hs-opentelemetry-api", "version": "1.0.0.0" },+ { "name": "hs-opentelemetry-sdk", "version": "1.0.0.0" },+ { "name": "hs-opentelemetry-propagator-w3c", "version": "1.0.0.0" },+ { "name": "hs-opentelemetry-exporter-in-memory", "version": "1.0.0.0" },+ { "name": "hs-opentelemetry-exporter-otlp", "version": "1.0.0.0" },+ { "name": "hs-opentelemetry-exporter-prometheus", "version": "1.0.0.0" },+ { "name": "hs-opentelemetry-otlp", "version": "1.0.0.0" },+ { "name": "hs-opentelemetry-semantic-conventions", "version": "1.40.0.0" }+ ] }+ ],+ "allowNewer": ["flux-test-support:ephemeral-pg"],+ "exceptions": [+ { "package": "tally-store-migrations", "selected": "0.4.0.0", "latestObserved": "0.5.0.0", "reason": "flux-migrations 0.17.0.0 bounds ^>=0.4.0.0", "recheckAfter": "2026-12-20" },+ { "package": "pg-migrate", "selected": "1.1.0.0", "latestObserved": "1.2.0.0", "reason": "flux-migrations and quill-migration bound ^>=1.1.0.0", "recheckAfter": "2026-12-20" },+ { "package": "effectful", "selected": "2.6.1.0", "latestObserved": "2.7.1.0", "reason": "flux 0.17.0.0 requires effectful <2.7", "recheckAfter": "2026-12-20" },+ { "package": "aeson", "selected": "2.2.5.1", "latestObserved": "2.3.2.0", "reason": "resolved transitively by the released runtime bounds", "recheckAfter": "2026-12-20" },+ { "package": "hw-brook-client", "selected": "5.3.0", "latestObserved": "5.3.0", "reason": "the head cohort uses a fork with unreleased correctness fixes", "recheckAfter": "2026-12-20" }+ ]+}
+ test/fixtures/plan-commit-change.json view
@@ -0,0 +1,8 @@+{+ "compiler-id": "ghc-9.12.4", "cabal-version": "3.16.1.0", "os": "linux", "arch": "x86_64",+ "install-plan": [+ { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+ { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+ { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "fedcba9876543210fedcba9876543210fedcba98", "subdir": "bar" } }, "flags": {} }+ ]+}
+ test/fixtures/plan-local-change.json view
@@ -0,0 +1,9 @@+{+ "compiler-id": "ghc-9.12.4", "cabal-version": "3.16.1.0", "os": "linux", "arch": "x86_64",+ "install-plan": [+ { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+ { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+ { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }, "flags": {} },+ { "type": "configured", "style": "local", "id": "kenshou-core", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0", "pkg-src": { "type": "local", "path": "/another/checkout/kenshou-core" }, "flags": {} }+ ]+}
+ test/fixtures/plan-released.json view
@@ -0,0 +1,34 @@+{+ "compiler-id": "ghc-9.12.4",+ "cabal-version": "3.16.1.0",+ "os": "linux",+ "arch": "x86_64",+ "install-plan": [+ { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+ {+ "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0",+ "pkg-src": { "type": "repo-tar" },+ "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "flags": { "fast": true }+ },+ {+ "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0",+ "pkg-src": {+ "type": "source-repo",+ "source-repo": {+ "type": "git", "location": "https://example.com/bar.git",+ "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar"+ }+ },+ "flags": {}+ },+ {+ "type": "configured", "style": "local", "id": "kenshou-core-lib", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0",+ "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {}+ },+ {+ "type": "configured", "style": "local", "id": "kenshou-core-test", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0",+ "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {}+ }+ ]+}
+ test/fixtures/plan-reordered.json view
@@ -0,0 +1,13 @@+{+ "compiler-id": "ghc-9.12.4",+ "cabal-version": "3.16.1.0",+ "os": "linux",+ "arch": "x86_64",+ "install-plan": [+ { "type": "configured", "style": "local", "id": "kenshou-core-test", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0", "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {} },+ { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }, "flags": {} },+ { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+ { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.0", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+ { "type": "configured", "style": "local", "id": "kenshou-core-lib", "pkg-name": "kenshou-core", "pkg-version": "0.1.0.0", "pkg-src": { "type": "local", "path": "/checkout/one/kenshou-core" }, "flags": {} }+ ]+}
+ test/fixtures/plan-version-change.json view
@@ -0,0 +1,8 @@+{+ "compiler-id": "ghc-9.12.4", "cabal-version": "3.16.1.0", "os": "linux", "arch": "x86_64",+ "install-plan": [+ { "type": "pre-existing", "id": "base", "pkg-name": "base", "pkg-version": "4.21.2.0" },+ { "type": "configured", "style": "global", "id": "foo", "pkg-name": "foo", "pkg-version": "1.0.1", "pkg-src": { "type": "repo-tar" }, "pkg-src-sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "flags": { "fast": true } },+ { "type": "configured", "style": "global", "id": "bar", "pkg-name": "bar", "pkg-version": "2.0.0", "pkg-src": { "type": "source-repo", "source-repo": { "type": "git", "location": "https://example.com/bar.git", "tag": "0123456789abcdef0123456789abcdef01234567", "subdir": "bar" } }, "flags": {} }+ ]+}
+ test/fixtures/plan/catalog-planned.json view
@@ -0,0 +1,64 @@+{+ "schema": "kenshou.scenario-list/v1",+ "suiteVersion": "0.1.0.0",+ "roles": [],+ "scenarios": [+ {"id":"selftest/kernel/concurrency/worker-echo","summary":"Worker echo","tier":"smoke","placement":"either"},+ {"id":"selftest/kernel/correctness/always-fail","summary":"Always fails","tier":"smoke","placement":"either"},+ {"id":"selftest/kernel/correctness/always-pass","summary":"Always passes","tier":"smoke","placement":"either"},+ {"id":"selftest/kernel/correctness/errors","summary":"Always errors","tier":"smoke","placement":"either"},+ {"id":"selftest/kernel/correctness/known-defect","summary":"Known defect","tier":"smoke","placement":"either"},+ {"id":"selftest/kernel/correctness/outcome","summary":"Selected outcome","tier":"smoke","placement":"either"},+ {"id":"selftest/kernel/correctness/postgres-roundtrip","summary":"PostgreSQL roundtrip","tier":"smoke","placement":"either","dimensions":{"pg.durability":{"values":["fsync-off","durable"],"default":"fsync-off"},"pg.version":{"values":["17","18"],"default":"18"}}},++ {"id":"quill/queue/correctness/create-delete","summary":"Queue lifecycle","tier":"smoke","placement":"either"},+ {"id":"quill/send/correctness/send-read","summary":"Send and read","tier":"standard","placement":"either"},+ {"id":"quill/send/benchmark/batch-throughput","summary":"Batch throughput","tier":"standard","placement":"cell","knobs":[{"name":"quill.batch-size","summary":"Batch size","type":"int","default":10,"allowed":{"kind":"one-of","values":[10,100]},"variants":[100]}],"dimensions":{"telemetry.tracing":{"values":["off","sdk-inmemory"],"default":"off"},"telemetry.metrics":{"values":["off","collect"],"default":"off"},"pg.durability":{"values":["durable"],"default":"durable"},"pg.version":{"values":["17","18"],"default":"18"}}},+ {"id":"quill/read/concurrency/visibility","summary":"Visibility concurrency","tier":"standard","placement":"either"},+ {"id":"quill/ack/correctness/archive","summary":"Archive acknowledgement","tier":"smoke","placement":"either"},+ {"id":"quill/vt/correctness/expiry-redelivers","summary":"Visibility timeout","tier":"standard","placement":"either"},+ {"id":"quill/fifo/concurrency/ordering","summary":"FIFO ordering","tier":"standard","placement":"either"},+ {"id":"quill/topics/correctness/routing","summary":"Topic routing","tier":"standard","placement":"either"},+ {"id":"quill/notify/correctness/wakeup","summary":"Notify wakeup","tier":"standard","placement":"either"},+ {"id":"quill/config/correctness/roundtrip","summary":"Configuration","tier":"smoke","placement":"either"},+ {"id":"quill/effectful/correctness/interpreter","summary":"Effect interpreter","tier":"smoke","placement":"either"},++ {"id":"tally/append/correctness/expected-version","summary":"Expected version","tier":"smoke","placement":"either"},+ {"id":"tally/append/benchmark/throughput","summary":"Append throughput","tier":"standard","placement":"cell","knobs":[{"name":"tally.batch-size","summary":"Batch size","type":"int","default":1,"allowed":{"kind":"one-of","values":[1,10,100]},"variants":[10,100]}],"dimensions":{"telemetry.tracing":{"values":["off","sdk-inmemory","sdk-otlp"],"default":"off"},"telemetry.metrics":{"values":["off","collect","serve-scraped"],"default":"off"},"pg.durability":{"values":["durable"],"default":"durable"},"pg.version":{"values":["17","18"],"default":"18"}}},+ {"id":"tally/read/correctness/stream","summary":"Read stream","tier":"smoke","placement":"either"},+ {"id":"tally/subscription/concurrency/checkpoint","summary":"Subscription checkpoint","tier":"standard","placement":"either"},+ {"id":"tally/consumer-group/concurrency/partition","summary":"Consumer group partition","tier":"standard","placement":"either"},+ {"id":"tally/dead-letter/correctness/retry","summary":"Dead letter retry","tier":"standard","placement":"either"},+ {"id":"tally/lifecycle/correctness/restart","summary":"Lifecycle restart","tier":"standard","placement":"local"},+ {"id":"tally/transaction/correctness/atomicity","summary":"Transaction atomicity","tier":"standard","placement":"either"},+ {"id":"tally/notifier/concurrency/wakeup","summary":"Notifier wakeup","tier":"standard","placement":"either"},+ {"id":"tally/retention/correctness/prune","summary":"Retention","tier":"extended","placement":"either"},+ {"id":"tally/metrics/correctness/truthful","summary":"Metrics truthful","tier":"standard","placement":"either","dimensions":{"telemetry.metrics":{"values":["off","collect","serve","serve-scraped"],"default":"off"}}},+ {"id":"tally/otel/correctness/spans","summary":"Tracing spans","tier":"standard","placement":"either","dimensions":{"telemetry.tracing":{"values":["off","noop","sdk-inmemory","sdk-otlp"],"default":"off"}}},++ {"id":"acme/core-runner/correctness/lifecycle","summary":"Runner lifecycle","tier":"standard","placement":"either"},+ {"id":"acme/core-ordering/concurrency/ordered","summary":"Ordering","tier":"standard","placement":"either"},+ {"id":"acme/core-batch/correctness/batches","summary":"Batching","tier":"standard","placement":"either"},+ {"id":"acme/quill-adapter/correctness/finalize","summary":"QUILL adapter","tier":"standard","placement":"either"},+ {"id":"acme/tally-adapter/correctness/checkpoint","summary":"Tally adapter","tier":"standard","placement":"either"},+ {"id":"acme/metrics/correctness/truthful","summary":"Metrics","tier":"standard","placement":"either"},++ {"id":"brook/adapter/correctness/delivery","summary":"Brook delivery","tier":"standard","placement":"either"},+ {"id":"brook/adapter/concurrency/rebalance","summary":"Brook rebalance","tier":"extended","placement":"either"},+ {"id":"brook/producer/benchmark/throughput","summary":"Brook producer throughput","tier":"standard","placement":"cell","dimensions":{"pg.durability":{"values":["durable"],"default":"durable"}}},++ {"id":"flux/command/correctness/decide","summary":"Command processing","tier":"smoke","placement":"either"},+ {"id":"flux/process-manager/correctness/react","summary":"Process manager","tier":"standard","placement":"either"},+ {"id":"flux/router/correctness/route","summary":"Router","tier":"standard","placement":"either"},+ {"id":"flux/outbox/concurrency/publish","summary":"Outbox","tier":"standard","placement":"either"},+ {"id":"flux/inbox/correctness/deduplicate","summary":"Inbox","tier":"standard","placement":"either"},+ {"id":"flux/queue/correctness/job","summary":"Job queue","tier":"standard","placement":"either"},+ {"id":"flux/workflow/correctness/resume","summary":"Workflow","tier":"standard","placement":"either"},+ {"id":"flux/timer/concurrency/claim","summary":"Timer","tier":"standard","placement":"either"},+ {"id":"flux/shard/concurrency/lease","summary":"Shard lease","tier":"standard","placement":"either"},+ {"id":"flux/wake/correctness/push","summary":"Push wake","tier":"standard","placement":"either"},++ {"id":"runtime/end-to-end/correctness/two-contexts","summary":"End to end","tier":"extended","placement":"either"},+ {"id":"runtime/failure-matrix/soak/recovery","summary":"Failure recovery soak","tier":"soak","placement":"cell"}+ ]+}
+ test/fixtures/plan/components.synthetic.json view
@@ -0,0 +1,89 @@+{+ "schema": "kenshou.component-graph/v1",+ "ignoredPackages": ["pg-migrate-test-support", "tally-test-support", "flux-dsl"],+ "components": [+ {"id":"postgresql","kind":"service","uri":null,"repository":"postgresql","sourceRoots":[],"packages":[],"selectors":[],"verifiedAgainst":"PostgreSQL 17 and 18","dependsOn":[]},+ {"id":"pg-migrate","kind":"library","uri":"mori://shinzui/pg-migrate","repository":"pg-migrate","sourceRoots":["src/","pg-migrate.cabal"],"packages":["pg-migrate","pg-migrate-embed","pg-migrate-cli","pg-migrate-import-codd","pg-migrate-import-hasql-migration"],"selectors":[],"verifiedAgainst":"pg-migrate 1.1.0.0","dependsOn":[{"to":"postgresql","kind":"runtime"}]},+ {"id":"ephemeral-pg","kind":"harness","uri":"mori://shinzui/ephemeral-pg","repository":"ephemeral-pg","sourceRoots":["src/","ephemeral-pg.cabal"],"packages":["ephemeral-pg"],"selectors":["selftest/kernel/**"],"verifiedAgainst":"ephemeral-pg 0.3.1.0","dependsOn":[{"to":"postgresql","kind":"runtime"}]},+ {"id":"hs-opentelemetry","kind":"library","uri":"mori://iand675/hs-opentelemetry","repository":"hs-opentelemetry","sourceRoots":[],"packages":["hs-opentelemetry-api","hs-opentelemetry-sdk","hs-opentelemetry-exporter-otlp","hs-opentelemetry-exporter-in-memory","hs-opentelemetry-propagator-w3c","hs-opentelemetry-semantic-conventions"],"selectors":["selftest/telemetry/**"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"hs-opentelemetry 1.0 family","dependsOn":[]},+ {"id":"quill-hs","kind":"library","uri":"mori://shinzui/quill-hs","repository":"quill-hs","sourceRoots":["quill-core/","quill-hasql/","quill-effectful/","quill-config/","quill-migration/"],"packages":["quill-core","quill-hasql","quill-effectful","quill-config","quill-migration"],"selectors":["quill/**"],"verifiedAgainst":"quill-hs 0.6.1.0","dependsOn":[{"to":"pg-migrate","kind":"build"},{"to":"hs-opentelemetry","kind":"build"},{"to":"postgresql","kind":"runtime"}]},+ {"id":"tally-store-migrations","kind":"migrations","uri":"mori://shinzui/tally/packages/tally-store-migrations","repository":"tally","sourceRoots":["tally-store-migrations/"],"packages":["tally-store-migrations"],"selectors":[],"verifiedAgainst":"tally-store-migrations 0.4.0.0","dependsOn":[{"to":"pg-migrate","kind":"build"},{"to":"postgresql","kind":"runtime"}]},+ {"id":"tally-store","kind":"library","uri":"mori://shinzui/tally/packages/tally-store","repository":"tally","sourceRoots":["tally-store/"],"packages":["tally-store"],"selectors":["tally/**"],"verifiedAgainst":"tally-store 0.8.0.1","dependsOn":[{"to":"tally-store-migrations","kind":"runtime"},{"to":"postgresql","kind":"runtime"}]},+ {"id":"tally-cli","kind":"library","uri":"mori://shinzui/tally/packages/tally-cli","repository":"tally","sourceRoots":["tally-cli/"],"packages":["tally-cli"],"selectors":[],"verifiedAgainst":"tally-cli 0.2.0.6","dependsOn":[{"to":"tally-store","kind":"build"}]},+ {"id":"tally-otel","kind":"library","uri":"mori://shinzui/tally/packages/tally-otel","repository":"tally","sourceRoots":["tally-otel/"],"packages":["tally-otel"],"selectors":["tally/otel/**"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"tally-otel 0.2.0.8","dependsOn":[{"to":"tally-store","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+ {"id":"tally-metrics","kind":"library","uri":"mori://shinzui/tally/packages/tally-metrics","repository":"tally","sourceRoots":["tally-metrics/"],"packages":["tally-metrics"],"selectors":["tally/metrics/**","flux/outbox/correctness/telemetry-contract","flux/outbox/benchmark/**","flux/inbox/benchmark/**"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"tally-metrics 0.1.0.8","dependsOn":[{"to":"tally-store","kind":"build"},{"to":"tally-cli","kind":"build"}]},+ {"id":"acme-core","kind":"library","uri":"mori://shinzui/acme/packages/acme-core","repository":"acme","sourceRoots":["acme-core/src/","acme-core/acme-core.cabal"],"packages":["acme-core"],"selectors":[],"verifiedAgainst":"acme-core 0.9.0.3","changeImplies":null,+ "subcomponents":[+ {"id":"contract","pathPrefixes":["acme-core/src/Acme/Core.hs","acme-core/src/Acme/Core/","acme-core/src/Acme/Adapter.hs","acme-core/src/Acme/Adapter/","acme-core/src/Acme/Handler.hs","acme-core/src/Acme/Policy.hs"],"selectors":["acme/**"],"uses":[]},+ {"id":"telemetry","pathPrefixes":["acme-core/src/Acme/Telemetry.hs","acme-core/src/Acme/Telemetry/"],"selectors":["acme/**"],"uses":[],"changeImplies":{"minDimensionPolicy":"telemetry-corners"}},+ {"id":"runner","pathPrefixes":["acme-core/src/Acme/App.hs","acme-core/src/Acme/Internal/","acme-core/src/Acme/Batch.hs","acme-core/src/Acme/Stream.hs"],"selectors":["acme/**","brook/**"],"uses":["contract","telemetry"]}+ ],"dependsOn":[{"to":"hs-opentelemetry","kind":"build"}]},+ {"id":"acme-metrics","kind":"library","uri":"mori://shinzui/acme/packages/acme-metrics","repository":"acme","sourceRoots":["acme-metrics/"],"packages":["acme-metrics"],"selectors":["acme/metrics/**","flux/queue/correctness/worker-metrics-contract"],"changeImplies":{"minDimensionPolicy":"telemetry-corners"},"verifiedAgainst":"acme-metrics 0.9.0.3","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract","runner"]}]},+ {"id":"acme-quill-adapter","kind":"library","uri":"mori://shinzui/acme-quill-adapter","repository":"acme-quill-adapter","sourceRoots":["src/","acme-quill-adapter.cabal"],"packages":["acme-quill-adapter"],"selectors":["acme/quill-adapter/**"],"verifiedAgainst":"acme-quill-adapter 0.16.0.0","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract","telemetry"]},{"to":"quill-hs","kind":"build"}]},+ {"id":"acme-tally-adapter","kind":"library","uri":"mori://shinzui/tally/packages/acme-tally-adapter","repository":"tally","sourceRoots":["acme-tally-adapter/"],"packages":["acme-tally-adapter"],"selectors":["acme/tally-adapter/**"],"verifiedAgainst":"acme-tally-adapter 0.5.1.2","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract","telemetry","runner"]},{"to":"tally-store","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+ {"id":"hw-brook-client","kind":"library","uri":"mori://haskell-works/hw-brook-client","repository":"hw-brook-client","sourceRoots":[],"packages":["hw-brook-client"],"selectors":["brook/**"],"verifiedAgainst":"hw-brook-client 5.3.0","dependsOn":[]},+ {"id":"hw-brook-streamly","kind":"library","uri":"mori://shinzui/hw-brook-streamly","repository":"hw-brook-streamly","sourceRoots":["src/"],"packages":["hw-brook-streamly"],"selectors":["brook/**"],"verifiedAgainst":"hw-brook-streamly 0.2.0.0","dependsOn":[{"to":"hw-brook-client","kind":"build"}]},+ {"id":"brook-effectful","kind":"library","uri":"mori://shinzui/brook-effectful","repository":"brook-effectful","sourceRoots":["src/"],"packages":["brook-effectful"],"selectors":["brook/**"],"verifiedAgainst":"brook-effectful 0.3.1.0","dependsOn":[{"to":"hw-brook-client","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+ {"id":"acme-brook-adapter","kind":"library","uri":"mori://shinzui/acme-brook-adapter","repository":"acme-brook-adapter","sourceRoots":["src/"],"packages":["acme-brook-adapter"],"selectors":["brook/**"],"verifiedAgainst":"acme-brook-adapter 0.9.0.1","dependsOn":[{"to":"acme-core","kind":"build","toSubs":["contract"]},{"to":"brook-effectful","kind":"build"},{"to":"hw-brook-streamly","kind":"build"},{"to":"hw-brook-client","kind":"build"},{"to":"hs-opentelemetry","kind":"build"}]},+ {"id":"keiki","kind":"library","uri":"mori://shinzui/keiki","repository":"keiki","sourceRoots":["keiki/","keiki-codec-json/"],"packages":["keiki","keiki-codec-json"],"selectors":[],"verifiedAgainst":"keiki 0.9.1.0","dependsOn":[]},+ {"id":"flux-core","kind":"library","uri":"mori://shinzui/flux/packages/flux-core","repository":"flux","sourceRoots":["flux-core/"],"packages":["flux-core"],"selectors":[],"verifiedAgainst":"flux-core 0.17.0.0","dependsOn":[{"to":"keiki","kind":"build"},{"to":"tally-store","kind":"build"}]},+ {"id":"flux","kind":"library","uri":"mori://shinzui/flux/packages/flux","repository":"flux","sourceRoots":["flux/src/","flux/flux.cabal"],"packages":["flux"],"selectors":[],"verifiedAgainst":"flux 0.17.0.0",+ "subcomponents":[+ {"id":"command","pathPrefixes":["flux/src/Flux/Command.hs","flux/src/Flux/Command/","flux/src/Flux/ReplayAudit.hs"],"selectors":["flux/command/**"],"uses":["snapshot","telemetry"]},+ {"id":"snapshot","pathPrefixes":["flux/src/Flux/Snapshot.hs","flux/src/Flux/Snapshot/"],"selectors":["flux/snapshot/**"],"uses":[]},+ {"id":"projection","pathPrefixes":["flux/src/Flux/Projection.hs","flux/src/Flux/Projection/","flux/src/Flux/ReadModel.hs","flux/src/Flux/ReadModel/"],"selectors":["flux/projection/**","flux/command/**"],"uses":["command","telemetry"]},+ {"id":"process-manager","pathPrefixes":["flux/src/Flux/ProcessManager.hs","flux/src/Flux/ProcessManager/"],"selectors":["flux/process-manager/**"],"uses":["command","projection","timer","telemetry"]},+ {"id":"router","pathPrefixes":["flux/src/Flux/Router.hs","flux/src/Flux/Router/"],"selectors":["flux/router/**"],"uses":["command","process-manager","projection","telemetry"]},+ {"id":"outbox","pathPrefixes":["flux/src/Flux/Outbox.hs","flux/src/Flux/Outbox/"],"selectors":["flux/outbox/**"],"uses":["telemetry"]},+ {"id":"inbox","pathPrefixes":["flux/src/Flux/Inbox.hs","flux/src/Flux/Inbox/"],"selectors":["flux/inbox/**"],"uses":["command","process-manager","telemetry"]},+ {"id":"workflow","pathPrefixes":["flux/src/Flux/Workflow.hs","flux/src/Flux/Workflow/"],"selectors":["flux/workflow/**"],"uses":["timer","wake","snapshot","telemetry"]},+ {"id":"timer","pathPrefixes":["flux/src/Flux/Timer.hs","flux/src/Flux/Timer/"],"selectors":["flux/timer/**"],"uses":["telemetry"]},+ {"id":"shard","pathPrefixes":["flux/src/Flux/Subscription/"],"selectors":["flux/shard/**"],"uses":[]},+ {"id":"wake","pathPrefixes":["flux/src/Flux/Wake.hs"],"selectors":["flux/wake/**"],"uses":[]},+ {"id":"telemetry","pathPrefixes":["flux/src/Flux/Telemetry.hs"],"selectors":["flux/**"],"uses":[],"changeImplies":{"minDimensionPolicy":"telemetry-corners"}}+ ],"dependsOn":[{"to":"flux-core","kind":"build"},{"to":"keiki","kind":"build"},{"to":"tally-store","kind":"build"},{"to":"hs-opentelemetry","kind":"build"},{"to":"acme-core","kind":"build","toSubs":["contract"],"fromSubs":["process-manager","router","inbox"]},{"to":"flux-migrations","kind":"runtime"}]},+ {"id":"flux-quill","kind":"library","uri":"mori://shinzui/flux/packages/flux-quill","repository":"flux","sourceRoots":["flux-quill/src/","flux-quill/flux-quill.cabal"],"packages":["flux-quill"],"selectors":["flux/queue/**"],"verifiedAgainst":"flux-quill 0.17.0.0","dependsOn":[{"to":"flux-core","kind":"build"},{"to":"quill-hs","kind":"build"},{"to":"acme-quill-adapter","kind":"build"},{"to":"acme-core","kind":"build","toSubs":["contract","runner","telemetry"]},{"to":"hs-opentelemetry","kind":"build"}]},+ {"id":"flux-migrations","kind":"migrations","uri":"mori://shinzui/flux/packages/flux-migrations","repository":"flux","sourceRoots":["flux-migrations/"],"packages":["flux-migrations"],"selectors":[],"verifiedAgainst":"flux-migrations 0.17.0.0","dependsOn":[{"to":"tally-store-migrations","kind":"build"},{"to":"pg-migrate","kind":"build"},{"to":"postgresql","kind":"runtime"}]},+ {"id":"flux-test-support","kind":"harness","uri":"mori://shinzui/flux/packages/flux-test-support","repository":"flux","sourceRoots":["flux-test-support/"],"packages":["flux-test-support"],"selectors":["selftest/kernel/**"],"verifiedAgainst":"flux-test-support 0.17.0.0","dependsOn":[{"to":"ephemeral-pg","kind":"build"},{"to":"flux-migrations","kind":"build"},{"to":"tally-store","kind":"build"},{"to":"tally-store-migrations","kind":"build"},{"to":"pg-migrate","kind":"build"}]},+ {"id":"kenshou-harness","kind":"harness","uri":"mori://shinzui/flux-runtime-kenshou/packages/kenshou-core","repository":"flux-runtime-kenshou","sourceRoots":["kenshou-core/"],"packages":["kenshou-core"],"selectors":["selftest/kernel/correctness/postgres-roundtrip"],"verifiedAgainst":"kenshou-core 0.1.0.0","subcomponents":[{"id":"kernel","pathPrefixes":[],"selectors":["selftest/kernel/**"],"uses":[]}],"dependsOn":[{"to":"pg-migrate","kind":"build"},{"to":"tally-store-migrations","kind":"build"},{"to":"flux-migrations","kind":"build"},{"to":"quill-hs","kind":"build"},{"to":"ephemeral-pg","kind":"build"},{"to":"tally-store","kind":"runtime","fromSubs":["kernel"]}]},+ {"id":"flux-ops","kind":"library","uri":"mori://shinzui/flux/packages/flux-ops","repository":"flux","sourceRoots":["flux-ops/src/","flux-ops/flux-ops.cabal"],"packages":["flux-ops"],"selectors":["runtime/ops/**"],"verifiedAgainst":"flux-ops 0.17.0.0","dependsOn":[{"to":"flux","kind":"build"},{"to":"flux-migrations","kind":"build"},{"to":"flux-quill","kind":"build"},{"to":"tally-store","kind":"build"}]},+ {"id":"runtime-assembly","kind":"assembly","uri":"mori://shinzui/flux-runtime-kenshou/packages/kenshou-runtime","repository":"flux-runtime-kenshou","sourceRoots":["kenshou-runtime/"],"packages":["kenshou-runtime"],"selectors":["runtime/**"],"verifiedAgainst":"kenshou-runtime 0.1.0.0","dependsOn":[{"to":"flux","kind":"build"},{"to":"flux-core","kind":"build"},{"to":"keiki","kind":"build"},{"to":"tally-store","kind":"build"},{"to":"brook-effectful","kind":"build"},{"to":"hw-brook-client","kind":"build"},{"to":"kenshou-harness","kind":"build"},{"to":"flux-quill","kind":"build"},{"to":"acme-core","kind":"build"},{"to":"acme-brook-adapter","kind":"build"},{"to":"flux-ops","kind":"build"},{"to":"hs-opentelemetry","kind":"build"},{"to":"tally-otel","kind":"build"},{"to":"quill-hs","kind":"build"},{"to":"flux-migrations","kind":"runtime"}]}+ ],+ "repositoryPaths": [+ {"prefix":"kenshou-core/src/Kenshou/Plan/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"kenshou-core/data/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"kenshou-cli/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"kenshou-remote/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"kenshou-evidence/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"schemas/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"suites/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"policies/","effect":{"kind":"selectors","selectors":["selftest/**"]}},+ {"prefix":"kenshou-quill/","effect":{"kind":"selectors","selectors":["quill/**"]}},+ {"prefix":"kenshou-tally/","effect":{"kind":"selectors","selectors":["tally/**"]}},+ {"prefix":"kenshou-acme/","effect":{"kind":"selectors","selectors":["acme/**"]}},+ {"prefix":"kenshou-brook/","effect":{"kind":"selectors","selectors":["brook/**","runtime/**"]}},+ {"prefix":"kenshou-flux/","effect":{"kind":"selectors","selectors":["flux/**","runtime/**"]}},+ {"prefix":"kenshou-runtime/","effect":{"kind":"selectors","selectors":["runtime/**"]}},+ {"prefix":"cohort/","effect":{"kind":"cohort"}},+ {"prefix":"kenshou-core/","effect":{"kind":"all"}},+ {"prefix":"kenshou-measure/","effect":{"kind":"all"}},+ {"prefix":"kenshou-check/","effect":{"kind":"all"}},+ {"prefix":"kenshou-diagnose/","effect":{"kind":"all"}},+ {"prefix":"kenshou-telemetry/","effect":{"kind":"all"}},+ {"prefix":"cabal.project","effect":{"kind":"all"}},+ {"prefix":"flake.","effect":{"kind":"all"}},+ {"prefix":"nix/","effect":{"kind":"all"}},+ {"prefix":"docs/","effect":{"kind":"nothing"}},+ {"prefix":"agents/","effect":{"kind":"nothing"}},+ {"prefix":".seihou/","effect":{"kind":"nothing"}},+ {"prefix":".github/","effect":{"kind":"nothing"}},+ {"prefix":".claude/","effect":{"kind":"nothing"}},+ {"prefix":".agents/","effect":{"kind":"nothing"}},+ {"prefix":"mori/","effect":{"kind":"nothing"}},+ {"prefix":"README.md","effect":{"kind":"nothing"}},+ {"prefix":"mori.dhall","effect":{"kind":"nothing"}},+ {"prefix":"mina.kdl","effect":{"kind":"nothing"}},+ {"prefix":"Justfile","effect":{"kind":"nothing"}},+ {"prefix":"process-compose.yaml","effect":{"kind":"nothing"}}+ ]+}
+ test/fixtures/plan/plan-missing-edge.json view
@@ -0,0 +1,6 @@+{+ "install-plan": [+ {"id":"krk-str-0.8.0.1-8c49cfa0","pkg-name":"tally-store","pkg-version":"0.8.0.1","component-name":"lib","depends":[]},+ {"id":"quill-hsql-0.6.1.0-c3f074ba","pkg-name":"quill-hasql","pkg-version":"0.6.1.0","component-name":"lib","depends":["krk-str-0.8.0.1-8c49cfa0"]}+ ]+}
+ test/fixtures/valid-project/cohort/active.project view
@@ -0,0 +1,1 @@+import: released.project
+ test/golden/manifest.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.artifact-manifest/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","createdAt":"2026-09-20T17:20:03.902Z","algorithm":"sha256","files":[{"path":"run-spec.json","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","bytes":82,"mediaType":"application/json"}]}
+ test/golden/perf-calibration-acceptance.synthetic.json view
@@ -0,0 +1,368 @@+{+ "schema": "kenshou.perf-calibration-acceptance/v1",+ "batchId": "0199a3c3-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "aa": [+ {+ "seed": 1,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/1/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 2,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/2/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 3,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/3/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 4,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/4/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 5,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/5/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 6,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/6/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 7,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/7/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 8,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/8/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 9,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/9/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 10,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/10/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 11,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/11/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 12,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/12/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 13,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/13/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 14,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/14/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 15,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/15/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 16,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/16/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 17,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/17/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 18,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/18/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 19,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/19/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 20,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/aa/20/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ }+ ],+ "slowdown": [+ {+ "seed": 1,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/1/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 2,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/2/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 3,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/3/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 4,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/4/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 5,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/5/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 6,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/6/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 7,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/7/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 8,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/8/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 9,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/9/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 10,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/10/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 11,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/11/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 12,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/12/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 13,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/13/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 14,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/14/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 15,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/15/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 16,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/16/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 17,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/17/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 18,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/18/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 19,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/19/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ },+ {+ "seed": 20,+ "receipts": [+ {+ "uri": "gs://fixture-evidence/slowdown/20/receipt.json",+ "sha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+ }+ ]+ }+ ]+}
+ test/golden/perf-calibration-definition.synthetic.json view
@@ -0,0 +1,41 @@+{+ "schema": "kenshou.perf-calibration-definition/v1",+ "batchId": "0199a3c3-7b1e-7c44-9d0a-3f5e2a61b7c9",+ "project": "mori://fixture/runtime",+ "profile": "synthetic-core",+ "workloadSha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "toolchainSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "harnessSemanticsSha256": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",+ "baselineRevision": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "slowdownRevision": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "injectionMechanism": "Synthetic codec control only; no runtime evidence.",+ "targetMetric": "op.latency.p99",+ "targetRelativeEffect": 0.5,+ "targetAbsoluteEffect": 1000000,+ "seeds": [+ 1,+ 2,+ 3,+ 4,+ 5,+ 6,+ 7,+ 8,+ 9,+ 10,+ 11,+ 12,+ 13,+ 14,+ 15,+ 16,+ 17,+ 18,+ 19,+ 20+ ],+ "infrastructureRetryCap": 1,+ "expiresAt": "2026-10-12T00:00:00Z",+ "machineProfile": "synthetic-machine-no-live-evidence",+ "healthPolicySha256": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"+}
+ test/golden/perf-calibration-reference.synthetic.json view
@@ -0,0 +1,6 @@+{+ "definitionUri": "gs://fixture-evidence/definition.json",+ "definitionSha256": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "acceptanceUri": "gs://fixture-evidence/acceptance.json",+ "acceptanceSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+}
+ test/golden/perf-result.not-applicable.fixture.json view
@@ -0,0 +1,17 @@+{+ "schema": "kenshou.perf-result/v1",+ "requestSha256": "sha256:7a0072d3819fddd8d6ef0278182b4a696a7006922e4f93610d3ff3d8338f59ac",+ "preparedSha256": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "purpose": "calibration",+ "coverage": "not-applicable",+ "changeCoverage": "incomplete",+ "performanceVerdict": "not-applicable",+ "infrastructureState": "healthy",+ "blockingRuntimeFailures": [],+ "gatingEligible": false,+ "configurations": [],+ "exclusions": [],+ "uncoveredScenarios": [],+ "evidenceVerified": true,+ "exitCode": 0+}
+ test/golden/plan-summary.minimal.json view
@@ -0,0 +1,24 @@+{+ "schema": "kenshou.plan-summary/v1",+ "planId": "0199a3f2-7c1e-7b8a-9d41-3e5f6a7b8c9d",+ "entries": [+ {+ "ordinal": 1,+ "scenario": "selftest/kernel/correctness/always-pass",+ "status": "completed",+ "attempts": [+ {+ "runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011",+ "startedAt": "2026-09-21T14:18:40Z",+ "finishedAt": "2026-09-21T14:18:41Z",+ "outcome": "passed",+ "knownDefect": false,+ "childExitCode": 0+ }+ ]+ }+ ],+ "counts": {"passed": 1},+ "worstOutcome": "passed",+ "exitCode": 0+}
+ test/golden/resources/run-spec.resources.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/postgres-roundtrip","scenarioRevision":1,"knobs":{"selftest.events":3},"dimensions":{"pg.durability":"durable","pg.version":"18"},"seed":7,"phases":{"warmUpSeconds":0,"steadySeconds":0,"drainSeconds":0},"timeoutSeconds":120,"environment":{"placement":"local","machineProfile":"local/darwin-aarch64","postgres":{"mode":"ephemeral","settings":{}},"telemetry":null,"resources":{"broker":{"backend":"external","brokers":["10.0.0.5:9092"]},"object-store":{"bucket":"fixture"}}},"cohortExpectation":null,"comparison":null,"labels":{}}
+ test/golden/resources/worker-init.resources.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.worker-init/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/postgres-roundtrip","role":"selftest/worker","instance":"worker-0","seed":7,"knobs":{},"dimensions":{},"postgres":null,"resources":{"broker":{"backend":"external","brokers":["10.0.0.5:9092"]},"object-store":{"bucket":"fixture"}},"telemetry":null,"outDir":"/tmp/out","args":{}}
+ test/golden/run-plan.minimal.json view
@@ -0,0 +1,44 @@+{+ "schema": "kenshou.run-plan/v1",+ "planId": "0199a3f2-7c1e-7b8a-9d41-3e5f6a7b8c9d",+ "createdAt": "2026-09-21T14:03:22Z",+ "suite": null,+ "graphDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",+ "cohort": {"name": "released", "planHash": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},+ "inputs": {"changed": ["kenshou-harness"]},+ "policy": {+ "maxTier": "standard",+ "kinds": ["correctness"],+ "placement": "local",+ "dimensionPolicy": "default-only",+ "knobPolicy": "defaults",+ "trials": 3,+ "seed": 42+ },+ "changes": [{"component": "kenshou-harness", "source": "named", "detail": "--changed kenshou-harness"}],+ "runs": [+ {+ "ordinal": 1,+ "runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011",+ "estimateMinutes": 1,+ "reasons": [{"component": "kenshou-harness", "source": "named", "via": ["kenshou-harness"], "selector": "selftest/kernel/correctness/postgres-roundtrip", "distance": 0}],+ "trial": null,+ "spec": {+ "schema": "kenshou.run-spec/v1",+ "runId": "0199a3f2-7c20-7f00-8a11-0c0d0e0f1011",+ "scenario": "selftest/kernel/correctness/postgres-roundtrip",+ "scenarioRevision": 1,+ "knobs": {},+ "dimensions": {},+ "seed": 42,+ "environment": {"placement": "local"},+ "cohortExpectation": null,+ "comparison": null,+ "labels": {}+ }+ }+ ],+ "skipped": [],+ "warnings": [],+ "estimateMinutes": 1+}
+ test/golden/run-result.known-defect.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-result/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e56","scenario":"selftest/kernel/correctness/known-defect","scenarioRevision":1,"outcome":"failed","blocking":false,"exitCode":0,"seed":7,"spec":{"path":"run-spec.json","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"cohort":{},"fingerprint":{},"compatibility":{"algorithm":"kenshou.compat-key/v1","comparisonKey":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","seriesKey":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","inputs":{}},"summaries":{},"observations":[]}
+ test/golden/run-result.passed.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-result/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/always-pass","scenarioRevision":1,"outcome":"passed","blocking":false,"exitCode":0,"seed":7,"spec":{"path":"run-spec.json","sha256":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"cohort":{},"fingerprint":{},"compatibility":{"algorithm":"kenshou.compat-key/v1","comparisonKey":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","seriesKey":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","inputs":{}},"summaries":{},"observations":[]}
+ test/golden/run-spec.effective.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/kernel/correctness/postgres-roundtrip","scenarioRevision":1,"knobs":{"selftest.events":3},"dimensions":{"pg.durability":"durable","pg.version":"18"},"seed":7,"phases":{"warmUpSeconds":0,"steadySeconds":0,"drainSeconds":0},"timeoutSeconds":120,"environment":{"placement":"local","machineProfile":"local/darwin-aarch64","postgres":{"mode":"ephemeral","settings":{}},"telemetry":null},"cohortExpectation":null,"comparison":null,"labels":{}}
+ test/golden/run-spec.external.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","scenario":"selftest/kernel/correctness/postgres-roundtrip","environment":{"placement":"local","postgres":{"mode":"external","connectionStringEnv":"KENSHOU_PG_URL"}}}
+ test/golden/run-spec.minimal.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-spec/v1","scenario":"selftest/kernel/correctness/always-pass"}
+ test/golden/scenario-list.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.scenario-list/v1","suiteVersion":"0.1.0.0","scenarios":[{"id":"selftest/kernel/correctness/always-pass","layer":"selftest","component":"kernel","kind":"correctness","name":"always-pass","revision":1,"summary":"Always reports passed.","tier":"smoke","placement":"either","knownDefect":null}],"roles":[{"name":"selftest/echo","summary":"Echoes custom worker messages."}]}
+ test/golden/worker-messages.jsonl view
@@ -0,0 +1,4 @@+{"v":1,"type":"ready"}+{"v":1,"type":"custom","name":"echo","payload":{"n":1}}+{"v":1,"type":"stop","graceMillis":2000}+{"v":1,"type":"done","detail":null}