kenshou-core-0.1.0.0: test/Kenshou/PerfPreparedSpec.hs
{-# LANGUAGE FieldSelectors #-}
module Kenshou.PerfPreparedSpec (spec) where
import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.Aeson.Types (parseEither)
import Data.ByteString qualified as ByteString
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Either (isLeft)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified
import Kenshou.ConsumerBaseline qualified as ConsumerBaseline
import Kenshou.Core.Canonical (sha256Hex)
import Kenshou.Core.Cohort qualified as Cohort
import Kenshou.Core.Manifest qualified as Manifest
import Kenshou.Perf.Artifacts
import Kenshou.Perf.Calibration qualified as Calibration
import Kenshou.Perf.Contracts (PerfContracts (..), noPerfContracts)
import Kenshou.Perf.Prepared hiding (plan)
import Kenshou.Perf.Source qualified as Source
import System.Directory (createFileLink)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec
spec :: Spec
spec = describe "performance preparation integrity" do
it "validates the pinned synthetic protocol fixture and rejects changed build keys" do
consumerContracts <- loadConsumerContracts
prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right
validatePrepared consumerContracts prepared `shouldBe` Right ()
let baselineArm = (prepared :: PerfPrepared).baseline
validatePrepared consumerContracts prepared {baseline = baselineArm {buildKey = "sha256:changed"}} `shouldSatisfy` isLeft
decoded <- right (eitherDecode (encode prepared) :: Either String PerfPrepared)
toJSON decoded `shouldBe` toJSON prepared
it "admits resolved-cohort identities, replays legacy identities and rejects a rehashed alias" do
consumerContracts <- loadConsumerContracts
prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right
(request, template, baselineSource, candidateSource) <- right (parseEither (withObject "Inputs" \fields -> (,,,) <$> fields .: "request" <*> fields .: "template" <*> fields .: "baselineSource" <*> fields .: "candidateSource") (prepared :: PerfPrepared).inputs)
baselineDescriptor <- right (Source.descriptorForSnapshot request template "baseline" baselineSource)
candidateDescriptor <- right (Source.descriptorForSnapshot request template "candidate" candidateSource)
let bind descriptor arm = do
let digest = Data.Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor)))
lock <- case arm.sourceLock of
Object fields -> pure (Object (KeyMap.insert "cohort" (toJSON descriptor.descriptorName) (KeyMap.insert "descriptorSha256" (String digest) fields)))
_ -> fail "fixture source lock is not an object"
payload <- case arm.payload of
Object fields -> do
original <- maybe (fail "fixture payload has no identity") pure (KeyMap.lookup "cohortIdentity" fields)
identity <- case fromJSON original of Error problem -> fail problem; Success value -> pure value
let changed = (identity :: Cohort.CohortIdentity) {Cohort.identityCohort = descriptor.descriptorName, Cohort.identityDescriptorSha256 = digest}
pure (Object (KeyMap.insert "cohortIdentity" (toJSON changed) (KeyMap.insert "flakeAttr" (String ("packages.x86_64-linux.kenshou-" <> Cohort.unCohortName descriptor.descriptorName)) fields)))
_ -> fail "fixture payload is not an object"
pure arm {descriptor = descriptor, sourceLock = lock, payload = payload, buildKey = buildKeyFor prepared.harnessRevision descriptor lock arm.buildInventory arm.boundsInventory}
baseline <- bind baselineDescriptor prepared.baseline
candidate <- bind candidateDescriptor prepared.candidate
let modern = prepared {baseline = baseline, candidate = candidate}
validatePrepared consumerContracts modern `shouldBe` Right ()
validatePrepared consumerContracts prepared `shouldBe` Right ()
validatePrepared consumerContracts modern {candidate = prepared.candidate} `shouldSatisfy` isLeft
alias <- bind baselineDescriptor {Cohort.descriptorName = Cohort.CohortName "perf-unrelated-alias", Cohort.descriptorProjectFile = "perf-unrelated-alias.project"} baseline
validatePrepared consumerContracts modern {baseline = alias, candidate = alias} `shouldSatisfy` isLeft
it "binds actual inventories to the planning toolchain while allowing catalog-only promotion" do
consumerContracts <- loadConsumerContracts
prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right
let inventory = (prepared :: PerfPrepared).baseline.buildInventory
case (prepared.inputs, inventory.toolchain) of
(Object inputs, Object toolchain) -> do
let expected = Object (KeyMap.delete "harnessRevision" toolchain)
pinned = prepared {inputs = Object (KeyMap.insert "buildToolchain" expected inputs)}
changed = inventory {toolchain = Object (KeyMap.insert "harnessRevision" (String "another catalog commit") toolchain)}
validatePrepared consumerContracts pinned `shouldBe` Right ()
Calibration.toolchainDigest inventory `shouldBe` Calibration.toolchainDigest changed
validatePrepared consumerContracts prepared {inputs = Object (KeyMap.insert "buildToolchain" (object ["strictBounds" .= False]) inputs)} `shouldSatisfy` isLeft
_ -> expectationFailure "invalid synthetic inventory fixture"
it "rejects bound lifting inside a resealed inventory even when its toolchain claims strict bounds" do
consumerContracts <- loadConsumerContracts
prepared <- eitherDecodeFileStrict' "../fixtures/consumer-baseline/kenshou-core/test/golden/perf-prepared.fixture.json" >>= right
let arm = (prepared :: PerfPrepared).baseline
corrupt = arm.buildInventory {packages = [entry {settings = object ["jailbreak" .= True, "configureFlags" .= ([] :: [Text])]} | entry <- arm.buildInventory.packages]}
changed = arm {buildInventory = corrupt, buildKey = buildKeyFor prepared.harnessRevision arm.descriptor arm.sourceLock corrupt arm.boundsInventory}
validatePrepared consumerContracts prepared {baseline = changed, candidate = changed} `shouldSatisfy` isLeft
it "allows only target source/version changes; target flags still must match" do
compareInventories ["runtime"] inventory inventory {packages = [runtime {version = "2", source = String "candidate"}, dependency]} `shouldBe` Right ()
compareInventories ["runtime"] inventory inventory {packages = [runtime {flags = Map.singleton "feature" True}, dependency]} `shouldSatisfy` isLeft
compareInventories ["runtime"] inventory inventory {packages = [runtime {settings = String "different"}, dependency]} `shouldSatisfy` isLeft
it "rejects unrelated source, version, toolchain and package-set drift" do
mapM_
(\changed -> compareInventories ["runtime"] inventory changed `shouldSatisfy` isLeft)
[inventory {packages = [runtime, dependency {source = String "other"}]}, inventory {packages = [runtime, dependency {version = "2"}]}, inventory {toolchain = String "other"}, inventory {packages = [runtime]}]
it "rejects missing targets and ambiguous package identities" do
compareInventories ["missing"] inventory inventory `shouldSatisfy` isLeft
compareInventories ["runtime"] inventory inventory {packages = [runtime, dependency, dependency]} `shouldSatisfy` isLeft
it "identifies harness sources by commit and includes transitive flags/revisions" do
resolved <- right (cabalInventory "/workspace/harness" revision plan)
length resolved.packages `shouldBe` 2
map (.flags) resolved.packages `shouldContain` [Map.singleton "feature" True]
cabalInventory "/workspace/other" revision plan `shouldSatisfy` isLeft
cabalInventory "/workspace/harness" "main" plan `shouldSatisfy` isLeft
eitherDecode (encode resolved) `shouldBe` Right resolved
it "renders a standalone bounds project and rejects blanket exceptions" do
descriptor <- syntheticCohort
project <- right (renderBoundsProject noPerfContracts ["/harness/kenshou-core/kenshou-core.cabal"] descriptor)
project `shouldNotContainText` "active.project"
renderBoundsProject noPerfContracts ["/harness/package.cabal"] descriptor {Cohort.descriptorAllowNewer = ["all"]} `shouldSatisfy` isLeft
renderBoundsProject noPerfContracts ["/unsafe path/package.cabal"] descriptor `shouldSatisfy` isLeft
it "appends the suite's lane stanzas for the cohorts it selects" do
descriptor <- syntheticCohort
let laneName = Cohort.CohortName "acme-lane"
contracts = noPerfContracts {laneStanzas = \selected -> if selected.descriptorName == laneName then ["package acme-suite", " flags: -extras"] else []}
lane <- right (renderBoundsProject contracts ["/harness/acme-suite/acme-suite.cabal"] descriptor {Cohort.descriptorName = laneName})
lane `shouldSatisfy` Data.Text.isInfixOf "package acme-suite\n flags: -extras"
full <- right (renderBoundsProject contracts ["/harness/acme-suite/acme-suite.cabal"] descriptor)
full `shouldNotContainText` "flags: -extras"
it "verifies the complete sealed file set and refuses an omitted file" $ withSystemTempDirectory "perf-seal" \root -> do
writeFile (root </> "input.json") "{}"
sealPerfDirectory root `shouldReturn` Right ()
verifyPerfManifest root `shouldReturn` Right ()
writeFile (root </> "unlisted.json") "{}"
verifyPerfManifest root >>= (`shouldSatisfy` isLeft)
it "refuses duplicate and escaping paths before opening a listed file" $ withSystemTempDirectory "perf-seal" \root -> do
writeFile (root </> "input.json") "{}"
sealPerfDirectory root `shouldReturn` Right ()
original <- eitherDecodeFileStrict' (root </> "manifest.json") >>= right
encodeFile (root </> "manifest.json") (original :: Manifest.Manifest) {Manifest.files = original.files <> original.files}
verifyPerfManifest root >>= (`shouldSatisfy` isLeft)
encodeFile (root </> "manifest.json") original {Manifest.files = map (\file -> file {Manifest.path = "../outside.json"}) original.files}
verifyPerfManifest root >>= (`shouldSatisfy` isLeft)
it "refuses symbolic links when sealing or verifying evidence" $ withSystemTempDirectory "perf-seal" \root -> do
writeFile (root </> "input.json") "{}"
sealPerfDirectory root `shouldReturn` Right ()
createFileLink (root </> "input.json") (root </> "link.json")
sealPerfDirectory root >>= (`shouldSatisfy` isLeft)
verifyPerfManifest root >>= (`shouldSatisfy` isLeft)
runtime, dependency :: InventoryPackage
runtime = InventoryPackage "runtime" "1" (String "baseline") Map.empty (object [])
dependency = InventoryPackage "dependency" "1" (String "fixed") Map.empty (object [])
inventory :: BuildInventory
inventory = BuildInventory "nix" "ghc-9.12.4" "x86_64-linux" (String "pins") [dependency, runtime]
revision :: Text
revision = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
plan :: Value
plan =
object
[ "compiler-id" .= ("ghc-9.12.4" :: Text),
"cabal-version" .= ("3.16" :: Text),
"os" .= ("linux" :: Text),
"arch" .= ("x86_64" :: Text),
"install-plan"
.= [ object ["type" .= ("configured" :: Text), "pkg-name" .= ("kenshou-core" :: Text), "pkg-version" .= ("0.1" :: Text), "pkg-src" .= object ["type" .= ("local" :: Text), "path" .= ("/workspace/harness/kenshou-core" :: Text)]],
object ["type" .= ("configured" :: Text), "pkg-name" .= ("dependency" :: Text), "pkg-version" .= ("1" :: Text), "pkg-src" .= object ["type" .= ("repo-tar" :: Text)], "flags" .= Map.singleton ("feature" :: Text) True, "pkg-revision" .= (1 :: Int), "pkg-cabal-sha256" .= ("description" :: Text), "pkg-src-sha256" .= ("archive" :: Text)]
]
]
right :: (Show e) => Either e a -> IO a
right = either (\problem -> expectationFailure (show problem) >> fail "expected Right") pure
shouldNotContainText :: Text -> Text -> Expectation
shouldNotContainText actual needle = actual `shouldSatisfy` (not . Data.Text.isInfixOf needle)
loadConsumerContracts :: IO PerfContracts
loadConsumerContracts = ConsumerBaseline.consumerContracts
syntheticCohort :: IO Cohort.CohortDescriptor
syntheticCohort = ByteString.readFile "test/fixtures/perf/cohort.json" >>= right . eitherDecodeStrict'