kenshou-evidence-0.1.0.0: src/Kenshou/Evidence/Record.hs
module Kenshou.Evidence.Record
( RecordInput (..),
RecordOptions (..),
RecordOutcome (..),
RecordError (..),
buildRunRecord,
recordRun,
recordRunWith,
recordComparison,
)
where
import Control.Applicative ((<|>))
import Control.Monad (forM, unless, when)
import Data.Aeson (FromJSON (..), Result (..), ToJSON (..), Value (..), fromJSON, object, withObject, (.:), (.:?), (.=))
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.Aeson.Types (Parser)
import Data.List (nub, sortOn)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe, mapMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, defaultTimeLocale, formatTime, getCurrentTime)
import Kenshou.Core.Cohort (CohortIdentity (..), CohortName (..), PackageSource (..), PlanHash (..), ResolvedComponent (..), ResolvedPackage (..))
import Kenshou.Core.Id (Kind (..), LayerId, RunId, ScenarioId (..), renderLayer, renderRunId, renderScenarioId)
import Kenshou.Core.Outcome (Outcome (..), renderOutcome)
import Kenshou.Core.RunSpec (EnvironmentSpec (..), RunSpec (..), SpecPlacement (..))
import Kenshou.Evidence.Bundle (BundleWriteError, BundleWriteResult (..), writeComparisonRecord, writeRunRecord)
import Kenshou.Evidence.Frontmatter (ComparisonEvidence (..), EvidenceRecord (..), recordFromDocument)
import Kenshou.Evidence.Publish (PublishError, PublishOptions (..), UploadMode, publishComparisonData, publishRunData)
import Kenshou.Evidence.Source (CellEvidence (..), ComparisonSource (..), ComparisonView (..), RunResultView (..), RunSource (..), SourceError, VerifiedFile (..), loadComparisonSource, loadRunSource)
import Kenshou.Evidence.Store (ObjectStore)
import Kenshou.Evidence.Types (ComponentRef (..), DataKind (..), DataLink (..), Purpose (..), Sha256, SubjectKind (..), mkRevision, mkSha256)
import Kenshou.Evidence.Types qualified as EvidenceTypes
import Okf.Bundle (conceptDocument, conceptSourcePath, walkBundle)
import Okf.ConceptId (parseConceptId, renderConceptLink)
data RecordInput = RecordInput
{ purpose :: !Purpose,
generatedAt :: !UTCTime,
allowDirty :: !Bool,
subjectOverride :: !(Maybe (Text, SubjectKind)),
produced :: ![Text],
previousRun :: !(Maybe Text),
-- | The evidence subject each registered layer declares.
subjects :: !(Map LayerId Text),
-- | The subject for a layer that declares none.
fallbackSubject :: !(Maybe Text)
}
deriving stock (Eq, Show)
data RecordError
= RecordError !Text
| SourceFailure !SourceError
| PublishFailure !PublishError
| BundleFailure !BundleWriteError
deriving stock (Eq, Show)
data RecordOptions = RecordOptions
{ bundleRoot :: !FilePath,
dataBaseUri :: !Text,
purpose :: !Purpose,
uploadMode :: !UploadMode,
deepVerify :: !Bool,
allowDirty :: !Bool,
linkLogs :: !Bool,
subjectOverride :: !(Maybe (Text, SubjectKind)),
produced :: ![Text],
subjects :: !(Map LayerId Text),
fallbackSubject :: !(Maybe Text)
}
deriving stock (Eq, Show)
data RecordOutcome = Recorded !FilePath | AlreadyRecorded !FilePath
deriving stock (Eq, Show)
recordRun :: ObjectStore -> RecordOptions -> FilePath -> IO (Either RecordError RecordOutcome)
recordRun store options directory = do
now <- getCurrentTime
recordRunWith writeRunRecord now store options directory
recordRunWith :: (FilePath -> EvidenceRecord -> IO (Either BundleWriteError BundleWriteResult)) -> UTCTime -> ObjectStore -> RecordOptions -> FilePath -> IO (Either RecordError RecordOutcome)
recordRunWith writer now store options directory = do
loaded <- loadRunSource directory
case loaded of
Left err -> pure (Left (SourceFailure err))
Right source -> case preflight source of
Left err -> pure (Left err)
Right () -> do
let publishing = PublishOptions options.dataBaseUri options.uploadMode options.deepVerify options.linkLogs
published <- publishRunData store publishing directory source
case published of
Left err -> pure (Left (PublishFailure err))
Right links -> do
let input = RecordInput options.purpose now options.allowDirty options.subjectOverride options.produced Nothing options.subjects options.fallbackSubject
case buildRunRecord input source links of
Left err -> pure (Left err)
Right record -> do
written <- writer options.bundleRoot record
pure case written of
Left err -> Left (BundleFailure err)
Right (RecordCreated path) -> Right (Recorded path)
Right (RecordPresent path) -> Right (AlreadyRecorded path)
where
preflight source = do
kenshou <- jsonField "kenshou" source.result.resultFingerprint
revision <- jsonField "revision" kenshou
_ <- either (Left . RecordError) Right (mkRevision revision)
dirty <- jsonField "dirty" kenshou
when (dirty && not options.allowDirty) (Left (RecordError "dirty harness requires --allow-dirty"))
recordComparison :: ObjectStore -> RecordOptions -> FilePath -> IO (Either RecordError RecordOutcome)
recordComparison store options path = do
loaded <- loadComparisonSource path
case loaded of
Left err -> pure (Left (SourceFailure err))
Right source -> do
arms <- loadRecordedArms options.bundleRoot (source.view.baselineRuns <> source.view.candidateRuns)
case arms of
Left err -> pure (Left err)
Right records -> case comparisonFields options source records of
Left err -> pure (Left err)
Right build -> do
let publishing = PublishOptions options.dataBaseUri options.uploadMode options.deepVerify options.linkLogs
published <- publishComparisonData store publishing path source
case published of
Left err -> pure (Left (PublishFailure err))
Right link -> do
now <- getCurrentTime
let record = build now link
written <- writeComparisonRecord options.bundleRoot record
pure case written of
Left err -> Left (BundleFailure err)
Right (RecordCreated relative) -> Right (Recorded relative)
Right (RecordPresent relative) -> Right (AlreadyRecorded relative)
loadRecordedArms :: FilePath -> [RunId] -> IO (Either RecordError [(FilePath, EvidenceRecord)])
loadRecordedArms root identifiers = do
found <- walkBundle root
pure do
concepts <- either (Left . RecordError . Text.pack . show) Right found
let records = [(conceptSourcePath concept, record) | concept <- concepts, Right record <- [recordFromDocument (conceptDocument concept)]]
let byId = Map.fromListWith (<>) [(record.runId, [(path, record)]) | (path, record) <- records]
forM identifiers $ \identifier -> case Map.lookup (renderRunId identifier) byId of
Just [entry] -> Right entry
Just _ -> Left (RecordError ("recorded arm has duplicate concepts: " <> renderRunId identifier))
Nothing -> Left (RecordError ("comparison arm is not recorded: " <> renderRunId identifier))
comparisonFields :: RecordOptions -> ComparisonSource -> [(FilePath, EvidenceRecord)] -> Either RecordError (UTCTime -> DataLink -> ComparisonEvidence)
comparisonFields options source arms = do
unless (null options.produced) (Left (RecordError "--produced applies only to run records"))
let view = source.view
pairCount = view.pairCount
(baselines, candidates) = splitAt pairCount arms
allRecords = map snd arms
unless (length baselines == pairCount && length candidates == pairCount) (Left (RecordError "comparison arm count differs from source"))
scenario <- case nub (map (.scenario) allRecords) of
[single] -> Right single
_ -> Left (RecordError "comparison arms must have one scenario")
axis <- case view.variedFactors of
[single] -> Right single
_ -> Left (RecordError "comparison evidence requires one varied factor")
(factor, factorName) <- factorForAxis axis
baselineValue <- armValue axis baselines
candidateValue <- armValue axis candidates
when (baselineValue == candidateValue) (Left (RecordError "comparison factor values must differ"))
revisionText <- maybe (Left (RecordError "comparison harness revision is unavailable")) Right view.harnessRevision
harnessRevision <- either (Left . RecordError) Right (mkRevision revisionText)
harnessDirty <- maybe (Left (RecordError "comparison harness dirty state is unavailable")) Right view.harnessDirty
when (harnessDirty && not options.allowDirty) (Left (RecordError "dirty harness requires --allow-dirty"))
first <- case candidates of
(_, candidate) : _ -> Right candidate
[] -> Left (RecordError "comparison has no candidate arm")
let subject = maybe first.subject fst options.subjectOverride
subjectKind = maybe first.subjectKind snd options.subjectOverride
unless ("mori://" `Text.isPrefixOf` subject) (Left (RecordError "subject must be a canonical Mori URI"))
let outcome = case view.verdict of
"pass" -> Passed
"regression" -> Failed
"inconclusive" -> Inconclusive
_ -> InfrastructureFailure
comparison =
object
( [ "verdict" .= view.verdict,
"factor" .= factor,
"baselineValue" .= baselineValue,
"candidateValue" .= candidateValue,
"design" .= view.design,
"baselineRuns" .= map (('/' :) . fst) baselines,
"candidateRuns" .= map (('/' :) . fst) candidates
]
<> maybe [] (\name -> ["factorName" .= name]) factorName
)
build now link =
ComparisonEvidence
{ title = renderScenarioId scenario <> " comparison " <> view.verdict,
description = "Recorded comparison of " <> Text.pack (show pairCount) <> " paired runs with digest-pinned data.",
generatedAt = utcText now,
runId = renderRunId view.comparisonId,
purpose = if harnessDirty then Investigation else options.purpose,
scenario,
tier = first.tier,
placement = first.placement,
outcome,
startedAt = utcText view.startedAt,
finishedAt = utcText view.finishedAt,
subject,
subjectKind,
harnessRevision,
harnessDirty,
computations = ["VC-3"],
dataLinks = [link],
comparison,
body = "The paired comparison produced the recorded verdict under [VC-3](/computations/paired-comparison.md). Its raw document is linked by digest in the frontmatter.\n"
}
pure build
factorForAxis :: Text -> Either RecordError (Text, Maybe Text)
factorForAxis axis
| axis == "cohort" = Right ("cohort", Nothing)
| Just name <- Text.stripPrefix "dim:" axis, not (Text.null name) = Right ("dimension", Just name)
| Just name <- Text.stripPrefix "knob:" axis, not (Text.null name) = Right ("knob", Just name)
| otherwise = Left (RecordError ("unsupported comparison factor: " <> axis))
armValue :: Text -> [(FilePath, EvidenceRecord)] -> Either RecordError Value
armValue axis records = case nub (mapMaybe value (map snd records)) of
[single] | length records == length (mapMaybe value (map snd records)) -> Right single
_ -> Left (RecordError ("comparison arm has missing or inconsistent " <> axis <> " values"))
where
value record
| axis == "cohort" = Just (String record.cohort)
| Just name <- Text.stripPrefix "dim:" axis = String <$> lookup name record.dimensions
| Just name <- Text.stripPrefix "knob:" axis = lookup name record.knobs
| otherwise = Nothing
data FingerprintFields = FingerprintFields
{ os :: !Text,
arch :: !Text,
cpuModel :: !(Maybe Text),
cores :: !Integer,
memoryBytes :: !(Maybe Integer),
ghc :: !Text,
postgres :: !Text,
harnessRevision :: !(Maybe Text),
harnessDirty :: !(Maybe Bool)
}
instance FromJSON FingerprintFields where
parseJSON = withObject "fingerprint" \value -> do
host <- value .: "host"
runtime <- value .: "runtime"
kenshou <- value .: "kenshou"
pg <- value .:? "postgres" :: Parser (Maybe Value)
postgres <- case pg of
Just (Object fields) -> fields .: "serverVersion"
_ -> pure "none"
FingerprintFields
<$> host .: "os"
<*> host .: "arch"
<*> host .:? "cpuModel"
<*> host .: "logicalCores"
<*> host .:? "memoryBytes"
<*> runtime .: "ghc"
<*> pure postgres
<*> kenshou .:? "revision"
<*> kenshou .:? "dirty"
buildRunRecord :: RecordInput -> RunSource -> [DataLink] -> Either RecordError EvidenceRecord
buildRunRecord input source dataLinks = do
let linkedKinds = map (.kind) dataLinks
unless (all (`elem` linkedKinds) [ManifestData, RunSpecData, RunResultData]) (Left (RecordError "run data must link its manifest, spec and result"))
case source.spec.environment.placement of
RunLocal -> when (source.cellEvidence /= Nothing || CellManifestData `elem` linkedKinds) (Left (RecordError "local run cannot link cell evidence"))
RunOnCell -> do
unless (source.cellEvidence /= Nothing && length (filter (== CellManifestData) linkedKinds) == 1) (Left (RecordError "cell run must link one outer cell manifest"))
nested <- case [link.uri | link <- dataLinks, link.kind == ManifestData] of
[uri] -> Right uri
_ -> Left (RecordError "cell run must link one nested manifest")
outer <- case [link | link <- dataLinks, link.kind == CellManifestData] of
[link] -> Right link
_ -> Left (RecordError "cell run must link one outer manifest")
let suffix = "/output/" <> renderRunId source.result.resultRunId <> "/manifest.json"
base <- maybe (Left (RecordError "nested manifest URI does not identify this cell run")) Right (Text.stripSuffix suffix nested)
unless (outer.uri == base <> "/manifest.json") (Left (RecordError "outer cell manifest URI differs from nested run prefix"))
case source.cellEvidence of
Just cell -> unless (outer.digest == cell.cellManifestFile.digest && outer.bytes == cell.cellManifestFile.bytes) (Left (RecordError "outer cell manifest link differs from verified source"))
Nothing -> Left (RecordError "cell run has no verified outer manifest")
subject <- resolveSubject input.subjectOverride input.subjects input.fallbackSubject source.result.resultScenario.layer
unless ("mori://" `Text.isPrefixOf` subject) (Left (RecordError "subject must be a canonical Mori URI"))
unless (all (Text.isPrefixOf "mori://") input.produced) (Left (RecordError "produced artifacts must use canonical Mori URIs"))
fingerprint <- decoded "fingerprint" source.result.resultFingerprint :: Either RecordError FingerprintFields
revisionText <- maybe (Left (RecordError "harness revision is unavailable")) Right fingerprint.harnessRevision
harnessRevision <- either (Left . RecordError) Right (mkRevision revisionText)
harnessDirty <- maybe (Left (RecordError "harness dirty state is unavailable")) Right fingerprint.harnessDirty
when (harnessDirty && not input.allowDirty) (Left (RecordError "dirty harness requires --allow-dirty"))
cpuModel <- maybe (Left (RecordError "CPU model is unavailable")) Right fingerprint.cpuModel
memoryBytes <- maybe (Left (RecordError "physical memory size is unavailable")) Right fingerprint.memoryBytes
when (fingerprint.cores < 0 || memoryBytes < 0) (Left (RecordError "invalid environment size"))
cellFields <- case source.cellEvidence of
Nothing -> Right []
Just _ -> do
cellFingerprint <- jsonField "cell" source.result.resultFingerprint :: Either RecordError Value
cell <- jsonField "cell" cellFingerprint :: Either RecordError Text
cellRun <- jsonField "cellRun" cellFingerprint :: Either RecordError RunId
gce <- jsonFieldOptional "gce" cellFingerprint :: Either RecordError (Maybe Value)
gceFields <- case gce of
Nothing -> Right []
Just details -> do
machineType <- jsonField "machineType" details :: Either RecordError Text
zone <- jsonField "zone" details :: Either RecordError Text
pure ["machineType" .= machineType, "zone" .= zone]
pure (["cell" .= cell, "cellRun" .= renderRunId cellRun] <> gceFields)
let effectivePurpose = if harnessDirty then Investigation else input.purpose
scenario = source.result.resultScenario
cohortIdentity = source.result.resultCohort
cohort = unCohortName cohortIdentity.identityCohort
subjectKind = maybe SubjectProject snd input.subjectOverride
environment = object (["os" .= fingerprint.os, "arch" .= fingerprint.arch, "cpuModel" .= cpuModel, "cores" .= fingerprint.cores, "memoryBytes" .= memoryBytes, "ghc" .= fingerprint.ghc, "postgres" .= fingerprint.postgres] <> cellFields)
effectiveOutcome = maybe source.result.resultOutcome (.cellEffectiveOutcome) source.cellEvidence
title = renderScenarioId scenario <> " " <> renderOutcome effectiveOutcome <> " on " <> cohort
description = "Recorded " <> renderScenarioId scenario <> " run against cohort " <> cohort <> " with digest-pinned data."
computations = "VC-1" : ["VC-2" | scenario.kind `elem` [Benchmark, Soak], hasMeasurementSummary source.result.resultSummaries]
solverPlanHash <- kernelDigest (unPlanHash cohortIdentity.identityPlanHash)
compatibilityKey <- do
comparisonKey <- jsonField "comparisonKey" source.result.resultCompatibility
kernelDigest comparisonKey
components <- concat <$> traverse componentRefs cohortIdentity.identityComponents
knownDefects <- case source.result.resultKnownDefect of
Nothing -> Right []
Just value@(Object fields) -> case KeyMap.lookup "defects" fields of
Nothing -> (: []) <$> jsonField "reference" value
Just raw -> do
defects <- decoded "defects" raw :: Either RecordError [Value]
traverse (jsonField "reference") defects
Just value -> (: []) <$> jsonField "reference" value
knobs <- case toJSON source.spec of
Object fields -> case KeyMap.lookup "knobs" fields of
Just (Object entries) -> Right $ sortOn fst [(Key.toText key, value) | (key, value) <- KeyMap.toList entries]
_ -> Left (RecordError "run spec has no resolved knobs")
_ -> Left (RecordError "run spec is not an object")
firstComputation <- either (Left . RecordError . Text.pack . show) Right (parseConceptId "computations/run-outcome")
pure
EvidenceRecord
{ title,
description,
generatedAt = utcText input.generatedAt,
runId = renderRunId source.result.resultRunId,
purpose = effectivePurpose,
scenario,
tier = source.result.resultTier,
placement = case source.spec.environment.placement of RunLocal -> "local"; RunOnCell -> "cell",
outcome = effectiveOutcome,
startedAt = utcText source.result.resultStartedAt,
finishedAt = utcText source.result.resultEndedAt,
subject,
subjectKind,
harnessRevision,
harnessDirty,
computations,
dataLinks,
cohort,
solverPlanHash,
components,
environment,
seed = source.result.resultSeed,
compatibilityKey,
knobs,
dimensions = source.spec.dimensions,
knownDefects,
produced = input.produced,
previousRun = input.previousRun,
body = "This run produced the recorded outcome under " <> renderConceptLink firstComputation "VC-1" <> ". Its raw data is linked by digest in the frontmatter.\n"
}
componentRefs :: ResolvedComponent -> Either RecordError [ComponentRef]
componentRefs component = forM component.resolvedComponentPackages $ \package -> do
(source, revision) <- case package.resolvedPackageSource of
FromHackage _ -> Right (EvidenceTypes.FromHackage, Nothing)
FromGit _ rawRevision _ -> do
parsed <- either (Left . RecordError) Right (mkRevision rawRevision)
Right (EvidenceTypes.FromGit, Just parsed)
FromBoot -> Left (RecordError ("boot package is not a runtime component: " <> package.resolvedPackageName))
FromLocalPath _ -> Left (RecordError ("local-path package cannot be recorded as a released runtime component: " <> package.resolvedPackageName))
unless ("mori://" `Text.isPrefixOf` component.resolvedComponentMoriUri) (Left (RecordError "component has no canonical Mori project URI"))
pure ComponentRef {project = component.resolvedComponentMoriUri, package = package.resolvedPackageName, version = package.resolvedPackageVersion, source, revision}
kernelDigest :: Text -> Either RecordError Sha256
kernelDigest value = do
raw <- maybe (Left (RecordError "kernel digest is missing its sha256: prefix")) Right (Text.stripPrefix "sha256:" value)
either (Left . RecordError) Right (mkSha256 raw)
jsonField :: (FromJSON value) => Text -> Value -> Either RecordError value
jsonField name value = case value of
Object fields -> case KeyMap.lookup (Key.fromText name) fields of
Nothing -> Left (RecordError ("missing JSON field " <> name))
Just raw -> decoded name raw
_ -> Left (RecordError "expected a JSON object")
jsonFieldOptional :: (FromJSON value) => Text -> Value -> Either RecordError (Maybe value)
jsonFieldOptional name value = case value of
Object fields -> traverse (decoded name) (KeyMap.lookup (Key.fromText name) fields)
_ -> Left (RecordError "expected a JSON object")
decoded :: (FromJSON value) => Text -> Value -> Either RecordError value
decoded label value = case fromJSON value of
Error message -> Left (RecordError (label <> ": " <> Text.pack message))
Success parsed -> Right parsed
utcText :: UTCTime -> Text
utcText = Text.pack . formatTime defaultTimeLocale "%Y-%m-%dT%H:%M:%S%QZ"
-- | The subject of a run record: the override, else the subject the registered
-- bundle of the run's layer declares, else the configured fallback.
resolveSubject :: Maybe (Text, SubjectKind) -> Map LayerId Text -> Maybe Text -> LayerId -> Either RecordError Text
resolveSubject override subjects fallback layer = case (fst <$> override) <|> Map.lookup layer subjects <|> fallback of
Just subject -> Right subject
Nothing -> Left (RecordError ("no evidence subject for layer " <> renderLayer layer <> "; pass --subject or set evidence.default-subject"))
hasMeasurementSummary :: Maybe Value -> Bool
hasMeasurementSummary (Just (Object summaries)) = case KeyMap.lookup "measurements" summaries of
Just (Object measurements) -> case KeyMap.lookup "measurements" measurements of
Just (Object _) -> True
_ -> False
_ -> False
hasMeasurementSummary _ = False