kenshou-core-0.1.0.0: src/Kenshou/Perf/Plan.hs
{-# LANGUAGE FieldSelectors #-}
module Kenshou.Perf.Plan
( RequiredMetric (..),
PerfProfile (..),
ProfileCatalog (..),
PlannedProfile (..),
PerfPlan (..),
validateProfile,
workloadDigest,
planPerformance,
conservativeChanges,
)
where
import Control.Monad (unless)
import Data.Aeson
import Data.List (nub, sort, sortOn)
import Data.List.NonEmpty qualified as NonEmpty
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)
import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), renderDimensions, resolveDimensions)
import Kenshou.Core.Id (Kind (..), ScenarioId (..), renderScenarioId)
import Kenshou.Core.Knob
import Kenshou.Perf.Contracts (PerfContracts (..))
import Kenshou.Perf.Request
import Kenshou.Plan.Catalog qualified as Catalog
import Kenshou.Plan.Change
import Kenshou.Plan.Components
data RequiredMetric = RequiredMetric {name :: Text, unit :: Text, minimumSteadySamples :: Int}
deriving stock (Eq, Show)
data PerfProfile = PerfProfile
{ name :: Text,
revision :: Int,
scenario :: Text,
scenarioRevision :: Int,
knobs :: Map Text Value,
dimensions :: Map Text Text,
workloadPurpose :: Text,
warmupSeconds :: Int,
steadyCount :: Int,
cooldownSeconds :: Int,
timeoutSeconds :: Int,
cachePolicy :: Text,
policy :: FilePath,
requiredMetrics :: [RequiredMetric],
capabilities :: [Text],
machineProfile :: Text,
trialSeconds :: Int,
resetSeconds :: Int,
sealSeconds :: Int,
minimumPairs :: Int,
replacementPairs :: Int,
calibration :: Maybe Value
}
deriving stock (Eq, Show)
newtype ProfileCatalog = ProfileCatalog {profiles :: [PerfProfile]}
deriving stock (Eq, Show)
data PlannedProfile = PlannedProfile
{ profile :: PerfProfile,
workloadSha256 :: Text,
effectiveKnobs :: ResolvedKnobs,
reasons :: [Value],
estimateSeconds :: Integer
}
deriving stock (Eq, Show)
data PerfPlan = PerfPlan
{ request :: PerfRequest,
requestSha256 :: Text,
graphSha256 :: Text,
catalogSha256 :: Text,
profileCatalogSha256 :: Text,
selected :: [PlannedProfile],
excluded :: [(Text, Text)],
affectedScenarios :: [Text],
uncoveredScenarios :: [Text],
unmappedPaths :: [FilePath],
documentationProof :: [FilePath],
coverage :: Text,
changeCoverage :: Text,
estimateSeconds :: Integer,
gatingEligible :: Bool
}
deriving stock (Eq, Show)
-- Calibration references are deliberately excluded from the workload identity.
-- Attaching evidence to an unchanged workload must not create a digest cycle.
workloadDigest :: PerfProfile -> Text -> Text
workloadDigest profile policyDigest = sha256Hex (canonicalEncode (object ["workload" .= profile {calibration = Nothing}, "policySha256" .= policyDigest]))
validateProfile :: PerfContracts -> [Catalog.ScenarioInfo] -> PerfProfile -> Either Text ResolvedKnobs
validateProfile contracts catalog profile = do
scenario <- case filter ((== profile.scenario) . renderScenarioId . (.id)) catalog of
[value] -> Right value
_ -> Left ("profile names an unknown or duplicate scenario: " <> profile.scenario)
require (scenario.id.kind == Benchmark && scenario.revision == profile.scenarioRevision) "profile scenario revision/kind mismatch"
require (profile.revision > 0 && not (Text.null profile.name) && Text.all (`elem` (['a' .. 'z'] <> ['0' .. '9'] <> ['-'])) profile.name) "invalid profile identity"
require (profile.workloadPurpose == "fixed-load-latency") "unsupported workload purpose; paced throughput is not capacity"
-- Only reviewed revisions implement count-bound steady work, not timed phases.
require ((profile.scenario, profile.scenarioRevision) `elem` contracts.reviewedPhaseContracts) "scenario phase contract has not been reviewed"
require (profile.warmupSeconds == 0 && profile.cooldownSeconds == 0 && profile.timeoutSeconds == 180) "scenario revision does not honor requested phases/timeout"
require (profile.cachePolicy == "cold-reset") "unsupported cache policy"
require (profile.minimumPairs >= 5 && profile.replacementPairs >= 0 && profile.replacementPairs <= 2) "invalid fixed pair/replacement allowance"
require (profile.trialSeconds > 0 && profile.resetSeconds >= 0 && profile.sealSeconds >= 0) "cost estimate needs positive trial time and nonnegative overheads"
require (not (null profile.requiredMetrics) && nub (map (.name) profile.requiredMetrics) == map (.name) profile.requiredMetrics) "required metrics must be nonempty and unique"
require (all (\metric -> not (Text.null metric.name) && not (Text.null metric.unit) && metric.minimumSteadySamples > 0) profile.requiredMetrics) "invalid required metric"
require (all ((<= profile.steadyCount) . (.minimumSteadySamples)) profile.requiredMetrics) "required sample count exceeds planned work"
require (not (Text.null profile.machineProfile) && not (null profile.capabilities)) "profile needs a machine profile and cell capabilities"
assignments <- traverse (\(key, value) -> (,RawJson value) <$> mkKnobName key) (Map.toAscList profile.knobs)
resolved <- either (Left . Text.pack . show) Right (resolveKnobs scenario.knobs assignments)
dimensionValues <- either (Left . Text.pack . show) Right (resolveDimensions scenario.dimensions (Map.toAscList profile.dimensions))
require (dimensionValues.pgDurability /= Just PgFsyncOff) "performance profiles require durable PostgreSQL"
rawKnob <- mkKnobName "measure.raw-samples"
case contracts.steadyCountKnob of
Just countKnob -> require (Map.lookup countKnob (resolvedKnobsMap resolved) == Just (VInt (fromIntegral profile.steadyCount))) "profile steady count disagrees with effective workload"
Nothing -> pure ()
require (Map.lookup rawKnob (resolvedKnobsMap resolved) == Just (VText "full")) "performance profiles require full raw samples"
rateKnob <- mkKnobName "bench.rate"
let offeredRate = case Map.lookup rateKnob (resolvedKnobsMap resolved) of Just (VInt rate) -> rate; _ -> 0
require (offeredRate > 0 && fromIntegral profile.trialSeconds * offeredRate >= fromIntegral profile.steadyCount) "trial estimate is shorter than the declared offered workload"
pure resolved
where
require condition message = unless condition (Left (profile.name <> ": " <> message))
-- Paths absent from the graph are not a documentation proof. Only explicit
-- documentation names outside all source roots can be excluded safely.
conservativeChanges :: ComponentGraph -> Text -> [FilePath] -> [Change] -> ([Change], [FilePath], [FilePath])
conservativeChanges graph repository paths mapped = (changes, unknown, documentation)
where
components = filter ((== repository) . (.repository)) graph.components
inRoot path = any (\component -> any (\root -> Text.pack root `Text.isPrefixOf` Text.pack path) component.sourceRoots) components
isDocumentation path = not (inRoot path) && ("docs/" `Text.isPrefixOf` Text.pack path || path `elem` ["README.md", "CHANGELOG.md", "LICENSE", "LICENSE.md"])
documentation = filter isDocumentation paths
unknown = filter (\path -> not (inRoot path) && not (isDocumentation path)) paths
changes =
if null components || not (null unknown)
then mapped <> [Change (ComponentRef (ComponentId "all") Nothing) Everything "unmapped upstream input; conservative selection"]
else mapped
-- The admission map is populated only by an evidence verifier, never by a
-- catalog's self-asserted calibrated flag. It maps workload digest to verified
-- calibration evidence digest. A catalog link cannot populate this map without
-- full receipt retrieval, raw report replay and acceptance-batch verification.
planPerformance :: PerfContracts -> PerfRequest -> ComponentGraph -> [Catalog.ScenarioInfo] -> ProfileCatalog -> [Change] -> [FilePath] -> [FilePath] -> Map Text Text -> Map Text Text -> Either Text PerfPlan
planPerformance contracts request graph catalog profiles changes unknown documentation policyDigests admitted = do
validateRequest request
unless (nub profileNames == profileNames) (Left "duplicate performance profile")
unless (all (`elem` profileNames) request.profiles) (Left "unknown requested performance profile")
checked <- traverse check (sortOn (.name) profiles.profiles)
let affected = selectScenarios graph catalog changes
affectedNames = sort (map (renderScenarioId . (.id) . (.scenario)) affected)
candidates = filter (wanted affectedNames) checked
(selected, excluded, _) = foldl admit ([], [], 0) candidates
covered = map (.scenario) (map (.profile) selected)
telemetryMissing = [renderScenarioId item.scenario.id | item <- affected, item.minPolicy == Just "telemetry-corners", not (any (coversTelemetry item) selected)]
uncovered = nub (filter (`notElem` covered) affectedNames <> telemetryMissing)
notApplicable = request.control == "none" && null changes && null unknown && not (null documentation) && null request.profiles
scopeComplete = if request.coverageMode == "selected-profiles" then sort (map (.name) (map (.profile) selected)) == sort request.profiles else null uncovered && not (null selected)
scope = if notApplicable then "not-applicable" else if scopeComplete && null excluded then "complete" else "incomplete"
pure
PerfPlan
{ request,
requestSha256 = requestDigest request,
graphSha256 = graphDigest graph,
catalogSha256 = sha256Hex (canonicalEncode (toJSON catalog)),
profileCatalogSha256 = sha256Hex (canonicalEncode (toJSON profiles)),
selected,
excluded,
affectedScenarios = affectedNames,
uncoveredScenarios = uncovered,
unmappedPaths = unknown,
documentationProof = documentation,
coverage = scope,
changeCoverage = if notApplicable then "not-applicable" else if null uncovered && not (null affectedNames) then "complete" else "incomplete",
estimateSeconds = sum (map (.estimateSeconds) selected),
gatingEligible = request.purpose == "gating" && scope == "complete" && all (\item -> Map.member item.workloadSha256 admitted) selected
}
where
profileNames = map (.name) profiles.profiles
check profile = do
effective <- validateProfile contracts catalog profile
scenario <- case filter ((== profile.scenario) . renderScenarioId . (.id)) catalog of
[value] -> Right value
_ -> Left "profile scenario is missing"
effectiveDimensions <- either (Left . Text.pack . show) Right (resolveDimensions scenario.dimensions (Map.toAscList profile.dimensions))
let normalized =
(profile :: PerfProfile)
{ knobs = Map.fromList [(renderKnobName name, toJSON value) | (name, value) <- Map.toAscList (resolvedKnobsMap effective)],
dimensions = Map.fromList (renderDimensions effectiveDimensions)
}
policyDigest <- maybe (Left ("missing policy digest: " <> Text.pack profile.policy)) Right (Map.lookup (Text.pack profile.policy) policyDigests)
pure (normalized, (effective, workloadDigest normalized policyDigest))
wanted affected (profile, _) = (null request.profiles || profile.name `elem` request.profiles) && (request.control == "aa" || request.coverageMode == "selected-profiles" || profile.scenario `elem` affected)
admit (selected, excluded, spent) (profile, (effective, digest))
| request.purpose /= "calibration" && Map.notMember digest admitted = (selected, excluded <> [(profile.name, "uncalibrated")], spent)
| spent + cost > fromIntegral request.budgetMinutes * 60 = (selected, excluded <> [(profile.name, "over-budget")], spent)
| otherwise = (selected <> [PlannedProfile profile digest effective reasons cost], excluded, spent + cost)
where
cost = 2 * (fromIntegral profile.minimumPairs + fromIntegral profile.replacementPairs) * (fromIntegral profile.trialSeconds + fromIntegral profile.resetSeconds + fromIntegral profile.sealSeconds)
reasons =
[toJSON reason | item <- selectScenarios graph catalog changes, renderScenarioId item.scenario.id == profile.scenario, reason <- NonEmpty.toList item.reasons]
<> [ object
[ "component" .= ("request/profile/" <> profile.name),
"source" .= ("named" :: Text),
"via" .= [if request.control == "aa" then "aa-control" else "selected-profiles" :: Text],
"selector" .= profile.scenario,
"distance" .= (0 :: Int)
]
| profile.name `elem` request.profiles
]
coversTelemetry item selected =
selected.profile.scenario == renderScenarioId item.scenario.id
&& Map.lookup "telemetry.tracing" selected.profile.dimensions `elem` [Just "sdk-inmemory", Just "sdk-otlp"]
&& Map.lookup "telemetry.metrics" selected.profile.dimensions `elem` [Just "collect", Just "serve", Just "serve-scraped"]
instance ToJSON RequiredMetric where
toJSON metric = object ["name" .= metric.name, "unit" .= metric.unit, "minimumSteadySamples" .= metric.minimumSteadySamples]
instance FromJSON RequiredMetric where
parseJSON = withObject "RequiredMetric" \value -> do
strictObject ["name", "unit", "minimumSteadySamples"] value
RequiredMetric <$> value .: "name" <*> value .: "unit" <*> value .: "minimumSteadySamples"
instance ToJSON PerfProfile where
toJSON profile =
object
[ "name" .= profile.name,
"revision" .= profile.revision,
"scenario" .= profile.scenario,
"scenarioRevision" .= profile.scenarioRevision,
"knobs" .= profile.knobs,
"dimensions" .= profile.dimensions,
"workloadPurpose" .= profile.workloadPurpose,
"warmupSeconds" .= profile.warmupSeconds,
"steadyCount" .= profile.steadyCount,
"cooldownSeconds" .= profile.cooldownSeconds,
"timeoutSeconds" .= profile.timeoutSeconds,
"cachePolicy" .= profile.cachePolicy,
"policy" .= profile.policy,
"requiredMetrics" .= profile.requiredMetrics,
"capabilities" .= profile.capabilities,
"machineProfile" .= profile.machineProfile,
"trialSeconds" .= profile.trialSeconds,
"resetSeconds" .= profile.resetSeconds,
"sealSeconds" .= profile.sealSeconds,
"minimumPairs" .= profile.minimumPairs,
"replacementPairs" .= profile.replacementPairs,
"calibration" .= profile.calibration
]
instance FromJSON PerfProfile where
parseJSON = withObject "PerfProfile" \value -> do
strictObject ["name", "revision", "scenario", "scenarioRevision", "knobs", "dimensions", "workloadPurpose", "warmupSeconds", "steadyCount", "cooldownSeconds", "timeoutSeconds", "cachePolicy", "policy", "requiredMetrics", "capabilities", "machineProfile", "trialSeconds", "resetSeconds", "sealSeconds", "minimumPairs", "replacementPairs", "calibration"] value
PerfProfile
<$> value .: "name"
<*> value .: "revision"
<*> value .: "scenario"
<*> value .: "scenarioRevision"
<*> value .: "knobs"
<*> value .: "dimensions"
<*> value .: "workloadPurpose"
<*> value .: "warmupSeconds"
<*> value .: "steadyCount"
<*> value .: "cooldownSeconds"
<*> value .: "timeoutSeconds"
<*> value .: "cachePolicy"
<*> value .: "policy"
<*> value .: "requiredMetrics"
<*> value .: "capabilities"
<*> value .: "machineProfile"
<*> value .: "trialSeconds"
<*> value .: "resetSeconds"
<*> value .: "sealSeconds"
<*> value .: "minimumPairs"
<*> value .: "replacementPairs"
<*> value .:? "calibration"
instance ToJSON ProfileCatalog where
toJSON catalog = object ["schema" .= ("kenshou.perf-catalog/v1" :: Text), "profiles" .= catalog.profiles]
instance FromJSON ProfileCatalog where
parseJSON = withObject "ProfileCatalog" \value -> do
strictObject ["schema", "profiles"] value
schema <- value .: "schema"
unless (schema == ("kenshou.perf-catalog/v1" :: Text)) (fail "unsupported performance catalog schema")
ProfileCatalog <$> value .: "profiles"
instance ToJSON PlannedProfile where
toJSON planned = object ["profile" .= planned.profile, "workloadSha256" .= planned.workloadSha256, "effectiveKnobs" .= planned.effectiveKnobs, "reasons" .= planned.reasons, "estimateSeconds" .= planned.estimateSeconds]
instance ToJSON PerfPlan where
toJSON plan =
object
[ "schema" .= ("kenshou.perf-plan/v1" :: Text),
"request" .= plan.request,
"requestSha256" .= plan.requestSha256,
"graphSha256" .= plan.graphSha256,
"catalogSha256" .= plan.catalogSha256,
"profileCatalogSha256" .= plan.profileCatalogSha256,
"selected" .= plan.selected,
"excluded" .= [object ["profile" .= name, "reason" .= reason] | (name, reason) <- plan.excluded],
"affectedScenarios" .= plan.affectedScenarios,
"uncoveredScenarios" .= plan.uncoveredScenarios,
"unmappedPaths" .= plan.unmappedPaths,
"documentationProof" .= plan.documentationProof,
"coverage" .= plan.coverage,
"changeCoverage" .= plan.changeCoverage,
"estimateSeconds" .= plan.estimateSeconds,
"gatingEligible" .= plan.gatingEligible
]
instance FromJSON PlannedProfile where
parseJSON = withObject "PlannedProfile" \value -> do
strictObject ["profile", "workloadSha256", "effectiveKnobs", "reasons", "estimateSeconds"] value
planned <- PlannedProfile <$> value .: "profile" <*> value .: "workloadSha256" <*> value .: "effectiveKnobs" <*> value .: "reasons" <*> value .: "estimateSeconds"
unless (planned.estimateSeconds > 0) (fail "invalid planned configuration estimate")
pure planned
instance FromJSON PerfPlan where
parseJSON = withObject "PerfPlan" \value -> do
strictObject ["schema", "request", "requestSha256", "graphSha256", "catalogSha256", "profileCatalogSha256", "selected", "excluded", "affectedScenarios", "uncoveredScenarios", "unmappedPaths", "documentationProof", "coverage", "changeCoverage", "estimateSeconds", "gatingEligible"] value
schema <- value .: "schema"
unless (schema == ("kenshou.perf-plan/v1" :: Text)) (fail "unsupported performance plan schema")
omissions <-
value .: "excluded"
>>= traverse
( withObject "ExcludedProfile" \item -> do
strictObject ["profile", "reason"] item
(,) <$> item .: "profile" <*> item .: "reason"
)
plan <- PerfPlan <$> value .: "request" <*> value .: "requestSha256" <*> value .: "graphSha256" <*> value .: "catalogSha256" <*> value .: "profileCatalogSha256" <*> value .: "selected" <*> pure omissions <*> value .: "affectedScenarios" <*> value .: "uncoveredScenarios" <*> value .: "unmappedPaths" <*> value .: "documentationProof" <*> value .: "coverage" <*> value .: "changeCoverage" <*> value .: "estimateSeconds" <*> value .: "gatingEligible"
unless (plan.requestSha256 == requestDigest plan.request) (fail "performance plan request digest mismatch")
unless (plan.estimateSeconds == sum (map (.estimateSeconds) plan.selected) && plan.estimateSeconds <= fromIntegral plan.request.budgetMinutes * 60) (fail "performance plan estimate mismatch or budget exceeded")
unless (all (`elem` ["complete", "incomplete", "not-applicable"]) [plan.coverage, plan.changeCoverage]) (fail "unknown performance coverage")
unless (nub (map (.name) (map (.profile) plan.selected)) == map (.name) (map (.profile) plan.selected)) (fail "duplicate selected performance profile")
unless (not plan.gatingEligible || plan.request.purpose == "gating" && plan.coverage == "complete" && not (null plan.selected)) (fail "invalid gating eligibility")
pure plan