packages feed

kenshou-core-0.1.0.0: src/Kenshou/Perf/Calibration.hs

{-# LANGUAGE FieldSelectors #-}

module Kenshou.Perf.Calibration
  ( CalibrationReference (..),
    CalibrationDefinition (..),
    CalibrationSlot (..),
    CalibrationAcceptance (..),
    CalibrationReceipt (..),
    VerifiedCalibrationTrial (..),
    validateCalibration,
    validateCalibrationDefinition,
    toolchainDigest,
  )
where

import Control.Monad (unless)
import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.List (nub, sort)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime)
import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)
import Kenshou.Core.Id (RunId)
import Kenshou.Perf.Prepared (BuildInventory (..))
import Kenshou.Perf.Request (canonicalProject, immutableRevision, strictObject)

data CalibrationReference = CalibrationReference {definitionUri :: Text, definitionSha256 :: Text, acceptanceUri :: Text, acceptanceSha256 :: Text}
  deriving stock (Eq, Show)

data CalibrationDefinition = CalibrationDefinition
  { batchId :: RunId,
    project :: Text,
    profile :: Text,
    machineProfile :: Text,
    healthPolicySha256 :: Text,
    workloadSha256 :: Text,
    toolchainSha256 :: Text,
    harnessSemanticsSha256 :: Text,
    baselineRevision :: Text,
    slowdownRevision :: Text,
    injectionMechanism :: Text,
    targetMetric :: Text,
    targetRelativeEffect :: Double,
    targetAbsoluteEffect :: Double,
    seeds :: [Integer],
    infrastructureRetryCap :: Int,
    expiresAt :: UTCTime
  }
  deriving stock (Eq, Show)

data CalibrationReceipt = CalibrationReceipt {uri :: Text, sha256 :: Text}
  deriving stock (Eq, Show)

data CalibrationSlot = CalibrationSlot {seed :: Integer, receipts :: [CalibrationReceipt]}
  deriving stock (Eq, Show)

data CalibrationAcceptance = CalibrationAcceptance {batchId :: RunId, aa :: [CalibrationSlot], slowdown :: [CalibrationSlot]}
  deriving stock (Eq, Show)

-- These observations are constructed only after the ordinary full report replay.
-- They are not accepted as a calibration input document or catalog assertion.
data VerifiedCalibrationTrial = VerifiedCalibrationTrial
  { sessionId :: RunId,
    seed :: Integer,
    control :: Text,
    project :: Text,
    baselineRevision :: Text,
    candidateRevision :: Text,
    profile :: Text,
    trialMachineProfile :: Text,
    trialHealthPolicySha256 :: Text,
    workloadSha256 :: Text,
    trialToolchainSha256 :: Text,
    trialHarnessSemanticsSha256 :: Text,
    verdict :: Text,
    infrastructureState :: Text,
    runtimeBlockers :: [Text],
    createdAt :: UTCTime,
    elapsedSeconds :: Double
  }
  deriving stock (Eq, Show)

-- Commit identity belongs to preparation, not this toolchain identity: attaching
-- a calibration link in a new catalog commit must not invalidate itself.
toolchainDigest :: BuildInventory -> Text
toolchainDigest inventory = sha256Hex (canonicalEncode (object ["compiler" .= inventory.compiler, "platform" .= inventory.platform, "toolchain" .= stable]))
  where
    stable = case inventory.toolchain of Object value -> Object (KeyMap.delete "harnessRevision" value); value -> value

validateCalibration :: UTCTime -> UTCTime -> CalibrationDefinition -> CalibrationAcceptance -> [[VerifiedCalibrationTrial]] -> [[VerifiedCalibrationTrial]] -> Either Text Double
validateCalibration now declaredAt definition acceptance aaTrials slowdownTrials = do
  validateCalibrationDefinition now declaredAt definition
  require (acceptance.batchId == definition.batchId) "calibration batch identity differs"
  mapM_ (\slots -> require (map (.seed) slots == definition.seeds && all (not . null . (.receipts)) slots) "acceptance omitted or reordered a declared slot") [acceptance.aa, acceptance.slowdown]
  let references = concatMap (.receipts) (acceptance.aa <> acceptance.slowdown)
      trials = concat (aaTrials <> slowdownTrials)
  require (nub (map (.uri) references) == map (.uri) references && nub (map (.sessionId) trials) == map (.sessionId) trials) "duplicate calibration evidence or session"
  aaAccepted <- check "aa" acceptance.aa aaTrials
  slowdownAccepted <- check "none" acceptance.slowdown slowdownTrials
  require (length (filter ((== "failed") . (.infrastructureState)) trials) <= definition.infrastructureRetryCap) "calibration infrastructure retry cap exceeded"
  require (counts "pass" aaAccepted >= 19 && counts "regression" aaAccepted == 0 && counts "inconclusive" aaAccepted <= 1) "A/A statistical acceptance failed"
  require (counts "regression" slowdownAccepted >= 19 && counts "pass" slowdownAccepted == 0 && counts "inconclusive" slowdownAccepted <= 1) "slowdown sensitivity acceptance failed"
  let elapsed = sort (map (sum . map (.elapsedSeconds)) aaTrials)
      p95 = elapsed !! 18 -- nearest rank ceil(0.95 * 20), including every retained retry
  require (p95 <= 900) "A/A warm turnaround p95 exceeds fifteen minutes"
  pure p95
  where
    require condition message = unless condition (Left message)
    counts outcome = length . filter ((== outcome) . (.verdict))
    check control slots groups = do
      require (length groups == 20 && length groups == length slots) "missing calibration trial group"
      traverse (checkSlot control) (zip slots groups)
    checkSlot control (slot, trials) = do
      require (not (null trials) && length trials == length slot.receipts) "calibration receipt/trial count differs"
      mapM_ (checkTrial slot.seed control) trials
      require (all ((== "failed") . (.infrastructureState)) (init trials) && (.infrastructureState) (last trials) == "healthy") "statistical outcomes were discarded or slot never completed"
      pure (last trials)
    checkTrial seed control trial = do
      require (trial.createdAt >= declaredAt && trial.createdAt <= definition.expiresAt) "trial is outside the frozen batch lifetime"
      require ((trial.seed, trial.control, trial.project, trial.profile, trial.trialMachineProfile, trial.trialHealthPolicySha256, trial.workloadSha256, trial.trialToolchainSha256, trial.trialHarnessSemanticsSha256) == (seed, control, definition.project, definition.profile, definition.machineProfile, definition.healthPolicySha256, definition.workloadSha256, definition.toolchainSha256, definition.harnessSemanticsSha256)) "calibration workload, seed, toolchain or harness drift"
      require (trial.baselineRevision == definition.baselineRevision && trial.candidateRevision == if control == "aa" then definition.baselineRevision else definition.slowdownRevision) "calibration source revision differs"
      require (null trial.runtimeBlockers) "runtime blocker invalidates calibration"
      require (trial.infrastructureState `elem` ["healthy", "failed"] && trial.verdict `elem` ["pass", "regression", "inconclusive", "not-run"]) "unsupported calibration outcome"
      require (trial.infrastructureState /= "healthy" || trial.verdict /= "not-run") "unexecuted slot is not a statistical outcome"
      require (not (isNaN trial.elapsedSeconds || isInfinite trial.elapsedSeconds) && trial.elapsedSeconds >= 0) "invalid retained calibration timing"

finitePositive :: Double -> Bool
finitePositive value = not (isNaN value || isInfinite value) && value > 0

validateCalibrationDefinition :: UTCTime -> UTCTime -> CalibrationDefinition -> Either Text ()
validateCalibrationDefinition now declaredAt definition = do
  require (canonicalProject definition.project && all immutableRevision [definition.baselineRevision, definition.slowdownRevision] && definition.baselineRevision /= definition.slowdownRevision) "invalid calibration source identity"
  require (length definition.seeds == 20 && nub definition.seeds == definition.seeds && all (\seed -> seed >= 0 && seed <= 9007199254740991) definition.seeds) "calibration needs twenty unique declared seeds"
  require (definition.infrastructureRetryCap >= 0 && definition.infrastructureRetryCap <= 20) "invalid calibration infrastructure retry cap"
  require (declaredAt <= now && declaredAt < definition.expiresAt && now <= definition.expiresAt) "calibration expired or predates its declaration"
  require (not (Text.null definition.injectionMechanism) && not (Text.null definition.targetMetric) && finitePositive definition.targetRelativeEffect && finitePositive definition.targetAbsoluteEffect) "missing or invalid injected effect"
  require (not (Text.null definition.profile) && not (Text.null definition.machineProfile) && not ("pending" `Text.isInfixOf` definition.machineProfile) && all digest [definition.healthPolicySha256, definition.workloadSha256, definition.toolchainSha256, definition.harnessSemanticsSha256]) "calibration needs immutable workload, toolchain and harness digests"
  where
    require condition message = unless condition (Left message)
    digest value = Text.length value == 71 && "sha256:" `Text.isPrefixOf` value && Text.all (`elem` (['0' .. '9'] <> ['a' .. 'f'])) (Text.drop 7 value)

instance ToJSON CalibrationReference where
  toJSON value = object ["definitionUri" .= value.definitionUri, "definitionSha256" .= value.definitionSha256, "acceptanceUri" .= value.acceptanceUri, "acceptanceSha256" .= value.acceptanceSha256]

instance FromJSON CalibrationReference where
  parseJSON = withObject "CalibrationReference" \v -> do
    strictObject ["definitionUri", "definitionSha256", "acceptanceUri", "acceptanceSha256"] v
    CalibrationReference <$> v .: "definitionUri" <*> v .: "definitionSha256" <*> v .: "acceptanceUri" <*> v .: "acceptanceSha256"

instance ToJSON CalibrationDefinition where
  toJSON value = object ["schema" .= ("kenshou.perf-calibration-definition/v1" :: Text), "batchId" .= value.batchId, "project" .= value.project, "profile" .= value.profile, "machineProfile" .= value.machineProfile, "healthPolicySha256" .= value.healthPolicySha256, "workloadSha256" .= value.workloadSha256, "toolchainSha256" .= value.toolchainSha256, "harnessSemanticsSha256" .= value.harnessSemanticsSha256, "baselineRevision" .= value.baselineRevision, "slowdownRevision" .= value.slowdownRevision, "injectionMechanism" .= value.injectionMechanism, "targetMetric" .= value.targetMetric, "targetRelativeEffect" .= value.targetRelativeEffect, "targetAbsoluteEffect" .= value.targetAbsoluteEffect, "seeds" .= value.seeds, "infrastructureRetryCap" .= value.infrastructureRetryCap, "expiresAt" .= value.expiresAt]

instance FromJSON CalibrationDefinition where
  parseJSON = withObject "CalibrationDefinition" \v -> do
    strictObject ["schema", "batchId", "project", "profile", "machineProfile", "healthPolicySha256", "workloadSha256", "toolchainSha256", "harnessSemanticsSha256", "baselineRevision", "slowdownRevision", "injectionMechanism", "targetMetric", "targetRelativeEffect", "targetAbsoluteEffect", "seeds", "infrastructureRetryCap", "expiresAt"] v
    schema <- v .: "schema"
    unless (schema == ("kenshou.perf-calibration-definition/v1" :: Text)) (fail "unsupported calibration definition")
    CalibrationDefinition <$> v .: "batchId" <*> v .: "project" <*> v .: "profile" <*> v .: "machineProfile" <*> v .: "healthPolicySha256" <*> v .: "workloadSha256" <*> v .: "toolchainSha256" <*> v .: "harnessSemanticsSha256" <*> v .: "baselineRevision" <*> v .: "slowdownRevision" <*> v .: "injectionMechanism" <*> v .: "targetMetric" <*> v .: "targetRelativeEffect" <*> v .: "targetAbsoluteEffect" <*> v .: "seeds" <*> v .: "infrastructureRetryCap" <*> v .: "expiresAt"

instance ToJSON CalibrationReceipt where
  toJSON value = object ["uri" .= value.uri, "sha256" .= value.sha256]

instance FromJSON CalibrationReceipt where
  parseJSON = withObject "CalibrationReceipt" \v -> do
    strictObject ["uri", "sha256"] v
    CalibrationReceipt <$> v .: "uri" <*> v .: "sha256"

instance ToJSON CalibrationSlot where
  toJSON value = object ["seed" .= value.seed, "receipts" .= value.receipts]

instance FromJSON CalibrationSlot where
  parseJSON = withObject "CalibrationSlot" \v -> do
    strictObject ["seed", "receipts"] v
    CalibrationSlot <$> v .: "seed" <*> v .: "receipts"

instance ToJSON CalibrationAcceptance where
  toJSON value = object ["schema" .= ("kenshou.perf-calibration-acceptance/v1" :: Text), "batchId" .= value.batchId, "aa" .= value.aa, "slowdown" .= value.slowdown]

instance FromJSON CalibrationAcceptance where
  parseJSON = withObject "CalibrationAcceptance" \v -> do
    strictObject ["schema", "batchId", "aa", "slowdown"] v
    schema <- v .: "schema"
    unless (schema == ("kenshou.perf-calibration-acceptance/v1" :: Text)) (fail "unsupported calibration acceptance")
    CalibrationAcceptance <$> v .: "batchId" <*> v .: "aa" <*> v .: "slowdown"