packages feed

kenshou-measure-0.1.0.0: src/Kenshou/Measure/Compare.hs

module Kenshou.Measure.Compare
  ( Verdict (..),
    MetricStatus (..),
    MetricComparison (..),
    Comparison (..),
    CompareError (..),
    compareRuns,
    compareMetricPairs,
    decideVerdict,
    verdictExitCode,
  )
where

import Data.Aeson
import Data.Aeson.Key (fromText)
import Data.Aeson.KeyMap qualified as KeyMap
import Data.List (find, nub)
import Data.List.NonEmpty (NonEmpty)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (catMaybes, fromMaybe, mapMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, getCurrentTime)
import Data.Time.Format (defaultTimeLocale, parseTimeM)
import Kenshou.Core.Fingerprint (HostFingerprint (..), collectHostFingerprint)
import Kenshou.Core.Id (newRunId, renderRunId)
import Kenshou.Measure.Compare.Compatibility
import Kenshou.Measure.Compare.Ordering
import Kenshou.Measure.Compare.Policy
import Kenshou.Measure.Health
import Kenshou.Measure.Metrics
import Kenshou.Measure.Stats
import Kenshou.Measure.Summary

data Verdict = VerdictPass | VerdictRegression | VerdictInconclusive | VerdictInfrastructureFailure deriving stock (Eq, Ord, Show)

data MetricStatus = MetricPass | MetricRegression | MetricInconclusive deriving stock (Eq, Ord, Show)

data MetricComparison = MetricComparison
  { metric :: Text,
    unit :: Text,
    pairs :: [(Double, Double)],
    ratio :: Interval,
    delta :: Interval,
    relativeLimit :: Double,
    absoluteFloor :: Double,
    baselineCv :: Double,
    status :: MetricStatus,
    message :: Text
  }
  deriving stock (Eq, Show)

data Comparison = Comparison
  { comparisonId :: Text,
    startedAt :: UTCTime,
    finishedAt :: UTCTime,
    harnessRevision :: Maybe Text,
    harnessDirty :: Maybe Bool,
    design :: Text,
    baselineRuns :: [Text],
    candidateRuns :: [Text],
    policy :: Policy,
    variedFactors :: [VaryingAxis],
    pairCount :: Int,
    metrics :: Map Text MetricComparison,
    reasons :: [Text],
    verdict :: Verdict
  }
  deriving stock (Eq, Show)

newtype CompareError = CompareError Text deriving stock (Eq, Show)

instance ToJSON Verdict where toJSON = String . verdictText

instance ToJSON MetricStatus where toJSON = String . statusText

instance ToJSON MetricComparison where
  toJSON value =
    object
      [ "metric" .= value.metric,
        "unit" .= value.unit,
        "pairs" .= [object ["baseline" .= baseline, "candidate" .= candidate] | (baseline, candidate) <- value.pairs],
        "ratio" .= value.ratio,
        "delta" .= value.delta,
        "relativeLimit" .= value.relativeLimit,
        "absoluteFloor" .= value.absoluteFloor,
        "baselineCv" .= value.baselineCv,
        "status" .= value.status,
        "message" .= value.message
      ]

instance ToJSON Comparison where
  toJSON value =
    object
      [ "schema" .= ("kenshou.comparison/v1" :: Text),
        "comparisonId" .= value.comparisonId,
        "startedAt" .= value.startedAt,
        "finishedAt" .= value.finishedAt,
        "harnessRevision" .= value.harnessRevision,
        "harnessDirty" .= value.harnessDirty,
        "design" .= value.design,
        "baselineRuns" .= value.baselineRuns,
        "candidateRuns" .= value.candidateRuns,
        "algorithm" .= object ["name" .= ("paired-bootstrap-t-envelope" :: Text), "version" .= (1 :: Int), "generator" .= ("splitmix" :: Text), "iterations" .= value.policy.bootstrapIterations, "seed" .= value.policy.resamplingSeed, "confidenceLevel" .= value.policy.confidenceLevel],
        "policy" .= value.policy,
        "variedFactors" .= value.variedFactors,
        "pairCount" .= value.pairCount,
        "metrics" .= value.metrics,
        "reasons" .= value.reasons,
        "verdict" .= value.verdict,
        "exitCode" .= verdictExitCode value.verdict
      ]

data RunData = RunData
  { runId :: Text,
    result :: Value,
    inputs :: Value,
    fingerprint :: Value,
    summary :: MeasurementSummary,
    outcome :: Text,
    startedAt :: Maybe UTCTime
  }

compareRuns :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison)
compareRuns policy axes baselineDirs candidateDirs
  | null baselineDirs || length baselineDirs /= length candidateDirs = pure (Left (CompareError "baseline and candidate runs must form the same non-zero number of pairs"))
  | otherwise = do
      baselineResults <- traverse loadRun baselineDirs
      candidateResults <- traverse loadRun candidateDirs
      case (sequence baselineResults, sequence candidateResults) of
        (Left err, _) -> pure (Left err)
        (_, Left err) -> pure (Left err)
        (Right baselines, Right candidates) -> compareLoaded baselines candidates
  where
    compareLoaded baselines candidates = case compatibilityProblem axes (baselines <> candidates) of
      Just message -> pure (Left (CompareError message))
      Nothing -> case varyingProblem axes baselines candidates of
        Just message -> pure (Left (CompareError message))
        Nothing -> do
          startedAt <- getCurrentTime
          identifier <- renderRunId <$> newRunId
          fingerprint <- collectHostFingerprint
          let referenceFingerprint = maybe Null (.fingerprint) (headMay baselines)
              environmentChanged = any ((/= referenceFingerprint) . (.fingerprint)) (tailSafe baselines <> candidates)
              badOutcomes = any ((`elem` ["errored", "infrastructure-failure"]) . (.outcome)) (baselines <> candidates)
              hardHealth = any (hasSeverity Hard . (.summary)) (baselines <> candidates)
              softHealth = any (hasSeverity Soft . (.summary)) (baselines <> candidates)
              checkpointAsymmetry = any (uncurry (checkpointDiffers policy.maxCheckpointAsymmetry)) (zip baselines candidates)
              metricComparisons = compareMetrics policy baselines candidates
              interleaving = validateInterleaving (catMaybes (concat (zipWith trialRows [0 ..] (zip baselines candidates))))
              lowGrade = any ((/= policy.requireGrade) . (.grade) . (.summary)) (baselines <> candidates)
              reasons =
                ["machine profiles differ" | environmentChanged]
                  <> ["an input run has an infrastructure outcome" | badOutcomes]
                  <> ["an input run has a hard health observation" | hardHealth]
                  <> ["fewer than the required number of pairs" | length baselines < policy.minimumPairs]
                  <> ["runs are not interleaved" | policy.requireInterleaving && either (const True) (const False) interleaving]
                  <> ["an input run is below the required evidence grade" | lowGrade]
                  <> ["an input run has a soft health observation" | softHealth]
                  <> ["checkpoint overlap differs too much within a pair" | checkpointAsymmetry]
              statuses = fmap (.status) (Map.elems metricComparisons)
              verdict = decideVerdict (environmentChanged || badOutcomes || hardHealth) reasons statuses
          finishedAt <- getCurrentTime
          pure (Right (Comparison identifier startedAt finishedAt fingerprint.revision fingerprint.dirty (comparisonDesign baselines candidates) (map (.runId) baselines) (map (.runId) candidates) policy (toList axes) (length baselines) metricComparisons reasons verdict))

comparisonDesign :: [RunData] -> [RunData] -> Text
comparisonDesign baselines candidates = case traverse firstArm (zip baselines candidates) of
  Just arms
    | arms == take (length arms) (cycle [Baseline, Candidate]) -> "abba"
    | arms == take (length arms) (cycle [Candidate, Baseline]) -> "baab"
  _ -> "sequential"
  where
    firstArm (baseline, candidate) = do
      baselineStart <- baseline.startedAt
      candidateStart <- candidate.startedAt
      if baselineStart < candidateStart then Just Baseline else if candidateStart < baselineStart then Just Candidate else Nothing

loadRun :: FilePath -> IO (Either CompareError RunData)
loadRun directory = do
  decoded <- eitherDecodeFileStrict' (directory <> "/run-result.json") :: IO (Either String Value)
  case decoded of
    Left message -> pure (Left (CompareError (Text.pack message)))
    Right result -> do
      summarized <- case jsonAt ["summaries", "measurements", "measurements"] result of
        Just value -> pure case fromJSON value of Error message -> Left (SummaryError (Text.pack message)); Success summary -> Right summary
        Nothing -> summarizeRunDir directory
      pure do
        summary <- either (Left . CompareError . (\(SummaryError message) -> message)) Right summarized
        inputs <- maybe (Left (CompareError "run result has no compatibility inputs")) Right (compatibilityInputs result)
        fingerprint <- maybe (Left (CompareError "run result has no environment fingerprint")) Right (environmentFingerprint result)
        runId <- maybe (Left (CompareError "run result has no run ID")) Right (textAt ["runId"] result)
        let outcome = fromMaybe "unknown" (textAt ["outcome"] result)
            started = textAt ["timings", "startedAt"] result >>= parseUtc
        Right RunData {runId, result, inputs, fingerprint, summary, outcome, startedAt = started}

compatibilityProblem :: NonEmpty VaryingAxis -> [RunData] -> Maybe Text
compatibilityProblem _ [] = Just "no runs"
compatibilityProblem axes (first : rest) = case [message | run <- rest, Left message <- [compatibleExcept axes first.inputs run.inputs]] of
  message : _ -> Just message
  [] -> Nothing

varyingProblem :: NonEmpty VaryingAxis -> [RunData] -> [RunData] -> Maybe Text
varyingProblem axes baselines candidates = findProblem (toList axes)
  where
    findProblem [] = Nothing
    findProblem (VaryControl : rest) =
      let baselineValues = nub (fmap (.inputs) baselines)
          candidateValues = nub (fmap (.inputs) candidates)
       in if length baselineValues /= 1 || baselineValues /= candidateValues
            then Just "control arms must have identical compatibility inputs"
            else findProblem rest
    findProblem (axis : rest) =
      let baselineValues = nub (fmap (varyingValue axis . (.inputs)) baselines)
          candidateValues = nub (fmap (varyingValue axis . (.inputs)) candidates)
       in if length baselineValues /= 1 || length candidateValues /= 1 || baselineValues == candidateValues
            then Just "each varying axis must have one distinct value per arm"
            else findProblem rest

compareMetrics :: Policy -> [RunData] -> [RunData] -> Map Text MetricComparison
compareMetrics policy baselines candidates = Map.fromList (mapMaybe compareOne commonNames)
  where
    allMetricMaps = fmap ((.metrics) . (.summary)) (baselines <> candidates)
    commonNames = case allMetricMaps of [] -> []; first : rest -> filter (\name -> all (Map.member name) rest) (Map.keys first)
    compareOne name = do
      rule <- find (\candidate -> matchesRule candidate.match name) policy.metricRules
      pairedValues <- traverse (pairMetric name) (zip baselines candidates)
      let unit = fromMaybe "" do
            baseline <- headMay baselines
            (.unit) <$> Map.lookup name baseline.summary.metrics
      pure (name, compareMetricPairs policy name unit rule pairedValues)
    pairMetric name (baseline, candidate) = do
      baselineMetric <- Map.lookup name baseline.summary.metrics
      candidateMetric <- Map.lookup name candidate.summary.metrics
      pure (baselineMetric.value, candidateMetric.value)

compareMetricPairs :: Policy -> Text -> Text -> MetricRule -> [(Double, Double)] -> MetricComparison
compareMetricPairs policy name unit rule pairedValues =
  let baselineValues = fmap fst pairedValues
      adverseRatios = fmap (adverseRatio rule.direction) pairedValues
      adverseDeltas = fmap (adverseDelta rule.direction) pairedValues
      ratioBootstrap = bootstrapInterval policy.resamplingSeed policy.bootstrapIterations policy.confidenceLevel geometricMean adverseRatios
      logT = studentTInterval policy.confidenceLevel (fmap log adverseRatios)
      ratioT = Interval (exp logT.low) (exp logT.estimate) (exp logT.high)
      ratioInterval = intervalEnvelope ratioBootstrap ratioT
      deltaInterval = intervalEnvelope (bootstrapInterval (policy.resamplingSeed + 1) policy.bootstrapIterations policy.confidenceLevel arithmeticMean adverseDeltas) (studentTInterval policy.confidenceLevel adverseDeltas)
      tooWide = ratioInterval.low <= 0 || ratioInterval.high / ratioInterval.low > 1 + policy.maxCiRelativeWidth
      comparisonStatus
        | tooWide = MetricInconclusive
        | ratioInterval.low > 1 + rule.relativeLimit && deltaInterval.low > rule.absoluteFloor = MetricRegression
        | ratioInterval.high <= 1 + rule.relativeLimit || deltaInterval.high <= rule.absoluteFloor = MetricPass
        | otherwise = MetricInconclusive
      comparisonMessage = case comparisonStatus of MetricPass -> "within policy limits"; MetricRegression -> "both relative and absolute regression limits exceeded"; MetricInconclusive -> "confidence interval crosses a policy limit or is too wide"
   in MetricComparison name unit pairedValues ratioInterval deltaInterval rule.relativeLimit rule.absoluteFloor (coefficientOfVariation baselineValues) comparisonStatus comparisonMessage
  where
    adverseRatio LowerIsBetter (baseline, candidate) = candidate / max 1e-12 baseline
    adverseRatio HigherIsBetter (baseline, candidate) = baseline / max 1e-12 candidate
    adverseDelta LowerIsBetter (baseline, candidate) = candidate - baseline
    adverseDelta HigherIsBetter (baseline, candidate) = baseline - candidate

hasSeverity :: Severity -> MeasurementSummary -> Bool
hasSeverity wanted summary = any ((== wanted) . (.severity)) summary.health

checkpointDiffers :: Double -> RunData -> RunData -> Bool
checkpointDiffers limit baseline candidate = abs (checkpointOverlap baseline.summary - checkpointOverlap candidate.summary) > limit

checkpointOverlap :: MeasurementSummary -> Double
checkpointOverlap summary = fromMaybe 0 do
  health <- find ((== "checkpoint-in-window") . (.gate)) summary.health
  case health.evidence of
    Object value -> case KeyMap.lookup "overlapFraction" value of Just (Number number) -> Just (realToFrac number); _ -> Nothing
    _ -> Nothing

decideVerdict :: Bool -> [Text] -> [MetricStatus] -> Verdict
decideVerdict infrastructureIssue reasons statuses
  | infrastructureIssue = VerdictInfrastructureFailure
  | not (null reasons) || null statuses = VerdictInconclusive
  | MetricRegression `elem` statuses = VerdictRegression
  | MetricInconclusive `elem` statuses = VerdictInconclusive
  | otherwise = VerdictPass

trialRows :: Int -> (RunData, RunData) -> [Maybe (Arm, Int, UTCTime)]
trialRows pair (baseline, candidate) = [fmap (Baseline,pair,) baseline.startedAt, fmap (Candidate,pair,) candidate.startedAt]

tailSafe :: [value] -> [value]
tailSafe [] = []; tailSafe (_ : rest) = rest

textAt :: [Text] -> Value -> Maybe Text
textAt [] (String value) = Just value
textAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= textAt rest
textAt _ _ = Nothing

jsonAt :: [Text] -> Value -> Maybe Value
jsonAt [] value = Just value
jsonAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= jsonAt rest
jsonAt _ _ = Nothing

headMay :: [value] -> Maybe value
headMay [] = Nothing
headMay (value : _) = Just value

parseUtc :: Text -> Maybe UTCTime
parseUtc = parseTimeM True defaultTimeLocale "%Y-%m-%dT%H:%M:%S%QZ" . Text.unpack

verdictExitCode :: Verdict -> Int
verdictExitCode VerdictPass = 0
verdictExitCode VerdictRegression = 1
verdictExitCode VerdictInconclusive = 3
verdictExitCode VerdictInfrastructureFailure = 4

verdictText :: Verdict -> Text
verdictText VerdictPass = "pass"
verdictText VerdictRegression = "regression"
verdictText VerdictInconclusive = "inconclusive"
verdictText VerdictInfrastructureFailure = "infrastructure-failure"

statusText :: MetricStatus -> Text
statusText MetricPass = "pass"
statusText MetricRegression = "regression"
statusText MetricInconclusive = "inconclusive"

toList :: NonEmpty value -> [value]
toList = foldr (:) []