kenshou-measure-0.1.0.0: src/Kenshou/Measure/Compare.hs
module Kenshou.Measure.Compare
( Verdict (..),
MetricStatus (..),
MetricComparison (..),
Comparison (..),
CompareError (..),
compareRuns,
compareMetricPairs,
decideVerdict,
verdictExitCode,
)
where
import Data.Aeson
import Data.Aeson.Key (fromText)
import Data.Aeson.KeyMap qualified as KeyMap
import Data.List (find, nub)
import Data.List.NonEmpty (NonEmpty)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (catMaybes, fromMaybe, mapMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, getCurrentTime)
import Data.Time.Format (defaultTimeLocale, parseTimeM)
import Kenshou.Core.Fingerprint (HostFingerprint (..), collectHostFingerprint)
import Kenshou.Core.Id (newRunId, renderRunId)
import Kenshou.Measure.Compare.Compatibility
import Kenshou.Measure.Compare.Ordering
import Kenshou.Measure.Compare.Policy
import Kenshou.Measure.Health
import Kenshou.Measure.Metrics
import Kenshou.Measure.Stats
import Kenshou.Measure.Summary
data Verdict = VerdictPass | VerdictRegression | VerdictInconclusive | VerdictInfrastructureFailure deriving stock (Eq, Ord, Show)
data MetricStatus = MetricPass | MetricRegression | MetricInconclusive deriving stock (Eq, Ord, Show)
data MetricComparison = MetricComparison
{ metric :: Text,
unit :: Text,
pairs :: [(Double, Double)],
ratio :: Interval,
delta :: Interval,
relativeLimit :: Double,
absoluteFloor :: Double,
baselineCv :: Double,
status :: MetricStatus,
message :: Text
}
deriving stock (Eq, Show)
data Comparison = Comparison
{ comparisonId :: Text,
startedAt :: UTCTime,
finishedAt :: UTCTime,
harnessRevision :: Maybe Text,
harnessDirty :: Maybe Bool,
design :: Text,
baselineRuns :: [Text],
candidateRuns :: [Text],
policy :: Policy,
variedFactors :: [VaryingAxis],
pairCount :: Int,
metrics :: Map Text MetricComparison,
reasons :: [Text],
verdict :: Verdict
}
deriving stock (Eq, Show)
newtype CompareError = CompareError Text deriving stock (Eq, Show)
instance ToJSON Verdict where toJSON = String . verdictText
instance ToJSON MetricStatus where toJSON = String . statusText
instance ToJSON MetricComparison where
toJSON value =
object
[ "metric" .= value.metric,
"unit" .= value.unit,
"pairs" .= [object ["baseline" .= baseline, "candidate" .= candidate] | (baseline, candidate) <- value.pairs],
"ratio" .= value.ratio,
"delta" .= value.delta,
"relativeLimit" .= value.relativeLimit,
"absoluteFloor" .= value.absoluteFloor,
"baselineCv" .= value.baselineCv,
"status" .= value.status,
"message" .= value.message
]
instance ToJSON Comparison where
toJSON value =
object
[ "schema" .= ("kenshou.comparison/v1" :: Text),
"comparisonId" .= value.comparisonId,
"startedAt" .= value.startedAt,
"finishedAt" .= value.finishedAt,
"harnessRevision" .= value.harnessRevision,
"harnessDirty" .= value.harnessDirty,
"design" .= value.design,
"baselineRuns" .= value.baselineRuns,
"candidateRuns" .= value.candidateRuns,
"algorithm" .= object ["name" .= ("paired-bootstrap-t-envelope" :: Text), "version" .= (1 :: Int), "generator" .= ("splitmix" :: Text), "iterations" .= value.policy.bootstrapIterations, "seed" .= value.policy.resamplingSeed, "confidenceLevel" .= value.policy.confidenceLevel],
"policy" .= value.policy,
"variedFactors" .= value.variedFactors,
"pairCount" .= value.pairCount,
"metrics" .= value.metrics,
"reasons" .= value.reasons,
"verdict" .= value.verdict,
"exitCode" .= verdictExitCode value.verdict
]
data RunData = RunData
{ runId :: Text,
result :: Value,
inputs :: Value,
fingerprint :: Value,
summary :: MeasurementSummary,
outcome :: Text,
startedAt :: Maybe UTCTime
}
compareRuns :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison)
compareRuns policy axes baselineDirs candidateDirs
| null baselineDirs || length baselineDirs /= length candidateDirs = pure (Left (CompareError "baseline and candidate runs must form the same non-zero number of pairs"))
| otherwise = do
baselineResults <- traverse loadRun baselineDirs
candidateResults <- traverse loadRun candidateDirs
case (sequence baselineResults, sequence candidateResults) of
(Left err, _) -> pure (Left err)
(_, Left err) -> pure (Left err)
(Right baselines, Right candidates) -> compareLoaded baselines candidates
where
compareLoaded baselines candidates = case compatibilityProblem axes (baselines <> candidates) of
Just message -> pure (Left (CompareError message))
Nothing -> case varyingProblem axes baselines candidates of
Just message -> pure (Left (CompareError message))
Nothing -> do
startedAt <- getCurrentTime
identifier <- renderRunId <$> newRunId
fingerprint <- collectHostFingerprint
let referenceFingerprint = maybe Null (.fingerprint) (headMay baselines)
environmentChanged = any ((/= referenceFingerprint) . (.fingerprint)) (tailSafe baselines <> candidates)
badOutcomes = any ((`elem` ["errored", "infrastructure-failure"]) . (.outcome)) (baselines <> candidates)
hardHealth = any (hasSeverity Hard . (.summary)) (baselines <> candidates)
softHealth = any (hasSeverity Soft . (.summary)) (baselines <> candidates)
checkpointAsymmetry = any (uncurry (checkpointDiffers policy.maxCheckpointAsymmetry)) (zip baselines candidates)
metricComparisons = compareMetrics policy baselines candidates
interleaving = validateInterleaving (catMaybes (concat (zipWith trialRows [0 ..] (zip baselines candidates))))
lowGrade = any ((/= policy.requireGrade) . (.grade) . (.summary)) (baselines <> candidates)
reasons =
["machine profiles differ" | environmentChanged]
<> ["an input run has an infrastructure outcome" | badOutcomes]
<> ["an input run has a hard health observation" | hardHealth]
<> ["fewer than the required number of pairs" | length baselines < policy.minimumPairs]
<> ["runs are not interleaved" | policy.requireInterleaving && either (const True) (const False) interleaving]
<> ["an input run is below the required evidence grade" | lowGrade]
<> ["an input run has a soft health observation" | softHealth]
<> ["checkpoint overlap differs too much within a pair" | checkpointAsymmetry]
statuses = fmap (.status) (Map.elems metricComparisons)
verdict = decideVerdict (environmentChanged || badOutcomes || hardHealth) reasons statuses
finishedAt <- getCurrentTime
pure (Right (Comparison identifier startedAt finishedAt fingerprint.revision fingerprint.dirty (comparisonDesign baselines candidates) (map (.runId) baselines) (map (.runId) candidates) policy (toList axes) (length baselines) metricComparisons reasons verdict))
comparisonDesign :: [RunData] -> [RunData] -> Text
comparisonDesign baselines candidates = case traverse firstArm (zip baselines candidates) of
Just arms
| arms == take (length arms) (cycle [Baseline, Candidate]) -> "abba"
| arms == take (length arms) (cycle [Candidate, Baseline]) -> "baab"
_ -> "sequential"
where
firstArm (baseline, candidate) = do
baselineStart <- baseline.startedAt
candidateStart <- candidate.startedAt
if baselineStart < candidateStart then Just Baseline else if candidateStart < baselineStart then Just Candidate else Nothing
loadRun :: FilePath -> IO (Either CompareError RunData)
loadRun directory = do
decoded <- eitherDecodeFileStrict' (directory <> "/run-result.json") :: IO (Either String Value)
case decoded of
Left message -> pure (Left (CompareError (Text.pack message)))
Right result -> do
summarized <- case jsonAt ["summaries", "measurements", "measurements"] result of
Just value -> pure case fromJSON value of Error message -> Left (SummaryError (Text.pack message)); Success summary -> Right summary
Nothing -> summarizeRunDir directory
pure do
summary <- either (Left . CompareError . (\(SummaryError message) -> message)) Right summarized
inputs <- maybe (Left (CompareError "run result has no compatibility inputs")) Right (compatibilityInputs result)
fingerprint <- maybe (Left (CompareError "run result has no environment fingerprint")) Right (environmentFingerprint result)
runId <- maybe (Left (CompareError "run result has no run ID")) Right (textAt ["runId"] result)
let outcome = fromMaybe "unknown" (textAt ["outcome"] result)
started = textAt ["timings", "startedAt"] result >>= parseUtc
Right RunData {runId, result, inputs, fingerprint, summary, outcome, startedAt = started}
compatibilityProblem :: NonEmpty VaryingAxis -> [RunData] -> Maybe Text
compatibilityProblem _ [] = Just "no runs"
compatibilityProblem axes (first : rest) = case [message | run <- rest, Left message <- [compatibleExcept axes first.inputs run.inputs]] of
message : _ -> Just message
[] -> Nothing
varyingProblem :: NonEmpty VaryingAxis -> [RunData] -> [RunData] -> Maybe Text
varyingProblem axes baselines candidates = findProblem (toList axes)
where
findProblem [] = Nothing
findProblem (VaryControl : rest) =
let baselineValues = nub (fmap (.inputs) baselines)
candidateValues = nub (fmap (.inputs) candidates)
in if length baselineValues /= 1 || baselineValues /= candidateValues
then Just "control arms must have identical compatibility inputs"
else findProblem rest
findProblem (axis : rest) =
let baselineValues = nub (fmap (varyingValue axis . (.inputs)) baselines)
candidateValues = nub (fmap (varyingValue axis . (.inputs)) candidates)
in if length baselineValues /= 1 || length candidateValues /= 1 || baselineValues == candidateValues
then Just "each varying axis must have one distinct value per arm"
else findProblem rest
compareMetrics :: Policy -> [RunData] -> [RunData] -> Map Text MetricComparison
compareMetrics policy baselines candidates = Map.fromList (mapMaybe compareOne commonNames)
where
allMetricMaps = fmap ((.metrics) . (.summary)) (baselines <> candidates)
commonNames = case allMetricMaps of [] -> []; first : rest -> filter (\name -> all (Map.member name) rest) (Map.keys first)
compareOne name = do
rule <- find (\candidate -> matchesRule candidate.match name) policy.metricRules
pairedValues <- traverse (pairMetric name) (zip baselines candidates)
let unit = fromMaybe "" do
baseline <- headMay baselines
(.unit) <$> Map.lookup name baseline.summary.metrics
pure (name, compareMetricPairs policy name unit rule pairedValues)
pairMetric name (baseline, candidate) = do
baselineMetric <- Map.lookup name baseline.summary.metrics
candidateMetric <- Map.lookup name candidate.summary.metrics
pure (baselineMetric.value, candidateMetric.value)
compareMetricPairs :: Policy -> Text -> Text -> MetricRule -> [(Double, Double)] -> MetricComparison
compareMetricPairs policy name unit rule pairedValues =
let baselineValues = fmap fst pairedValues
adverseRatios = fmap (adverseRatio rule.direction) pairedValues
adverseDeltas = fmap (adverseDelta rule.direction) pairedValues
ratioBootstrap = bootstrapInterval policy.resamplingSeed policy.bootstrapIterations policy.confidenceLevel geometricMean adverseRatios
logT = studentTInterval policy.confidenceLevel (fmap log adverseRatios)
ratioT = Interval (exp logT.low) (exp logT.estimate) (exp logT.high)
ratioInterval = intervalEnvelope ratioBootstrap ratioT
deltaInterval = intervalEnvelope (bootstrapInterval (policy.resamplingSeed + 1) policy.bootstrapIterations policy.confidenceLevel arithmeticMean adverseDeltas) (studentTInterval policy.confidenceLevel adverseDeltas)
tooWide = ratioInterval.low <= 0 || ratioInterval.high / ratioInterval.low > 1 + policy.maxCiRelativeWidth
comparisonStatus
| tooWide = MetricInconclusive
| ratioInterval.low > 1 + rule.relativeLimit && deltaInterval.low > rule.absoluteFloor = MetricRegression
| ratioInterval.high <= 1 + rule.relativeLimit || deltaInterval.high <= rule.absoluteFloor = MetricPass
| otherwise = MetricInconclusive
comparisonMessage = case comparisonStatus of MetricPass -> "within policy limits"; MetricRegression -> "both relative and absolute regression limits exceeded"; MetricInconclusive -> "confidence interval crosses a policy limit or is too wide"
in MetricComparison name unit pairedValues ratioInterval deltaInterval rule.relativeLimit rule.absoluteFloor (coefficientOfVariation baselineValues) comparisonStatus comparisonMessage
where
adverseRatio LowerIsBetter (baseline, candidate) = candidate / max 1e-12 baseline
adverseRatio HigherIsBetter (baseline, candidate) = baseline / max 1e-12 candidate
adverseDelta LowerIsBetter (baseline, candidate) = candidate - baseline
adverseDelta HigherIsBetter (baseline, candidate) = baseline - candidate
hasSeverity :: Severity -> MeasurementSummary -> Bool
hasSeverity wanted summary = any ((== wanted) . (.severity)) summary.health
checkpointDiffers :: Double -> RunData -> RunData -> Bool
checkpointDiffers limit baseline candidate = abs (checkpointOverlap baseline.summary - checkpointOverlap candidate.summary) > limit
checkpointOverlap :: MeasurementSummary -> Double
checkpointOverlap summary = fromMaybe 0 do
health <- find ((== "checkpoint-in-window") . (.gate)) summary.health
case health.evidence of
Object value -> case KeyMap.lookup "overlapFraction" value of Just (Number number) -> Just (realToFrac number); _ -> Nothing
_ -> Nothing
decideVerdict :: Bool -> [Text] -> [MetricStatus] -> Verdict
decideVerdict infrastructureIssue reasons statuses
| infrastructureIssue = VerdictInfrastructureFailure
| not (null reasons) || null statuses = VerdictInconclusive
| MetricRegression `elem` statuses = VerdictRegression
| MetricInconclusive `elem` statuses = VerdictInconclusive
| otherwise = VerdictPass
trialRows :: Int -> (RunData, RunData) -> [Maybe (Arm, Int, UTCTime)]
trialRows pair (baseline, candidate) = [fmap (Baseline,pair,) baseline.startedAt, fmap (Candidate,pair,) candidate.startedAt]
tailSafe :: [value] -> [value]
tailSafe [] = []; tailSafe (_ : rest) = rest
textAt :: [Text] -> Value -> Maybe Text
textAt [] (String value) = Just value
textAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= textAt rest
textAt _ _ = Nothing
jsonAt :: [Text] -> Value -> Maybe Value
jsonAt [] value = Just value
jsonAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= jsonAt rest
jsonAt _ _ = Nothing
headMay :: [value] -> Maybe value
headMay [] = Nothing
headMay (value : _) = Just value
parseUtc :: Text -> Maybe UTCTime
parseUtc = parseTimeM True defaultTimeLocale "%Y-%m-%dT%H:%M:%S%QZ" . Text.unpack
verdictExitCode :: Verdict -> Int
verdictExitCode VerdictPass = 0
verdictExitCode VerdictRegression = 1
verdictExitCode VerdictInconclusive = 3
verdictExitCode VerdictInfrastructureFailure = 4
verdictText :: Verdict -> Text
verdictText VerdictPass = "pass"
verdictText VerdictRegression = "regression"
verdictText VerdictInconclusive = "inconclusive"
verdictText VerdictInfrastructureFailure = "infrastructure-failure"
statusText :: MetricStatus -> Text
statusText MetricPass = "pass"
statusText MetricRegression = "regression"
statusText MetricInconclusive = "inconclusive"
toList :: NonEmpty value -> [value]
toList = foldr (:) []