kenshou-cli-0.1.0.0: src/Kenshou/Cli/Attest.hs
module Kenshou.Cli.Attest (runOutcomeRecomputer, measurementRecomputer, pairedComparisonRecomputer) where
import Control.Monad (forM)
import Data.Aeson (Result (..), Value (..), eitherDecodeFileStrict', fromJSON, toJSON)
import Data.Aeson.KeyMap qualified as KeyMap
import Data.List.NonEmpty qualified as NonEmpty
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Evidence.Attest (Recomputation (..), Recomputer (..), coreRecomputers)
import Kenshou.Measure.Compare (CompareError (..), compareRuns)
import Kenshou.Measure.Compare.Compatibility (VaryingAxis, parseVaryingAxis)
import Kenshou.Measure.Compare.Policy (Policy)
import Kenshou.Measure.Summary (SummaryError (..), summarizeRunDir)
import System.FilePath ((</>))
runOutcomeRecomputer :: Recomputer
runOutcomeRecomputer = Recomputer "run-outcome" 1 $ \root -> case coreRecomputers of
(core : _) -> core.recompute root
[] -> pure (Left "the core outcome oracle is unavailable")
measurementRecomputer :: Recomputer
measurementRecomputer = Recomputer "kenshou-summary" 1 $ \root -> do
measured <- summarizeRunDir root
case measured of
Left (SummaryError message) -> pure (Left message)
Right summary -> do
stored <- eitherDecodeFileStrict' (root </> "run-result.json") :: IO (Either String Value)
pure case stored of
Left message -> Left (Text.pack message)
Right document ->
Right
Recomputation
{ agreesWithDocuments = storedMeasurements document == Just (toJSON summary),
outcome = Nothing,
comparisonVerdict = Nothing,
detail = "recomputed kenshou.measurements/v1 from the sealed samples and series"
}
storedMeasurements :: Value -> Maybe Value
storedMeasurements (Object result) = do
Object summaries <- KeyMap.lookup "summaries" result
Object measurements <- KeyMap.lookup "measurements" summaries
KeyMap.lookup "measurements" measurements
storedMeasurements _ = Nothing
pairedComparisonRecomputer :: Recomputer
pairedComparisonRecomputer = Recomputer "paired-bootstrap-t-envelope" 1 $ \root -> do
stored <- eitherDecodeFileStrict' (root </> "comparison.json") :: IO (Either String Value)
case stored of
Left message -> pure (Left (Text.pack message))
Right document -> case comparisonInputs document of
Left message -> pure (Left message)
Right (policy, axes, pairs) -> do
let baselines = [root </> "baseline" </> show index | index <- [0 .. pairs - 1]]
candidates = [root </> "candidate" </> show index | index <- [0 .. pairs - 1]]
verified <- forM (baselines <> candidates) $ \path -> do
measured <- summarizeRunDir path
result <- eitherDecodeFileStrict' (path </> "run-result.json") :: IO (Either String Value)
pure case (measured, result) of
(Right summary, Right value) | storedMeasurements value == Just (toJSON summary) -> Right ()
(Left (SummaryError message), _) -> Left message
(_, Left message) -> Left (Text.pack message)
_ -> Left "arm measurements differ from the sealed samples and series"
case sequence verified of
Left message -> pure (Left message)
Right _ -> do
replayed <- compareRuns policy axes baselines candidates
pure case replayed of
Left (CompareError message) -> Left message
Right comparison ->
let actual = toJSON comparison
stableKeys = ["design", "baselineRuns", "candidateRuns", "algorithm", "policy", "variedFactors", "pairCount", "metrics", "reasons", "verdict", "exitCode"]
agrees = all (\key -> field key actual == field key document) stableKeys
in Right
Recomputation
{ agreesWithDocuments = agrees,
outcome = Nothing,
comparisonVerdict = case field "verdict" actual of Just (String verdict) -> Just verdict; _ -> Nothing,
detail = "recomputed arm summaries and the paired comparison from sealed data"
}
where
field name (Object value) = KeyMap.lookup name value
field _ _ = Nothing
comparisonInputs :: Value -> Either Text (Policy, NonEmpty.NonEmpty VaryingAxis, Int)
comparisonInputs (Object document) = do
rawPolicy <- maybe (Left "comparison has no policy") Right (KeyMap.lookup "policy" document)
policy <- case fromJSON rawPolicy of
Error message -> Left (Text.pack message)
Success value -> Right value
rawAxes <- maybe (Left "comparison has no varied factors") Right (KeyMap.lookup "variedFactors" document)
axisNames <- case fromJSON rawAxes of
Error message -> Left (Text.pack message)
Success value -> Right value
axes <- traverse parseVaryingAxis axisNames >>= maybe (Left "comparison has no varied factors") Right . NonEmpty.nonEmpty
rawPairs <- maybe (Left "comparison has no pair count") Right (KeyMap.lookup "pairCount" document)
pairs <- case fromJSON rawPairs of
Error message -> Left (Text.pack message)
Success value -> Right value
if pairs > 0 then Right (policy, axes, pairs) else Left "comparison pair count must be positive"
comparisonInputs _ = Left "comparison document is not an object"