kenshou-measure-0.1.0.0: src/Kenshou/Measure/Cli.hs
module Kenshou.Measure.Cli
( compareCommand,
summarizeCommand,
)
where
import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString.Lazy qualified as LazyByteString
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text
import Kenshou.Core.Cli
import Kenshou.Measure.Compare
import Kenshou.Measure.Compare.Compatibility
import Kenshou.Measure.Compare.Policy
import Kenshou.Measure.Metrics
import Kenshou.Measure.Stats
import Kenshou.Measure.Summary
import Options.Applicative
import System.Exit (ExitCode (..))
import System.IO (stderr)
data SummarizeOptions = SummarizeOptions {runDir :: FilePath, json :: Bool, verify :: Bool}
data CompareOptions = CompareOptions
{ baselines :: [FilePath],
candidates :: [FilePath],
policySource :: InputSource,
vary :: [Text],
output :: Maybe FilePath,
json :: Bool
}
summarizeCommand :: CliCommand
summarizeCommand = CliCommand "summarize" "Recompute and verify run measurements" Analysis False (runSummarize <$> summarizeParser)
compareCommand :: CliCommand
compareCommand = CliCommand "compare" "Compare paired baseline and candidate runs" Analysis False (runCompare <$> compareParser)
summarizeParser :: Parser SummarizeOptions
summarizeParser =
SummarizeOptions
<$> strArgument (metavar "RUN_DIR" <> help "Completed run directory")
<*> switch (long "json" <> help "Write JSON to standard output")
<*> switch (long "verify" <> help "Compare recomputed figures with run-result.json")
compareParser :: Parser CompareOptions
compareParser =
CompareOptions
<$> some (strOption (long "baseline" <> metavar "DIR" <> help "Baseline run directory; repeat once per pair"))
<*> some (strOption (long "candidate" <> metavar "DIR" <> help "Candidate run directory; repeat once per pair"))
<*> (parseInputSource <$> strOption (long "policy" <> metavar "FILE" <> help "Comparison policy file, or - for stdin"))
<*> many (Text.pack <$> strOption (long "vary" <> metavar "AXIS" <> help "Allowed varying axis: cohort, dim:NAME, or knob:NAME"))
<*> optional (strOption (long "out" <> metavar "FILE" <> help "Write the comparison JSON document"))
<*> switch (long "json" <> help "Write JSON to standard output")
runSummarize :: SummarizeOptions -> CliEnv -> IO ExitCode
runSummarize options _ = do
result <- summarizeRunDir options.runDir
case result of
Left (SummaryError message) -> Text.hPutStrLn stderr ("kenshou summarize: " <> message) >> pure (ExitFailure 4)
Right summary -> do
let document = toJSON summary
if options.json then LazyByteString.putStr (encode document <> "\n") else renderSummary summary
if not options.verify then pure ExitSuccess else verifyStored options.runDir document
verifyStored :: FilePath -> Value -> IO ExitCode
verifyStored runDir recomputed = do
decoded <- eitherDecodeFileStrict' (runDir <> "/run-result.json") :: IO (Either String Value)
case decoded of
Left message -> Text.hPutStrLn stderr ("kenshou summarize: " <> Text.pack message) >> pure (ExitFailure 4)
Right result -> case storedMeasurements result of
Nothing -> Text.hPutStrLn stderr "kenshou summarize: run-result.json has no stored measurements" >> pure (ExitFailure 1)
Just stored | stored == recomputed -> pure ExitSuccess
Just _ -> Text.hPutStrLn stderr "kenshou summarize: stored measurements differ from recomputed measurements" >> pure (ExitFailure 1)
storedMeasurements :: Value -> Maybe Value
storedMeasurements (Object root) = do
Object summaries <- KeyMap.lookup "summaries" root
Object measurements <- KeyMap.lookup "measurements" summaries
KeyMap.lookup "measurements" measurements
storedMeasurements _ = Nothing
runCompare :: CompareOptions -> CliEnv -> IO ExitCode
runCompare options _ = do
policyBytes <- readInputSource options.policySource
case decodePolicy policyBytes of
Left message -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)
Right policy -> case traverse parseVaryingAxis (if null options.vary then ["cohort"] else options.vary) of
Left message -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)
Right [] -> pure (ExitFailure 2)
Right (axis : axes) -> do
result <- compareRuns policy (axis :| axes) options.baselines options.candidates
case result of
Left (CompareError message) -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)
Right comparison -> do
let bytes = encode comparison
maybe (pure ()) (\path -> LazyByteString.writeFile path (bytes <> "\n")) options.output
if options.json then LazyByteString.putStr (bytes <> "\n") else renderComparison comparison
pure case verdictExitCode comparison.verdict of 0 -> ExitSuccess; code -> ExitFailure code
renderSummary :: MeasurementSummary -> IO ()
renderSummary summary = do
Text.putStrLn ("grade: " <> summary.grade)
mapM_ (\(name, metric) -> Text.putStrLn (name <> ": " <> Text.pack (show metric.value) <> " " <> metric.unit)) (toList summary.metrics)
renderComparison :: Comparison -> IO ()
renderComparison comparison = do
Text.putStrLn ("comparison " <> comparison.comparisonId <> " verdict: " <> verdictName comparison.verdict)
mapM_ (\(_, metric) -> Text.putStrLn (metric.metric <> " " <> statusName metric.status <> " ratio=" <> Text.pack (show metric.ratio.estimate))) (toList comparison.metrics)
mapM_ (Text.putStrLn . ("reason: " <>)) comparison.reasons
verdictName :: Verdict -> Text
verdictName VerdictPass = "pass"
verdictName VerdictRegression = "regression"
verdictName VerdictInconclusive = "inconclusive"
verdictName VerdictInfrastructureFailure = "infrastructure-failure"
statusName :: MetricStatus -> Text
statusName MetricPass = "pass"
statusName MetricRegression = "regression"
statusName MetricInconclusive = "inconclusive"
toList :: Map key value -> [(key, value)]
toList = Map.toList