packages feed

kenshou-cli-0.1.0.0: src/Kenshou/Cli/Command/Overhead.hs

module Kenshou.Cli.Command.Overhead
  ( OverheadOptions (..),
    overheadCommand,
    overheadParser,
    parseInputs,
    loadPolicy,
    findResumeDirectory,
    verdictText,
  )
where

import Control.Exception (IOException, try)
import Data.Aeson qualified as Aeson
import Data.ByteString.Lazy.Char8 qualified as LazyByteString
import Data.List (isPrefixOf, sort)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text.IO
import Data.Word (Word64)
import Kenshou.Core.Bundle (lookupScenario)
import Kenshou.Core.Cli
import Kenshou.Core.Id (ScenarioId, newRunId, parseScenarioId, renderRunId)
import Kenshou.Core.Knob (KnobError (..), KnobName, RawKnob, parseAssignment)
import Kenshou.Core.RunSpec (RunSpec (..))
import Kenshou.Diagnose.Leak (analyseRunDirectory, defaultLeakSpec)
import Kenshou.Measure.Compare (Verdict (..), compareRuns)
import Kenshou.Telemetry.Overhead
import Kenshou.Telemetry.Overhead.Policy
import Options.Applicative
import System.Directory (createDirectoryIfMissing, doesFileExist, listDirectory)
import System.Environment (getExecutablePath)
import System.Exit (ExitCode (..))
import System.FilePath (takeFileName, (</>))
import System.IO (IOMode (..), stderr, withFile)
import System.Process (StdStream (..), createProcess, proc, std_err, std_out, waitForProcess)

data OverheadOptions = OverheadOptions
  { scenario :: Text,
    arms :: [Text],
    mode :: OverheadMode,
    control :: Bool,
    trials :: Int,
    knobs :: [Text],
    dimensions :: [Text],
    policy :: Maybe InputSource,
    seed :: Word64,
    settleSeconds :: Int,
    retries :: Int,
    resume :: Bool,
    analyseOnly :: Bool,
    json :: Bool,
    out :: FilePath
  }

overheadCommand :: CliCommand
overheadCommand = CliCommand "overhead" "Run interleaved telemetry overhead arms" Execution False (overheadHandler <$> overheadParser)

overheadParser :: Parser OverheadOptions
overheadParser =
  OverheadOptions
    <$> (Text.pack <$> strArgument (metavar "SCENARIO" <> help "Registered benchmark scenario"))
    <*> some (Text.pack <$> strOption (long "arms" <> metavar "FACTOR=VALUE,..." <> help "Telemetry factor and ordered arm values"))
    <*> option (eitherReader parseMode) (long "mode" <> metavar "one-factor|full" <> value OneFactor <> showDefaultWith (const "one-factor") <> help "Arm expansion mode")
    <*> switch (long "control" <> help "Add an identical baseline A/A arm")
    <*> option auto (long "trials" <> metavar "N" <> value 3 <> showDefault <> help "Valid paired blocks")
    <*> many (Text.pack <$> strOption (long "set" <> metavar "NAME=VALUE" <> help "Set a fixed scenario knob"))
    <*> many (Text.pack <$> strOption (long "dim" <> metavar "NAME=VALUE" <> help "Set a fixed dimension"))
    <*> optional (option (parseInputSource <$> str) (long "policy" <> metavar "FILE" <> help "Read kenshou.overhead-policy/v1; use - for stdin"))
    <*> option auto (long "seed" <> metavar "N" <> value 7 <> showDefault <> help "Replayable ordering seed")
    <*> option auto (long "settle-seconds" <> metavar "S" <> value 2 <> showDefault <> help "Pause between child runs")
    <*> option auto (long "retries" <> metavar "R" <> value 1 <> showDefault <> help "Retries for infrastructure failures")
    <*> switch (long "resume" <> help "Resume the latest matching overhead directory")
    <*> switch (long "analyse-only" <> help "Rebuild a report without running child slots")
    <*> switch (long "json" <> help "Print only the overhead report JSON")
    <*> strOption (long "out" <> metavar "DIR" <> help "Output root or overhead invocation directory")

overheadHandler :: OverheadOptions -> CliEnv -> IO ExitCode
overheadHandler options environment = case parseScenarioId options.scenario of
  Left err -> usage err
  Right scenarioId -> case lookupScenario environment.registry scenarioId of
    Nothing -> usage ("unknown scenario " <> options.scenario)
    Just scenario -> do
      parsed <- pure (parseInputs options)
      case parsed of
        Left err -> usage err
        Right (factors, fixedKnobs, fixedDimensions) -> do
          selectedPolicy <- loadPolicy options.policy
          case selectedPolicy of
            Left err -> usage err
            Right policy -> do
              resumed <- if options.resume || options.analyseOnly then findResumeDirectory options.out scenarioId else pure Nothing
              case (options.resume || options.analyseOnly, resumed) of
                (True, Nothing) -> usage "no matching overhead state was found under --out"
                (_, Just directory) -> continue policy directory
                (_, Nothing) -> do
                  identifier <- renderRunId <$> newRunId
                  let directory = options.out </> ("overhead-" <> Text.unpack identifier)
                      request = OverheadRequest identifier scenarioId factors options.mode options.control options.trials fixedKnobs fixedDimensions options.seed options.settleSeconds options.retries
                  case planOverhead request scenario of
                    Left (UsageError err) -> usage err
                    Right plan -> do
                      createDirectoryIfMissing True directory
                      runPlan policy directory plan Nothing
  where
    continue policy directory = do
      loaded <- loadOverheadState directory
      case loaded of
        Left err -> usage ("invalid overhead state: " <> Text.pack err)
        Right state -> runPlan policy directory state.plan (Just state)

    runPlan policy directory plan existing = do
      executable <- getExecutablePath
      let leakCheck runDirectory = fmap (either (const Nothing) (Just . Aeson.toJSON)) (analyseRunDirectory runDirectory defaultLeakSpec plan.seed)
          hooks = OverheadHooks (runChildProcess executable directory) compareRuns (Just leakCheck)
      state <- if options.analyseOnly then maybe (fail "missing overhead state") pure existing else executeOverhead hooks plan directory
      report <- analyseOverhead hooks policy plan state directory
      if options.json
        then LazyByteString.putStrLn (Aeson.encode report)
        else do
          mapM_ printComparison report.comparisons
          Text.IO.putStrLn ("verdict " <> verdictText report.verdict <> "  " <> Text.pack directory)
      pure case overheadVerdictExitCode report.verdict of 0 -> ExitSuccess; code -> ExitFailure code

    printComparison comparison =
      Text.IO.putStrLn (comparison.candidate <> " vs " <> comparison.baseline <> "  " <> verdictText comparison.verdict)

usage :: Text -> IO ExitCode
usage message = Text.IO.hPutStrLn stderr ("kenshou: " <> message) >> pure (ExitFailure 2)

parseInputs :: OverheadOptions -> Either Text (Map Text [Text], [(KnobName, RawKnob)], Map Text Text)
parseInputs options = do
  factorPairs <- traverse parseArms options.arms
  let factorNames = fmap fst factorPairs
  if length factorNames /= Map.size (Map.fromList [(name, ()) | name <- factorNames])
    then Left "each --arms factor may be specified only once"
    else pure ()
  fixedKnobs <- traverse parseKnob options.knobs
  fixedDimensions <- Map.fromList <$> traverse parseAssignmentText options.dimensions
  pure (Map.fromList factorPairs, fixedKnobs, fixedDimensions)
  where
    parseKnob input = case parseAssignment input of Left (KnobError err) -> Left err; Right parsed -> Right parsed

parseArms :: Text -> Either Text (Text, [Text])
parseArms input = do
  (rawName, rawValues) <- parseAssignmentText input
  name <- case rawName of
    "tracing" -> Right "telemetry.tracing"
    "metrics" -> Right "telemetry.metrics"
    "telemetry.tracing" -> Right rawName
    "telemetry.metrics" -> Right rawName
    _ -> Left ("unknown overhead factor " <> rawName)
  let values = Text.splitOn "," rawValues
  if any Text.null values then Left (name <> " contains an empty arm value") else Right (name, values)

parseAssignmentText :: Text -> Either Text (Text, Text)
parseAssignmentText input = case Text.breakOn "=" input of
  (name, assignment) | not (Text.null name) && not (Text.null assignment) -> Right (name, Text.drop 1 assignment)
  _ -> Left ("expected NAME=VALUE, got " <> input)

parseMode :: String -> Either String OverheadMode
parseMode "one-factor" = Right OneFactor
parseMode "full" = Right FullFactorial
parseMode input = Left ("unknown mode " <> input)

loadPolicy :: Maybe InputSource -> IO (Either Text OverheadPolicy)
loadPolicy source = do
  bytes <- readInputSource (maybe (InputFile "policies/telemetry-overhead.json") id source)
  pure (either (Left . Text.pack) Right (decodeOverheadPolicy bytes))

findResumeDirectory :: FilePath -> ScenarioId -> IO (Maybe FilePath)
findResumeDirectory root scenarioId = do
  direct <- doesFileExist (root </> "state.json")
  if direct
    then matching root
    else do
      namesResult <- try (listDirectory root) :: IO (Either IOException [FilePath])
      let candidates = case namesResult of Left _ -> []; Right names -> reverse (sort [root </> name | name <- names, "overhead-" `isPrefixOf` takeFileName name])
      firstMatching candidates
  where
    matching directory = do
      loaded <- loadOverheadState directory
      pure case loaded of Right state | state.plan.scenarioId == scenarioId -> Just directory; _ -> Nothing
    firstMatching [] = pure Nothing
    firstMatching (directory : rest) = matching directory >>= maybe (firstMatching rest) (pure . Just)

runChildProcess :: FilePath -> FilePath -> RunSpec -> FilePath -> IO ExitCode
runChildProcess executable invocation spec runsRoot = do
  createDirectoryIfMissing True (invocation </> ".specs")
  createDirectoryIfMissing True (invocation </> "logs")
  let identifier = maybe "unassigned" (Text.unpack . renderRunId) spec.runId
      specPath = invocation </> ".specs" </> identifier <> ".json"
      logPath = invocation </> "logs" </> identifier <> ".log"
  Aeson.encodeFile specPath spec
  withFile logPath WriteMode \handle -> do
    (_, _, _, process) <- createProcess (proc executable ["run", "--spec", specPath, "--out", runsRoot]) {std_out = UseHandle handle, std_err = UseHandle handle}
    waitForProcess process

verdictText :: Verdict -> Text
verdictText VerdictPass = "pass"
verdictText VerdictRegression = "regression"
verdictText VerdictInconclusive = "inconclusive"
verdictText VerdictInfrastructureFailure = "infrastructure-failure"