kenshou-cli-0.1.0.0: src/Kenshou/Cli/Command/Plan.hs
{-# LANGUAGE FieldSelectors #-}
module Kenshou.Cli.Command.Plan (planCommand) where
import Data.Aeson qualified as Aeson
import Data.Bifunctor (first)
import Data.ByteString.Lazy.Char8 qualified as LazyByteString
import Data.List (nubBy, partition)
import Data.List.NonEmpty qualified as NonEmpty
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text.IO
import Kenshou.Cli.Cohort (resolveDefaultCohortIdentity)
import Kenshou.Core.Bundle (allScenarios)
import Kenshou.Core.Cli
import Kenshou.Core.Cohort qualified as Cohort
import Kenshou.Core.Id qualified as Id
import Kenshou.Core.Knob (KnobSpec (..), renderKnobName)
import Kenshou.Core.RunSpec (SpecPlacement (..))
import Kenshou.Core.Scenario (Tier (..))
import Kenshou.Plan.Catalog (ScenarioInfo (..), decodeCatalog, fromScenario)
import Kenshou.Plan.Change
import Kenshou.Plan.Change.Cohort
import Kenshou.Plan.Change.Git
import Kenshou.Plan.Components
import Kenshou.Plan.Components qualified as Components
import Kenshou.Plan.Components.Check
import Kenshou.Plan.Policy
import Kenshou.Plan.RunPlan
import Kenshou.Plan.Selector (Selector, parseSelector, renderSelector)
import Kenshou.Plan.Suite
import Options.Applicative hiding (value)
import System.Exit (ExitCode (..))
import System.IO (stderr)
import System.Random (randomIO)
data PlanOptions = PlanOptions
{ graphShow :: Bool,
graphCheck :: Bool,
json :: Bool,
graphSource :: Maybe InputSource,
catalogSource :: Maybe InputSource,
suiteName :: Maybe Text,
suiteSource :: Maybe InputSource,
changed :: [Text],
cohortFrom :: Maybe Text,
cohortTo :: Maybe Text,
since :: Maybe Text,
upstreamDiffs :: [Text],
allScenariosFlag :: Bool,
includes :: [Text],
excludes :: [Text],
explain :: Bool,
outputPath :: Maybe FilePath,
maxTierText :: Maybe Text,
kindTexts :: [Text],
placementText :: Maybe Text,
dimensionPolicyText :: Maybe Text,
knobPolicyText :: Maybe Text,
trialCount :: Maybe Int,
budget :: Maybe Int,
seedValue :: Maybe Integer,
knobPins :: [Text],
dimensionPins :: [Text]
}
planCommand :: CliCommand
planCommand =
CliCommand
{ name = "plan",
description = "Select and plan verification runs",
group = Discovery,
hidden = False,
parser = runPlan <$> planParser
}
planParser :: Parser PlanOptions
planParser =
PlanOptions
<$> switch (long "graph-show" <> help "Print the checked-in component graph")
<*> switch (long "graph-check" <> help "Check graph validity, build edges, and selectors")
<*> switch (long "json" <> help "Write machine-readable JSON")
<*> optional (option (parseInputSource <$> str) (long "graph" <> metavar "FILE" <> help "Override the component graph; use - for stdin"))
<*> optional (option (parseInputSource <$> str) (long "catalog" <> metavar "FILE" <> help "Plan against a scenario catalog; use - for stdin"))
<*> optional (Text.pack <$> strOption (long "suite" <> metavar "NAME" <> help "Apply suites/NAME.json"))
<*> optional (option (parseInputSource <$> str) (long "suite-file" <> metavar "FILE" <> help "Apply a suite document; use - for stdin"))
<*> many (Text.pack <$> strOption (long "changed" <> metavar "COMPONENT[,COMPONENT]" <> help "Name changed components or sub-components"))
<*> optional (Text.pack <$> strOption (long "cohort-from" <> metavar "COHORT" <> help "Baseline cohort name, file, or git object"))
<*> optional (Text.pack <$> strOption (long "cohort-to" <> metavar "COHORT" <> help "Candidate cohort name, file, or git object"))
<*> optional (Text.pack <$> strOption (long "since" <> metavar "REV" <> help "Map repository changes since a revision"))
<*> many (Text.pack <$> strOption (long "upstream-diff" <> metavar "REPO=PATH@REVA..REVB" <> help "Map paths changed in an upstream checkout"))
<*> switch (long "all" <> help "Select every catalog scenario")
<*> many (Text.pack <$> strOption (long "select" <> metavar "SELECTOR" <> help "Keep matching scenarios"))
<*> many (Text.pack <$> strOption (long "exclude" <> metavar "SELECTOR" <> help "Remove matching scenarios"))
<*> switch (long "explain" <> help "Explain changes, dependency paths, and selections")
<*> optional (strOption (long "out" <> metavar "FILE" <> help "Write the run plan to a file"))
<*> optional (Text.pack <$> strOption (long "max-tier" <> metavar "TIER" <> help "Maximum cost tier"))
<*> many (Text.pack <$> strOption (long "kind" <> metavar "KIND" <> help "Include a scenario kind; repeatable"))
<*> optional (Text.pack <$> strOption (long "placement" <> metavar "local|cell" <> help "Target placement"))
<*> optional (Text.pack <$> strOption (long "dimension-policy" <> metavar "POLICY" <> help "default-only, telemetry-corners, pairwise, or full"))
<*> optional (Text.pack <$> strOption (long "knob-policy" <> metavar "POLICY" <> help "defaults or declared-variants"))
<*> optional (option auto (long "trials" <> metavar "N" <> help "Benchmark trials (minimum 3)"))
<*> optional (option auto (long "budget-minutes" <> metavar "N" <> help "Maximum estimated run time"))
<*> optional (option auto (long "seed" <> metavar "N" <> help "Deterministic plan seed"))
<*> many (Text.pack <$> strOption (long "set" <> metavar "NAME=VALUE" <> help "Pin a knob value"))
<*> many (Text.pack <$> strOption (long "dim" <> metavar "NAME=VALUE" <> help "Pin a dimension value"))
runPlan :: PlanOptions -> CliEnv -> IO ExitCode
runPlan options environment
| stdinCount options > 1 = usage "at most one document input may use standard input"
| options.graphShow && options.graphCheck = usage "choose only one of --graph-show or --graph-check"
| options.suiteName /= Nothing && options.suiteSource /= Nothing = usage "choose only one of --suite or --suite-file"
| otherwise = do
graphResult <- loadGraph environment options.graphSource
catalogResult <- loadCatalog environment options.catalogSource
suiteResult <- loadSuite options
case (graphResult, catalogResult, suiteResult) of
(Left err, _, _) -> usage err
(_, Left err, _) -> usage err
(_, _, Left err) -> usage err
(Right graph, Right catalog, Right suite)
| options.graphShow -> showGraph options.json graph
| options.graphCheck -> checkGraph graph catalog
| otherwise -> planSelection options suite graph catalog
planSelection :: PlanOptions -> Maybe Suite -> ComponentGraph -> [ScenarioInfo] -> IO ExitCode
planSelection options suite graph catalog = case (traverse (parseSelector) options.includes, traverse parseSelector options.excludes) of
(Left err, _) -> usage err
(_, Left err) -> usage err
(Right includes, Right excludes) -> do
gathered <- gatherChanges options graph
case gathered of
Left err -> usage err
Right (changes, directSelectors, warnings)
| null changes && null directSelectors && not options.allScenariosFlag && not (maybe False ((== SuiteAll) . (.mode) . (.selection)) suite) -> usage "say what changed with --changed, --cohort-from/--cohort-to, --since, --upstream-diff, --all, or an all-mode suite"
| otherwise -> do
let componentSelected = if options.allScenariosFlag then selectAll catalog else selectScenarios graph catalog changes
pathSelected = selectBySelectors Since "repository path" directSelectors catalog
changedSelected = deduplicate (componentSelected <> pathSelected)
suiteSelected = maybe changedSelected (\suiteValue -> applySuite suiteValue catalog changedSelected) suite
selected = applySelectors includes excludes (deduplicate suiteSelected)
unmatchedPins = [name | raw <- options.knobPins, let (name, _) = splitAssignment raw, all (\selection -> not (declares name selection.scenario)) selected]
if not (null unmatchedPins)
then usage ("no selected scenario declares knob: " <> Text.intercalate ", " unmatchedPins)
else
if options.explain
then renderSelection True changes warnings selected >> pure ExitSuccess
else writeRunPlan options suite graph changes warnings selected
where
declares name scenario = any ((== name) . renderKnobName . (.name)) scenario.knobs
writeRunPlan :: PlanOptions -> Maybe Suite -> ComponentGraph -> [Change] -> [Warning] -> [Selected] -> IO ExitCode
writeRunPlan options suite graph changes warnings selected = do
seedResult <- makeSeed options.seedValue
cohortResult <- first (("unable to resolve cohort identity: " <>) . Text.pack . show) <$> resolveDefaultCohortIdentity
case (seedResult, cohortResult, (\seed -> makePolicy options suite seed) =<< seedResult) of
(Left err, _, _) -> usage err
(_, Left err, _) -> usage err
(_, _, Left err) -> usage err
(Right _, Right cohort, Right policy) -> do
let context =
PlanContext
{ suite = fmap (.name) suite,
graphDigest = Components.graphDigest graph,
cohortName = Cohort.unCohortName cohort.identityCohort,
cohortPlanHash = Cohort.unPlanHash cohort.identityPlanHash,
inputs = PlanInputs (inputValue options),
changes,
warnings = fmap (.message) warnings
}
plan <- stampPlan (buildPlan context policy selected)
let bytes = Aeson.encode plan
case options.outputPath of
Nothing -> LazyByteString.putStrLn bytes
Just path -> LazyByteString.writeFile path bytes
Text.IO.hPutStrLn stderr ("planned " <> Text.pack (show (length plan.runs)) <> " runs, estimate " <> Text.pack (show plan.estimateMinutes) <> " min, " <> Text.pack (show (length plan.skipped)) <> " skipped")
pure ExitSuccess
makeSeed :: Maybe Integer -> IO (Either Text Id.Seed)
makeSeed (Just value)
| value < 0 = pure (Left "seed must be non-negative")
| otherwise = pure (Id.mkSeed (fromInteger value))
makeSeed Nothing = do
value <- randomIO
pure (Id.mkSeed (value `mod` 9007199254740992))
makePolicy :: PlanOptions -> Maybe Suite -> Id.Seed -> Either Text PlanPolicy
makePolicy options suite seed = do
base <- maybe (Right (defaultPlanPolicy seed)) (suitePolicy seed) suite
maxTier <- maybe (Right base.maxTier) parseTierValue options.maxTierText
let kindInputs = concatMap (filter (not . Text.null) . fmap Text.strip . Text.splitOn ",") options.kindTexts
kinds <- if null kindInputs then Right base.kinds else Set.fromList <$> traverse Id.parseKind kindInputs
placement <- maybe (Right base.placement) parsePlacementValue options.placementText
dimensionPolicy <- maybe (Right base.dimensionPolicy) (maybe (Left "unknown dimension policy") Right . parseDimensionPolicy) options.dimensionPolicyText
knobPolicy <- maybe (Right base.knobPolicy) (maybe (Left "unknown knob policy") Right . parseKnobPolicy) options.knobPolicyText
let trials = maybe base.trials (\value -> value) options.trialCount
if trials < 3 then Left "--trials must be at least 3" else pure ()
case options.budget of Just value | value < 0 -> Left "--budget-minutes must be non-negative"; _ -> pure ()
pinnedKnobs <- traverse parseAssignmentText options.knobPins
pinnedDimensions <- traverse parseAssignmentText options.dimensionPins
pure
base
{ maxTier,
kinds,
placement,
dimensionPolicy,
knobPolicy,
trials,
budgetMinutes = options.budget <|> base.budgetMinutes,
pinnedKnobs,
pinnedDimensions
}
parseTierValue :: Text -> Either Text Tier
parseTierValue "smoke" = Right TierSmoke
parseTierValue "standard" = Right TierStandard
parseTierValue "extended" = Right TierExtended
parseTierValue "soak" = Right TierSoak
parseTierValue value = Left ("unknown tier " <> value)
parsePlacementValue :: Text -> Either Text SpecPlacement
parsePlacementValue "local" = Right RunLocal
parsePlacementValue "cell" = Right RunOnCell
parsePlacementValue value = Left ("unknown placement " <> value)
parseAssignmentText :: Text -> Either Text (Text, Text)
parseAssignmentText raw = case splitAssignment raw of
(name, value) | Text.null name || Text.null value -> Left "expected NAME=VALUE"
pair -> Right pair
splitAssignment :: Text -> (Text, Text)
splitAssignment raw = let (name, rest) = Text.breakOn "=" raw in (name, Text.drop 1 rest)
inputValue :: PlanOptions -> Aeson.Value
inputValue options =
Aeson.object
[ "changed" Aeson..= options.changed,
"cohortFrom" Aeson..= options.cohortFrom,
"cohortTo" Aeson..= options.cohortTo,
"since" Aeson..= options.since,
"upstreamDiffs" Aeson..= options.upstreamDiffs,
"all" Aeson..= options.allScenariosFlag,
"select" Aeson..= options.includes,
"exclude" Aeson..= options.excludes
]
gatherChanges :: PlanOptions -> ComponentGraph -> IO (Either Text ([Change], [Selector], [Warning]))
gatherChanges options graph = do
cohortResult <- gatherCohort
sinceResult <- maybe (pure (Right ([], [], []))) (changesSince graph ".") options.since
upstreamResults <- traverse gatherUpstream options.upstreamDiffs
pure do
named <- traverse (parseRef graph) namedInputs
(cohortChanges, cohortWarnings) <- cohortResult
(sinceChanges, sinceSelectors, sinceWarnings) <- sinceResult
upstream <- sequence upstreamResults
let upstreamChanges = concatMap fst upstream
upstreamWarnings = [Warning "ignored-upstream-paths" (Text.pack (show ignored) <> " upstream paths were outside component source roots") | (_, ignored) <- upstream, ignored > 0]
namedChanges = [Change reference Named ("--changed " <> renderRef reference) | reference <- named]
Right (namedChanges <> cohortChanges <> sinceChanges <> upstreamChanges, sinceSelectors, cohortWarnings <> sinceWarnings <> upstreamWarnings)
where
namedInputs = concatMap (filter (not . Text.null) . fmap Text.strip . Text.splitOn ",") options.changed
gatherCohort = case (options.cohortFrom, options.cohortTo) of
(Nothing, Nothing) -> pure (Right ([], []))
(Just from, Just to) -> do
old <- readCohortPackages "." (parseCohortInput from)
new <- readCohortPackages "." (parseCohortInput to)
pure ((uncurry (diffCohorts graph)) <$> ((,) <$> old <*> new))
_ -> pure (Left "--cohort-from and --cohort-to must be supplied together")
gatherUpstream raw = case parseUpstreamDiff raw of
Left err -> pure (Left err)
Right input -> changesFromUpstream graph input
deduplicate :: [Selected] -> [Selected]
deduplicate = nubBy (\left right -> left.scenario.id == right.scenario.id)
renderSelection :: Bool -> [Change] -> [Warning] -> [Selected] -> IO ()
renderSelection explain changes warnings selected = do
mapM_ (Text.IO.hPutStrLn stderr . ("warning: " <>) . (.message)) warnings
if explain
then do
mapM_ renderChange changes
mapM_ renderSelected selected
Text.IO.putStrLn ("selected " <> Text.pack (show (length selected)) <> " scenarios")
else mapM_ (Text.IO.putStrLn . Id.renderScenarioId . (.id) . (.scenario)) selected
where
renderChange changeValue = Text.IO.putStrLn ("changed " <> renderRef changeValue.ref <> " " <> changeValue.detail)
renderSelected selectedValue = do
let reason = NonEmpty.head selectedValue.reasons
Text.IO.putStrLn ("selected " <> Id.renderScenarioId selectedValue.scenario.id <> " via " <> Text.intercalate " <- " (fmap renderRef reason.via) <> " selector " <> renderSelector reason.selector)
loadGraph :: CliEnv -> Maybe InputSource -> IO (Either Text ComponentGraph)
loadGraph environment Nothing = pure (firstGraph (decodeGraph environment.componentGraph))
loadGraph _ (Just source) = firstGraph . decodeGraph <$> readInputSource source
firstGraph :: Either GraphError ComponentGraph -> Either Text ComponentGraph
firstGraph = either (Left . (.errorText)) Right
loadCatalog :: CliEnv -> Maybe InputSource -> IO (Either Text [ScenarioInfo])
loadCatalog environment Nothing = pure (Right (fmap fromScenario (allScenarios environment.registry)))
loadCatalog _ (Just source) = decodeCatalog <$> readInputSource source
loadSuite :: PlanOptions -> IO (Either Text (Maybe Suite))
loadSuite options = case (options.suiteName, options.suiteSource) of
(Nothing, Nothing) -> pure (Right Nothing)
(Just name, Nothing) -> fmap Just <$> readSuite ("suites/" <> Text.unpack name <> ".json")
(Nothing, Just source) -> fmap Just . decodeSuite <$> readInputSource source
(Just _, Just _) -> pure (Left "choose only one of --suite or --suite-file")
showGraph :: Bool -> ComponentGraph -> IO ExitCode
showGraph asJson graph = do
if asJson
then LazyByteString.putStrLn (Aeson.encode graph)
else do
let subCount = sum (fmap (length . (.subcomponents)) graph.components)
edges = concatMap (.dependsOn) graph.components
buildCount = length (filter ((== Build) . (.kind)) edges)
Text.IO.putStrLn (Text.pack (show (length graph.components)) <> " components, " <> Text.pack (show subCount) <> " sub-components, " <> Text.pack (show (length edges)) <> " edges (" <> Text.pack (show buildCount) <> " build, " <> Text.pack (show (length edges - buildCount)) <> " runtime)")
mapM_ renderComponent graph.components
pure ExitSuccess
where
renderComponent componentValue = Text.IO.putStrLn (Components.unComponentId componentValue.id <> " " <> Text.pack (show componentValue.kind) <> " " <> Text.intercalate "," componentValue.packages)
checkGraph :: ComponentGraph -> [ScenarioInfo] -> IO ExitCode
checkGraph graph catalog = do
driftResult <- checkAgainstPlanJson graph "dist-newstyle/cache/plan.json"
case driftResult of
Left err -> usage ("cannot read Cabal plan: " <> err)
Right drift -> do
let lint = lintAgainstCatalog graph catalog
(driftErrors, driftInfo) = partition isDriftError drift
(lintErrors, lintWarnings) = partition isLintError lint
Text.IO.putStrLn ("component graph: " <> Text.pack (show (length graph.components)) <> " components; digest " <> Components.graphDigest graph)
mapM_ (Text.IO.putStrLn . ("error: " <>) . Text.pack . show) driftErrors
mapM_ (Text.IO.putStrLn . ("error: " <>) . Text.pack . show) lintErrors
mapM_ (Text.IO.putStrLn . ("info: " <>) . Text.pack . show) driftInfo
mapM_ (Text.IO.putStrLn . ("warning: " <>) . Text.pack . show) lintWarnings
pure if null driftErrors && null lintErrors then ExitSuccess else ExitFailure 1
stdinCount :: PlanOptions -> Int
stdinCount options = length [() | Just InputStdin <- [options.graphSource, options.catalogSource, options.suiteSource]]
usage :: Text -> IO ExitCode
usage message = Text.IO.hPutStrLn stderr ("kenshou: " <> message) >> pure (ExitFailure 2)