kenshou-core-0.1.0.0: src/Kenshou/Plan/RunPlan.hs
{-# LANGUAGE FieldSelectors #-}
module Kenshou.Plan.RunPlan
( PlanInputs (..),
PlanContext (..),
TrialInfo (..),
PlannedRun (..),
SkipReason (..),
Skipped (..),
SkeletonRun (..),
PlanSkeleton (..),
RunPlan (..),
buildPlan,
stampPlan,
)
where
import Data.Aeson (ToJSON (..), Value, object, (.=))
import Data.List (sortOn)
import Data.List.NonEmpty (NonEmpty)
import Data.List.NonEmpty qualified as NonEmpty
import Data.Map.Strict qualified as Map
import Data.Set qualified as Set
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, getCurrentTime)
import Kenshou.Core.Id
import Kenshou.Core.Knob (RawKnob (..))
import Kenshou.Core.RunSpec
import Kenshou.Core.RunSpec qualified as RunSpec
import Kenshou.Core.Scenario (Placement (..), renderPlacement, renderTier)
import Kenshou.Plan.Catalog (ScenarioInfo (..))
import Kenshou.Plan.Change
import Kenshou.Plan.Matrix
import Kenshou.Plan.Policy
import System.Random.SplitMix (nextWord64)
newtype PlanInputs = PlanInputs {value :: Value}
deriving stock (Eq, Show)
data PlanContext = PlanContext
{ suite :: Maybe Text,
graphDigest :: Text,
cohortName :: Text,
cohortPlanHash :: Text,
inputs :: PlanInputs,
changes :: [Change],
warnings :: [Text]
}
deriving stock (Eq, Show)
data TrialInfo = TrialInfo {group :: Text, arm :: Text, index :: Int, of_ :: Int}
deriving stock (Eq, Show)
data PlannedRun = PlannedRun
{ ordinal :: Int,
runId :: RunId,
estimateMinutes :: Int,
reasons :: NonEmpty Reason,
trial :: Maybe TrialInfo,
spec :: RunSpec
}
deriving stock (Eq, Show)
data SkipReason
= SkipTier
| SkipKind
| SkipPlacement
| SkipExcluded
| SkipUnsupportedDimension
| SkipBenchmarkWithoutDurable
| SkipUnsupportedKnob
| SkipOverBudget
deriving stock (Eq, Ord, Show)
data Skipped = Skipped {scenario :: ScenarioId, reason :: SkipReason, detail :: Text}
deriving stock (Eq, Show)
data SkeletonRun = SkeletonRun
{ estimateMinutes :: Int,
reasons :: NonEmpty Reason,
trial :: Maybe TrialInfo,
spec :: RunSpec
}
deriving stock (Eq, Show)
data PlanSkeleton = PlanSkeleton
{ context :: PlanContext,
policy :: PlanPolicy,
runs :: [SkeletonRun],
skipped :: [Skipped],
estimateMinutes :: Int
}
deriving stock (Eq, Show)
data RunPlan = RunPlan
{ planId :: RunId,
createdAt :: UTCTime,
context :: PlanContext,
policy :: PlanPolicy,
runs :: [PlannedRun],
skipped :: [Skipped],
estimateMinutes :: Int
}
deriving stock (Eq, Show)
buildPlan :: PlanContext -> PlanPolicy -> [Selected] -> PlanSkeleton
buildPlan context policy selected =
PlanSkeleton
{ context,
policy,
runs = kept,
skipped = initialSkipped <> budgetSkipped,
estimateMinutes = sum (fmap (.estimateMinutes) kept)
}
where
ordered = sortOn selectedKey selected
expanded = fmap expandOne ordered
candidates = concatMap fst expanded
initialSkipped = concatMap snd expanded
(kept, budgetSkipped) = applyBudget policy.budgetMinutes candidates
selectedKey value =
( minimum (fmap (.distance) (NonEmpty.toList value.reasons)),
value.scenario.tier,
kindRank value.scenario.id.kind,
renderScenarioId value.scenario.id
)
kindRank Correctness = (0 :: Int)
kindRank Concurrency = 1
kindRank Benchmark = 2
kindRank Soak = 3
expandOne selectedValue
| selectedValue.scenario.tier > policy.maxTier = ([], [skip SkipTier ("tier " <> renderTier selectedValue.scenario.tier <> " exceeds " <> renderTier policy.maxTier)])
| selectedValue.scenario.id.kind `Set.notMember` policy.kinds = ([], [skip SkipKind "scenario kind is not enabled"])
| not (placementMatches policy.placement selectedValue.scenario.placement) = ([], [skip SkipPlacement ("scenario placement is " <> renderPlacement selectedValue.scenario.placement)])
| otherwise = case expandScenario policy selectedValue of
([], matrixSkips) -> ([], fmap (matrixSkip selectedValue.scenario.id) matrixSkips)
(configs, _) -> (makeRuns selectedValue configs, [])
where
skip reason detail = Skipped selectedValue.scenario.id reason detail
matrixSkip scenarioId problem = Skipped scenarioId (matrixReason problem.reason) problem.detail
matrixReason "benchmark-without-durable" = SkipBenchmarkWithoutDurable
matrixReason "unsupported-dimension" = SkipUnsupportedDimension
matrixReason _ = SkipUnsupportedKnob
makeRuns selectedValue configs
| selectedValue.scenario.id.kind == Benchmark = concatMap makeTrial [0 .. policy.trials - 1]
| otherwise = fmap (makeRun selectedValue Nothing 0) configs
where
makeTrial trialIndex =
[ makeRun selectedValue (Just (TrialInfo (renderScenarioId selectedValue.scenario.id) ("arm-" <> Text.pack (show armIndex)) trialIndex policy.trials)) trialIndex config
| (armIndex, config) <- zip [0 :: Int ..] (if even trialIndex then configs else reverse configs)
]
makeRun selectedValue trial trialIndex config =
SkeletonRun
{ estimateMinutes = Map.findWithDefault 10 selectedValue.scenario.tier policy.tierMinutes,
reasons = selectedValue.reasons,
trial,
spec =
(minimalRunSpec selectedValue.scenario.id)
{ RunSpec.scenarioRevision = Just selectedValue.scenario.revision,
RunSpec.knobs = [(name, RawJson (toJSON value)) | (name, value) <- Map.toAscList config.knobs],
RunSpec.dimensions = Map.toAscList config.dimensions,
RunSpec.seed = Just (derivedSeed policy.seed selectedValue.scenario.id trialIndex),
RunSpec.environment = defaultEnvironment policy.placement,
RunSpec.cohortExpectation = Just (CohortExpectation (Just context.cohortName) context.cohortPlanHash),
RunSpec.comparison = fmap toComparison trial
}
}
toComparison trial = ComparisonMembership trial.group trial.arm trial.index (trial.index + 1)
applyBudget :: Maybe Int -> [SkeletonRun] -> ([SkeletonRun], [Skipped])
applyBudget Nothing runs = (runs, [])
applyBudget (Just budget) runs = go 0 [] [] (groupRuns runs)
where
go _ kept skipped [] = (reverse kept, reverse skipped)
go used kept skipped ([] : rest) = go used kept skipped rest
go used kept skipped (group@(first : _) : rest)
| used + groupCost <= budget = go (used + groupCost) (reverse group <> kept) skipped rest
| otherwise =
let scenarioId = first.spec.scenario
item = Skipped scenarioId SkipOverBudget ("group costs " <> Text.pack (show groupCost) <> " minutes with " <> Text.pack (show (budget - used)) <> " remaining")
in go used kept (item : skipped) rest
where
groupCost = sum (fmap (.estimateMinutes) group)
groupRuns [] = []
groupRuns (first : rest) =
let (same, later) = span ((== first.spec.scenario) . (.scenario) . (.spec)) rest
in (first : same) : groupRuns later
stampPlan :: PlanSkeleton -> IO RunPlan
stampPlan skeleton = do
planId <- newRunId
createdAt <- getCurrentTime
runs <- traverse stamp (zip [1 :: Int ..] skeleton.runs)
pure
RunPlan
{ planId,
createdAt,
context = skeleton.context,
policy = skeleton.policy,
runs,
skipped = skeleton.skipped,
estimateMinutes = skeleton.estimateMinutes
}
where
stamp (ordinal, candidate) = do
runId <- newRunId
pure
PlannedRun
{ ordinal,
runId,
estimateMinutes = candidate.estimateMinutes,
reasons = candidate.reasons,
trial = candidate.trial,
spec = candidate.spec
}
derivedSeed :: Seed -> ScenarioId -> Int -> Seed
derivedSeed seed scenario trialIndex = case mkSeed (word `mod` 9007199254740992) of
Right value -> value
Left _ -> seed
where
(word, _) = nextWord64 (deriveGen seed (renderScenarioId scenario <> ":" <> Text.pack (show trialIndex)))
placementMatches :: SpecPlacement -> Placement -> Bool
placementMatches RunLocal PlaceLocal = True
placementMatches RunLocal PlaceEither = True
placementMatches RunOnCell PlaceCell = True
placementMatches RunOnCell PlaceEither = True
placementMatches _ _ = False
defaultEnvironment :: SpecPlacement -> EnvironmentSpec
defaultEnvironment placement = EnvironmentSpec placement Nothing Nothing Map.empty Map.empty Nothing
instance ToJSON RunPlan where
toJSON plan =
object
[ "schema" .= ("kenshou.run-plan/v1" :: Text),
"planId" .= plan.planId,
"createdAt" .= plan.createdAt,
"suite" .= plan.context.suite,
"graphDigest" .= plan.context.graphDigest,
"cohort" .= object ["name" .= plan.context.cohortName, "planHash" .= plan.context.cohortPlanHash],
"inputs" .= plan.context.inputs.value,
"policy" .= plan.policy,
"changes" .= plan.context.changes,
"runs" .= plan.runs,
"skipped" .= plan.skipped,
"warnings" .= plan.context.warnings,
"estimateMinutes" .= plan.estimateMinutes
]
instance ToJSON PlannedRun where
toJSON run = object ["ordinal" .= run.ordinal, "runId" .= run.runId, "estimateMinutes" .= run.estimateMinutes, "reasons" .= run.reasons, "trial" .= run.trial, "spec" .= run.spec]
instance ToJSON TrialInfo where
toJSON trial = object ["group" .= trial.group, "arm" .= trial.arm, "index" .= trial.index, "of" .= trial.of_]
instance ToJSON Skipped where
toJSON skipped = object ["scenario" .= skipped.scenario, "reason" .= renderSkipReason skipped.reason, "detail" .= skipped.detail]
renderSkipReason :: SkipReason -> Text
renderSkipReason SkipTier = "tier"
renderSkipReason SkipKind = "kind"
renderSkipReason SkipPlacement = "placement"
renderSkipReason SkipExcluded = "excluded"
renderSkipReason SkipUnsupportedDimension = "unsupported-dimension"
renderSkipReason SkipBenchmarkWithoutDurable = "benchmark-without-durable"
renderSkipReason SkipUnsupportedKnob = "unsupported-knob"
renderSkipReason SkipOverBudget = "over-budget"