kenshou-remote-0.1.0.0: src/Kenshou/Remote/Pair.hs
module Kenshou.Remote.Pair
( PairRequest (..),
PairState (..),
SliceResult (..),
pairPlan,
judgePairs,
)
where
import Data.List (nub, nubBy)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as TextEncoding
import Data.Word (Word64)
import Kenshou.Core.Canonical (sha256Hex)
import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), mkSeed, parseRunId, renderRunId, renderScenarioId)
import Kenshou.Core.RunSpec (ComparisonMembership (..), RunSpec (..))
import Kenshou.Measure.Compare.Ordering (Arm (..), PairedOrdering, TrialSlot (..), pairedSchedule)
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))
data PairRequest = PairRequest
{ candidate :: !Text,
baseline :: !Text,
pairs :: !Int,
ordering :: !PairedOrdering,
seed :: !Word64,
maxReplacements :: !Int
}
deriving stock (Eq, Show)
data PairState = PairValid | PairInvalid !Text
deriving stock (Eq, Show)
data SliceResult = SliceResult
{ group :: !Text,
pairIndex :: !Int,
arm :: !Arm,
passed :: !Bool
}
deriving stock (Eq, Show)
-- A planned benchmark may already have planner-generated trial repeats. The
-- cell schedule replaces those repeats with adjacent, same-seed pairs.
pairPlan :: PairRequest -> RunPlan -> Either Text RunPlan
pairPlan request source = do
if request.pairs < 3 then Left "paired comparisons need at least three pairs" else pure ()
if request.maxReplacements < 0 then Left "max replacements cannot be negative" else pure ()
if Text.null request.baseline || Text.null request.candidate || request.baseline == request.candidate
then Left "baseline and candidate payload labels must be distinct"
else pure ()
if null source.runs then Left "paired comparison plan has no runs" else pure ()
if any ((/= Benchmark) . (.kind) . (.scenario) . (.spec)) source.runs
then Left "paired comparisons require benchmark runs only"
else pure ()
let configurations = nubBy sameConfiguration source.runs
slots = pairedSchedule request.ordering (request.pairs + request.maxReplacements) request.seed
selected = [(configurationIndex, entry, slot) | (configurationIndex, entry) <- zip [0 :: Int ..] configurations, slot <- slots]
planned <- traverse makeRun (zip [1 :: Int ..] selected)
pure source {runs = planned, estimateMinutes = sum (fmap (.estimateMinutes) planned)}
where
sameConfiguration left right =
let first = left.spec
second = right.spec
in first.scenario == second.scenario
&& first.knobs == second.knobs
&& first.dimensions == second.dimensions
&& first.phases == second.phases
&& first.environment == second.environment
makeRun (ordinal, (configurationIndex, original, slot)) = do
identifier <- derivedRunId original.runId configurationIndex slot.position
pairSeed <- mkSeed (slot.pairSeed `mod` 9007199254740992)
let groupName = renderScenarioId original.spec.scenario <> "/config-" <> Text.pack (show configurationIndex)
armName = case slot.arm of Baseline -> request.baseline; Candidate -> request.candidate
trial = TrialInfo groupName armName slot.pairIndex (request.pairs + request.maxReplacements)
spec = original.spec {runId = Just identifier, seed = Just pairSeed, comparison = Just (ComparisonMembership groupName armName slot.pairIndex slot.position)}
pure original {ordinal, runId = identifier, trial = Just trial, spec}
derivedRunId :: RunId -> Int -> Int -> Either Text RunId
derivedRunId original configurationIndex position =
parseRunId (Text.take 18 (renderRunId original) <> "-a" <> Text.take 3 digest <> "-" <> Text.take 12 (Text.drop 3 digest))
where
digest = Text.drop 7 (sha256Hex (TextEncoding.encodeUtf8 (renderRunId original <> ":" <> Text.pack (show configurationIndex) <> ":" <> Text.pack (show position))))
judgePairs :: [SliceResult] -> [((Text, Int), PairState)]
judgePairs results = fmap judge keys
where
keys = nub [(result.group, result.pairIndex) | result <- results]
judge key =
let members = [result | result <- results, (result.group, result.pairIndex) == key]
baseline = filter ((== Baseline) . (.arm)) members
candidate = filter ((== Candidate) . (.arm)) members
in (key, if length baseline == 1 && length candidate == 1 && all (.passed) members then PairValid else PairInvalid "one or both cell trials failed or are missing")