packages feed

kenshou-remote-0.1.0.0: src/Kenshou/Remote/Perf/Attempt.hs

module Kenshou.Remote.Perf.Attempt
  ( TrialExecutor,
    executeAttempt,
    verifyAttempt,
    verifyAttemptInputs,
  )
where

import Control.Monad (unless)
import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.Aeson.Types (parseEither)
import Data.List (find, nub, sort)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Id (renderRunId, renderScenarioId)
import Kenshou.Core.Phase (zeroPhases)
import Kenshou.Core.RunSpec (ComparisonMembership (..), RunSpec (..))
import Kenshou.Measure.Compare
import Kenshou.Measure.Compare.Compatibility (VaryingAxis)
import Kenshou.Measure.Compare.Policy (Policy)
import Kenshou.Measure.Perf
import Kenshou.Perf.Plan (PerfProfile (..), PlannedProfile (..))
import Kenshou.Perf.Result
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))
import Kenshou.Remote.Perf.Journal (LeaseIdentity)
import Kenshou.Remote.Perf.Runner (ComparisonHooks (..))
import System.Directory (canonicalizePath, createDirectoryIfMissing, doesFileExist, renameFile)
import System.FilePath (isAbsolute, makeRelative, splitDirectories, (</>))

-- The adapter persists cell work before submission. Calling it again for the
-- same entry reconciles the existing low-level journal under the SAME lease.
-- It returns a verified nested run directory, retaining the outer cell tree.
type TrialExecutor = PlannedRun -> IO (Either Text FilePath)

data Observation = Observation PlannedRun (Maybe FilePath) [AdmissionProblem]

executeAttempt :: LeaseIdentity -> PlannedProfile -> RunPlan -> Policy -> NonEmpty VaryingAxis -> FilePath -> ComparisonHooks -> TrialExecutor -> IO ConfigurationResult
executeAttempt lease entry schedule policy axes directory callbacks execute = do
  createDirectoryIfMissing True directory
  let inputs = attemptInputs lease entry schedule
  immutable (directory </> "attempt.json") inputs
  validateSchedule entry schedule
  paths <- loadPaths directory
  validatePaths directory schedule paths
  failures <- loadFailures directory schedule paths
  atomicPaths directory paths
  collect paths failures [] [0 .. entry.profile.minimumPairs - 1] >>= continue entry.profile.minimumPairs
  where
    collect paths failures observations [] = pure (paths, failures, observations)
    collect paths failures observations (index : rest) = do
      let runs = filter (maybe False ((== index) . (.index)) . (.trial)) schedule.runs
      (changed, rejected, appended) <- runEntries paths failures runs
      collect changed rejected (observations <> appended) rest
    runEntries paths failures [] = pure (paths, failures, [])
    runEntries paths failures (run : rest) = do
      let identifier = renderRunId run.runId
      located <- case (Map.lookup identifier paths, Map.lookup identifier failures) of
        (Just relative, _) -> pure (Right (directory </> relative))
        (_, Just reason) -> pure (Left reason)
        _ -> execute run
      (changed, rejected, observation) <- case located of
        Left reason -> do
          let rejected = Map.insert identifier reason failures
          atomicJSON (directory </> "trial-failures.json") rejected
          pure (paths, rejected, Observation run Nothing [InvalidPerformanceEvidence reason])
        Right path -> do
          relative <- relativePath directory path
          let changed = Map.insert (renderRunId run.runId) relative paths
          atomicPaths directory changed
          observation <- admitTrial entry.profile run path
          mapM_ callbacks.recordRuntimeFailure (runtimeProblems observation)
          pure (changed, failures, observation)
      (final, finalFailures, appended) <- runEntries changed rejected rest
      pure (final, finalFailures, observation : appended)
    continue next (paths, failures, observations)
      | length (validPairs observations) >= entry.profile.minimumPairs = finish directory entry policy axes observations
      | next >= entry.profile.minimumPairs + entry.profile.replacementPairs = finish directory entry policy axes observations
      | otherwise = do
          callbacks.reserveReplacement next
          updated <- collect paths failures observations [next]
          continue (next + 1) updated

-- Replay uses only retained paths and rechecks every trial, including failures.
-- It never submits work or overwrites the original comparison bytes.
verifyAttempt :: PlannedProfile -> RunPlan -> Policy -> NonEmpty VaryingAxis -> FilePath -> ConfigurationResult -> IO Bool
verifyAttempt entry schedule policy axes directory retained = do
  validateSchedule entry schedule
  paths <- loadPaths directory
  validatePaths directory schedule paths
  failures <- loadFailures directory schedule paths
  observations <-
    traverse
      ( \run -> case Map.lookup (renderRunId run.runId) paths of
          Nothing -> pure (Observation run Nothing [InvalidPerformanceEvidence (Map.findWithDefault "trial has no retained result" (renderRunId run.runId) failures)])
          Just relative -> admitTrial entry.profile run (directory </> relative)
      )
      [run | run <- schedule.runs, Map.member (renderRunId run.runId) paths || Map.member (renderRunId run.runId) failures]
  recomputed <- finish directory entry policy axes observations
  pure (same recomputed retained)
  where
    same left right =
      left.profile == right.profile
        && left.workloadSha256 == right.workloadSha256
        && left.performanceVerdict == right.performanceVerdict
        && left.infrastructureState == right.infrastructureState
        && all (`elem` right.runtimeFailures) left.runtimeFailures
        && left.validPairs == right.validPairs
        && left.attempts == right.attempts
        && case (left.comparison, right.comparison) of
          (Just a, Just b) -> semanticComparison a == semanticComparison b
          (Nothing, Nothing) -> True
          _ -> False

verifyAttemptInputs :: LeaseIdentity -> PlannedProfile -> RunPlan -> FilePath -> IO ()
verifyAttemptInputs lease entry schedule directory = do
  validateSchedule entry schedule
  header <- eitherDecodeFileStrict' (directory </> "attempt.json") >>= either (ioError . userError) pure
  unless (header == attemptInputs lease entry schedule) (ioError (userError "retained attempt header differs from its journaled lease or pinned workload"))
  paths <- loadPaths directory
  validatePaths directory schedule paths
  _ <- loadFailures directory schedule paths
  pure ()

attemptInputs :: LeaseIdentity -> PlannedProfile -> RunPlan -> Value
attemptInputs lease entry schedule = object ["schema" .= ("kenshou.perf-attempt/v1" :: Text), "lease" .= lease, "workloadSha256" .= entry.workloadSha256, "plan" .= schedule]

admitTrial :: PerfProfile -> PlannedRun -> FilePath -> IO Observation
admitTrial profile run directory = do
  admitted <- verifyPerformanceRun profile directory
  result <- eitherDecodeFileStrict' (directory </> "run-result.json")
  spec <- eitherDecodeFileStrict' (directory </> "run-spec.json") :: IO (Either String RunSpec)
  let binding = do
        document <- either (Left . Text.pack) Right result
        effective <- either (Left . Text.pack) Right spec
        either (Left . Text.pack) Right (parseEither (withObject "runtime result" (.: "runId")) document) >>= \identifier ->
          unless (identifier == run.runId) (Left "retained result has a different run ID")
        unless
          ( effective.runId == Just run.runId
              && effective.scenario == run.spec.scenario
              && effective.seed == run.spec.seed
              && effective.scenarioRevision == Just profile.scenarioRevision
              && effective.phases == Just zeroPhases
              && effective.timeoutSeconds == Just profile.timeoutSeconds
              && knobs (toJSON effective) == knobs (toJSON run.spec)
              && Map.fromList effective.dimensions == Map.fromList run.spec.dimensions
          )
          (Left "retained runtime inputs differ from the pinned trial")
      problems = either id (const []) admitted <> either (\reason -> [InvalidPerformanceEvidence reason]) (const []) binding
  pure (Observation run (Just directory) problems)
  where
    knobs (Object fields) = KeyMap.lookup "knobs" fields
    knobs _ = Nothing

runtimeProblems :: Observation -> [Text]
runtimeProblems (Observation _ _ problems) = [reason | RuntimeWorkFailure reason <- problems]

validPairs :: [Observation] -> [Int]
validPairs observations = filter valid (sort (nub [trial.index | Observation run _ _ <- observations, Just trial <- [run.trial]]))
  where
    valid index = case [observation | observation@(Observation run _ _) <- observations, maybe False ((== index) . (.index)) run.trial] of
      [Observation a (Just _) ap, Observation b (Just _) bp] ->
        fmap (.arm) a.trial /= fmap (.arm) b.trial && all runtimeOnly (ap <> bp)
      _ -> False
    runtimeOnly (RuntimeWorkFailure _) = True
    runtimeOnly _ = False

finish :: FilePath -> PlannedProfile -> Policy -> NonEmpty VaryingAxis -> [Observation] -> IO ConfigurationResult
finish root entry policy axes observations = do
  let chosen = take entry.profile.minimumPairs (validPairs observations)
      path index arm = case find (\(Observation run _ _) -> maybe False (\trial -> trial.index == index && trial.arm == arm) run.trial) observations of
        Just (Observation _ (Just directory) _) -> directory
        _ -> error "valid pair lacks a retained arm"
      failures = nub (concatMap runtimeProblems observations)
      attempts = [object ["runId" .= run.runId, "path" .= fmap (makeRelative root) directory, "problems" .= map (Text.replace (Text.pack root) "." . Text.pack . show) problems] | Observation run directory problems <- observations]
      make verdict infrastructure comparison = ConfigurationResult entry.profile.name entry.workloadSha256 verdict infrastructure failures (length chosen) entry.profile.minimumPairs attempts comparison
  if length chosen < entry.profile.minimumPairs
    then pure (make "inconclusive" "failed" Nothing)
    else do
      compared <- compareRuns policy axes (map (`path` "baseline") chosen) (map (`path` "candidate") chosen)
      case compared of
        Left (CompareError _) -> pure (make "inconclusive" "failed" Nothing)
        Right comparison ->
          pure
            ( make
                (case comparison.verdict of VerdictPass -> "pass"; VerdictRegression -> "regression"; _ -> "inconclusive")
                (if comparison.verdict == VerdictInfrastructureFailure || length chosen < entry.profile.minimumPairs then "failed" else "healthy")
                (Just (toJSON comparison))
            )

validateSchedule :: PlannedProfile -> RunPlan -> IO ()
validateSchedule entry schedule = do
  let runs = schedule.runs
      identifiers = map (.runId) runs
      expected = entry.profile.minimumPairs + entry.profile.replacementPairs
      trials = [(trial.index, trial.arm) | run <- runs, Just trial <- [run.trial]]
      ordering = [(index, arm) | index <- [0 .. expected - 1], arm <- if even index then ["baseline", "candidate"] else ["candidate", "baseline"]]
      seeds = [nub [run.spec.seed | run <- runs, maybe False ((== index) . (.index)) run.trial] | index <- [0 .. expected - 1]]
  unless
    ( length runs == 2 * expected
        && nub identifiers == identifiers
        && trials == ordering
        && map (.ordinal) runs == [1 .. 2 * expected]
        && all ((== 1) . length) seeds
        && all (\run -> run.spec.runId == Just run.runId && maybe False ((== expected) . (.of_)) run.trial) runs
        && all memberMatches runs
    )
    (ioError (userError "attempt must contain the full fixed paired schedule"))
  unless (all matches runs) (ioError (userError "attempt workload differs from its pinned profile"))
  where
    memberMatches run = case (run.trial, run.spec.comparison) of
      (Just trial, Just membership) -> membership.group == trial.group && membership.arm == trial.arm && membership.trial == trial.index && membership.position == run.ordinal - 1
      _ -> False
    matches run =
      renderScenarioId run.spec.scenario == entry.profile.scenario
        && run.spec.scenarioRevision == Just entry.profile.scenarioRevision
        && run.spec.seed /= Nothing
        && run.spec.phases == Just zeroPhases
        && run.spec.timeoutSeconds == Just entry.profile.timeoutSeconds
        && Map.fromList run.spec.dimensions == entry.profile.dimensions
        && case toJSON run.spec of
          Object fields -> KeyMap.lookup "knobs" fields == Just (toJSON entry.profile.knobs)
          _ -> False

validatePaths :: FilePath -> RunPlan -> Map Text FilePath -> IO ()
validatePaths directory schedule paths = do
  unless
    (all (`elem` map (renderRunId . (.runId)) schedule.runs) (Map.keys paths))
    (ioError (userError "retained trial path names an unplanned run"))
  mapM_
    ( \relative -> do
        actual <- relativePath directory (directory </> relative)
        unless (actual == relative) (ioError (userError "retained trial path contains a directory alias"))
    )
    (Map.elems paths)

loadPaths :: FilePath -> IO (Map Text FilePath)
loadPaths directory = do
  let path = directory </> "trial-paths.json"
  exists <- doesFileExist path
  paths <- if exists then eitherDecodeFileStrict' path >>= either (ioError . userError) pure else pure Map.empty
  unless (all safe (Map.elems paths)) (ioError (userError "unsafe retained trial path"))
  pure paths
  where
    safe path = not (null path) && not (isAbsolute path) && all (`notElem` ["..", ".", ""]) (splitDirectories path)

loadFailures :: FilePath -> RunPlan -> Map Text FilePath -> IO (Map Text Text)
loadFailures directory schedule paths = do
  let path = directory </> "trial-failures.json"
  exists <- doesFileExist path
  failures <- if exists then eitherDecodeFileStrict' path >>= either (ioError . userError) pure else pure Map.empty
  unless (all (`elem` map (renderRunId . (.runId)) schedule.runs) (Map.keys failures) && all (`Map.notMember` paths) (Map.keys failures) && all (not . Text.null) (Map.elems failures)) (ioError (userError "retained trial failure has an unknown, conflicting or empty identity"))
  pure failures

relativePath :: FilePath -> FilePath -> IO FilePath
relativePath root path = do
  absoluteRoot <- canonicalizePath root
  absolutePath <- canonicalizePath path
  let relative = makeRelative absoluteRoot absolutePath
  unless (relative /= absolutePath && ".." `notElem` splitDirectories relative) (ioError (userError "trial evidence escaped its attempt directory"))
  pure relative

atomicPaths :: FilePath -> Map Text FilePath -> IO ()
atomicPaths directory paths = do
  atomicJSON (directory </> "trial-paths.json") paths

atomicJSON :: (ToJSON document) => FilePath -> document -> IO ()
atomicJSON destination document = do
  let temporary = destination <> ".next"
  encodeFile temporary document
  renameFile temporary destination

immutable :: FilePath -> Value -> IO ()
immutable path document = do
  exists <- doesFileExist path
  if exists
    then do
      retained <- eitherDecodeFileStrict' path >>= either (ioError . userError) pure
      unless (retained == document) (ioError (userError "attempt lease, workload or schedule changed"))
    else atomicJSON path document