packages feed

kenshou-remote-0.1.0.0: src/Kenshou/Remote/Perf/Cell.hs

module Kenshou.Remote.Perf.Cell
  ( CellAdapter (..),
    executeCellTrial,
    verifyCellTrial,
    retainedRuntimeProblems,
  )
where

import Control.Concurrent (threadDelay)
import Control.Exception (IOException, catch, displayException)
import Control.Monad (unless)
import Data.Aeson (Value, eitherDecodeFileStrict', encode)
import Data.ByteString.Lazy qualified as Bytes
import Data.Map.Strict (Map)
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Bundle (Registry)
import Kenshou.Core.Id (renderRunId)
import Kenshou.Core.RunSpec (EnvironmentSpec (..), RunSpec (..))
import Kenshou.Measure.Perf (AdmissionProblem (..), runtimeResultProblems, verifyPerformanceRun)
import Kenshou.Perf.Plan (PerfProfile)
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))
import Kenshou.Remote.Cell.Docs (CellBuckets (..), CellDescriptor (..), CellImages (..), CellNodes (..), CellOutcome (..), Requirements (..), ResetBlock (..), Submission (..), WorkObject (..))
import Kenshou.Remote.Cell.Index (CellManifestLink (..), CellRunIndex (..), EvidenceLink (..), RunLink (..), deriveCellRunIndex)
import Kenshou.Remote.Cell.Lease (CellRef (..), Lease (..), LeaseHandle, leaseSnapshot, withHeartbeat)
import Kenshou.Remote.Cell.Prepare (cellMachineProfile)
import Kenshou.Remote.Cell.RouteRules (CellCapabilities, RoutingRule)
import Kenshou.Remote.Cell.Session.Build
import Kenshou.Remote.Cell.Session.Journal
import Kenshou.Remote.Cell.Session.Resume
import Kenshou.Remote.Cell.Session.Runner
import Kenshou.Remote.Cell.Submit (workObjectFor)
import Kenshou.Remote.Cell.Watch (WatchEvent)
import Kenshou.Remote.Payload (PayloadDescriptor)
import Kenshou.Remote.Perf.Conditions (validateCellConditions, validateCellObservations)
import Kenshou.Remote.Store (Bucket (..), ObjectStore)
import System.Directory (createDirectoryIfMissing, doesFileExist)
import System.FilePath (takeDirectory, (</>))

data CellAdapter = CellAdapter
  { registry :: Registry,
    store :: ObjectStore,
    ref :: CellRef,
    descriptor :: CellDescriptor,
    healthPolicy :: Value,
    capabilities :: Maybe CellCapabilities,
    rules :: [RoutingRule],
    payloads :: Map Text PayloadDescriptor,
    options :: BuildOptions,
    storeLabel :: Text,
    emit :: WatchEvent -> IO ()
  }

-- Each trial has a separate low-level cell journal. Its work digest is checked
-- against the immutable paired schedule before any new or resumed submission.
executeCellTrial :: CellAdapter -> LeaseHandle -> RunPlan -> FilePath -> PerfProfile -> (Text -> IO ()) -> PlannedRun -> IO (Either Text FilePath)
executeCellTrial adapter handle schedule attempt profile observeRuntime run = do
  lease <- leaseSnapshot handle
  let directory = attempt </> "trials" </> Text.unpack (renderRunId run.runId)
      journalPath = directory </> "session.json"
      bytes = encode schedule {runs = [run], estimateMinutes = run.estimateMinutes}
      expected = (workObjectFor "application/json" bytes).sha256
      results = Bucket adapter.descriptor.buckets.results
  exists <- doesFileExist journalPath
  result <- withHeartbeat adapter.store adapter.ref handle \_ ->
    if exists
      then do
        saved <- readSessionJournal journalPath
        case saved of
          Left problem -> pure (Left problem)
          Right journal
            | journal.leaseId /= lease.leaseId || journal.planSha256 /= expected || journal.payloads /= adapter.payloads ->
                pure (Left "retained cell journal differs from the pinned trial, payloads or lease")
          Right _ -> continue results journalPath
      else do
        built <- buildSession adapter.registry adapter.descriptor adapter.capabilities adapter.rules adapter.payloads adapter.options adapter.storeLabel lease.leaseId bytes
        case built of
          Left problem -> pure (Left problem)
          Right session -> do
            createDirectoryIfMissing True directory
            Bytes.writeFile (directory </> "plan.json") bytes
            mapM_
              ( \(relative, content) -> do
                  createDirectoryIfMissing True (takeDirectory (directory </> relative))
                  Bytes.writeFile (directory </> relative) content
              )
              session.workFiles
            either (Left . Text.pack . show) Right <$> runPlannedSlices adapter.store adapter.ref results handle journalPath session.journal adapter.emit
  case result of
    Left problem -> pure (Left problem)
    Right journal -> do
      problems <- retainedRuntimeProblems profile run directory journal
      mapM_ observeRuntime problems
      verifyCellTrial adapter.healthPolicy adapter.descriptor run directory journal
  where
    continue results journalPath = do
      resumed <- resumeHeldSession adapter.store adapter.ref results handle journalPath adapter.emit
      case resumed of
        Left (ContinuationError (UnresolvedSlice _ SliceSubmitted)) -> threadDelay 2000000 >> continue results journalPath
        Left failure -> pure (Left (Text.pack (show failure)))
        Right journal -> pure (Right journal)

-- A bad outer health gate cannot erase a sealed nested runtime assertion or
-- failed/incomplete steady work. Reconstruct these before rejecting the pair.
retainedRuntimeProblems :: PerfProfile -> PlannedRun -> FilePath -> SessionJournal -> IO [Text]
retainedRuntimeProblems profile run directory journal = case journal.slices of
  [slice] | slice.state `elem` [SliceFetched, SliceVerified] -> do
    let tree = takeDirectory directory </> Text.unpack (renderRunId slice.cellRun) </> "tree"
    verified <- deriveCellRunIndex (Bucket journal.resultsBucket) Nothing tree
    case verified of
      Right index | index.cellRun == slice.cellRun && index.leaseId == journal.leaseId && index.cell == journal.cell && slice.manifestSha256 == Just index.cellManifest.sha256 && slice.cellOutcome == Just index.cellOutcome && any (\link -> link.runId == run.runId && link.path == "output/" <> renderRunId run.runId) index.runs -> do
        let nested = tree </> "output" </> Text.unpack (renderRunId run.runId)
        result <- eitherDecodeFileStrict' (nested </> "run-result.json")
        measured <- verifyPerformanceRun profile nested
        pure ([reason | RuntimeWorkFailure reason <- either (const []) runtimeResultProblems result <> either id (const []) measured])
      _ -> pure []
  _ -> pure []

-- Recheck the outer cell seal and health even when a low-level journal already
-- says verified. An exit-one runtime result is still returned for independent
-- runtime classification; failed cell health cannot become a valid trial.
verifyCellTrial :: Value -> CellDescriptor -> PlannedRun -> FilePath -> SessionJournal -> IO (Either Text FilePath)
verifyCellTrial healthPolicy descriptor run directory journal =
  catch (verifyTrial healthPolicy descriptor run directory journal) (\(problem :: IOException) -> pure (Left (Text.pack (displayException problem))))

verifyTrial :: Value -> CellDescriptor -> PlannedRun -> FilePath -> SessionJournal -> IO (Either Text FilePath)
verifyTrial healthPolicy descriptor run directory journal = case journal.slices of
  [slice] | slice.state == SliceVerified && slice.runIds == [run.runId] -> do
    let tree = takeDirectory directory </> Text.unpack (renderRunId slice.cellRun) </> "tree"
    verified <- deriveCellRunIndex (Bucket journal.resultsBucket) Nothing tree
    let evidencePresent = either (const False) (\index -> all (`elem` map (.kind) index.evidence) ["cell-health", "cell-reset-evidence"]) verified
    conditions <-
      if evidencePresent
        then do
          health <- eitherDecodeFileStrict' (tree </> "cell/health.json")
          reset <- eitherDecodeFileStrict' (tree </> "cell/reset-evidence.json")
          interval <- eitherDecodeFileStrict' (tree </> "cell/execution-window.json")
          let roles = [("driver", (case descriptor.instances.drivers of driver : _ -> driver; [] -> ""), descriptor.images.driver), ("postgres", descriptor.instances.postgres, descriptor.images.postgres), ("monitoring", descriptor.instances.monitoring, descriptor.images.monitoring)] <> case (slice.submission.reset.broker, descriptor.instances.broker) of (Just _, Just broker) -> [("broker", broker, descriptor.images.broker)]; _ -> []
          raw <- traverse (\(role, machine, image) -> do value <- eitherDecodeFileStrict' (tree </> "cell/observations" </> Text.unpack role <> ".json"); pure (role, machine, image, value)) roles
          let machines =
                descriptor.instances.drivers
                  <> [descriptor.instances.postgres, descriptor.instances.monitoring]
                  <> case (slice.submission.reset.broker, descriptor.instances.broker) of (Just _, Just broker) -> [broker]; _ -> []
          pure do
            healthDocument <- either (Left . Text.pack) Right health
            resetDocument <- either (Left . Text.pack) Right reset
            validateCellConditions slice.cellRun descriptor.name machines healthDocument resetDocument
            executionWindow <- either (Left . Text.pack) Right interval
            windows <- traverse (\(role, machine, image, value) -> (role,machine,image,) <$> either (Left . Text.pack) Right value) raw
            validateCellObservations slice.cellRun slice.submission.requires.minAgentVersion healthPolicy healthDocument executionWindow windows
        else pure (Left "outer cell is missing sealed health or reset evidence")
    effective <- eitherDecodeFileStrict' (tree </> "output" </> Text.unpack (renderRunId run.runId) </> "run-spec.json") :: IO (Either String RunSpec)
    pure do
      index <- either (Left . Text.pack . show) Right verified
      conditions
      spec <- either (Left . Text.pack) Right effective
      unless (spec.environment.machineProfile == Just (cellMachineProfile descriptor)) (Left "effective machine profile differs from the frozen cell descriptor")
      unless
        ( index.cellOutcome == Completed
            && index.leaseId == journal.leaseId
            && index.cell == journal.cell
            && index.cellRun == slice.cellRun
            && slice.manifestSha256 == Just index.cellManifest.sha256
        )
        (Left "outer cell identity, seal or health differs from its journal")
      pure (tree </> "output" </> Text.unpack (renderRunId run.runId))
  _ -> pure (Left "cell trial is incomplete, rejected or has unexpected runs")