packages feed

kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Prepare.hs

module Kenshou.Remote.Cell.Prepare
  ( Granularity (..),
    RejectReason (..),
    PrepareOptions (..),
    Prepared (..),
    Routed (..),
    PreparedRun (..),
    Slice (..),
    OtlpSink (..),
    SubmissionInputs (..),
    sliceRuns,
    slicePlan,
    submissionFor,
    prepareForCell,
    routePlan,
    routePlanWithoutPayload,
    cellMachineProfile,
  )
where

import Data.Aeson (eitherDecode, encode)
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Int (Int64)
import Data.List (find)
import Data.List.NonEmpty (NonEmpty (..))
import Data.List.NonEmpty qualified as NonEmpty
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe, isNothing)
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Bundle (Registry, lookupResource, lookupScenario)
import Kenshou.Core.Canonical (sha256Hex)
import Kenshou.Core.Cohort (CohortIdentity (..), PlanHash (..))
import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), resolveDimensions)
import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))
import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), renderRunId)
import Kenshou.Core.Resource (CellBinding (..), ResourceName, ResourceProvider (..))
import Kenshou.Core.Resource.Types (legacyKafkaName)
import Kenshou.Core.RunSpec (CohortExpectation (..), ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..))
import Kenshou.Core.Scenario (Placement (..), Scenario (..))
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..), TrialInfo (..))
import Kenshou.Remote.Cell.Docs (BrokerReset (..), CachePolicy (..), CellDescriptor (..), CellImages (..), CollectOptions (..), Limits (..), PgReset (..), Requirements (..), ResetBlock (..), Submission (..), WorkObject (..))
import Kenshou.Remote.Cell.Lease (CellRef (..))
import Kenshou.Remote.Cell.Legacy (legacyBrokerAvailable, legacyBrokerReset)
import Kenshou.Remote.Cell.RouteRules (CellCapabilities (..), RoutingRule (..), capabilityKnown, descriptorDigest, matchingRules)
import Kenshou.Remote.Payload (Bundle (..), CellPayload (..), PayloadDescriptor (..))
import Kenshou.Remote.Store (Bucket (..))

cellMachineProfile :: CellDescriptor -> Text
cellMachineProfile descriptor =
  "gcp/" <> descriptor.zone <> "/shape-" <> Text.take 16 (Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor.shape)))) <> "/images-" <> Text.take 16 (Text.drop 7 (sha256Hex (LazyByteString.toStrict (encode descriptor.images)))) <> "/pg" <> Text.pack (show descriptor.postgresMajor)

data Granularity = GranularityAuto | GranularityPlan | GranularityRun
  deriving stock (Eq, Show)

data RejectReason
  = PlacementLocalOnly
  | UnknownScenario
  | PgVersionMismatch Int Int
  | DurabilityNotDurable
  | NeedsServerControl Text
  | ResourceUnavailable ResourceName
  | PayloadLabelUnknown Text
  | ConflictingPostgresSetting Text
  | CollidingExtraPostgresName Text
  | RunIdMismatch
  | InvalidTimeout
  | MissingCapability Text
  | UnsupportedDimensions Text
  deriving stock (Eq, Show)

data PrepareOptions = PrepareOptions
  { coerceDurable :: !Bool,
    ephemeralOnDriver :: !Bool,
    pgSettings :: ![(Text, Text)],
    cachePolicy :: !CachePolicy
  }
  deriving stock (Eq, Show)

data Prepared = Prepared
  { accepted :: ![PreparedRun],
    rejected :: ![(PlannedRun, RejectReason)],
    warnings :: ![Text]
  }
  deriving stock (Eq, Show)

data Routed = Routed
  { perCell :: !(Map.Map Text RunPlan),
    local :: !(Maybe RunPlan),
    unroutable :: ![(PlannedRun, RejectReason)],
    warnings :: ![Text]
  }
  deriving stock (Eq, Show)

data PreparedRun = PreparedRun
  { ordinal :: !Int,
    runId :: !RunId,
    spec :: !RunSpec,
    payloadLabel :: !Text,
    reset :: !ResetBlock,
    timeoutSeconds :: !Int
  }
  deriving stock (Eq, Show)

data Slice = Slice
  { index :: !Int,
    payloadLabel :: !Text,
    reset :: !ResetBlock,
    entries :: !(NonEmpty PreparedRun),
    wallClockSeconds :: !Int
  }
  deriving stock (Eq, Show)

data OtlpSink = NullSink | FileSink
  deriving stock (Eq, Show)

data SubmissionInputs = SubmissionInputs
  { cellRun :: !RunId,
    leaseId :: !RunId,
    sessionId :: !RunId,
    planId :: !RunId,
    otlpSink :: !OtlpSink,
    rtsOptions :: !(Maybe Text),
    memoryMaxBytes :: !Int64,
    outputMaxBytes :: !Int64,
    minAgentVersion :: !Text,
    requiredCapabilities :: !(Maybe [Text])
  }
  deriving stock (Eq, Show)

-- Preflight a plan before it is sliced. Nothing is dropped silently: every
-- rejected entry remains paired with its original run identity.
prepareForCell :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Map.Map Text PayloadDescriptor -> PrepareOptions -> RunPlan -> Prepared
prepareForCell registry descriptor cachedCapabilities routingRules payloads options plan =
  prepareWithPayload registry descriptor cachedCapabilities routingRules (Just payloads) options plan

prepareWithPayload :: Registry -> CellDescriptor -> Maybe CellCapabilities -> [RoutingRule] -> Maybe (Map.Map Text PayloadDescriptor) -> PrepareOptions -> RunPlan -> Prepared
prepareWithPayload registry descriptor cachedCapabilities routingRules payloads options plan =
  Prepared
    [prepared | Right (prepared, _) <- results]
    [(entry, reason) | (entry, Left reason) <- zip plan.runs results]
    (concat [notices | Right (_, notices) <- results])
  where
    results = fmap prepare plan.runs
    activeCapabilities = case cachedCapabilities of
      Just cache | cache.cell == descriptor.name && cache.descriptorSha256 == descriptorDigest descriptor -> Just cache
      _ -> Nothing
    profile = cellMachineProfile descriptor

    prepare entry = do
      if maybe True (== entry.runId) entry.spec.runId then pure () else Left RunIdMismatch
      scenario <- maybe (Left UnknownScenario) Right (lookupScenario registry entry.spec.scenario)
      if scenario.placement == PlaceLocal then Left PlacementLocalOnly else pure ()
      let label = maybe "default" (\trial -> if maybe False (Map.member trial.arm) payloads then trial.arm else "default") entry.trial
      payload <- traverse (\available -> maybe (Left (PayloadLabelUnknown label)) Right (Map.lookup label available)) payloads
      let requirements = scenario.requires
          controlled = case requirements.postgres of
            Just requirement | requirement.needsServerControl -> Just "primary"
            _ -> fst <$> find (\(_, requirement) -> requirement.needsServerControl) requirements.extraPostgres
          driverLocal = maybe False (const (options.ephemeralOnDriver && entry.spec.scenario.kind `elem` [Correctness, Concurrency])) controlled
      case controlled of
        Just name | not driverLocal -> Left (NeedsServerControl name)
        _ -> pure ()
      bindings <- traverse cellBinding requirements.resources
      let rules = matchingRules routingRules entry.spec
          denied = find (\rule -> maybe False (\cache -> capabilityKnown cache rule.requires == Just False) activeCapabilities) rules
      case denied of
        Just rule -> Left (MissingCapability rule.requires)
        Nothing -> pure ()
      let originalSpec = entry.spec
          dimensions =
            if options.coerceDurable && not driverLocal && controlled == Nothing && (requirements.postgres /= Nothing || not (null requirements.extraPostgres))
              then ("pg.durability", "durable") : filter ((/= "pg.durability") . fst) originalSpec.dimensions
              else originalSpec.dimensions
      resolved <- either (Left . UnsupportedDimensions . Text.intercalate "; " . NonEmpty.toList) Right (resolveDimensions scenario.dimensions dimensions)
      let needsPostgres = requirements.postgres /= Nothing || not (null requirements.extraPostgres)
      if needsPostgres && not driverLocal && resolved.pgDurability /= Just PgDurable then Left DurabilityNotDurable else pure ()
      let requestedMajor = case resolved.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> descriptor.postgresMajor
      if needsPostgres && not driverLocal && requestedMajor /= descriptor.postgresMajor
        then Left (PgVersionMismatch requestedMajor descriptor.postgresMajor)
        else pure ()
      let extraVariables = fmap (variableName . fst) requirements.extraPostgres
      case find (\name -> length (filter (== name) extraVariables) > 1) extraVariables of
        Just name -> Left (CollidingExtraPostgresName name)
        Nothing -> pure ()
      settings <- if needsPostgres && not driverLocal then combinedSettings scenario originalSpec else Right Map.empty
      let prior = originalSpec.environment
          primary =
            if requirements.postgres == Nothing
              then Nothing
              else
                if driverLocal
                  then Just (case prior.postgres of Just ephemeral@(PostgresEphemeral _) -> ephemeral; _ -> PostgresEphemeral [])
                  else Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL"))
          extras =
            if driverLocal
              then Map.fromList [(name, case Map.lookup name prior.extraPostgres of Just ephemeral@(PostgresEphemeral _) -> ephemeral; _ -> PostgresEphemeral []) | (name, _) <- requirements.extraPostgres]
              else Map.fromList [(name, PostgresExternal (ConnFromEnv ("KENSHOU_CELL_PG_URL_" <> variableName name))) | (name, _) <- requirements.extraPostgres]
          env = EnvironmentSpec RunOnCell (Just profile) primary extras prior.resources prior.telemetry
          expectation = maybe originalSpec.cohortExpectation (\selected -> Just (CohortExpectation (Just selected.cohort) selected.cohortIdentity.identityPlanHash.unPlanHash)) payload
          labels = Map.insert "postgresPlacement" (if driverLocal then "driver-ephemeral" else "cell-server") originalSpec.labels
          spec = RunSpec (Just entry.runId) originalSpec.scenario originalSpec.scenarioRevision originalSpec.knobs dimensions originalSpec.seed originalSpec.phases originalSpec.timeoutSeconds env expectation originalSpec.comparison labels
          -- The released cell agent currently requires a PostgreSQL role reset
          -- even when the scenario itself has no database requirement.
          reset = legacyBrokerReset (ResetBlock options.cachePolicy (Just (PgReset descriptor.postgresMajor [] settings)) Nothing (Map.fromList [(name, value) | (name, binding) <- bindings, Just value <- [binding.reset]]))
          timeout = fromMaybe (max 60 (entry.estimateMinutes * 60)) originalSpec.timeoutSeconds
          notices =
            ["coerced pg.durability=durable for " <> renderRunId entry.runId | dimensions /= originalSpec.dimensions]
              <> ["capability " <> rule.requires <> " is unprobed for " <> renderRunId entry.runId <> "; run kenshou cell probe" | rule <- rules, maybe True (\cache -> capabilityKnown cache rule.requires == Nothing) activeCapabilities]
              <> ["payload not selected for " <> renderRunId entry.runId <> "; cohort expectation will be set at submission" | isNothing payloads]
      if timeout <= 0 then Left InvalidTimeout else Right (PreparedRun entry.ordinal entry.runId spec label reset timeout, notices)
    -- A required resource is available on the cell when its provider can bind it
    -- there and the cell reports the binding's capability; a cell broker image
    -- still counts for the resource the legacy broker fields describe.
    cellBinding name = do
      binding <- maybe (Left (ResourceUnavailable name)) Right (lookupResource registry name >>= (.cell))
      let reported = maybe Nothing (\cache -> capabilityKnown cache binding.capability) activeCapabilities
          available = reported == Just True || (name == legacyKafkaName && reported /= Just False && legacyBrokerAvailable descriptor)
      if available then Right (name, binding) else Left (ResourceUnavailable name)

    combinedSettings scenario spec = foldl add (Right Map.empty) sources
      where
        -- The cell image owns preloaded libraries. The external PostgreSQL
        -- adapter verifies required libraries against the running server.
        required = filter ((/= "shared_preload_libraries") . fst) (maybe [] (.settings) scenario.requires.postgres <> concatMap ((.settings) . snd) scenario.requires.extraPostgres)
        primary = case spec.environment.postgres of Just (PostgresEphemeral values) -> values; _ -> []
        extras = concatMap (\value -> case value of PostgresEphemeral values -> values; _ -> []) (Map.elems spec.environment.extraPostgres)
        sources = [("fsync", "on"), ("synchronous_commit", "on"), ("full_page_writes", "on")] <> required <> primary <> extras <> options.pgSettings
        add previous (name, value) = do
          settings <- previous
          case Map.lookup name settings of
            Nothing -> Right (Map.insert name value settings)
            Just existing | existing == value -> Right settings
            _ -> Left (ConflictingPostgresSetting name)

variableName :: Text -> Text
variableName = Text.map (\character -> if character == '-' then '_' else character) . Text.toUpper

-- Prefer the caller's cell order. A rejected run is either retained for local
-- execution or named as unroutable; its original identity never disappears.
routePlan :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Map.Map Text PayloadDescriptor -> PrepareOptions -> RunPlan -> Routed
routePlan registry cells routingRules payloads options plan = routePlanWith registry cells routingRules (Just payloads) options plan

routePlanWithoutPayload :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> PrepareOptions -> RunPlan -> Routed
routePlanWithoutPayload registry cells routingRules options plan = routePlanWith registry cells routingRules Nothing options plan

routePlanWith :: Registry -> NonEmpty (CellDescriptor, Maybe CellCapabilities) -> [RoutingRule] -> Maybe (Map.Map Text PayloadDescriptor) -> PrepareOptions -> RunPlan -> Routed
routePlanWith registry cells routingRules payloads options plan =
  Routed (Map.map makePlan assigned) (if null localRuns then Nothing else Just (makePlan localRuns)) rejectedRuns notices
  where
    choices = fmap routeEntry plan.runs
    assigned = foldl addCell Map.empty [(name, selected) | Right (name, selected, _) <- choices]
    localRuns = [entry | Left (entry, reason) <- choices, localReason reason]
    rejectedRuns = [(entry, reason) | Left (entry, reason) <- choices, not (localReason reason)]
    notices = concat [messages | Right (_, _, messages) <- choices]

    routeEntry entry = choose (NonEmpty.toList cells) Nothing
      where
        choose [] firstFailure = Left (entry, fromMaybe UnknownScenario firstFailure)
        choose ((descriptor, capabilities) : rest) firstFailure =
          let one = RunPlan plan.planId plan.createdAt plan.context plan.policy [entry] [] entry.estimateMinutes
              prepared = prepareWithPayload registry descriptor capabilities routingRules payloads options one
           in case prepared.accepted of
                [accepted] -> Right (descriptor.name, preparedEntry accepted, prepared.warnings)
                _ -> case prepared.rejected of
                  [(_, reason)] -> choose rest (Just (fromMaybe reason firstFailure))
                  _ -> choose rest (Just (fromMaybe UnknownScenario firstFailure))
        preparedEntry accepted = PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial accepted.spec

    addCell grouped (name, entry) = Map.insertWith (flip (<>)) name [entry] grouped
    makePlan entries = RunPlan plan.planId plan.createdAt plan.context plan.policy entries plan.skipped (sum (fmap (.estimateMinutes) entries))
    localReason PlacementLocalOnly = True
    localReason (NeedsServerControl _) = True
    localReason _ = False

sliceRuns :: Granularity -> [PreparedRun] -> Either Text [Slice]
sliceRuns granularity prepared = do
  if all valid prepared && unique (fmap (.runId) prepared) && unique (fmap (.ordinal) prepared)
    then pure ()
    else Left "prepared runs have an empty payload, invalid timeout, duplicate identity or invalid ordinal"
  groups <- case granularity of
    GranularityRun -> pure (fmap (:| []) prepared)
    GranularityAuto -> pure (autoGroups prepared)
    GranularityPlan -> case NonEmpty.nonEmpty prepared of
      Nothing -> pure []
      Just group
        | all (compatible (NonEmpty.head group)) (NonEmpty.tail group) -> pure [group]
        | otherwise -> Left "plan granularity requires one payload and reset for every run"
  traverse makeSlice (zip [0 ..] groups)
  where
    valid run = run.ordinal >= 0 && run.timeoutSeconds > 0 && not (Text.null run.payloadLabel) && maybe True (== run.runId) run.spec.runId
    makeSlice (index, group) =
      let runBudget :: PreparedRun -> Integer
          runBudget run = toInteger run.timeoutSeconds + if run.spec.scenario.kind == Soak then 600 else 0
          -- Soaks drain delayed work and compute resource verdicts after load ends.
          -- Finding 49 records a full inbox run exhausting the old shared margin.
          total = 300 + sum (fmap runBudget (NonEmpty.toList group))
       in if total > toInteger (maxBound :: Int)
            then Left "cell slice wall-clock limit is too large"
            else Right (Slice index (NonEmpty.head group).payloadLabel (NonEmpty.head group).reset group (fromInteger total))

slicePlan :: RunPlan -> Slice -> Either Text RunPlan
slicePlan plan slice = do
  selected <- traverse select (NonEmpty.toList slice.entries)
  pure (RunPlan plan.planId plan.createdAt plan.context plan.policy selected plan.skipped (sum (fmap (.estimateMinutes) selected)))
  where
    select prepared = case find ((== prepared.runId) . (.runId)) plan.runs of
      Nothing -> Left "slice contains a run absent from its plan"
      Just original
        | original.ordinal /= prepared.ordinal -> Left "slice ordinal differs from its plan"
        | original.spec.scenario /= prepared.spec.scenario -> Left "slice scenario differs from its plan"
        | otherwise -> Right (PlannedRun original.ordinal original.runId original.estimateMinutes original.reasons original.trial prepared.spec)

submissionFor :: CellRef -> SubmissionInputs -> PayloadDescriptor -> Slice -> WorkObject -> Either Text Submission
submissionFor ref inputs descriptor slice work = do
  case eitherDecode (encode descriptor) :: Either String PayloadDescriptor of
    Left failure -> Left ("invalid payload descriptor: " <> Text.pack failure)
    Right _ -> pure ()
  case eitherDecode (encode work) :: Either String WorkObject of
    Left failure -> Left ("invalid work object: " <> Text.pack failure)
    Right _ -> pure ()
  if work.mediaType /= "application/json" || work.bytes <= 0
    then Left "cell work must be a nonempty JSON run plan"
    else pure ()
  let bundle = descriptor.cell.bundle
      expectedPrefix = "gs://" <> ref.controlBucket.unBucket <> "/payloads/sha256/"
  if not (expectedPrefix `Text.isPrefixOf` bundle.uri)
    then Left "payload bundle is outside the cell control bucket"
    else pure ()
  if inputs.memoryMaxBytes <= 0 || inputs.outputMaxBytes <= 0 || Text.null inputs.minAgentVersion || maybe False Text.null inputs.rtsOptions
    then Left "invalid cell submission limits or options"
    else pure ()
  let env =
        Map.fromList
          ( [ ("KENSHOU_PAYLOAD_BUNDLE_SHA256", bundle.sha256),
              ("KENSHOU_PAYLOAD_STORE_PATH", descriptor.cell.storePath),
              ("KENSHOU_PAYLOAD_NAR_HASH", descriptor.cell.narHash),
              ("KENSHOU_PAYLOAD_COHORT", descriptor.cohort),
              ("KENSHOU_OTLP_SINK", case inputs.otlpSink of NullSink -> "null"; FileSink -> "file")
            ]
              <> maybe [] (\options -> [("GHCRTS", options)]) inputs.rtsOptions
          )
      labels =
        Map.fromList
          [ ("session", renderRunId inputs.sessionId),
            ("plan", renderRunId inputs.planId),
            ("slice", Text.pack (show slice.index)),
            ("payload", slice.payloadLabel)
          ]
      profileCollection = maybe False (any profileFlag . Text.words) inputs.rtsOptions
      traceCollection = inputs.otlpSink == FileSink
      collection = if profileCollection || traceCollection then Just (CollectOptions traceCollection profileCollection) else Nothing
      submission = Submission inputs.cellRun inputs.leaseId descriptor.cell work env slice.reset (Limits (fromIntegral slice.wallClockSeconds) inputs.memoryMaxBytes inputs.outputMaxBytes) (Requirements inputs.minAgentVersion inputs.requiredCapabilities) collection labels
  case eitherDecode (encode submission) :: Either String Submission of
    Left failure -> Left ("invalid cell submission: " <> Text.pack failure)
    Right _ -> Right submission
  where
    profileFlag flag = any (`Text.isPrefixOf` flag) ["-p", "-h", "-l"]

autoGroups :: [PreparedRun] -> [NonEmpty PreparedRun]
autoGroups [] = []
autoGroups (first : rest)
  | isolated first = (first :| []) : autoGroups rest
  | otherwise =
      let (same, later) = span (\candidate -> not (isolated candidate) && compatible first candidate) rest
       in (first :| same) : autoGroups later

compatible :: PreparedRun -> PreparedRun -> Bool
compatible first second = first.payloadLabel == second.payloadLabel && first.reset == second.reset

isolated :: PreparedRun -> Bool
isolated run = run.spec.scenario.kind `elem` [Benchmark, Soak]

unique :: (Eq value) => [value] -> Bool
unique [] = True
unique (first : rest) = first `notElem` rest && unique rest