kenshou-remote-0.1.0.0: src/Kenshou/Remote/Cell/Exec.hs
module Kenshou.Remote.Cell.Exec (resolveOnCell, resolveWorkJson, cellExec) where
import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)
import Control.Monad (when)
import Data.Aeson (Value (..), encode, object, (.=))
import Data.Aeson qualified as Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Foldable (toList)
import Data.List.NonEmpty qualified as NonEmpty
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as TextIO
import Kenshou.Core.Bundle (Registry, lookupScenario)
import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..), TracingArm (..), resolveDimensions)
import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))
import Kenshou.Core.Id (Kind (..), RunId, ScenarioId (..), parseRunId, renderScenarioId)
import Kenshou.Core.Knob (KnobSpec (..), RawKnob (..), mkKnobName)
import Kenshou.Core.Resource.Types (legacyKafkaName, renderResourceName)
import Kenshou.Core.RunSpec (ConnectionSource (..), EnvironmentSpec (..), PostgresSpec (..), RunSpec (..), SpecPlacement (..))
import Kenshou.Core.Scenario (Placement (..), Scenario (..))
import Kenshou.Plan.Execute (ExecuteOptions (..), executePlan)
import Kenshou.Plan.RunPlan (PlannedRun (..), RunPlan (..))
import Kenshou.Plan.Summary (summaryExitCode)
import Kenshou.Remote.Cell.Docs (CellBroker (..), CellEnvironment (..), CellPostgres (..), OtlpEndpoint (..), OtlpSinks (..))
import Kenshou.Remote.Cell.Health (withCellHealthFile)
import Kenshou.Remote.Cell.Legacy (brokerAsResource)
import Kenshou.Remote.Cell.Prepare (OtlpSink (..))
import System.Directory (Permissions (..), createDirectoryIfMissing, doesFileExist, getPermissions)
import System.Environment (lookupEnv, setEnv, unsetEnv)
import System.Exit (ExitCode (..))
import System.FilePath ((</>))
import System.IO (stderr)
import System.Process (readProcessWithExitCode)
-- Bind the generic environment in a prepared work plan to endpoints available
-- on this cell. Run IDs and plan metadata are preserved; no run is silently
-- dropped here.
resolveOnCell :: Registry -> CellEnvironment -> OtlpSink -> RunPlan -> Either Text RunPlan
resolveOnCell registry environment sink plan = do
runs <- traverse resolveRun plan.runs
pure (RunPlan plan.planId plan.createdAt plan.context plan.policy runs plan.skipped plan.estimateMinutes)
where
resolveRun entry = do
resolved <- resolveSpec registry environment sink entry.spec
pure (PlannedRun entry.ordinal entry.runId entry.estimateMinutes entry.reasons entry.trial resolved)
-- The executor accepts a narrow run-plan JSON document rather than decoding
-- every planning provenance type. Rewrite just the spec member of each entry.
resolveWorkJson :: Registry -> CellEnvironment -> OtlpSink -> Value -> Either Text Value
resolveWorkJson registry environment sink work = fst <$> resolveWorkDocument registry environment sink work
resolveWorkDocument :: Registry -> CellEnvironment -> OtlpSink -> Value -> Either Text (Value, [RunSpec])
resolveWorkDocument registry environment sink work = case work of
Object fields -> case (KeyMap.lookup "schema" fields, KeyMap.lookup "runs" fields) of
(Just (String "kenshou.run-plan/v1"), Just (Array runs)) -> do
resolved <- traverse resolveEntry runs
pure (Object (KeyMap.insert "runs" (Array (fmap fst resolved)) fields), toList (fmap snd resolved))
_ -> Left "cell work is not a kenshou.run-plan/v1 document"
_ -> Left "cell work is not a JSON object"
where
resolveEntry (Object entry) = do
identifier <- case KeyMap.lookup "runId" entry of
Just value -> (decodeValue "planned run ID" value :: Either Text RunId)
Nothing -> Left "cell work entry has no run ID"
spec <- case KeyMap.lookup "spec" entry of
Just value -> (decodeValue "planned run spec" value :: Either Text RunSpec)
Nothing -> Left "cell work entry has no run spec"
if maybe True (== identifier) spec.runId
then pure ()
else Left "cell work entry and spec have different run IDs"
resolved <- resolveSpec registry environment sink spec
pure (Object (KeyMap.insert "spec" (Aeson.toJSON resolved) entry), resolved)
resolveEntry _ = Left "cell work run is not an object"
decodeValue :: (Aeson.FromJSON document) => Text -> Value -> Either Text document
decodeValue label value = case Aeson.fromJSON value of
Aeson.Error failure -> Left (label <> ": " <> Text.pack failure)
Aeson.Success document -> Right document
resolveSpec :: Registry -> CellEnvironment -> OtlpSink -> RunSpec -> Either Text RunSpec
resolveSpec registry environment sink spec = do
scenario <- maybe (Left ("unknown cell scenario: " <> renderScenarioId spec.scenario)) Right (lookupScenario registry spec.scenario)
if scenario.placement == PlaceLocal
then Left ("scenario is local-only: " <> renderScenarioId scenario.id)
else pure ()
dimensions <- either (Left . Text.intercalate "; " . NonEmpty.toList) Right (resolveDimensions scenario.dimensions spec.dimensions)
postgres <- bindPostgres scenario dimensions
extras <- bindExtras scenario postgres
resources <- Map.fromList <$> traverse (bindResource scenario) scenario.requires.resources
telemetryKnobs <- bindOtlp scenario dimensions
knobs <- bindRemoteScenario telemetryKnobs
let prior = spec.environment
cellEnvironment = EnvironmentSpec RunOnCell prior.machineProfile postgres extras resources prior.telemetry
pure (RunSpec spec.runId spec.scenario spec.scenarioRevision knobs spec.dimensions spec.seed spec.phases spec.timeoutSeconds cellEnvironment spec.cohortExpectation spec.comparison spec.labels)
where
bindPostgres scenario dimensions = case scenario.requires.postgres of
Nothing -> pure spec.environment.postgres
Just requirement
| requirement.needsServerControl ->
if scenario.id.kind `elem` [Correctness, Concurrency]
then case spec.environment.postgres of
Just ephemeral@(PostgresEphemeral _) -> pure (Just ephemeral)
_ -> Left ("scenario needs driver-local PostgreSQL server control: " <> renderScenarioId scenario.id)
else Left ("benchmark or soak cannot control the cell PostgreSQL server: " <> renderScenarioId scenario.id)
| otherwise -> do
if dimensions.pgDurability /= Just PgDurable
then Left ("cell run requires pg.durability=durable: " <> renderScenarioId scenario.id)
else pure ()
let requested = case dimensions.pgVersion of Just Pg17 -> 17; Just Pg18 -> 18; Nothing -> 0
if requested /= environment.postgres.major
then Left ("cell PostgreSQL major differs from pg.version for " <> renderScenarioId scenario.id)
else pure (Just (PostgresExternal (ConnFromEnv "KENSHOU_CELL_PG_URL")))
bindExtras scenario primary =
if case primary of Just (PostgresEphemeral _) -> True; _ -> False
then Map.fromList <$> traverse driverExtra scenario.requires.extraPostgres
else do
let requirements = scenario.requires.extraPostgres
variableNames = fmap (variableName . fst) requirements
if any (.needsServerControl) (fmap snd requirements)
then Left ("extra PostgreSQL server needs driver-local control: " <> renderScenarioId scenario.id)
else pure ()
if length variableNames /= length (unique variableNames)
then Left "extra PostgreSQL names collide as environment variables"
else pure (Map.fromList [(name, PostgresExternal (ConnFromEnv ("KENSHOU_CELL_PG_URL_" <> variableName name))) | (name, _) <- requirements])
where
driverExtra (name, _) = case Map.lookup name spec.environment.extraPostgres of
Nothing -> pure (name, PostgresEphemeral [])
Just local@(PostgresEphemeral _) -> pure (name, local)
Just (PostgresExternal _) -> Left ("driver-local PostgreSQL requires every extra server to be ephemeral: " <> name)
-- The cell environment binds a resource by name; the legacy broker fields still
-- bind the resource they always described.
bindResource scenario name = case (Map.lookup name environment.resources, environment.broker) of
(Just value, _) -> pure (name, value)
(Nothing, Just broker) | name == legacyKafkaName -> (name,) <$> brokerAsResource broker
_ -> Left ("cell has no resource " <> renderResourceName name <> " for " <> renderScenarioId scenario.id)
bindOtlp scenario dimensions
| dimensions.tracing /= Just TracingSdkOtlp = pure spec.knobs
| otherwise = do
name <- mkKnobName "otel.endpoint"
if name `notElem` fmap (.name) scenario.knobs
then Left ("tracing scenario lacks otel.endpoint: " <> renderScenarioId scenario.id)
else pure ()
sinks <- maybe (Left "cell has no OTLP sink") Right environment.otlp
let endpoint = case sink of NullSink -> sinks.nullEndpoint.http; FileSink -> sinks.fileEndpoint.http
if Text.null endpoint
then Left "cell OTLP sink has no HTTP endpoint"
else pure ((name, RawText endpoint) : filter ((/= name) . fst) spec.knobs)
bindRemoteScenario knobs
| renderScenarioId spec.scenario /= "selftest/remote/correctness/cell-environment" = pure knobs
| otherwise = do
otlpName <- mkKnobName "remote.otlp-endpoint"
let selectedOtlp = case environment.otlp of
Nothing -> ""
Just sinks -> case sink of NullSink -> sinks.nullEndpoint.http; FileSink -> sinks.fileEndpoint.http
pure (assignWhenBlank otlpName selectedOtlp knobs)
assignWhenBlank name value knobs
| Text.null value || maybe False (not . blank) (lookup name knobs) = knobs
| otherwise = assign name value knobs
blank (RawText value) = Text.null value
blank (RawJson (String value)) = Text.null value
blank _ = False
assign name value knobs = (name, RawText value) : filter ((/= name) . fst) knobs
variableName :: Text -> Text
variableName = Text.map (\character -> if character == '-' then '_' else character) . Text.toUpper
unique :: (Eq value) => [value] -> [value]
unique [] = []
unique (first : rest) = first : unique (filter (/= first) rest)
-- The generic cell agent invokes this command on the first driver with the
-- submitted work and an environment document. Every child run receives the
-- resolved plan, the cell context and the same payload identity variables.
cellExec :: Registry -> FilePath -> FilePath -> IO ExitCode
cellExec registry workFile outDir = do
result <- try (executeOnDriver registry workFile outDir) :: IO (Either SomeException ExitCode)
case result of
Left failure | Just async <- (fromException failure :: Maybe SomeAsyncException) -> throwIO async
Left failure -> adapterFailure outDir (Text.pack (displayException failure))
Right code -> pure code
executeOnDriver :: Registry -> FilePath -> FilePath -> IO ExitCode
executeOnDriver registry workFile outDir = do
driverIndex <- integerEnv "CELL_DRIVER_INDEX" 0
driverCount <- integerEnv "CELL_DRIVER_COUNT" 1
when (driverIndex >= driverCount) (ioError (userError "CELL_DRIVER_INDEX exceeds CELL_DRIVER_COUNT"))
let cellDirectory = outDir </> "kenshou-cell"
createDirectoryIfMissing True cellDirectory
if driverIndex /= 0
then do
LazyByteString.writeFile (outDir </> "idle-driver.json") (encode (object ["schema" .= ("kenshou.cell-idle-driver/v1" :: Text), "index" .= driverIndex, "count" .= driverCount]))
pure ExitSuccess
else do
environmentPath <- requiredEnv "CELL_ENV_FILE"
environment <- Aeson.eitherDecodeFileStrict' environmentPath >>= either (ioError . userError . ("invalid CELL_ENV_FILE: " <>)) pure
cellRun <- requiredEnv "CELL_RUN_ID" >>= either (ioError . userError . Text.unpack) pure . parseRunId . Text.pack
when (cellRun /= environment.runId) (ioError (userError "CELL_RUN_ID differs from the cell environment"))
scratch <- requiredEnv "CELL_SCRATCH_DIR"
sink <-
requiredEnv "KENSHOU_OTLP_SINK" >>= \case
"null" -> pure NullSink
"file" -> pure FileSink
_ -> ioError (userError "KENSHOU_OTLP_SINK must be null or file")
identity <- requiredEnv "KENSHOU_COHORT_IDENTITY"
present <- doesFileExist identity
when (not present) (ioError (userError "payload cohort identity file is missing"))
payload <- requiredPayloadIdentity
work <- Aeson.eitherDecodeFileStrict' workFile >>= either (ioError . userError . ("invalid cell work: " <>)) pure
(resolved, specs) <- either (ioError . userError . Text.unpack) pure (resolveWorkDocument registry environment sink work)
let planPath = cellDirectory </> "plan.resolved.json"
contextPath = cellDirectory </> "context.json"
noticePath = cellDirectory </> "health-notices.jsonl"
cachePath = scratch </> "cache"
tempPath = scratch </> "tmp"
createDirectoryIfMissing True cachePath
createDirectoryIfMissing True tempPath
LazyByteString.writeFile planPath (encode resolved)
LazyByteString.writeFile noticePath ""
setEnv "KENSHOU_CELL_PG_URL" (Text.unpack environment.postgres.connectionString)
mapM_ (setExtraPostgres environment.postgres.connectionString) specs
setEnv "KENSHOU_CELL_FINGERPRINT" contextPath
setEnv "KENSHOU_HEALTH_NOTICES" noticePath
setEnv "XDG_CACHE_HOME" cachePath
setEnv "TMPDIR" tempPath
faultEnabled <- case environment.faultHook of
Nothing -> unsetEnv "KENSHOU_CELL_FAULT_HOOK" >> pure False
Just hook -> do
available <- doesFileExist (Text.unpack hook)
executableHook <- if available then (.executable) <$> getPermissions (Text.unpack hook) else pure False
if executableHook then setEnv "KENSHOU_CELL_FAULT_HOOK" (Text.unpack hook) else unsetEnv "KENSHOU_CELL_FAULT_HOOK"
pure executableHook
pgPartman <- probePgPartman environment.postgres.connectionString
LazyByteString.writeFile contextPath (encode (cellContext environment driverIndex driverCount sink faultEnabled pgPartman payload))
if driverCount > 1
then setEnv "KENSHOU_CLOCK_SKEW_BOUND_MICROS" (show (maybe 50000 id environment.clockSkewBoundMicros))
else unsetEnv "KENSHOU_CLOCK_SKEW_BOUND_MICROS"
healthFile <- lookupEnv "CELL_HEALTH_FILE"
let runPlan = executePlan (ExecuteOptions planPath outDir False False Nothing Nothing)
withHealth = withCellHealthFile healthFile noticePath cellRun runPlan
summary <- if faultEnabled then healAll environment.faultHook >> withHealth `finally` healAll environment.faultHook else withHealth
pure (summaryExitCode summary)
healAll :: Maybe Text -> IO ()
healAll Nothing = pure ()
healAll (Just hook) = do
(code, _, errors) <- readProcessWithExitCode (Text.unpack hook) ["heal-all"] ""
when (code /= ExitSuccess) (ioError (userError ("cell fault hook heal-all failed: " <> errors)))
setExtraPostgres :: Text -> RunSpec -> IO ()
setExtraPostgres connection spec = mapM_ setOne (Map.elems spec.environment.extraPostgres)
where
setOne (PostgresExternal (ConnFromEnv name)) | "KENSHOU_CELL_PG_URL_" `Text.isPrefixOf` name = setEnv (Text.unpack name) (Text.unpack connection)
setOne _ = pure ()
probePgPartman :: Text -> IO (Maybe Text)
probePgPartman connection = do
(code, output, errors) <- readProcessWithExitCode "psql" ["-d", Text.unpack connection, "-Atqc", "SELECT COALESCE((SELECT default_version FROM pg_available_extensions WHERE name = 'pg_partman'), '')"] ""
case code of
ExitSuccess -> pure case Text.strip (Text.pack output) of
"" -> Nothing
version -> Just version
_ -> ioError (userError ("PostgreSQL extension probe failed: " <> errors))
cellContext :: CellEnvironment -> Int -> Int -> OtlpSink -> Bool -> Maybe Text -> Map.Map Text Text -> Value
cellContext environment driverIndex driverCount sink faultEnabled pgPartman payload =
object
[ "schema" .= ("kenshou.cell-context/v1" :: Text),
"cell" .= environment.cell,
"cellRun" .= environment.runId,
"leaseId" .= environment.leaseId,
"driver" .= object ["index" .= driverIndex, "count" .= driverCount],
"postgres" .= object ["major" .= environment.postgres.major, "host" .= environment.postgres.host, "port" .= environment.postgres.port, "placement" .= ("cell-server" :: Text), "extraPostgres" .= ("shared-server" :: Text), "extensions" .= object ["pg_partman" .= object ["available" .= maybe False (const True) pgPartman, "version" .= pgPartman]]],
"broker" .= environment.broker,
"capabilities"
.= object
[ "postgres.major" .= Text.pack (show environment.postgres.major),
"postgres.pg_partman" .= maybe False (const True) pgPartman,
"postgres.second-server" .= False,
"postgres.control-hook" .= False,
"broker" .= maybe "none" (\broker -> broker.implementation <> " " <> broker.version) environment.broker,
"otlp.null" .= maybe False (not . Text.null . (.nullEndpoint.http)) environment.otlp,
"otlp.file" .= maybe False (not . Text.null . (.fileEndpoint.http)) environment.otlp,
"fault-hook" .= faultEnabled
],
"otlpSink" .= (case sink of NullSink -> "null" :: Text; FileSink -> "file"),
"faultHook" .= faultEnabled,
"clock" .= object ["skewBoundMicros" .= environment.clockSkewBoundMicros],
"payload"
.= object
[ "bundleSha256" .= field "KENSHOU_PAYLOAD_BUNDLE_SHA256",
"storePath" .= field "KENSHOU_PAYLOAD_STORE_PATH",
"narHash" .= field "KENSHOU_PAYLOAD_NAR_HASH",
"cohort" .= field "KENSHOU_PAYLOAD_COHORT",
"harness" .= object ["revision" .= field "KENSHOU_HARNESS_REVISION", "dirty" .= (field "KENSHOU_HARNESS_DIRTY" == "true")]
]
]
where
field name = Map.findWithDefault "" name payload
requiredPayloadIdentity :: IO (Map.Map Text Text)
requiredPayloadIdentity = do
values <- Map.fromList <$> traverse load (filter (/= "KENSHOU_COHORT_IDENTITY") payloadIdentityNames)
when (Map.lookup "KENSHOU_HARNESS_DIRTY" values `notElem` [Just "true", Just "false"]) (ioError (userError "KENSHOU_HARNESS_DIRTY must be true or false"))
pure values
where
load name = (name,) . Text.pack <$> requiredEnv (Text.unpack name)
requiredEnv :: String -> IO String
requiredEnv name =
lookupEnv name >>= \case
Just value | not (null value) -> pure value
_ -> ioError (userError (name <> " is required by the cell payload"))
integerEnv :: String -> Int -> IO Int
integerEnv name fallback =
lookupEnv name >>= \case
Nothing -> pure fallback
Just raw -> case reads raw of
[(value, "")] | value >= 0 -> pure value
_ -> ioError (userError (name <> " must be a non-negative integer"))
adapterFailure :: FilePath -> Text -> IO ExitCode
adapterFailure outDir detail = do
let directory = outDir </> "kenshou-cell"
reason :: Text
reason
| "CELL_ENV_FILE" `Text.isInfixOf` detail = "cell-environment-missing"
| any (`Text.isInfixOf` detail) payloadIdentityNames = "payload-identity-missing"
| otherwise = "cell-exec-failed"
createDirectoryIfMissing True directory
LazyByteString.writeFile (directory </> "adapter-error.json") (encode (object ["schema" .= ("kenshou.cell-adapter-error/v1" :: Text), "reason" .= reason, "detail" .= detail]))
TextIO.hPutStrLn stderr ("kenshou cell exec: " <> detail)
pure (ExitFailure 4)
payloadIdentityNames :: [Text]
payloadIdentityNames =
[ "KENSHOU_COHORT_IDENTITY",
"KENSHOU_HARNESS_REVISION",
"KENSHOU_HARNESS_DIRTY",
"KENSHOU_PAYLOAD_BUNDLE_SHA256",
"KENSHOU_PAYLOAD_STORE_PATH",
"KENSHOU_PAYLOAD_NAR_HASH",
"KENSHOU_PAYLOAD_COHORT"
]