kenshou-remote-0.1.0.0: src/Kenshou/Remote/Perf/Conditions.hs
module Kenshou.Remote.Perf.Conditions (validateCellConditions, validateCellObservations, validateHealthPolicy) where
import Control.Monad (unless)
import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.Aeson.Types (Parser, parseEither)
import Data.List (nub, sort)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, addUTCTime, diffUTCTime)
import Kenshou.Core.Id (RunId)
import Kenshou.Perf.Request (strictObject)
data Sample = Sample {at :: UTCTime, total :: Integer, idle :: Integer, steal :: Integer, disk :: Double}
-- Canonical algorithm: mori://shinzui/load-testing-infra, project-relative
-- nixos/pkgs/cell-agent/src/src/health.rs and runner.rs (artifact URIs pending).
-- The immutable execution interval supplies the actual payload start; nested
-- runtime timestamps and mutable status timestamps cannot substitute for it.
validateCellObservations :: RunId -> Text -> Value -> Value -> Value -> [(Text, Text, Text, Value)] -> Either Text ()
validateCellObservations runId agentVersion policy health interval windows = either (Left . Text.pack) Right do
let roles = [role | (role, _, _, _) <- windows]
machines = [machine | (_, machine, _, _) <- windows]
unless (sort roles `elem` [sort ["driver", "postgres", "monitoring"], sort ["driver", "postgres", "monitoring", "broker"]] && nub machines == machines) (Left "raw observation role coverage is missing or duplicated")
parseEither replay interval
where
replay = withObject "execution window" \v -> do
strictObject ["schema", "runId", "resetStarted", "runStarted", "runFinished"] v
schema <- v .: "schema"
identifier <- v .: "runId"
resetStarted <- v .: "resetStarted"
started <- v .: "runStarted"
finished <- v .: "runFinished"
unless (schema == ("cell.execution-window/v1" :: Text) && identifier == runId && resetStarted <= addUTCTime (-10) started && started < finished) (fail "invalid sealed execution interval")
(healthStart, healthEnd, gates) <- withObject "health report" (\h -> (,,) <$> h .: "windowStart" <*> h .: "windowEnd" <*> h .: "gates") health
unless (healthStart == resetStarted && healthEnd == finished) (fail "health and execution intervals differ")
limits <- healthLimits policy
mapM_ (window resetStarted started finished limits gates) windows
window resetStarted started finished (busyLimit, stealLimit, startLimit, runLimit) gates (role, machine, image, document) =
withObject
"raw machine observations"
( \v -> do
strictObject ["schema", "machine", "image", "agentVersion", "systemClosure", "from", "to", "samples", "maintenanceEvents", "samplingErrors"] v
schema <- v .: "schema"
actualMachine <- v .: "machine"
actualImage <- v .: "image"
actualVersion <- v .: "agentVersion"
closure <- v .: "systemClosure"
from <- v .: "from"
to <- v .: "to"
maintenance <- v .: "maintenanceEvents" :: Parser [Value]
errors <- v .: "samplingErrors" :: Parser [Value]
unless (schema == ("cell.observations/v1" :: Text) && actualMachine == machine && actualImage == image && actualVersion == agentVersion && "/nix/store/" `Text.isPrefixOf` closure && Text.length closure > 11 && abs (diffUTCTime from resetStarted) <= 0.001 && abs (diffUTCTime to finished) <= 0.001 && null maintenance && null errors) (fail "raw machine identity, window, maintenance or sampling errors invalidate the trial")
samples <- v .: "samples" >>= traverse sample
unless (not (null samples) && all (\s -> s.at <= to) samples && and [a.at < b.at && diffUTCTime b.at a.at <= 2 && a.total < b.total && a.idle <= b.idle && a.steal <= b.steal | (a, b) <- zip samples (drop 1 samples)]) (fail "raw samples are missing, unordered, interrupted or have invalid counters")
let settle = filter (\s -> s.at >= addUTCTime (-10) started && s.at <= started) samples
running = filter (\s -> s.at >= started && s.at <= finished) samples
(firstSettle, lastSettle) <- endpoints settle
(firstRun, lastRun) <- endpoints running
unless (diffUTCTime lastSettle.at firstSettle.at >= 8 && diffUTCTime firstRun.at started <= 2 && diffUTCTime finished lastRun.at <= 2) (fail "raw settle or run sample coverage is incomplete")
busy <- fraction True firstSettle lastSettle
steals <- traverse (\(prior, latest) -> fraction False (rolling prior latest) latest) [(take i running, latest) | (i, latest) <- zip [1 ..] (drop 1 running)]
let settleBusy = if role `elem` ["monitoring", "broker"] then 0 else busy
runSteal = if role == "monitoring" then 0 else maximum (0 : steals)
startFree = firstRun.disk
runFree = minimum (map (.disk) running)
unless (settleBusy <= busyLimit && runSteal <= stealLimit && startFree >= startLimit && runFree >= runLimit) (fail "raw observations violate the frozen health policy")
checkGate gates machine "background-load" (object ["settleCpuPercent" .= settleBusy, "runStealCpuPercent" .= runSteal]) (object ["settleCpuMaxPercent" .= busyLimit, "runStealCpuMaxPercent" .= stealLimit]) finished
checkGate gates machine "storage-pressure" (object ["startFreePercent" .= startFree, "runFreePercent" .= runFree, "outputExceeded" .= False]) (object ["startFreeMinPercent" .= startLimit, "runFreeMinPercent" .= runLimit, "outputExceeded" .= False]) finished
)
document
sample = withObject "health sample" \v -> do
strictObject ["at", "cpuTotalTicks", "cpuIdleTicks", "cpuStealTicks", "diskFreePercent"] v
value <- Sample <$> v .: "at" <*> v .: "cpuTotalTicks" <*> v .: "cpuIdleTicks" <*> v .: "cpuStealTicks" <*> v .: "diskFreePercent"
unless (value.total >= 0 && value.idle >= 0 && value.steal >= 0 && value.idle + value.steal <= value.total && validPercent value.disk) (fail "invalid health counters or disk percentage")
pure value
endpoints [] = fail "missing health samples"
endpoints (first : rest) = pure (first, foldl (\_ next -> next) first rest)
rolling prior latest = case reverse (filter (\s -> diffUTCTime latest.at s.at >= 30) prior) of
first : _ -> first
[] -> case prior of first : _ -> first; [] -> latest
fraction countIdle first latest = do
let total = latest.total - first.total
counter = if countIdle then latest.idle - first.idle else latest.steal - first.steal
unless (total > 0 && counter >= 0 && counter <= total) (fail "CPU counters did not advance consistently")
pure (100 * fromIntegral (if countIdle then total - counter else counter) / fromIntegral total :: Double)
checkGate :: [Value] -> Text -> Text -> Value -> Value -> UTCTime -> Parser ()
checkGate gates machine name observed threshold finished = do
matching <- filter (\(m, n, _, _, _) -> m == machine && n == name) <$> traverse (withObject "gate" \v -> (,,,,) <$> v .: "machine" <*> v .: "name" <*> v .: "observed" <*> v .: "threshold" <*> v .: "at") gates
case matching of
[(_, _, actual, actualThreshold, at)] | near actual observed && actualThreshold == threshold && at == finished -> pure ()
_ -> fail "reported health gate differs from raw observations or frozen policy"
near (Number a) (Number b) = abs (realToFrac a - realToFrac b :: Double) <= 0.000001
near (Object a) (Object b) = a == b || (length a == length b && and [maybe False (near value) (KeyMap.lookup key b) | (key, value) <- KeyMap.toList a])
near a b = a == b
validateHealthPolicy :: Value -> Either Text ()
validateHealthPolicy = either (Left . Text.pack) (const (Right ())) . parseEither healthLimits
healthLimits :: Value -> Parser (Double, Double, Double, Double)
healthLimits = withObject "cell policy" \v -> do
schema <- v .: "schema"
unless (schema == ("cell.policy/v1" :: Text)) (fail "unsupported cell health policy")
limits <- v .: "health" >>= withObject "health limits" (\h -> (,,,) <$> h .: "settleCpuMaxPercent" <*> h .: "runStealCpuMaxPercent" <*> h .: "startFreeDiskMinPercent" <*> h .: "runFreeDiskMinPercent")
let (a, b, c, d) = limits
unless (all validPercent [a, b, c, d]) (fail "invalid frozen health thresholds")
pure limits
validPercent :: Double -> Bool
validPercent value = not (isNaN value || isInfinite value) && value >= 0 && value <= 100
-- Canonical owner: mori://shinzui/load-testing-infra, artifact URI pending;
-- project-relative nixos/pkgs/cell-agent/src/src/health.rs. Check the reported
-- observations against their thresholds, rather than trusting passed alone.
validateCellConditions :: RunId -> Text -> [Text] -> Value -> Value -> Either Text ()
validateCellConditions runId cell machines health reset = either (Left . Text.pack) Right do
parseEither healthDocument health
parseEither resetDocument reset
where
names = ["host-maintenance", "background-load", "storage-pressure", "incomplete-reset", "version-mismatch"]
healthDocument = withObject "cell health" \v -> do
schema <- v .: "schema"
identifier <- v .: "runId"
passed <- v .: "passed"
start <- v .: "windowStart" :: Parser UTCTime
end <- v .: "windowEnd" :: Parser UTCTime
unless (schema == ("cell.health/v1" :: Text) && identifier == runId && passed && start <= end && not (null machines)) (fail "cell health identity, window or outcome is invalid")
gates <- v .: "gates" >>= traverse (gate start end)
unless (sort gates == sort [(machine, name) | machine <- machines, name <- names]) (fail "cell health gate coverage is missing, duplicated or unexpected")
gate start end = withObject "cell health gate" \v -> do
machine <- v .: "machine"
name <- v .: "name"
status <- v .: "status"
at <- v .: "at" :: Parser UTCTime
observed <- v .: "observed"
threshold <- v .: "threshold"
unless (status == ("ok" :: Text) && at >= start && at <= end) (fail "cell health gate tripped or lies outside its window")
valid <- case name :: Text of
"host-maintenance" -> pure (observed == String "NONE" && threshold == String "NONE")
"incomplete-reset" -> pure (observed == Bool True && threshold == Bool True)
"version-mismatch" -> pure (observed == Bool True && threshold == Bool True)
"background-load" -> do
busy <- percent "settleCpuPercent" observed
steal <- percent "runStealCpuPercent" observed
busyLimit <- percent "settleCpuMaxPercent" threshold
stealLimit <- percent "runStealCpuMaxPercent" threshold
pure (busy <= busyLimit && steal <= stealLimit)
"storage-pressure" -> do
initial <- percent "startFreePercent" observed
final <- percent "runFreePercent" observed
initialLimit <- percent "startFreeMinPercent" threshold
finalLimit <- percent "runFreeMinPercent" threshold
exceeded <- withObject "disk observation" (.: "outputExceeded") observed
expectedExceeded <- withObject "disk threshold" (.: "outputExceeded") threshold
pure (initial >= initialLimit && final >= finalLimit && not exceeded && not expectedExceeded)
_ -> fail "unknown cell health gate"
unless valid (fail "cell health observation violates its threshold")
pure (machine, name)
percent key = withObject "health percentage" \v -> do
value <- v .: key :: Parser Double
unless (not (isNaN value || isInfinite value) && value >= 0 && value <= 100) (fail "invalid cell health percentage")
pure value
resetDocument = withObject "cell reset" \v -> do
schema <- v .: "schema"
identifier <- v .: "runId"
owner <- v .: "cell"
cache <- v .: "cachePolicy"
verified <- v .: "verified"
start <- v .: "startedAt" :: Parser UTCTime
end <- v .: "finishedAt" :: Parser UTCTime
steps <- v .: "steps" :: Parser [Value]
unless (schema == ("cell.reset-evidence/v1" :: Text) && identifier == runId && owner == cell && cache == ("cold" :: Text) && verified && start <= end && not (null steps)) (fail "cell reset identity, cache policy, window or verification is invalid")
mapM_
( withObject "reset step" \step -> do
status <- step .: "status"
from <- step .: "startedAt" :: Parser UTCTime
to <- step .: "finishedAt" :: Parser UTCTime
unless (status == ("ok" :: Text) && from >= start && to <= end && from <= to) (fail "cell reset step failed or lies outside its window")
)
steps