kenshou-remote-0.1.0.0: src/Kenshou/Remote/Perf/Runner.hs
module Kenshou.Remote.Perf.Runner
( RunnerHooks (..),
ComparisonHooks (..),
Admission (..),
AdmissionHooks (..),
runPerformance,
)
where
import Control.Exception (IOException, mask_, onException, try)
import Control.Monad (unless, when)
import Data.Aeson (toJSON)
import Data.IORef
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, addUTCTime, diffUTCTime)
import Kenshou.Perf.Contracts (PerfContracts)
import Kenshou.Perf.Plan (PerfPlan (..), PerfProfile (..), PlannedProfile (..))
import Kenshou.Perf.Prepared qualified as Prepared
import Kenshou.Perf.Request (PerfRequest (..))
import Kenshou.Perf.Result
import Kenshou.Remote.Perf.Journal
import System.Timeout (timeout)
data Admission = Admitted LeaseIdentity | Occupied | AdmissionFailed Text
deriving stock (Eq, Show)
-- Replacement and runtime observations are persisted before another trial can
-- be dispatched. The cell adapter also persists its low-level work/submission
-- journal before effects and reconciles that journal when resuming an attempt.
data ComparisonHooks = ComparisonHooks
{ reserveReplacement :: Int -> IO (),
recordRuntimeFailure :: Text -> IO ()
}
-- Persist the proposed ID before CAS; withdraw only on a definite non-claim.
-- An unresolved proposal is reconciled read-only on resume, never replaced.
data AdmissionHooks = AdmissionHooks {proposeClaim :: LeaseIdentity -> IO (), withdrawClaim :: IO ()}
data RunnerHooks = RunnerHooks
{ -- | The suite's performance contracts, against which the preparation is
-- re-validated before any work resumes.
contracts :: PerfContracts,
clock :: IO UTCTime,
admit :: PlannedProfile -> Int -> Maybe (Text, LeaseIdentity, UTCTime) -> AdmissionHooks -> IO Admission,
ownsLease :: LeaseIdentity -> IO Bool,
canRunConfiguration :: LeaseIdentity -> PlannedProfile -> IO Bool,
compareConfiguration :: LeaseIdentity -> PlannedProfile -> Int -> Bool -> ComparisonHooks -> IO ConfigurationResult,
verifyCompleted :: ConfigurationResult -> IO Bool,
cancelOwnedWork :: LeaseIdentity -> IO (),
releaseOwnedLease :: LeaseIdentity -> IO Bool,
recoverRuntimeFailures :: LeaseIdentity -> IO [(Text, Text)],
reportMetadata :: PerfSession -> PerfResult -> IO PerfResult,
publishAndVerify :: PerfSession -> PerfResult -> IO Bool
}
-- Caller holds the session's exclusive orchestration lock. All inputs have
-- already passed manifest and payload retrieval verification. Admission hooks
-- include their bounded waiting; measurement callbacks must never pool pairs
-- across different lease identities or extend the supplied session deadline.
runPerformance :: RunnerHooks -> Prepared.PerfPrepared -> FilePath -> IO PerfResult
runPerformance hooks prepared path = do
loaded <- readPerfSession path >>= require
require (validateSessionInputs hooks.contracts prepared loaded)
retained <- require (journalState loaded)
mapM_ verify (Map.elems retained.completed)
current <- newIORef loaded
let checkpoint action = mask_ do
now <- hooks.clock
before <- readIORef current
changed <- require (appendEvent (SessionEvent now action) before)
writePerfSession path changed
writeIORef current changed
state = readIORef current >>= require . journalState
cleanup = do
saved <- state
let proposed = case saved.pendingClaim of Just (_, identity, _) -> Just identity; Nothing -> Nothing
case (case saved.lease of Just identity -> Just identity; Nothing -> proposed) of
Nothing -> pure ()
Just identity -> do
now <- hooks.clock
let deadline = case saved.pendingClaim of Just (_, _, at) -> Just (addUTCTime (fromIntegral (prepared.plan.request.budgetMinutes * 60 + 60)) at); Nothing -> saved.cleanupDeadline
micros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` now)) deadline
released <- timeout micros (hooks.cancelOwnedWork identity >> hooks.releaseOwnedLease identity >>= \released -> when released (recover identity) >> pure released)
case saved.lease of
Just _ -> checkpoint (LeaseLost "client interrupted; owned cleanup requested")
Nothing -> when (released == Just True) (checkpoint LeaseClaimReleased)
recover identity = do
faults <- hooks.recoverRuntimeFailures identity
mapM_ (\(profile, reason) -> do observed <- state; unless (reason `elem` Map.findWithDefault [] profile observed.runtimeBlockers) (checkpoint (RuntimeBlocked profile reason))) faults
bounded :: String -> IO value -> IO value
bounded label action = do
observed <- state
started <- hooks.clock
let micros = maybe 10000000 (min 10000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` started)) observed.executionDeadline
timeout micros action >>= maybe (ioError (userError (label <> " exceeded its bounded transport allowance"))) pure
execute = do
saved <- state
now <- hooks.clock
session <- readIORef current
case saved.lease of
Just identity -> do
owned <- bounded "lease ownership check" (hooks.ownsLease identity)
if not owned
then do
released <- bounded "lost lease reconciliation" (hooks.releaseOwnedLease identity)
unless released (ioError (userError "lost lease still has unsettled owned work"))
bounded "sealed lost-lease observations" (recover identity)
checkpoint (LeaseLost "lease ownership lost; owned work reconciled")
execute
else case saved.activeComparison of
Just (name, index) -> do
entry <- require (selected session name)
runOne identity entry index True
Nothing -> case filter (\entry -> Map.notMember entry.profile.name saved.completed) session.plan.selected of
[] -> release identity
entry : _ -> do
let index = Map.findWithDefault 0 entry.profile.name saved.attempts
compatible <- bounded "configuration compatibility check" (hooks.canRunConfiguration identity entry)
if not compatible
then release identity >> execute
else case appendEvent (SessionEvent now (ComparisonStarted entry.profile.name index)) session of
Left _ -> release identity
Right _ -> checkpoint (ComparisonStarted entry.profile.name index) >> runOne identity entry index False
Nothing -> case filter (\entry -> Map.notMember entry.profile.name saved.completed) session.plan.selected of
[] -> pure ()
entry : _ -> do
let admissionLeft = session.plan.request.waitSeconds - saved.admissionSpentSeconds
measured = saved.executionDeadline /= Nothing
enough = maybe True (\deadline -> cost entry <= floor (diffUTCTime deadline now)) saved.executionDeadline
initialAdmissionAt = case [event.at | event <- session.events, AdmissionReserved _ <- [event.action]] of first : _ -> first; [] -> now
wallLeft = max 0 (session.plan.request.waitSeconds - ceiling (diffUTCTime now initialAdmissionAt))
seconds = if measured then maybe 1 (min 10 . max 0 . floor . (`diffUTCTime` now)) saved.executionDeadline else case saved.pendingClaim of Just _ -> 10; Nothing -> min 10 (min admissionLeft wallLeft)
if not enough || seconds <= 0
then case saved.pendingClaim of Nothing -> pure (); Just _ -> ioError (userError "unresolved cell claim exceeds the original session budget")
else do
unless (measured || saved.pendingClaim /= Nothing) (checkpoint (AdmissionReserved seconds))
let callbacks = AdmissionHooks (checkpoint . LeaseClaimProposed entry.profile.name) (checkpoint LeaseClaimWithdrawn)
granted <- timeout (seconds * 1000000) (hooks.admit entry seconds saved.pendingClaim callbacks)
receivedAt <- hooks.clock
observed <- state
admission <- case granted of
Nothing | observed.pendingClaim == Nothing -> pure Occupied
Nothing -> ioError (userError "cell admission transport exceeded its reserved allowance; proposed claim retained")
Just response -> do
when (receivedAt > addUTCTime (fromIntegral seconds) now) (ioError (userError "cell admission returned after its reserved allowance"))
pure response
case admission of
Admitted identity -> checkpoint (LeaseAcquired identity) >> execute
Occupied -> do
unless (observed.pendingClaim == Nothing) (ioError (userError "occupied admission left an unresolved claim"))
if measured then pure () else execute
AdmissionFailed problem -> ioError (userError (Text.unpack problem))
runOne identity entry index resuming = do
saved <- state
now <- hooks.clock
let micros = maybe 0 (max 0 . floor . (* 1000000) . (`diffUTCTime` now)) saved.executionDeadline
reserve pair = do
session <- readIORef current
let action = ReplacementScheduled entry.profile.name index pair
unless (action `elem` map (.action) session.events) (checkpoint action)
blocked reason = do
observed <- state
unless (reason `elem` Map.findWithDefault [] entry.profile.name observed.runtimeBlockers) (checkpoint (RuntimeBlocked entry.profile.name reason))
callbacks = ComparisonHooks reserve blocked
result <- timeout micros (hooks.compareConfiguration identity entry index resuming callbacks)
case result of
Nothing -> do
cleanupAt <- hooks.clock
let cleanupMicros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` cleanupAt)) saved.cleanupDeadline
_ <- timeout cleanupMicros (hooks.cancelOwnedWork identity)
checkpoint (ComparisonCompleted (ConfigurationResult entry.profile.name entry.workloadSha256 "inconclusive" "pending" [] 0 entry.profile.minimumPairs [] Nothing))
release identity
Just comparison -> checkpoint (ComparisonCompleted comparison) >> execute
release identity = do
saved <- state
now <- hooks.clock
let micros = maybe 60000000 (min 60000000 . max 0 . floor . (* 1000000) . (`diffUTCTime` now)) saved.cleanupDeadline
released <- timeout micros (hooks.releaseOwnedLease identity >>= \released -> when released (recover identity) >> pure released)
if released == Just True then checkpoint LeaseReleased else ioError (userError "owned lease release failed or exceeded cleanup grace")
execution <- try @IOException (execute `onException` cleanup)
session <- readIORef current
saved <- require (journalState session)
let unfinished entry =
ConfigurationResult
entry.profile.name
entry.workloadSha256
"not-run"
(either (const "failed") (const "pending") execution)
(Map.findWithDefault [] entry.profile.name saved.runtimeBlockers)
0
entry.profile.minimumPairs
(map toJSON session.events)
Nothing
completed = [Map.findWithDefault (unfinished entry) entry.profile.name saved.completed | entry <- session.plan.selected]
preliminary <- require (aggregateResult session.plan session.preparedSha256 (either (const False) (const True) execution) completed)
publicationStarted <- hooks.clock
let publicationMicros =
maybe
(session.plan.request.budgetMinutes * 60000000)
(min (session.plan.request.budgetMinutes * 60000000) . max 0 . floor . (* 1000000) . (`diffUTCTime` publicationStarted))
saved.executionDeadline
publication <-
try @IOException
( timeout publicationMicros do
reported <- hooks.reportMetadata session preliminary
require (validateResult reported)
unless (reported {stageSeconds = Map.empty, artifactReferences = []} == preliminary) (ioError (userError "report metadata changed a performance decision"))
published <- hooks.publishAndVerify session reported
pure (reported, published)
)
let (reported, published) = either (const (preliminary, False)) (maybe (preliminary, False) id) publication
verifiedAt <- hooks.clock
let withinBudget = maybe True (verifiedAt <=) saved.executionDeadline
final <- require (aggregateResult session.plan session.preparedSha256 (published && withinBudget && either (const False) (const True) execution) completed)
pure final {stageSeconds = reported.stageSeconds, artifactReferences = reported.artifactReferences}
where
verify result = hooks.verifyCompleted result >>= \valid -> unless valid (ioError (userError "retained completed configuration failed evidence verification"))
cost entry = 2 * entry.profile.minimumPairs * (entry.profile.trialSeconds + entry.profile.resetSeconds + entry.profile.sealSeconds)
selected session name = case filter ((== name) . (.name) . (.profile)) session.plan.selected of
[entry] -> Right entry
_ -> Left "active comparison has no pinned profile"
require :: Either Text a -> IO a
require = either (ioError . userError . Text.unpack) pure