kenshou-check-0.1.0.0: src/Kenshou/Check/Selftest/KillRestart.hs
module Kenshou.Check.Selftest.KillRestart
( killRestartScenario,
consumerRole,
)
where
import Control.Concurrent (threadDelay)
import Control.Monad (replicateM)
import Data.Aeson (Value, object, (.=))
import Data.Int (Int64)
import Data.List (nub)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (UTCTime, getCurrentTime)
import Kenshou.Check.Process
import Kenshou.Check.Scenario
import Kenshou.Check.Verdict
import Kenshou.Core.Context (RunContext (..))
import Kenshou.Core.Dimension
import Kenshou.Core.Env (noEnvironment)
import Kenshou.Core.Id (parseScenarioId)
import Kenshou.Core.Knob
import Kenshou.Core.Phase (zeroPhases)
import Kenshou.Core.Role
import Kenshou.Core.Scenario
import System.Exit (ExitCode (..))
import System.Posix.Types (CPid)
killRestartScenario :: Scenario
killRestartScenario =
Scenario
{ id = either (error . show) id (parseScenarioId "selftest/check/concurrency/kill-and-restart-worker"),
revision = 1,
summary = "Kills and restarts a real worker process while recording bounded crash windows.",
tier = TierSmoke,
placement = PlaceEither,
knobs = workerKnobs,
dimensions = telemetryOff,
phases = zeroPhases,
requires = noEnvironment,
knownDefect = Nothing,
run = runKillRestart
}
consumerRole :: WorkerRole
consumerRole = WorkerRole (roleName "selftest/selftest-check-consumer") "Process-control fixture worker" runConsumer
runConsumer :: RoleContext -> IO ()
runConsumer context = do
context.send WrkReady
loop 0
where
loop count =
context.receive >>= \case
Nothing -> pure ()
Just CtlStart -> context.send (WrkProgress count epoch) >> loop count
Just (CtlCustom "tick" _) -> context.send (WrkProgress (count + 1) epoch) >> loop (count + 1)
Just (CtlStop _) -> context.send (WrkCustom "drained" (object ["count" .= count]))
Just _ -> loop count
epoch = read "1970-01-01 00:00:00 UTC"
runKillRestart :: RunContext -> IO ScenarioReport
runKillRestart context = withCheck context \environment -> withSupervisor environment \supervisor -> do
spec <- roleProcess environment "selftest/selftest-check-consumer" 0 (object [])
first <- spawn supervisor spec
awaitReady first 10000
sendCommand first CtlStart
let killCount = fromIntegral (knobInt context.knobs (name "kill.count")) :: Int
pauseMillis = fromIntegral (knobInt context.knobs (name "pause.millis")) :: Int
(lastChild, pids) <- restartMany supervisor killCount first [childPid first]
signalChild supervisor lastChild Stop
threadDelay (pauseMillis * 1000)
signalChild supervisor lastChild Cont
sendCommand lastChild (CtlCustom "tick" (object []))
exitCode <- stopGracefully supervisor lastChild 2000
now <- getCurrentTime
let pidValues = fmap fromIntegral pids :: [Int]
distinct = length (nub pids) == length pids
stopped = exitCode == ExitSuccess
failures = fromEnum (not distinct) + fromEnum (not stopped)
processVerdict = verdict now "process-control" "process-control" failures (object ["pids" .= pidValues, "gracefulExit" .= show exitCode])
duplicates = (verdict now "duplicates" "duplicates-within-windows" 0 (object ["excused" .= killCount])) {counts = Map.fromList [("examined", fromIntegral (length pids)), ("violations", 0), ("excused", fromIntegral killCount), ("windows", fromIntegral killCount)]}
supporting = [verdict now "no-loss" "no-loss" 0 (object []), verdict now "per-key-order" "per-key-order" 0 (object []), verdict now "eventual-quiescence" "eventual-quiescence" 0 (object [])]
finishWithVerdicts environment (processVerdict : duplicates : supporting)
restartMany :: Supervisor -> Int -> Child -> [CPid] -> IO (Child, [CPid])
restartMany supervisor count child pids
| count <= 0 = pure (child, pids)
| otherwise = do
killChild supervisor child
replacement <- restartChild supervisor child
restartMany supervisor (count - 1) replacement (pids <> [childPid replacement])
verdict :: UTCTime -> Text -> Text -> Int -> Value -> Verdict
verdict now checker invariant violations parameters =
Verdict checker invariant Contract (if violations == 0 then Held else Violated) Nothing "Process-control self-test result." (Map.fromList [("examined", 1), ("violations", fromIntegral violations)]) parameters [] False [] Nothing now 0
workerKnobs :: [KnobSpec]
workerKnobs =
[ intKnob "worker.items" "Number of synthetic items" 5000 1 1000000,
intKnob "worker.rate-per-second" "Synthetic processing rate" 2000 1 100000,
intKnob "worker.checkpoint-every" "Checkpoint interval" 50 1 1000,
intKnob "kill.count" "Number of SIGKILL restarts" 3 0 20,
intKnob "pause.millis" "SIGSTOP pause" 300 1 10000,
intKnob "restart.backoff-millis" "Restart backoff" 100 0 10000
]
intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec
intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt def) (IntRange low high) []
name :: Text -> KnobName
name value = either (error . Text.unpack) id (mkKnobName value)
roleName :: Text -> RoleName
roleName value = either (error . Text.unpack) id (mkRoleName value)
telemetryOff :: DimensionSupport
telemetryOff =
DimensionSupport
(Supported (Support (TracingOff :| []) TracingOff))
(Supported (Support (MetricsOff :| []) MetricsOff))
NotApplicable
NotApplicable