kenshou-diagnose-0.1.0.0: src/Kenshou/Diagnose/SelfTest/Leak.hs
module Kenshou.Diagnose.SelfTest.Leak
( leakingWorkerScenario,
stableWorkerScenario,
)
where
import Control.Concurrent (MVar, ThreadId, killThread, newEmptyMVar, takeMVar, threadDelay)
import Control.Exception (bracket, evaluate, finally)
import Control.Monad (forM, forM_, replicateM, void)
import Data.ByteString qualified as ByteString
import Data.ByteString.Short (ShortByteString)
import Data.ByteString.Short qualified as Short
import Data.IORef
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Context (RunContext (..))
import Kenshou.Core.Dimension
import Kenshou.Core.Env (noEnvironment)
import Kenshou.Core.Id (Kind (Soak), ScenarioId, parseScenarioId)
import Kenshou.Core.Knob
import Kenshou.Core.Phase qualified as Core
import Kenshou.Core.Scenario
import Kenshou.Diagnose.Leak
import Kenshou.Diagnose.Leak.MajorGcProbe (withMajorGcProbe)
import Kenshou.Diagnose.Series (SeriesBinding (..))
import Kenshou.Diagnose.Threads (forkLabelled)
import Kenshou.Measure.Clock (Nanos (..))
import Kenshou.Measure.Knobs (measureKnobs)
import Kenshou.Measure.Phase qualified as Measure
import Kenshou.Measure.Sampler
import Kenshou.Measure.Session (MeasureEnv (..), measureEnvFromRunContext)
import System.IO (Handle, IOMode (ReadMode), hClose, openFile)
leakingWorkerScenario :: Scenario
leakingWorkerScenario =
Scenario
{ id = scenarioId "selftest/diagnose/soak/leaking-worker",
revision = 1,
summary = "Proves that bounded heap, thread, and descriptor growth is detected.",
tier = TierSmoke,
placement = PlaceEither,
knobs = leakKnobs <> samplingKnobs,
dimensions = telemetryOff,
phases = Core.PhasePlan 5 40 0,
requires = noEnvironment,
knownDefect = Nothing,
run = runLeakingWorker
}
stableWorkerScenario :: Scenario
stableWorkerScenario =
Scenario
{ id = scenarioId "selftest/diagnose/soak/stable-worker",
revision = 1,
summary = "Proves that high allocation into a bounded retained ring remains stable.",
tier = TierSmoke,
placement = PlaceEither,
knobs = stableKnobs <> samplingKnobs,
dimensions = telemetryOff,
phases = Core.PhasePlan 10 35 0,
requires = noEnvironment,
knownDefect = Nothing,
run = runStableWorker
}
data LeakResources
= HeapResources (IORef [ShortByteString])
| ThreadResources (MVar ()) (IORef [ThreadId])
| DescriptorResources (IORef [Handle])
runLeakingWorker :: RunContext -> IO ScenarioReport
runLeakingWorker context = do
let kind = knobText context.knobs (name "leak.kind")
duration = fromIntegral (knobInt context.knobs (name "run.duration-s"))
warmup = min duration (fromIntegral (knobInt context.knobs (name "run.warmup-s")))
majorGcMs = fromIntegral (knobInt context.knobs (name "diagnose.major-gc-interval-ms"))
bytesPerSecond = fromIntegral (knobInt context.knobs (name "leak.bytes-per-second"))
unitsPerSecond = fromIntegral (knobInt context.knobs (name "leak.units-per-second"))
chunkBytes = fromIntegral (knobInt context.knobs (name "leak.chunk-bytes"))
targetRate = if kind == "heap" then fromIntegral bytesPerSecond else fromIntegral unitsPerSecond
bracket (newResources kind) cleanupResources $ \resources -> do
_ <- withMajorGcProbe context majorGcMs $ withSelfTestSampling context \clock -> do
Measure.enterPhase clock Measure.WarmUp
produceFor warmup (produce kind resources bytesPerSecond unitsPerSecond chunkBytes)
Measure.enterPhase clock Measure.Steady
produceFor (duration - warmup) (produce kind resources bytesPerSecond unitsPerSecond chunkBytes)
report <- judgeLeaks context (leakingSpec kind majorGcMs duration warmup targetRate)
pure (assessLeak kind targetRate report)
runStableWorker :: RunContext -> IO ScenarioReport
runStableWorker context = do
let duration = fromIntegral (knobInt context.knobs (name "run.duration-s"))
warmup = min duration (fromIntegral (knobInt context.knobs (name "run.warmup-s")))
majorGcMs = fromIntegral (knobInt context.knobs (name "diagnose.major-gc-interval-ms"))
bytesPerSecond = fromIntegral (knobInt context.knobs (name "alloc.bytes-per-second"))
ringBytes = fromIntegral (knobInt context.knobs (name "retain.ring-bytes"))
chunkBytes = 4096
retained <- newIORef []
allocated <- newIORef (0 :: Int)
_ <- withMajorGcProbe context majorGcMs $ withSelfTestSampling context \clock -> do
Measure.enterPhase clock Measure.WarmUp
stableFor warmup retained allocated bytesPerSecond ringBytes chunkBytes
Measure.enterPhase clock Measure.Steady
stableFor (duration - warmup) retained allocated bytesPerSecond ringBytes chunkBytes
report <- judgeLeaks context (stableSpec majorGcMs duration warmup)
total <- readIORef allocated
let heap = findProbe "heap.live-bytes" report
enoughAllocation = total > 5 * ringBytes
pure case heap of
Nothing -> inconclusiveBecause "heap.live-bytes was not reported"
Just probe
| probe.verdict == InsufficientData -> inconclusiveBecause ("heap evidence is insufficient: " <> probe.reason)
| probe.verdict == Stable && enoughAllocation -> passed
| otherwise -> failedWith ["stable-worker-verdict"] ("heap verdict=" <> showText probe.verdict <> ", allocated=" <> showText total)
withSelfTestSampling :: RunContext -> (Measure.PhaseClock -> IO value) -> IO value
withSelfTestSampling context action = do
environment <- measureEnvFromRunContext context
clock <- Measure.newPhaseClock environment.onPhase (Measure.PhasePlan (Nanos 0) (Measure.SteadyCount 0) (Nanos 0))
sampling <-
startSampling
SamplerConfig
{ runDir = environment.runDir,
origin = environment.origin,
intervalMs = 500,
postgres = Nothing,
extraSamplers = [],
phaseClock = clock,
declareArtifact = environment.declareArtifact,
logLine = environment.logLine
}
action clock `finally` void (stopSampling sampling)
newResources :: Text -> IO LeakResources
newResources "heap" = HeapResources <$> newIORef []
newResources "threads" = ThreadResources <$> newEmptyMVar <*> newIORef []
newResources "fds" = DescriptorResources <$> newIORef []
newResources other = ioError (userError ("unknown leak.kind " <> Text.unpack other))
cleanupResources :: LeakResources -> IO ()
cleanupResources (HeapResources retained) = writeIORef retained []
cleanupResources (ThreadResources _ threads) = readIORef threads >>= mapM_ killThread
cleanupResources (DescriptorResources handles) = readIORef handles >>= mapM_ hClose
produce :: Text -> LeakResources -> Int -> Int -> Int -> Int -> IO ()
produce "heap" (HeapResources retained) bytesPerSecond _ chunkBytes step = do
chunks <- freshChunks bytesPerSecond chunkBytes step
modifyIORef' retained (chunks <>)
produce "threads" (ThreadResources blocker threads) _ unitsPerSecond _ _ = do
spawned <- replicateM unitsPerSecond (forkLabelled "kenshou:selftest:leaked-thread" (waitForever blocker))
modifyIORef' threads (spawned <>)
produce "fds" (DescriptorResources handles) _ unitsPerSecond _ _ = do
opened <- replicateM unitsPerSecond (openFile "/dev/null" ReadMode)
modifyIORef' handles (opened <>)
produce _ _ _ _ _ _ = ioError (userError "leak resource does not match leak.kind")
waitForever :: MVar () -> IO ()
waitForever blocker = do
_ <- takeMVar blocker
pure ()
produceFor :: Int -> (Int -> IO ()) -> IO ()
produceFor seconds action = forM_ [0 .. seconds - 1] \step -> action step >> threadDelay 1_000_000
stableFor :: Int -> IORef [ShortByteString] -> IORef Int -> Int -> Int -> Int -> IO ()
stableFor seconds retained allocated bytesPerSecond ringBytes chunkBytes =
forM_ [0 .. seconds - 1] \step -> do
chunks <- freshChunks bytesPerSecond chunkBytes step
previous <- readIORef retained
let bounded = take (max 1 (ringBytes `div` chunkBytes)) (chunks <> previous)
_ <- evaluate (sum (fmap Short.length bounded))
writeIORef retained bounded
modifyIORef' allocated (+ sum (fmap Short.length chunks))
threadDelay 1_000_000
freshChunks :: Int -> Int -> Int -> IO [ShortByteString]
freshChunks bytesPerSecond chunkBytes step = do
let count = max 1 ((bytesPerSecond + chunkBytes - 1) `div` chunkBytes)
byte = fromIntegral (step `mod` 251)
forM [1 .. count] \index -> do
let remaining = bytesPerSecond - (index - 1) * chunkBytes
size = max 1 (min chunkBytes remaining)
chunk = Short.toShort (ByteString.replicate size byte)
Short.length chunk `seq` pure chunk
leakingSpec :: Text -> Double -> Int -> Int -> Double -> LeakSpec
leakingSpec kind majorGcMs duration warmup targetRate =
defaultLeakSpec
{ probes = fmap tune defaultLeakSpec.probes,
warmupCutSeconds = if majorGcMs > 0 then 0 else fromIntegral warmup,
minPoints = max 5 (min 20 (duration - warmup - 2)),
minDurationSeconds = max 5 (fromIntegral (duration - warmup) * 0.7),
envelopeWindowSeconds = 1,
resamples = 300
}
where
target = targetProbe kind
tune probe
| probe.name == target =
probe
{ binding = if kind == "heap" && majorGcMs > 0 then SeriesBinding "rts-major.csv" "t_mono_ns" "live_bytes" Map.empty else probe.binding,
floorPerHour = targetRate * 1800,
minGrowth = targetRate * fromIntegral (duration - warmup) * 0.25,
minRelativeGrowth = 0
}
| otherwise = probe {floorPerHour = 1.0e30, minGrowth = 1.0e30, minRelativeGrowth = 0}
stableSpec :: Double -> Int -> Int -> LeakSpec
stableSpec majorGcMs duration warmup =
defaultLeakSpec
{ probes = fmap tune defaultLeakSpec.probes,
warmupCutSeconds = if majorGcMs > 0 then 0 else fromIntegral warmup,
minPoints = max 5 (min 20 (duration - warmup - 2)),
minDurationSeconds = max 5 (fromIntegral (duration - warmup) * 0.7),
envelopeWindowSeconds = 1,
resamples = 300
}
where
tune :: ProbeSpec -> ProbeSpec
tune probe
| probe.name == "heap.live-bytes" = probe {binding = if majorGcMs > 0 then SeriesBinding "rts-major.csv" "t_mono_ns" "live_bytes" Map.empty else probe.binding}
| otherwise = ProbeSpec probe.name probe.unit probe.binding probe.aggregation Informational probe.floorPerHour probe.minGrowth probe.minRelativeGrowth probe.limit
assessLeak :: Text -> Double -> LeakReport -> ScenarioReport
assessLeak kind expectedRate report = case findProbe (targetProbe kind) report of
Nothing -> inconclusiveBecause ("target probe was not reported: " <> targetProbe kind)
Just probe
| probe.verdict == InsufficientData -> inconclusiveBecause ("target probe evidence is insufficient: " <> probe.reason)
| otherwise ->
let actualRate = (/ 3600) <$> probe.slopePerHour
tolerance = if kind == "heap" then 0.15 else 0.20
rateOk = maybe False (\actual -> abs (actual - expectedRate) <= expectedRate * tolerance) actualRate
intervalOk = case (probe.slopePerHour, probe.intervalPerHour) of
(Just slope, Just (low, high)) -> low <= slope && slope <= high
_ -> False
unrelated = [candidate.probe | candidate <- report.probes, candidate.probe /= probe.probe, candidate.expectation == Bounded, candidate.verdict == LeakSuspected]
failures = ["target-verdict" | probe.verdict /= LeakSuspected] <> ["target-rate" | not rateOk] <> ["target-interval" | not intervalOk] <> ["unrelated-probe" | not (null unrelated)]
in if null failures
then passed
else failedWith failures ("expected " <> targetProbe kind <> " at " <> showText expectedRate <> "/s; actual=" <> showText actualRate <> "; unrelated=" <> showText unrelated)
findProbe :: Text -> LeakReport -> Maybe ProbeReport
findProbe wanted report = case filter ((== wanted) . (.probe)) report.probes of
value : _ -> Just value
[] -> Nothing
targetProbe :: Text -> Text
targetProbe "heap" = "heap.live-bytes"
targetProbe "threads" = "haskell.threads"
targetProbe "fds" = "os.fds"
targetProbe other = error ("unknown leak kind " <> Text.unpack other)
leakKnobs :: [KnobSpec]
leakKnobs =
[ textKnob "leak.kind" "Resource retained by the worker" "heap" ["heap", "threads", "fds"],
intKnob "leak.bytes-per-second" "Heap bytes retained each second" 1_048_576 65_536 67_108_864,
intKnob "leak.units-per-second" "Threads or descriptors retained each second" 5 1 1_000,
intKnob "leak.chunk-bytes" "Size of each retained heap chunk" 4_096 256 1_048_576,
intKnob "run.duration-s" "Run duration" 45 20 3_600,
intKnob "run.warmup-s" "Warm-up duration" 5 0 600,
intKnob "diagnose.major-gc-interval-ms" "Forced major-GC interval, or zero to disable" 1_000 0 60_000
]
stableKnobs :: [KnobSpec]
stableKnobs =
[ intKnob "alloc.bytes-per-second" "Bytes allocated each second" 8_388_608 65_536 67_108_864,
intKnob "retain.ring-bytes" "Maximum bytes retained in the ring" 33_554_432 65_536 536_870_912,
intKnob "run.duration-s" "Run duration" 45 20 3_600,
intKnob "run.warmup-s" "Warm-up duration" 10 0 600,
intKnob "diagnose.major-gc-interval-ms" "Forced major-GC interval, or zero to disable" 1_000 0 60_000
]
samplingKnobs :: [KnobSpec]
samplingKnobs = fmap setInterval (measureKnobs Soak)
where
setInterval :: KnobSpec -> KnobSpec
setInterval spec
| renderKnobName spec.name == "measure.sample-interval-ms" = KnobSpec spec.name spec.summary spec.knobType (VInt 500) spec.allowed spec.variants
| otherwise = spec
telemetryOff :: DimensionSupport
telemetryOff =
DimensionSupport
(Supported (Support (TracingOff :| []) TracingOff))
(Supported (Support (MetricsOff :| []) MetricsOff))
NotApplicable
NotApplicable
intKnob :: Text -> Text -> Int -> Int -> Int -> KnobSpec
intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt (fromIntegral def)) (IntRange (fromIntegral low) (fromIntegral high)) []
textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec
textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []
textKnob knobName _ _ [] = error ("no values for " <> Text.unpack knobName)
name :: Text -> KnobName
name = either (error . show) id . mkKnobName
scenarioId :: Text -> ScenarioId
scenarioId = either (error . show) id . parseScenarioId
showText :: (Show value) => value -> Text
showText = Text.pack . show