kenshou-diagnose (empty) → 0.1.0.0
raw patch · 49 files changed
+3166/−0 lines, 49 filesdep +aesondep +asyncdep +base
Dependencies added: aeson, async, base, bytestring, containers, directory, filepath, ghc-debug-stub, hasql, hasql-pool, hspec, kenshou-core, kenshou-diagnose, kenshou-measure, process, splitmix, stm, temporary, text, time, unix, uuid, vector
Files
- CHANGELOG.md +12/−0
- LICENSE +28/−0
- kenshou-diagnose.cabal +144/−0
- src/Kenshou/Diagnose.hs +10/−0
- src/Kenshou/Diagnose/Context.hs +42/−0
- src/Kenshou/Diagnose/Document.hs +80/−0
- src/Kenshou/Diagnose/Leak.hs +320/−0
- src/Kenshou/Diagnose/Leak/MajorGcProbe.hs +43/−0
- src/Kenshou/Diagnose/LockGraph.hs +65/−0
- src/Kenshou/Diagnose/Pool.hs +60/−0
- src/Kenshou/Diagnose/Postgres.hs +169/−0
- src/Kenshou/Diagnose/Profile.hs +227/−0
- src/Kenshou/Diagnose/Profile/EventlogGuard.hs +45/−0
- src/Kenshou/Diagnose/Profile/GhcDebug.hs +39/−0
- src/Kenshou/Diagnose/Progress.hs +52/−0
- src/Kenshou/Diagnose/Render.hs +59/−0
- src/Kenshou/Diagnose/SelfTest.hs +20/−0
- src/Kenshou/Diagnose/SelfTest/Leak.hs +305/−0
- src/Kenshou/Diagnose/SelfTest/Stall.hs +285/−0
- src/Kenshou/Diagnose/Series.hs +113/−0
- src/Kenshou/Diagnose/Series/Catalog.hs +43/−0
- src/Kenshou/Diagnose/Stall.hs +201/−0
- src/Kenshou/Diagnose/Stall/Classify.hs +42/−0
- src/Kenshou/Diagnose/Stall/Types.hs +77/−0
- src/Kenshou/Diagnose/Stats.hs +151/−0
- src/Kenshou/Diagnose/Threads.hs +114/−0
- test/Kenshou/Diagnose/LeakSpec.hs +97/−0
- test/Kenshou/Diagnose/ProfileSpec.hs +18/−0
- test/Kenshou/Diagnose/RenderSpec.hs +33/−0
- test/Kenshou/Diagnose/StallSpec.hs +152/−0
- test/Kenshou/Diagnose/StatsSpec.hs +33/−0
- test/Main.hs +16/−0
- test/fixtures/run-leaking/manifest.json +1/−0
- test/fixtures/run-leaking/run-result.json +1/−0
- test/fixtures/run-leaking/series/rts.csv +32/−0
- test/fixtures/run-stalled/diagnosis/stall-1.json +1/−0
- test/fixtures/run-stalled/run-result.json +1/−0
- test/fixtures/series/leak.csv +7/−0
- test/fixtures/snapshots/blocked-indefinitely.json +1/−0
- test/fixtures/snapshots/deadlock.json +1/−0
- test/fixtures/snapshots/idle-spin.json +1/−0
- test/fixtures/snapshots/lock-wait.json +1/−0
- test/fixtures/snapshots/pool-starvation.json +1/−0
- test/fixtures/snapshots/postgres-unavailable.json +1/−0
- test/fixtures/snapshots/unknown.json +1/−0
- test/golden/diagnose-leak.txt +9/−0
- test/golden/diagnose-profile.txt +4/−0
- test/golden/diagnose-stall.txt +7/−0
- test/golden/leak-diagnosis.json +1/−0
+ CHANGELOG.md view
@@ -0,0 +1,12 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of the diagnostics toolkit: heap-leak verdicts on live bytes after major collections, stall capture, lock graphs, and profiling guards.+- Stall captures name advisory locks with the labels bundles contribute, and the watchdog takes its configuration from the registry.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+ may be used to endorse or promote products derived from this software without+ specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ kenshou-diagnose.cabal view
@@ -0,0 +1,144 @@+cabal-version: 3.0+name: kenshou-diagnose+version: 0.1.0.0+synopsis: Leak and stall diagnostics for Kenshou scenarios+description: Offline leak verdicts, live stall capture, profiling recipes, and diagnostic self-tests.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+ test/fixtures/run-leaking/**/*.csv+ test/fixtures/run-leaking/**/*.json+ test/fixtures/run-stalled/**/*.json+ test/fixtures/series/*.csv+ test/fixtures/snapshots/*.json+ test/golden/*.json+ test/golden/*.txt++flag ghc-debug+ description: Enable the optional ghc-debug socket stub+ default: False+ manual: True++common warnings+ ghc-options:+ -Wall+ -Wcompat+ -Widentities+ -Wincomplete-record-updates+ -Wincomplete-uni-patterns+ -Wpartial-fields+ -Wredundant-constraints++ default-language: GHC2024+ default-extensions:+ BlockArguments+ DuplicateRecordFields+ ImportQualifiedPost+ NoFieldSelectors+ OverloadedLabels+ OverloadedRecordDot+ OverloadedStrings++library+ import: warnings+ hs-source-dirs: src+ exposed-modules:+ Kenshou.Diagnose+ Kenshou.Diagnose.Context+ Kenshou.Diagnose.Document+ Kenshou.Diagnose.Leak+ Kenshou.Diagnose.Leak.MajorGcProbe+ Kenshou.Diagnose.LockGraph+ Kenshou.Diagnose.Pool+ Kenshou.Diagnose.Postgres+ Kenshou.Diagnose.Profile+ Kenshou.Diagnose.Profile.EventlogGuard+ Kenshou.Diagnose.Profile.GhcDebug+ Kenshou.Diagnose.Progress+ Kenshou.Diagnose.Render+ Kenshou.Diagnose.SelfTest+ Kenshou.Diagnose.SelfTest.Leak+ Kenshou.Diagnose.SelfTest.Stall+ Kenshou.Diagnose.Series+ Kenshou.Diagnose.Series.Catalog+ Kenshou.Diagnose.Stall+ Kenshou.Diagnose.Stall.Classify+ Kenshou.Diagnose.Stall.Types+ Kenshou.Diagnose.Stats+ Kenshou.Diagnose.Threads++ build-depends:+ aeson >=2.2 && <2.3,+ async >=2.2 && <2.3,+ base >=4.21 && <4.22,+ bytestring >=0.12 && <0.13,+ containers >=0.7 && <0.8,+ directory >=1.3 && <1.4,+ filepath >=1.5 && <1.6,+ hasql >=1.10 && <1.11,+ hasql-pool >=1.4 && <1.5,+ kenshou-core ^>=0.1.0.0,+ kenshou-measure ^>=0.1.0.0,+ process >=1.6 && <1.7,+ splitmix >=0.1 && <0.2,+ stm >=2.5 && <2.6,+ text >=2.1 && <2.2,+ time >=1.14 && <1.15,+ unix >=2.8 && <2.9,+ uuid >=1.3 && <1.4,+ vector >=0.13 && <0.14,++ if flag(ghc-debug)+ cpp-options: -DWITH_GHC_DEBUG+ build-depends: ghc-debug-stub ==0.8.0.0++ if !os(osx)+ extra-libraries: stdc++++test-suite kenshou-diagnose-test+ import: warnings+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Main.hs+ other-modules:+ Kenshou.Diagnose.LeakSpec+ Kenshou.Diagnose.ProfileSpec+ Kenshou.Diagnose.RenderSpec+ Kenshou.Diagnose.StallSpec+ Kenshou.Diagnose.StatsSpec++ ghc-options:+ -threaded+ -rtsopts+ -with-rtsopts=-N+ -with-rtsopts=-T++ build-depends:+ aeson,+ async,+ base >=4.21 && <4.22,+ bytestring,+ containers,+ directory >=1.3 && <1.4,+ filepath,+ hasql,+ hspec >=2.11 && <2.12,+ kenshou-core ^>=0.1.0.0,+ kenshou-diagnose,+ temporary,+ text,+ time >=1.14 && <1.15,+ unix >=2.8 && <2.9,+ vector,++source-repository head+ type: git+ location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Diagnose.hs view
@@ -0,0 +1,10 @@+module Kenshou.Diagnose+ ( module Kenshou.Diagnose.Leak,+ module Kenshou.Diagnose.Leak.MajorGcProbe,+ module Kenshou.Diagnose.Stall,+ )+where++import Kenshou.Diagnose.Leak+import Kenshou.Diagnose.Leak.MajorGcProbe+import Kenshou.Diagnose.Stall
+ src/Kenshou/Diagnose/Context.hs view
@@ -0,0 +1,42 @@+module Kenshou.Diagnose.Context+ ( runDirectory,+ runIdentity,+ runSeed,+ scenarioKind,+ steadyWindow,+ publishDiagnosis,+ logWarn,+ )+where++import Data.Aeson qualified as Aeson+import Data.Text (Text)+import Data.Word (Word64)+import Kenshou.Core.Context qualified as Core+import Kenshou.Core.Id (Kind, ScenarioId (..), renderRunId, renderScenarioId, unSeed)+import Kenshou.Core.Log (Severity (Warning))+import Kenshou.Core.Phase (PhasePlan (..))++runDirectory :: Core.RunContext -> FilePath+runDirectory context = context.outDir++runIdentity :: Core.RunContext -> (Text, Text)+runIdentity context = (renderRunId context.runId, renderScenarioId context.scenario)++runSeed :: Core.RunContext -> Word64+runSeed context = unSeed context.seed++scenarioKind :: Core.RunContext -> Kind+scenarioKind context = context.scenario.kind++steadyWindow :: Core.RunContext -> IO (Maybe (Double, Double))+steadyWindow context =+ let start = context.phases.warmUpSeconds+ end = start + context.phases.steadySeconds+ in pure (if end > start then Just (start, end) else Nothing)++publishDiagnosis :: Core.RunContext -> Text -> Aeson.Value -> IO ()+publishDiagnosis context = Core.putSummary context Core.Diagnosis++logWarn :: Core.RunContext -> Text -> IO ()+logWarn context = Core.observe context "kenshou-diagnose" Warning
+ src/Kenshou/Diagnose/Document.hs view
@@ -0,0 +1,80 @@+module Kenshou.Diagnose.Document+ ( Diagnosis (..),+ DiagnosisKind (..),+ Generator (..),+ encodeDiagnosis,+ decodeDiagnosis,+ diagnosisKindText,+ )+where++import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.ByteString.Lazy (ByteString)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)++data DiagnosisKind = LeakDiagnosis | StallDiagnosis | ThreadDumpDiagnosis | ProfileDiagnosis+ deriving stock (Eq, Show)++data Generator = Generator+ { package :: Text,+ version :: Text,+ algorithm :: Text+ }+ deriving stock (Eq, Show)++data Diagnosis = Diagnosis+ { kind :: DiagnosisKind,+ runId :: Text,+ scenario :: Text,+ generatedAt :: UTCTime,+ generator :: Generator,+ body :: Value+ }+ deriving stock (Eq, Show)++diagnosisKindText :: DiagnosisKind -> Text+diagnosisKindText LeakDiagnosis = "leak"+diagnosisKindText StallDiagnosis = "stall"+diagnosisKindText ThreadDumpDiagnosis = "thread-dump"+diagnosisKindText ProfileDiagnosis = "profile"++parseDiagnosisKind :: Text -> Parser DiagnosisKind+parseDiagnosisKind "leak" = pure LeakDiagnosis+parseDiagnosisKind "stall" = pure StallDiagnosis+parseDiagnosisKind "thread-dump" = pure ThreadDumpDiagnosis+parseDiagnosisKind "profile" = pure ProfileDiagnosis+parseDiagnosisKind other = fail ("unknown diagnosis kind " <> Text.unpack other)++instance ToJSON Generator where+ toJSON value = object ["package" .= value.package, "version" .= value.version, "algorithm" .= value.algorithm]++instance FromJSON Generator where+ parseJSON = withObject "Generator" \value -> Generator <$> value .: "package" <*> value .: "version" <*> value .: "algorithm"++instance ToJSON Diagnosis where+ toJSON value =+ object+ [ "schema" .= ("kenshou.diagnosis/v1" :: Text),+ "kind" .= diagnosisKindText value.kind,+ "runId" .= value.runId,+ "scenario" .= value.scenario,+ "generatedAt" .= value.generatedAt,+ "generator" .= value.generator,+ "body" .= value.body+ ]++instance FromJSON Diagnosis where+ parseJSON = withObject "Diagnosis" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.diagnosis/v1" :: Text)+ then fail "unsupported diagnosis schema"+ else Diagnosis <$> (value .: "kind" >>= parseDiagnosisKind) <*> value .: "runId" <*> value .: "scenario" <*> value .: "generatedAt" <*> value .: "generator" <*> value .: "body"++encodeDiagnosis :: Diagnosis -> ByteString+encodeDiagnosis = encode++decodeDiagnosis :: ByteString -> Either String Diagnosis+decodeDiagnosis = eitherDecode
+ src/Kenshou/Diagnose/Leak.hs view
@@ -0,0 +1,320 @@+module Kenshou.Diagnose.Leak+ ( LeakVerdict (..),+ Expectation (..),+ Aggregation (..),+ ProbeSpec (..),+ LeakSpec (..),+ ProbeReport (..),+ LeakReport (..),+ defaultLeakSpec,+ loadLeakPolicy,+ judgeLeaks,+ judgeLeaksWithWindow,+ analyseRunDirectory,+ analyseSeriesDirectory,+ majorGcSamples,+ judgeSeries,+ leakOutcome,+ )+where++import Data.Aeson+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, fromMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Data.Vector (Vector)+import Data.Vector qualified as Vector+import Data.Word (Word64)+import Kenshou.Core.Context qualified as Core+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Diagnose.Context qualified as Context+import Kenshou.Diagnose.Document+import Kenshou.Diagnose.Series+import Kenshou.Diagnose.Series.Catalog+import Kenshou.Diagnose.Stats+import System.Directory (doesFileExist)+import System.FilePath (takeDirectory, (</>))++data LeakVerdict = LeakSuspected | Stable | InsufficientData+ deriving stock (Eq, Ord, Show)++data Expectation = Bounded | Informational deriving stock (Eq, Show)++data Aggregation = WindowMin | WindowMedian deriving stock (Eq, Show)++data ProbeSpec = ProbeSpec+ { name :: !Text,+ unit :: !Text,+ binding :: !SeriesBinding,+ aggregation :: !Aggregation,+ expectation :: !Expectation,+ floorPerHour :: !Double,+ minGrowth :: !Double,+ minRelativeGrowth :: !Double,+ limit :: !(Maybe Double)+ }+ deriving stock (Eq, Show)++data LeakSpec = LeakSpec+ { probes :: ![ProbeSpec],+ warmupCutSeconds :: !Double,+ minPoints :: !Int,+ minDurationSeconds :: !Double,+ envelopeWindowSeconds :: !Double,+ resamples :: !Int,+ confidence :: !Double+ }+ deriving stock (Eq, Show)++data ProbeReport = ProbeReport+ { probe :: !Text,+ process :: !Text,+ unit :: !Text,+ expectation :: !Expectation,+ basis :: !Text,+ points :: !Int,+ durationSeconds :: !Double,+ first :: !(Maybe Double),+ last :: !(Maybe Double),+ medianLevel :: !(Maybe Double),+ slopePerHour :: !(Maybe Double),+ intervalPerHour :: !(Maybe (Double, Double)),+ secondHalfSlopePerHour :: !(Maybe Double),+ growthOverWindow :: !(Maybe Double),+ projectedHoursToLimit :: !(Maybe Double),+ verdict :: !LeakVerdict,+ reason :: !Text,+ sourceFile :: !FilePath,+ sourceColumn :: !Text+ }+ deriving stock (Eq, Show)++data LeakReport = LeakReport+ { verdict :: !LeakVerdict,+ window :: !(Maybe (Double, Double)),+ seed :: !Word64,+ policy :: !Text,+ probes :: ![ProbeReport]+ }+ deriving stock (Eq, Show)++defaultLeakSpec :: LeakSpec+defaultLeakSpec =+ LeakSpec+ { probes =+ [ bounded "heap.live-bytes" "bytes" WindowMin 1_048_576 2_097_152 0.01,+ bounded "process.native-bytes" "bytes" WindowMedian 8_388_608 8_388_608 0.05,+ bounded "haskell.threads" "count" WindowMedian 1 5 0,+ bounded "os.threads" "count" WindowMedian 1 5 0,+ bounded "os.fds" "count" WindowMedian 1 5 0,+ bounded "pg.connections" "count" WindowMedian 1 5 0,+ informational "pg.relation-bytes" "bytes" WindowMin,+ informational "pg.dead-tuples" "count" WindowMin+ ],+ warmupCutSeconds = 300,+ minPoints = 30,+ minDurationSeconds = 1200,+ envelopeWindowSeconds = 60,+ resamples = 1000,+ confidence = 0.95+ }+ where+ catalog name = fromMaybe (error ("missing default catalog binding " <> Text.unpack name)) (Map.lookup name defaultCatalog)+ bounded name unit aggregation slopeFloor growth relative = ProbeSpec name unit (catalog (if name == "process.native-bytes" then "process.rss-bytes" else name)) aggregation Bounded slopeFloor growth relative Nothing+ informational name unit aggregation = ProbeSpec name unit (catalog name) aggregation Informational 0 0 0 Nothing++loadLeakPolicy :: FilePath -> IO (Either Text LeakSpec)+loadLeakPolicy path = do+ decoded <- eitherDecodeFileStrict' path+ pure case decoded of+ Left err -> Left (Text.pack err)+ Right spec -> Right spec++judgeLeaks :: Core.RunContext -> LeakSpec -> IO LeakReport+judgeLeaks context spec = do+ window <- Context.steadyWindow context+ judgeLeaksWithWindow context window spec++-- | Use the effective measurement window when a scenario overrides its+-- registered phase duration at run time.+judgeLeaksWithWindow :: Core.RunContext -> Maybe (Double, Double) -> LeakSpec -> IO LeakReport+judgeLeaksWithWindow context window spec = do+ report <- analyse (Context.runDirectory context) spec (Context.runSeed context) (pure window)+ generatedAt <- getCurrentTime+ let (runId, scenario) = Context.runIdentity context+ document = Diagnosis LeakDiagnosis runId scenario generatedAt generator (toJSON report)+ relative = "diagnosis/leak.json"+ path <- Core.artifactPath context Core.DiagnosisDir "leak.json"+ LazyByteString.writeFile path (encodeDiagnosis document)+ Core.declareMediaType context relative "application/json"+ Context.publishDiagnosis context "leak" (toJSON report)+ pure report+ where+ generator = Generator "kenshou-diagnose" "0.1.0.0" "theil-sen+moving-block-bootstrap/1"++analyseRunDirectory :: FilePath -> LeakSpec -> Word64 -> IO (Either DiagnoseError LeakReport)+analyseRunDirectory runDirectory spec seed = do+ exists <- doesFileExist (runDirectory </> "run-result.json")+ if not exists+ then pure (Left (InvalidRunDirectory runDirectory "run-result.json is missing"))+ else Right <$> analyse runDirectory spec seed (pure Nothing)++analyseSeriesDirectory :: FilePath -> LeakSpec -> Word64 -> IO LeakReport+analyseSeriesDirectory runDirectory spec seed = analyse runDirectory spec seed (pure Nothing)++analyse :: FilePath -> LeakSpec -> Word64 -> IO (Maybe (Double, Double)) -> IO LeakReport+analyse runDirectory spec seed getWindow = do+ reports <- catMaybes <$> traverse (analyseProbe runDirectory spec seed) spec.probes+ window <- getWindow+ let bounded = [report.verdict | report <- reports, report.expectation == Bounded]+ overall+ | LeakSuspected `elem` bounded = LeakSuspected+ | null bounded || InsufficientData `elem` bounded = InsufficientData+ | otherwise = Stable+ pure (LeakReport overall window seed "policy" reports)++analyseProbe :: FilePath -> LeakSpec -> Word64 -> ProbeSpec -> IO (Maybe ProbeReport)+analyseProbe runDirectory spec seed probe = do+ (series, postMajor) <-+ if probe.name == "heap.live-bytes"+ then heapSeries runDirectory probe+ else do+ result <- if probe.name == "process.native-bytes" then nativeSeries runDirectory probe else if probe.name == "pg.connections" then connectionSeries runDirectory probe else ordinarySeries runDirectory probe+ pure (result, False)+ pure $ Just case series of+ Left err -> emptyReport probe (Text.pack (show err))+ Right raw ->+ let report = judgeSeries (if postMajor then spec {minPoints = min 10 spec.minPoints} else spec) seed probe raw+ in if postMajor then report {basis = "post-major-collection"} else report++heapSeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)), Bool)+heapSeries runDirectory probe = do+ let path = runDirectory </> "series" </> probe.binding.file+ unfiltered = probe.binding {filters = Map.empty}+ live <- readBinding path unfiltered+ majors <- readBinding path (unfiltered {valueColumn = "major_gcs"})+ pure case (live, majors) of+ (Right values, Right counts) -> (Right (majorGcSamples values counts), True)+ (Right _, Left (MissingColumn _ _)) -> (live, False)+ (Left err, _) -> (Left err, False)+ (_, Left err) -> (Left err, False)++majorGcSamples :: Vector (Double, Double) -> Vector (Double, Double) -> Vector (Double, Double)+majorGcSamples live majors = Vector.fromList (go Nothing (Vector.toList (Vector.zip live majors)))+ where+ go _ [] = []+ go previous (((time, value), (_, count)) : rest) =+ let selected = maybe False (count >) previous+ in (if selected then [(time, value)] else []) <> go (Just count) rest++ordinarySeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)))+ordinarySeries runDirectory probe =+ let binding = probe.binding+ path = runDirectory </> "series" </> binding.file+ in readBinding path binding++connectionSeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)))+connectionSeries runDirectory probe =+ fmap (Vector.fromList . Map.toAscList . Map.fromListWith (+) . Vector.toList) <$> ordinarySeries runDirectory probe++nativeSeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)))+nativeSeries runDirectory probe = do+ rss <- ordinarySeries runDirectory probe+ let memBinding = fromMaybe (error "runtime mem binding missing") (Map.lookup "runtime.mem-in-use-bytes" defaultCatalog)+ let childBinding = memBinding {file = takeDirectory probe.binding.file </> memBinding.file}+ mem <- readBinding (runDirectory </> "series" </> childBinding.file) childBinding+ pure ((Vector.zipWith difference) <$> rss <*> mem)+ where+ difference (time, resident) (_, inUse) = (time, max 0 (resident - inUse))++judgeSeries :: LeakSpec -> Word64 -> ProbeSpec -> Vector (Double, Double) -> ProbeReport+judgeSeries spec seed probe raw+ | Vector.length reduced < spec.minPoints = emptyReport probe "too-few-points"+ | duration < spec.minDurationSeconds = emptyReport probe "too-short"+ | otherwise = case slopeWithInterval seed spec.resamples spec.confidence reduced of+ Nothing -> emptyReport probe "slope-unavailable"+ Just estimate -> finish estimate+ where+ cut = Vector.dropWhile (\(time, _) -> time < spec.warmupCutSeconds) raw+ reduced = case probe.aggregation of WindowMin -> windowMinima spec.envelopeWindowSeconds cut; WindowMedian -> windowMedians spec.envelopeWindowSeconds cut+ duration = if Vector.length reduced < 2 then 0 else fst (Vector.last reduced) - fst (Vector.head reduced)+ values = fmap snd (Vector.toList reduced)+ firstValue = snd <$> reduced Vector.!? 0+ lastValue = snd <$> reduced Vector.!? (Vector.length reduced - 1)+ medianValue = if null values then Nothing else Just (median values)+ growth = (-) <$> lastValue <*> firstValue+ finish estimate =+ let slopeHour = estimate.slope * 3600+ lowHour = estimate.low * 3600+ highHour = estimate.high * 3600+ half = Vector.drop (Vector.length reduced `div` 2) reduced+ second = (* 3600) . fst <$> theilSen half+ enoughGrowth = fromMaybe False ((\value level -> value >= probe.minGrowth && value >= probe.minRelativeGrowth * level) <$> growth <*> medianValue)+ plateau = maybe True (< probe.floorPerHour / 2) second+ (verdict, reason)+ | lowHour > probe.floorPerHour && enoughGrowth && not plateau = (LeakSuspected, "interval-above-floor")+ | lowHour > probe.floorPerHour && plateau = (Stable, "plateau-after-growth")+ | highHour < probe.floorPerHour || not enoughGrowth = (Stable, "below-growth-floor")+ | otherwise = (InsufficientData, "interval-straddles-floor")+ projected = do+ limit <- probe.limit+ level <- lastValue+ if slopeHour > 0 then Just ((limit - level) / slopeHour) else Nothing+ in ProbeReport probe.name "main" probe.unit probe.expectation (basisFor probe) (Vector.length reduced) duration firstValue lastValue medianValue (Just slopeHour) (Just (lowHour, highHour)) second growth projected verdict reason probe.binding.file probe.binding.valueColumn++emptyReport :: ProbeSpec -> Text -> ProbeReport+emptyReport probe reason = ProbeReport probe.name "main" probe.unit probe.expectation (basisFor probe) 0 0 Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing InsufficientData reason probe.binding.file probe.binding.valueColumn++basisFor :: ProbeSpec -> Text+basisFor probe = case probe.aggregation of WindowMin -> "window-minimum"; WindowMedian -> "window-median"++leakOutcome :: LeakReport -> Outcome+leakOutcome report = case report.verdict of LeakSuspected -> Failed; Stable -> Passed; InsufficientData -> Inconclusive++verdictText :: LeakVerdict -> Text+verdictText LeakSuspected = "leak-suspected"+verdictText Stable = "stable"+verdictText InsufficientData = "insufficient-data"++instance ToJSON LeakVerdict where toJSON = toJSON . verdictText++instance ToJSON Expectation where toJSON Bounded = String "bounded"; toJSON Informational = String "informational"++instance ToJSON Aggregation where toJSON WindowMin = String "window-min"; toJSON WindowMedian = String "window-median"++instance ToJSON ProbeReport where+ toJSON report = object ["probe" .= report.probe, "process" .= report.process, "unit" .= report.unit, "expectation" .= report.expectation, "basis" .= report.basis, "points" .= report.points, "durationSeconds" .= report.durationSeconds, "level" .= object ["first" .= report.first, "last" .= report.last, "median" .= report.medianLevel], "slopePerHour" .= report.slopePerHour, "intervalPerHour" .= fmap (\(low, high) -> [low, high]) report.intervalPerHour, "secondHalfSlopePerHour" .= report.secondHalfSlopePerHour, "growthOverWindow" .= report.growthOverWindow, "projectedHoursToLimit" .= report.projectedHoursToLimit, "verdict" .= report.verdict, "reason" .= report.reason, "source" .= object ["file" .= report.sourceFile, "column" .= report.sourceColumn]]++instance ToJSON LeakReport where+ toJSON report = object ["verdict" .= report.verdict, "window" .= fmap (\(start, end) -> object ["startSeconds" .= start, "endSeconds" .= end]) report.window, "parameters" .= object ["seed" .= report.seed, "policy" .= report.policy], "probes" .= report.probes]++instance FromJSON LeakSpec where+ parseJSON = withObject "LeakSpec" \value -> do+ schema <- value .:? "schema" .!= ("kenshou.leak-policy/v1" :: Text)+ if schema /= ("kenshou.leak-policy/v1" :: Text)+ then fail "unsupported leak policy schema"+ else LeakSpec <$> value .: "probes" <*> value .: "warmupCutSeconds" <*> value .: "minPoints" <*> value .: "minDurationSeconds" <*> value .: "envelopeWindowSeconds" <*> value .: "resamples" <*> value .: "confidence"++instance FromJSON ProbeSpec where+ parseJSON = withObject "ProbeSpec" \value -> do+ name <- value .: "name"+ unit <- value .: "unit"+ file <- value .: "file"+ timeColumn <- value .:? "timeColumn" .!= "t_mono_ns"+ valueColumn <- value .: "valueColumn"+ aggregationText <- value .: "aggregation"+ expectationText <- value .: "expectation"+ aggregation <- case (aggregationText :: Text) of "window-min" -> pure WindowMin; "window-median" -> pure WindowMedian; other -> fail ("unknown aggregation " <> Text.unpack other)+ expectation <- case (expectationText :: Text) of "bounded" -> pure Bounded; "informational" -> pure Informational; other -> fail ("unknown expectation " <> Text.unpack other)+ let filters = if file == ("rts.csv" :: FilePath) || file == "proc.csv" then Map.singleton "phase" "steady" else Map.empty+ ProbeSpec name unit (SeriesBinding file timeColumn valueColumn filters) aggregation expectation <$> value .: "floorPerHour" <*> value .: "minGrowth" <*> value .: "minRelativeGrowth" <*> value .:? "limit"++median :: [Double] -> Double+median [] = 0+median values = let ordered = quicksort values in ordered !! ((length ordered - 1) `div` 2)+ where+ quicksort [] = []+ quicksort (item : rest) = quicksort (filter (<= item) rest) <> [item] <> quicksort (filter (> item) rest)
+ src/Kenshou/Diagnose/Leak/MajorGcProbe.hs view
@@ -0,0 +1,43 @@+module Kenshou.Diagnose.Leak.MajorGcProbe+ ( withMajorGcProbe,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (withAsync)+import Control.Monad (forever, when)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import GHC.Clock (getMonotonicTimeNSec)+import GHC.Stats+import Kenshou.Core.Context qualified as Core+import Kenshou.Core.Id (Kind (Benchmark))+import Kenshou.Diagnose.Context qualified as Context+import System.IO (BufferMode (LineBuffering), IOMode (AppendMode), hSetBuffering, withFile)+import System.Mem (performMajorGC)++-- Forced major collections perturb latency and heap ageing. Scenario authors+-- must never enable this probe in benchmark evidence.+withMajorGcProbe :: Core.RunContext -> Double -> IO value -> IO value+withMajorGcProbe context intervalMilliseconds action+ | intervalMilliseconds <= 0 = action+ | Context.scenarioKind context == Benchmark = ioError (userError "withMajorGcProbe: benchmark scenarios cannot force major garbage collections")+ | otherwise = do+ path <- Core.artifactPath context Core.SeriesDir "rts-major.csv"+ Core.declareMediaType context "series/rts-major.csv" "text/csv"+ Text.writeFile path "t_mono_ns,live_bytes,major_gcs,gc_gen,pause_ns\n"+ withAsync (sampleLoop path) (const action)+ where+ delay = max 1 (floor (intervalMilliseconds * 1000))+ sampleLoop path = withFile path AppendMode \handle -> do+ hSetBuffering handle LineBuffering+ forever do+ threadDelay delay+ started <- getMonotonicTimeNSec+ performMajorGC+ stats <- getRTSStats+ ended <- getMonotonicTimeNSec+ let details = stats.gc+ when (details.gcdetails_gen > 0) $+ Text.hPutStrLn handle $+ Text.intercalate "," (fmap (Text.pack . show) [ended, details.gcdetails_live_bytes, fromIntegral stats.major_gcs, fromIntegral details.gcdetails_gen, ended - started])
+ src/Kenshou/Diagnose/LockGraph.hs view
@@ -0,0 +1,65 @@+module Kenshou.Diagnose.LockGraph+ ( GraphNode (..),+ GraphEdge (..),+ LockGraph (..),+ buildGraph,+ renderDot,+ renderText,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.!=), (.:), (.:?), (.=))+import Data.Graph (SCC (..), stronglyConnComp)+import Data.List (sort)+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Diagnose.Postgres++data GraphNode = GraphNode {pid :: !Int, applicationName :: !Text, state :: !Text, waitEvent :: !(Maybe Text), transactionAgeSeconds :: !(Maybe Double), query :: !Text}+ deriving stock (Eq, Show)++data GraphEdge = GraphEdge {waiter :: !Int, blocker :: !Int, label :: !Text}+ deriving stock (Eq, Ord, Show)++data LockGraph = LockGraph {nodes :: ![GraphNode], edges :: ![GraphEdge], cycles :: ![[Int]], roots :: ![Int]}+ deriving stock (Eq, Show)++buildGraph :: [ActivityRow] -> [LockRow] -> LockGraph+buildGraph activity _locks = LockGraph nodes edges foundCycles foundRoots+ where+ nodes = [GraphNode row.pid row.applicationName row.state row.waitEvent row.transactionAgeSeconds row.query | row <- activity]+ edges = [GraphEdge row.pid blocker (maybe "lock" id row.waitEvent) | row <- activity, blocker <- row.blockedBy]+ foundCycles = cycleComponents activity+ waiters = Set.fromList (fmap (.waiter) edges)+ blockers = Set.fromList (fmap (.blocker) edges)+ foundRoots = sort (Set.toList (blockers `Set.difference` waiters))++cycleComponents :: [ActivityRow] -> [[Int]]+cycleComponents rows = sort [sort component | CyclicSCC component <- stronglyConnComp [(row.pid, row.pid, row.blockedBy) | row <- rows]]++renderDot :: LockGraph -> Text+renderDot graph = "digraph waits {\n" <> Text.concat (fmap nodeLine graph.nodes) <> Text.concat (fmap edgeLine graph.edges) <> "}\n"+ where+ nodeLine node = " " <> shown node.pid <> " [label=\"" <> shown node.pid <> " " <> escape node.applicationName <> " " <> escape node.state <> "\"];\n"+ edgeLine edge = " " <> shown edge.waiter <> " -> " <> shown edge.blocker <> " [label=\"" <> escape edge.label <> "\"];\n"+ shown = Text.pack . show+ escape = Text.replace "\"" "\\\""++renderText :: LockGraph -> Text+renderText graph = Text.unlines (fmap cycleLine graph.cycles <> fmap edgeLine graph.edges)+ where+ cycleLine pids = "cycle: " <> Text.intercalate " -> " (fmap (Text.pack . show) (pids <> take 1 pids))+ edgeLine edge = Text.pack (show edge.waiter) <> " waits for " <> Text.pack (show edge.blocker) <> " (" <> edge.label <> ")"++instance ToJSON GraphNode where toJSON node = object ["pid" .= node.pid, "applicationName" .= node.applicationName, "state" .= node.state, "waitEvent" .= node.waitEvent, "transactionAgeSeconds" .= node.transactionAgeSeconds, "query" .= node.query]++instance FromJSON GraphNode where parseJSON = withObject "GraphNode" \value -> GraphNode <$> value .: "pid" <*> value .:? "applicationName" .!= "" <*> value .:? "state" .!= "" <*> value .:? "waitEvent" <*> value .:? "transactionAgeSeconds" <*> value .:? "query" .!= ""++instance ToJSON GraphEdge where toJSON edge = object ["waiter" .= edge.waiter, "blocker" .= edge.blocker, "label" .= edge.label]++instance FromJSON GraphEdge where parseJSON = withObject "GraphEdge" \value -> GraphEdge <$> value .: "waiter" <*> value .: "blocker" <*> value .:? "label" .!= "lock"++instance ToJSON LockGraph where toJSON graph = object ["nodes" .= graph.nodes, "edges" .= graph.edges, "cycles" .= graph.cycles, "roots" .= graph.roots, "dot" .= renderDot graph]++instance FromJSON LockGraph where parseJSON = withObject "LockGraph" \value -> LockGraph <$> value .:? "nodes" .!= [] <*> value .:? "edges" .!= [] <*> value .:? "cycles" .!= [] <*> value .:? "roots" .!= []
+ src/Kenshou/Diagnose/Pool.hs view
@@ -0,0 +1,60 @@+module Kenshou.Diagnose.Pool+ ( PoolStats (..),+ newPoolObserver,+ )+where++import Control.Applicative ((<|>))+import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.!=), (.:), (.:?), (.=))+import Data.IORef+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (UTCTime, diffUTCTime, getCurrentTime)+import Data.UUID (UUID)+import Hasql.Pool.Observation++data PoolStats = PoolStats+ { name :: !Text,+ size :: !Int,+ connecting :: !Int,+ ready :: !Int,+ inUse :: !Int,+ terminated :: !Int,+ saturatedSeconds :: !Double+ }+ deriving stock (Eq, Show)++data ObserverState = ObserverState {statuses :: Map UUID ConnectionStatus, saturatedSince :: Maybe UTCTime}++newPoolObserver :: Text -> Int -> IO (Observation -> IO (), IO PoolStats)+newPoolObserver name size = do+ state <- newIORef (ObserverState Map.empty Nothing)+ let observe (ConnectionObservation connection status) = do+ now <- getCurrentTime+ atomicModifyIORef' state \current ->+ let statuses = Map.insert connection status current.statuses+ inUse = length [() | InUseConnectionStatus <- Map.elems statuses]+ saturatedSince = if inUse >= size then current.saturatedSince <|> Just now else Nothing+ in (ObserverState statuses saturatedSince, ())+ snapshot = do+ now <- getCurrentTime+ current <- readIORef state+ let statuses = Map.elems current.statuses+ count predicate = length (filter predicate statuses)+ saturatedSeconds = maybe 0 (realToFrac . diffUTCTime now) current.saturatedSince+ pure (PoolStats name size (count isConnecting) (count isReady) (count isInUse) (count isTerminated) saturatedSeconds)+ pure (observe, snapshot)+ where+ isConnecting ConnectingConnectionStatus = True+ isConnecting _ = False+ isReady (ReadyForUseConnectionStatus _) = True+ isReady _ = False+ isInUse InUseConnectionStatus = True+ isInUse _ = False+ isTerminated (TerminatedConnectionStatus _) = True+ isTerminated _ = False++instance ToJSON PoolStats where toJSON stats = object ["name" .= stats.name, "size" .= stats.size, "connecting" .= stats.connecting, "ready" .= stats.ready, "inUse" .= stats.inUse, "terminated" .= stats.terminated, "saturatedSeconds" .= stats.saturatedSeconds]++instance FromJSON PoolStats where parseJSON = withObject "PoolStats" \value -> PoolStats <$> value .: "name" <*> value .: "size" <*> value .:? "connecting" .!= 0 <*> value .:? "ready" .!= 0 <*> value .:? "inUse" .!= 0 <*> value .:? "terminated" .!= 0 <*> value .:? "saturatedSeconds" .!= 0
+ src/Kenshou/Diagnose/Postgres.hs view
@@ -0,0 +1,169 @@+module Kenshou.Diagnose.Postgres+ ( ActivityRow (..),+ LockRow (..),+ StatementRate (..),+ PostgresSnapshot (..),+ AdvisoryKeySpec (..),+ AdvisoryLabel (..),+ captureActivity,+ captureLocks,+ captureStatementRate,+ captureSettings,+ capturePostgres,+ reconstructAdvisoryKey,+ pgMigrateLedgerLock,+ )+where++import Control.Concurrent (threadDelay)+import Data.Aeson+import Data.Bits (shiftL, (.|.))+import Data.Int (Int32, Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Data.Word (Word32, Word64)+import Hasql.Connection qualified as Connection+import Hasql.Decoders qualified as Decoders+import Hasql.Encoders qualified as Encoders+import Hasql.Session qualified as Session+import Hasql.Statement qualified as Statement+import Kenshou.Core.Contribution (AdvisoryKeySpec (..), AdvisoryLabel (..))++data ActivityRow = ActivityRow+ { pid :: !Int,+ applicationName :: !Text,+ state :: !Text,+ waitEventType :: !(Maybe Text),+ waitEvent :: !(Maybe Text),+ transactionAgeSeconds :: !(Maybe Double),+ queryAgeSeconds :: !(Maybe Double),+ blockedBy :: ![Int],+ query :: !Text+ }+ deriving stock (Eq, Show)++data LockRow = LockRow+ { pid :: !(Maybe Int),+ lockType :: !Text,+ mode :: !Text,+ granted :: !Bool,+ relation :: !(Maybe Text),+ transactionId :: !(Maybe Text),+ classId :: !(Maybe Word32),+ objectId :: !(Maybe Word32),+ objectSubId :: !(Maybe Int),+ waitAgeSeconds :: !(Maybe Double)+ }+ deriving stock (Eq, Show)++data StatementRate = StatementRate {source :: !Text, callsPerSecond :: !Double}+ deriving stock (Eq, Show)++data PostgresSnapshot = PostgresSnapshot+ { available :: !Bool,+ error :: !(Maybe Text),+ activity :: ![ActivityRow],+ locks :: ![LockRow],+ statementRate :: !(Maybe StatementRate),+ settings :: !Value+ }+ deriving stock (Eq, Show)++captureActivity :: Connection.Connection -> IO (Either Text [ActivityRow])+captureActivity connection = captureJson connection activitySql++captureLocks :: Connection.Connection -> IO (Either Text [LockRow])+captureLocks connection = captureJson connection locksSql++captureSettings :: Connection.Connection -> IO (Either Text Value)+captureSettings connection = captureJson connection settingsSql++captureStatementRate :: Connection.Connection -> Double -> IO (Either Text StatementRate)+captureStatementRate connection seconds = do+ first <- captureCounter connection statementsCounterSql+ case first of+ Left _ -> fallback+ Right left -> do+ threadDelay interval+ second <- captureCounter connection statementsCounterSql+ case second of+ Right right -> pure (Right (StatementRate "pg_stat_statements" ((right - left) / max 0.001 seconds)))+ Left _ -> fallback+ where+ interval = max 1 (floor (seconds * 1_000_000))+ fallback = do+ fallbackFirst <- captureCounter connection databaseCounterSql+ threadDelay interval+ fallbackSecond <- captureCounter connection databaseCounterSql+ pure $ StatementRate "pg_stat_database" . (/ max 0.001 seconds) <$> ((-) <$> fallbackSecond <*> fallbackFirst)++capturePostgres :: Connection.Connection -> Double -> IO PostgresSnapshot+capturePostgres connection spinSeconds = do+ activity <- captureActivity connection+ locks <- captureLocks connection+ rate <- captureStatementRate connection spinSeconds+ settings <- captureSettings connection+ pure case (activity, locks, settings) of+ (Right activityRows, Right lockRows, Right settingValue) -> PostgresSnapshot True Nothing activityRows lockRows (either (const Nothing) Just rate) settingValue+ _ -> PostgresSnapshot False (Just (Text.intercalate "; " (lefts [void activity, void locks, void settings]))) (either (const []) id activity) (either (const []) id locks) (either (const Nothing) Just rate) (either (const Null) id settings)+ where+ void = fmap (const ())+ lefts = foldr (\value rest -> either (: rest) (const rest) value) []++captureJson :: (FromJSON value) => Connection.Connection -> Text -> IO (Either Text value)+captureJson connection sql = do+ result <- Connection.use connection (Session.statement () (textStatement sql))+ pure case result of+ Left err -> Left (Text.pack (show err))+ Right encoded -> case eitherDecodeStrict' (Text.encodeUtf8 encoded) of Left err -> Left (Text.pack err); Right value -> Right value++captureCounter :: Connection.Connection -> Text -> IO (Either Text Double)+captureCounter connection sql = do+ result <- Connection.use connection (Session.statement () (textStatement sql))+ pure case result of Left err -> Left (Text.pack (show err)); Right value -> maybe (Left ("invalid counter: " <> value)) Right (readDouble value)++readDouble :: Text -> Maybe Double+readDouble value = case reads (Text.unpack value) of [(number, "")] -> Just number; _ -> Nothing++textStatement :: Text -> Statement.Statement () Text+textStatement sql = Statement.unpreparable sql Encoders.noParams (Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.text)))++activitySql :: Text+activitySql = "SELECT coalesce(jsonb_agg(to_jsonb(t)), '[]'::jsonb)::text FROM (SELECT pid, coalesce(application_name,'') AS \"applicationName\", coalesce(state,'') AS state, wait_event_type AS \"waitEventType\", wait_event AS \"waitEvent\", extract(epoch FROM clock_timestamp()-xact_start) AS \"transactionAgeSeconds\", extract(epoch FROM clock_timestamp()-query_start) AS \"queryAgeSeconds\", pg_blocking_pids(pid) AS \"blockedBy\", left(query,2000) AS query FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid()) t"++locksSql :: Text+locksSql = "SELECT coalesce(jsonb_agg(to_jsonb(t)), '[]'::jsonb)::text FROM (SELECT pid, locktype AS \"lockType\", mode, granted, relation::regclass::text AS relation, transactionid::text AS \"transactionId\", classid AS \"classId\", objid AS \"objectId\", objsubid AS \"objectSubId\", extract(epoch FROM clock_timestamp()-waitstart) AS \"waitAgeSeconds\" FROM pg_locks) t"++settingsSql :: Text+settingsSql = "SELECT jsonb_build_object('maxConnections', current_setting('max_connections')::int, 'deadlockTimeout', current_setting('deadlock_timeout'), 'statementTimeout', current_setting('statement_timeout'))::text"++statementsCounterSql, databaseCounterSql :: Text+statementsCounterSql = "SELECT coalesce(sum(calls),0)::text FROM pg_stat_statements(false) WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database())"+databaseCounterSql = "SELECT (xact_commit+xact_rollback)::text FROM pg_stat_database WHERE datname=current_database()"++reconstructAdvisoryKey :: Word32 -> Word32 -> Int64+reconstructAdvisoryKey high low = fromIntegral (((fromIntegral high :: Word64) `shiftL` 32) .|. fromIntegral low)++pgMigrateLedgerLock :: AdvisoryLabel+pgMigrateLedgerLock = AdvisoryLabel "pg-migrate ledger" (LiteralKey 0x70675F6D69677261)++instance ToJSON ActivityRow where+ toJSON row = object ["pid" .= row.pid, "applicationName" .= row.applicationName, "state" .= row.state, "waitEventType" .= row.waitEventType, "waitEvent" .= row.waitEvent, "transactionAgeSeconds" .= row.transactionAgeSeconds, "queryAgeSeconds" .= row.queryAgeSeconds, "blockedBy" .= row.blockedBy, "query" .= row.query]++instance FromJSON ActivityRow where+ parseJSON = withObject "ActivityRow" \value -> ActivityRow <$> value .: "pid" <*> value .:? "applicationName" .!= "" <*> value .:? "state" .!= "" <*> value .:? "waitEventType" <*> value .:? "waitEvent" <*> value .:? "transactionAgeSeconds" <*> value .:? "queryAgeSeconds" <*> value .:? "blockedBy" .!= [] <*> value .:? "query" .!= ""++instance ToJSON LockRow where+ toJSON row = object ["pid" .= row.pid, "lockType" .= row.lockType, "mode" .= row.mode, "granted" .= row.granted, "relation" .= row.relation, "transactionId" .= row.transactionId, "classId" .= row.classId, "objectId" .= row.objectId, "objectSubId" .= row.objectSubId, "waitAgeSeconds" .= row.waitAgeSeconds]++instance FromJSON LockRow where+ parseJSON = withObject "LockRow" \value -> LockRow <$> value .:? "pid" <*> value .:? "lockType" .!= "" <*> value .:? "mode" .!= "" <*> value .:? "granted" .!= False <*> value .:? "relation" <*> value .:? "transactionId" <*> value .:? "classId" <*> value .:? "objectId" <*> value .:? "objectSubId" <*> value .:? "waitAgeSeconds"++instance ToJSON StatementRate where toJSON rate = object ["source" .= rate.source, "callsPerSecond" .= rate.callsPerSecond]++instance FromJSON StatementRate where parseJSON = withObject "StatementRate" \value -> StatementRate <$> value .: "source" <*> value .: "callsPerSecond"++instance ToJSON PostgresSnapshot where toJSON snapshot = object ["available" .= snapshot.available, "error" .= snapshot.error, "activity" .= snapshot.activity, "locks" .= snapshot.locks, "statementRate" .= snapshot.statementRate, "settings" .= snapshot.settings]++instance FromJSON PostgresSnapshot where parseJSON = withObject "PostgresSnapshot" \value -> PostgresSnapshot <$> value .: "available" <*> value .:? "error" <*> value .:? "activity" .!= [] <*> value .:? "locks" .!= [] <*> value .:? "statementRate" <*> value .:? "settings" .!= Null
+ src/Kenshou/Diagnose/Profile.hs view
@@ -0,0 +1,227 @@+module Kenshou.Diagnose.Profile+ ( ProfileMode (..),+ ProfileRequest (..),+ ProfileReport (..),+ rtsFlags,+ runProfileSession,+ reexecWithWorkerRts,+ markPhase,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (race)+import Control.Exception (Exception, throwIO)+import Data.Aeson (ToJSON (..), object, toJSON, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Char (isAlphaNum, toUpper)+import Data.List (isPrefixOf)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (defaultTimeLocale, formatTime, getCurrentTime)+import Data.Word (Word64)+import Debug.Trace (traceMarkerIO)+import Kenshou.Diagnose.Document (Diagnosis (..), DiagnosisKind (ProfileDiagnosis), Generator (..), encodeDiagnosis)+import System.Directory+import System.Environment (getArgs, getEnvironment, getExecutablePath, lookupEnv)+import System.Exit (ExitCode (..))+import System.FilePath (takeFileName, (</>))+import System.Posix.Process (executeFile, getProcessID)+import System.Process+import Text.Read (readMaybe)++data ProfileMode = ClosureType | InfoTable | Eventlog ![Char] | Profiled !Char+ deriving stock (Eq, Show)++data ProfileRequest = ProfileRequest+ { mode :: !ProfileMode,+ scenario :: !Text,+ runArgs :: ![String],+ sessionRoot :: !FilePath,+ censusIntervalSeconds :: !Double,+ eventlogMaxBytes :: !Word64,+ workerRole :: !(Maybe Text)+ }+ deriving stock (Eq, Show)++data ProfileReport = ProfileReport+ { sessionDirectory :: !FilePath,+ mode :: !ProfileMode,+ variant :: !Text,+ flags :: ![String],+ binary :: !FilePath,+ runDirectory :: !(Maybe FilePath),+ manifestSha256 :: !(Maybe Text),+ eventlogBytes :: !Word64,+ truncated :: !Bool,+ exitCode :: !Int+ }+ deriving stock (Eq, Show)++newtype ProfileError = ProfileError Text deriving stock (Show)++instance Exception ProfileError++rtsFlags :: ProfileMode -> FilePath -> Double -> [String]+rtsFlags mode eventlog interval = profileFlags <> [eventFlags, "-ol" <> eventlog, "--eventlog-flush-interval=5"]+ where+ profileFlags = case mode of+ ClosureType -> ["-hT", "-i" <> show interval]+ InfoTable -> ["-hi", "-i" <> show interval]+ Eventlog _ -> []+ Profiled breakdown -> ["-h" <> [breakdown], "-i" <> show interval]+ eventFlags = case mode of+ Eventlog classes -> "-l-a" <> if null classes then "gu" else classes+ _ -> "-l-agu"++runProfileSession :: ProfileRequest -> IO ProfileReport+runProfileSession request = do+ now <- getCurrentTime+ let session = request.sessionRoot </> ("profile-" <> formatTime defaultTimeLocale "%Y%m%dT%H%M%SZ" now <> "-" <> modeSlug request.mode)+ eventlog = session </> "kenshou.eventlog"+ flags = rtsFlags request.mode eventlog request.censusIntervalSeconds+ variant = variantName request.mode+ createDirectoryIfMissing True session+ ensureDiskSpace session request.eventlogMaxBytes+ binary <- locateBinary request.mode+ baseEnvironment <- getEnvironment+ let profileEnvironment = case request.workerRole of+ Nothing -> [("KENSHOU_EVENTLOG_PATH", eventlog), ("KENSHOU_EVENTLOG_MAX_BYTES", show request.eventlogMaxBytes)]+ Just role -> workerVariables session request.eventlogMaxBytes flags role+ environment = replaceEnvironment profileEnvironment baseEnvironment+ mainRts = if request.workerRole == Nothing then ["+RTS"] <> flags <> ["-RTS"] else []+ arguments = ["run", Text.unpack request.scenario, "--out", session </> "run"] <> request.runArgs <> mainRts+ (_, _, _, process) <- createProcess (proc binary arguments) {env = Just environment}+ outcome <- race (waitForProcess process) (watchBackstop eventlog request.eventlogMaxBytes)+ (code, parentTruncated) <- case outcome of+ Left code -> pure (code, False)+ Right () -> terminateProcess process >> (,True) <$> waitForProcess process+ truncatedByChild <- doesFileExist (eventlog <> ".truncated")+ bytes <- fileBytes eventlog+ runDirectory <- findRunDirectory (session </> "run")+ manifestSha256 <- traverse hashManifest runDirectory+ let report = ProfileReport session request.mode variant flags binary runDirectory manifestSha256 bytes (parentTruncated || truncatedByChild) (exitCodeNumber code)+ runId = maybe "" (Text.pack . takeFileName) runDirectory+ document = Diagnosis ProfileDiagnosis runId request.scenario now (Generator "kenshou-diagnose" "0.1.0.0" "profile-session-v1") (toJSON report)+ LazyByteString.writeFile (session </> "profile.json") (encodeDiagnosis document)+ pure report++locateBinary :: ProfileMode -> IO FilePath+locateBinary mode = do+ let (arguments, buildCommand) = case mode of+ InfoTable -> (["--project-file=cabal.diagnose-info-table.project", "--builddir=dist-diagnose/info-table", "list-bin", "kenshou-cli:exe:kenshou"], "just diagnose-build-info-table")+ Profiled _ -> (["--project-file=cabal.diagnose-profiled.project", "--builddir=dist-diagnose/profiled", "list-bin", "kenshou-cli:exe:kenshou"], "just diagnose-build-profiled")+ _ -> (["list-bin", "kenshou-cli:exe:kenshou"], "cabal build kenshou-cli:exe:kenshou")+ (code, output, err) <- readProcessWithExitCode "cabal" arguments ""+ case code of+ ExitSuccess -> pure (trim output)+ _ -> throwIO (ProfileError ("profiling binary is unavailable; run `" <> buildCommand <> "`: " <> Text.pack err))++ensureDiskSpace :: FilePath -> Word64 -> IO ()+ensureDiskSpace path limit = do+ (code, output, err) <- readProcessWithExitCode "df" ["-Pk", path] ""+ case code of+ ExitSuccess -> case reverse (lines output) of+ row : _ -> case words row of+ _filesystem : _blocks : _used : available : _ -> case readMaybe available :: Maybe Integer of+ Just availableKiB | availableKiB * 1024 >= fromIntegral limit * 4 -> pure ()+ _ -> throwIO (ProfileError "profiling requires free disk space of at least four times the event-log limit")+ _ -> throwIO (ProfileError "unable to parse df output")+ [] -> throwIO (ProfileError "df returned no output")+ _ -> throwIO (ProfileError ("unable to inspect free disk space: " <> Text.pack err))++watchBackstop :: FilePath -> Word64 -> IO ()+watchBackstop path limit = do+ bytes <- fileBytes path+ if bytes >= limit * 2 then pure () else threadDelay 1_000_000 >> watchBackstop path limit++reexecWithWorkerRts :: Text -> IO ()+reexecWithWorkerRts role = do+ let variable = workerVariable role+ flags <- lookupEnv variable+ inherited <- lookupEnv "GHCRTS"+ directory <- lookupEnv "KENSHOU_WORKER_EVENTLOG_DIR"+ case (flags, inherited, directory) of+ (Just rawFlags, Nothing, Just eventlogDirectory) -> do+ executable <- getExecutablePath+ arguments <- getArgs+ environment <- getEnvironment+ pid <- getProcessID+ let eventlog = eventlogDirectory </> ("worker-" <> Text.unpack role <> "-" <> show pid <> ".eventlog")+ ghcRts = unwords [rawFlags, "-ol" <> eventlog]+ replacements = [("GHCRTS", ghcRts), ("KENSHOU_EVENTLOG_PATH", eventlog)]+ cleaned = filter ((`notElem` [variable, "KENSHOU_WORKER_EVENTLOG_DIR"]) . fst) environment+ executeFile executable False arguments (Just (replaceEnvironment replacements cleaned))+ _ -> pure ()++markPhase :: Text -> IO ()+markPhase = traceMarkerIO . Text.unpack . ("kenshou.phase:" <>)++workerVariables :: FilePath -> Word64 -> [String] -> Text -> [(String, String)]+workerVariables session limit flags role =+ [ (workerVariable role, unwords (filter (not . isPrefixOf "-ol") flags)),+ ("KENSHOU_WORKER_EVENTLOG_DIR", session),+ ("KENSHOU_EVENTLOG_MAX_BYTES", show limit)+ ]++workerVariable :: Text -> String+workerVariable role = "KENSHOU_WORKER_GHCRTS_" <> fmap normalize (Text.unpack role)+ where+ normalize character | isAlphaNum character = toUpper character+ normalize _ = '_'++variantName :: ProfileMode -> Text+variantName InfoTable = "info-table"+variantName (Profiled _) = "profiled"+variantName _ = "ordinary"++modeSlug :: ProfileMode -> String+modeSlug ClosureType = "closure-type"+modeSlug InfoTable = "info-table"+modeSlug (Eventlog _) = "eventlog"+modeSlug (Profiled _) = "profiled"++fileBytes :: FilePath -> IO Word64+fileBytes path = do+ exists <- doesFileExist path+ if exists then fromIntegral <$> getFileSize path else pure 0++findRunDirectory :: FilePath -> IO (Maybe FilePath)+findRunDirectory root = do+ exists <- doesDirectoryExist root+ if not exists+ then pure Nothing+ else do+ entries <- listDirectory root+ findM (doesFileExist . (</> "manifest.json")) [root </> entry | entry <- entries]++hashManifest :: FilePath -> IO Text+hashManifest runDirectory = do+ (code, output, err) <- readProcessWithExitCode "shasum" ["-a", "256", runDirectory </> "manifest.json"] ""+ case (code, words output) of+ (ExitSuccess, digest : _) -> pure (Text.pack digest)+ _ -> throwIO (ProfileError ("unable to hash run manifest: " <> Text.pack err))++replaceEnvironment :: [(String, String)] -> [(String, String)] -> [(String, String)]+replaceEnvironment replacements original = replacements <> filter ((`notElem` fmap fst replacements) . fst) original++findM :: (value -> IO Bool) -> [value] -> IO (Maybe value)+findM _ [] = pure Nothing+findM predicate (value : rest) = predicate value >>= \matches -> if matches then pure (Just value) else findM predicate rest++trim :: String -> String+trim = reverse . dropWhile (`elem` ['\n', '\r', ' ']) . reverse++exitCodeNumber :: ExitCode -> Int+exitCodeNumber ExitSuccess = 0+exitCodeNumber (ExitFailure code) = code++instance ToJSON ProfileMode where+ toJSON mode = case mode of+ ClosureType -> object ["name" .= ("closure-type" :: Text)]+ InfoTable -> object ["name" .= ("info-table" :: Text)]+ Eventlog classes -> object ["name" .= ("eventlog" :: Text), "classes" .= classes]+ Profiled breakdown -> object ["name" .= ("profiled" :: Text), "breakdown" .= [breakdown]]++instance ToJSON ProfileReport where+ toJSON report = object ["schema" .= ("kenshou.profile/v1" :: Text), "sessionDirectory" .= report.sessionDirectory, "mode" .= report.mode, "variant" .= report.variant, "flags" .= report.flags, "binary" .= report.binary, "runDirectory" .= report.runDirectory, "manifestSha256" .= report.manifestSha256, "eventlogBytes" .= report.eventlogBytes, "truncated" .= report.truncated, "exitCode" .= report.exitCode]
+ src/Kenshou/Diagnose/Profile/EventlogGuard.hs view
@@ -0,0 +1,45 @@+{-# LANGUAGE ForeignFunctionInterface #-}++module Kenshou.Diagnose.Profile.EventlogGuard+ ( withEventlogGuard,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (withAsync)+import Kenshou.Diagnose.Threads (labelMe)+import System.Directory (doesFileExist, getFileSize)+import System.Environment (lookupEnv)+import System.IO (hPutStrLn, stderr)+import Text.Read (readMaybe)++foreign import ccall safe "endEventLogging" endEventLogging :: IO ()++withEventlogGuard :: IO value -> IO value+withEventlogGuard action = do+ configuration <- guardConfiguration+ case configuration of+ Nothing -> action+ Just (path, limit) -> withAsync (guard path limit) (const action)++guardConfiguration :: IO (Maybe (FilePath, Integer))+guardConfiguration = do+ path <- lookupEnv "KENSHOU_EVENTLOG_PATH"+ rawLimit <- lookupEnv "KENSHOU_EVENTLOG_MAX_BYTES"+ pure do+ eventlogPath <- path+ limit <- rawLimit >>= readMaybe+ if limit > 0 then Just (eventlogPath, limit) else Nothing++guard :: FilePath -> Integer -> IO ()+guard path limit = do+ labelMe "kenshou:diagnose:eventlog-guard"+ waitForLimit+ endEventLogging+ writeFile (path <> ".truncated") "event logging stopped at configured byte limit\n"+ hPutStrLn stderr ("kenshou: event log reached " <> show limit <> " bytes; logging stopped")+ where+ waitForLimit = do+ exists <- doesFileExist path+ size <- if exists then getFileSize path else pure 0+ if size >= limit then pure () else threadDelay 1_000_000 >> waitForLimit
+ src/Kenshou/Diagnose/Profile/GhcDebug.hs view
@@ -0,0 +1,39 @@+{-# LANGUAGE CPP #-}++module Kenshou.Diagnose.Profile.GhcDebug+ ( withGhcDebugIfRequested,+ )+where++import System.Environment (lookupEnv)++#ifdef WITH_GHC_DEBUG+import Data.Word (Word16)+import GHC.Debug.Stub qualified as GhcDebug+import Text.Read (readMaybe)+#else+import System.IO (hPutStrLn, stderr)+#endif++withGhcDebugIfRequested :: IO value -> IO value+withGhcDebugIfRequested action = do+ socket <- lookupEnv "KENSHOU_GHC_DEBUG_SOCKET"+ tcp <- lookupEnv "KENSHOU_GHC_DEBUG_TCP"+#ifdef WITH_GHC_DEBUG+ case (socket, tcp >>= parseTcp) of+ (Just path, _) -> GhcDebug.withGhcDebugUnix path action+ (Nothing, Just (host, port)) -> GhcDebug.withGhcDebugTCP host port action+ _ -> action+#else+ case socket <> tcp of+ Nothing -> pure ()+ Just _ -> hPutStrLn stderr "kenshou: ghc-debug requested, but the executable was built without -fghc-debug"+ action+#endif++#ifdef WITH_GHC_DEBUG+parseTcp :: String -> Maybe (String, Word16)+parseTcp value = case break (== ':') value of+ (host, ':' : rawPort) -> (host,) <$> readMaybe rawPort+ _ -> Nothing+#endif
+ src/Kenshou/Diagnose/Progress.hs view
@@ -0,0 +1,52 @@+module Kenshou.Diagnose.Progress+ ( ProgressCounter,+ ProgressSnapshot (..),+ newProgressCounter,+ tick,+ tickBy,+ snapshotProgress,+ )+where++import Data.Aeson (FromJSON (..), ToJSON (..), object, withObject, (.:), (.=))+import Data.IORef+import Data.Text (Text)+import Data.Time (UTCTime, getCurrentTime)+import Data.Word (Word64)++data ProgressCounter = ProgressCounter+ { name :: !Text,+ required :: !Bool,+ value :: !(IORef Word64),+ lastAdvancedAt :: !(IORef UTCTime)+ }++data ProgressSnapshot = ProgressSnapshot+ { name :: !Text,+ required :: !Bool,+ value :: !Word64,+ lastAdvancedAt :: !UTCTime+ }+ deriving stock (Eq, Show)++newProgressCounter :: Text -> Bool -> IO ProgressCounter+newProgressCounter name required = do+ now <- getCurrentTime+ ProgressCounter name required <$> newIORef 0 <*> newIORef now++tick :: ProgressCounter -> IO ()+tick counter = tickBy counter 1++tickBy :: ProgressCounter -> Word64 -> IO ()+tickBy counter amount = do+ atomicModifyIORef' counter.value (\value -> (value + amount, ()))+ getCurrentTime >>= writeIORef counter.lastAdvancedAt++snapshotProgress :: ProgressCounter -> IO ProgressSnapshot+snapshotProgress counter = ProgressSnapshot counter.name counter.required <$> readIORef counter.value <*> readIORef counter.lastAdvancedAt++instance ToJSON ProgressSnapshot where+ toJSON snapshot = object ["name" .= snapshot.name, "required" .= snapshot.required, "value" .= snapshot.value, "lastAdvancedAt" .= snapshot.lastAdvancedAt]++instance FromJSON ProgressSnapshot where+ parseJSON = withObject "ProgressSnapshot" \value -> ProgressSnapshot <$> value .: "name" <*> value .: "required" <*> value .: "value" <*> value .: "lastAdvancedAt"
+ src/Kenshou/Diagnose/Render.hs view
@@ -0,0 +1,59 @@+module Kenshou.Diagnose.Render+ ( renderLeakReport,+ renderStallReport,+ renderProfileReport,+ )+where++import Data.Maybe (fromMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Diagnose.Leak+import Kenshou.Diagnose.LockGraph qualified as LockGraph+import Kenshou.Diagnose.Pool (PoolStats (..))+import Kenshou.Diagnose.Profile+import Kenshou.Diagnose.Stall.Types+import Kenshou.Diagnose.Threads (ThreadDump (..), ThreadEntry (..))++renderLeakReport :: LeakReport -> Text+renderLeakReport report = Text.unlines (header : fmap renderProbe report.probes)+ where+ header = "leak verdict: " <> leakVerdictText report.verdict+ renderProbe probe =+ Text.intercalate+ " "+ [ probe.probe <> "[" <> probe.process <> "]",+ leakVerdictText probe.verdict,+ "slope=" <> maybe "n/a" (\value -> showText value <> " " <> probe.unit <> "/hour") probe.slopePerHour,+ "reason=" <> probe.reason+ ]++renderStallReport :: StallReport -> Text+renderStallReport report =+ Text.unlines+ ( ["stall " <> Text.pack (show report.captureNumber) <> ": " <> stallClassText report.classification]+ <> fmap ("reason: " <>) report.reasons+ <> nonempty "wait graph:" (LockGraph.renderText report.snapshot.graph)+ <> ["thread " <> fromMaybe "<unlabelled>" entry.label <> ": " <> entry.status <> maybe "" (" " <>) entry.blockReason | entry <- report.snapshot.haskellThreads.entries, entry.status == "blocked"]+ <> ["pool " <> pool.name <> ": " <> showText pool.inUse <> "/" <> showText pool.size <> " in use, saturated " <> showText pool.saturatedSeconds <> "s" | pool <- report.snapshot.pools]+ )+ where+ nonempty _ "" = []+ nonempty label value = label : Text.lines value++renderProfileReport :: ProfileReport -> Text+renderProfileReport report =+ Text.unlines+ [ "profile session: " <> Text.pack report.sessionDirectory,+ "mode: " <> Text.pack (show report.mode),+ "event log: " <> showText report.eventlogBytes <> " bytes" <> if report.truncated then " (truncated)" else "",+ "wrapped exit: " <> showText report.exitCode+ ]++leakVerdictText :: LeakVerdict -> Text+leakVerdictText LeakSuspected = "leak-suspected"+leakVerdictText Stable = "stable"+leakVerdictText InsufficientData = "insufficient-data"++showText :: (Show value) => value -> Text+showText = Text.pack . show
+ src/Kenshou/Diagnose/SelfTest.hs view
@@ -0,0 +1,20 @@+module Kenshou.Diagnose.SelfTest (bundle) where++import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Id (selftestLayer)+import Kenshou.Diagnose.SelfTest.Leak (leakingWorkerScenario, stableWorkerScenario)+import Kenshou.Diagnose.SelfTest.Stall++bundle :: LayerBundle+bundle =+ mkBundle+ selftestLayer+ [ leakingWorkerScenario,+ stableWorkerScenario,+ deadlockedWorkersScenario,+ poolStarvedScenario,+ lockWaiterScenario,+ idleSpinnerScenario,+ healthyProgressScenario+ ]+ []
+ src/Kenshou/Diagnose/SelfTest/Leak.hs view
@@ -0,0 +1,305 @@+module Kenshou.Diagnose.SelfTest.Leak+ ( leakingWorkerScenario,+ stableWorkerScenario,+ )+where++import Control.Concurrent (MVar, ThreadId, killThread, newEmptyMVar, takeMVar, threadDelay)+import Control.Exception (bracket, evaluate, finally)+import Control.Monad (forM, forM_, replicateM, void)+import Data.ByteString qualified as ByteString+import Data.ByteString.Short (ShortByteString)+import Data.ByteString.Short qualified as Short+import Data.IORef+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Dimension+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (Kind (Soak), ScenarioId, parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Diagnose.Leak+import Kenshou.Diagnose.Leak.MajorGcProbe (withMajorGcProbe)+import Kenshou.Diagnose.Series (SeriesBinding (..))+import Kenshou.Diagnose.Threads (forkLabelled)+import Kenshou.Measure.Clock (Nanos (..))+import Kenshou.Measure.Knobs (measureKnobs)+import Kenshou.Measure.Phase qualified as Measure+import Kenshou.Measure.Sampler+import Kenshou.Measure.Session (MeasureEnv (..), measureEnvFromRunContext)+import System.IO (Handle, IOMode (ReadMode), hClose, openFile)++leakingWorkerScenario :: Scenario+leakingWorkerScenario =+ Scenario+ { id = scenarioId "selftest/diagnose/soak/leaking-worker",+ revision = 1,+ summary = "Proves that bounded heap, thread, and descriptor growth is detected.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = leakKnobs <> samplingKnobs,+ dimensions = telemetryOff,+ phases = Core.PhasePlan 5 40 0,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = runLeakingWorker+ }++stableWorkerScenario :: Scenario+stableWorkerScenario =+ Scenario+ { id = scenarioId "selftest/diagnose/soak/stable-worker",+ revision = 1,+ summary = "Proves that high allocation into a bounded retained ring remains stable.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = stableKnobs <> samplingKnobs,+ dimensions = telemetryOff,+ phases = Core.PhasePlan 10 35 0,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = runStableWorker+ }++data LeakResources+ = HeapResources (IORef [ShortByteString])+ | ThreadResources (MVar ()) (IORef [ThreadId])+ | DescriptorResources (IORef [Handle])++runLeakingWorker :: RunContext -> IO ScenarioReport+runLeakingWorker context = do+ let kind = knobText context.knobs (name "leak.kind")+ duration = fromIntegral (knobInt context.knobs (name "run.duration-s"))+ warmup = min duration (fromIntegral (knobInt context.knobs (name "run.warmup-s")))+ majorGcMs = fromIntegral (knobInt context.knobs (name "diagnose.major-gc-interval-ms"))+ bytesPerSecond = fromIntegral (knobInt context.knobs (name "leak.bytes-per-second"))+ unitsPerSecond = fromIntegral (knobInt context.knobs (name "leak.units-per-second"))+ chunkBytes = fromIntegral (knobInt context.knobs (name "leak.chunk-bytes"))+ targetRate = if kind == "heap" then fromIntegral bytesPerSecond else fromIntegral unitsPerSecond+ bracket (newResources kind) cleanupResources $ \resources -> do+ _ <- withMajorGcProbe context majorGcMs $ withSelfTestSampling context \clock -> do+ Measure.enterPhase clock Measure.WarmUp+ produceFor warmup (produce kind resources bytesPerSecond unitsPerSecond chunkBytes)+ Measure.enterPhase clock Measure.Steady+ produceFor (duration - warmup) (produce kind resources bytesPerSecond unitsPerSecond chunkBytes)+ report <- judgeLeaks context (leakingSpec kind majorGcMs duration warmup targetRate)+ pure (assessLeak kind targetRate report)++runStableWorker :: RunContext -> IO ScenarioReport+runStableWorker context = do+ let duration = fromIntegral (knobInt context.knobs (name "run.duration-s"))+ warmup = min duration (fromIntegral (knobInt context.knobs (name "run.warmup-s")))+ majorGcMs = fromIntegral (knobInt context.knobs (name "diagnose.major-gc-interval-ms"))+ bytesPerSecond = fromIntegral (knobInt context.knobs (name "alloc.bytes-per-second"))+ ringBytes = fromIntegral (knobInt context.knobs (name "retain.ring-bytes"))+ chunkBytes = 4096+ retained <- newIORef []+ allocated <- newIORef (0 :: Int)+ _ <- withMajorGcProbe context majorGcMs $ withSelfTestSampling context \clock -> do+ Measure.enterPhase clock Measure.WarmUp+ stableFor warmup retained allocated bytesPerSecond ringBytes chunkBytes+ Measure.enterPhase clock Measure.Steady+ stableFor (duration - warmup) retained allocated bytesPerSecond ringBytes chunkBytes+ report <- judgeLeaks context (stableSpec majorGcMs duration warmup)+ total <- readIORef allocated+ let heap = findProbe "heap.live-bytes" report+ enoughAllocation = total > 5 * ringBytes+ pure case heap of+ Nothing -> inconclusiveBecause "heap.live-bytes was not reported"+ Just probe+ | probe.verdict == InsufficientData -> inconclusiveBecause ("heap evidence is insufficient: " <> probe.reason)+ | probe.verdict == Stable && enoughAllocation -> passed+ | otherwise -> failedWith ["stable-worker-verdict"] ("heap verdict=" <> showText probe.verdict <> ", allocated=" <> showText total)++withSelfTestSampling :: RunContext -> (Measure.PhaseClock -> IO value) -> IO value+withSelfTestSampling context action = do+ environment <- measureEnvFromRunContext context+ clock <- Measure.newPhaseClock environment.onPhase (Measure.PhasePlan (Nanos 0) (Measure.SteadyCount 0) (Nanos 0))+ sampling <-+ startSampling+ SamplerConfig+ { runDir = environment.runDir,+ origin = environment.origin,+ intervalMs = 500,+ postgres = Nothing,+ extraSamplers = [],+ phaseClock = clock,+ declareArtifact = environment.declareArtifact,+ logLine = environment.logLine+ }+ action clock `finally` void (stopSampling sampling)++newResources :: Text -> IO LeakResources+newResources "heap" = HeapResources <$> newIORef []+newResources "threads" = ThreadResources <$> newEmptyMVar <*> newIORef []+newResources "fds" = DescriptorResources <$> newIORef []+newResources other = ioError (userError ("unknown leak.kind " <> Text.unpack other))++cleanupResources :: LeakResources -> IO ()+cleanupResources (HeapResources retained) = writeIORef retained []+cleanupResources (ThreadResources _ threads) = readIORef threads >>= mapM_ killThread+cleanupResources (DescriptorResources handles) = readIORef handles >>= mapM_ hClose++produce :: Text -> LeakResources -> Int -> Int -> Int -> Int -> IO ()+produce "heap" (HeapResources retained) bytesPerSecond _ chunkBytes step = do+ chunks <- freshChunks bytesPerSecond chunkBytes step+ modifyIORef' retained (chunks <>)+produce "threads" (ThreadResources blocker threads) _ unitsPerSecond _ _ = do+ spawned <- replicateM unitsPerSecond (forkLabelled "kenshou:selftest:leaked-thread" (waitForever blocker))+ modifyIORef' threads (spawned <>)+produce "fds" (DescriptorResources handles) _ unitsPerSecond _ _ = do+ opened <- replicateM unitsPerSecond (openFile "/dev/null" ReadMode)+ modifyIORef' handles (opened <>)+produce _ _ _ _ _ _ = ioError (userError "leak resource does not match leak.kind")++waitForever :: MVar () -> IO ()+waitForever blocker = do+ _ <- takeMVar blocker+ pure ()++produceFor :: Int -> (Int -> IO ()) -> IO ()+produceFor seconds action = forM_ [0 .. seconds - 1] \step -> action step >> threadDelay 1_000_000++stableFor :: Int -> IORef [ShortByteString] -> IORef Int -> Int -> Int -> Int -> IO ()+stableFor seconds retained allocated bytesPerSecond ringBytes chunkBytes =+ forM_ [0 .. seconds - 1] \step -> do+ chunks <- freshChunks bytesPerSecond chunkBytes step+ previous <- readIORef retained+ let bounded = take (max 1 (ringBytes `div` chunkBytes)) (chunks <> previous)+ _ <- evaluate (sum (fmap Short.length bounded))+ writeIORef retained bounded+ modifyIORef' allocated (+ sum (fmap Short.length chunks))+ threadDelay 1_000_000++freshChunks :: Int -> Int -> Int -> IO [ShortByteString]+freshChunks bytesPerSecond chunkBytes step = do+ let count = max 1 ((bytesPerSecond + chunkBytes - 1) `div` chunkBytes)+ byte = fromIntegral (step `mod` 251)+ forM [1 .. count] \index -> do+ let remaining = bytesPerSecond - (index - 1) * chunkBytes+ size = max 1 (min chunkBytes remaining)+ chunk = Short.toShort (ByteString.replicate size byte)+ Short.length chunk `seq` pure chunk++leakingSpec :: Text -> Double -> Int -> Int -> Double -> LeakSpec+leakingSpec kind majorGcMs duration warmup targetRate =+ defaultLeakSpec+ { probes = fmap tune defaultLeakSpec.probes,+ warmupCutSeconds = if majorGcMs > 0 then 0 else fromIntegral warmup,+ minPoints = max 5 (min 20 (duration - warmup - 2)),+ minDurationSeconds = max 5 (fromIntegral (duration - warmup) * 0.7),+ envelopeWindowSeconds = 1,+ resamples = 300+ }+ where+ target = targetProbe kind+ tune probe+ | probe.name == target =+ probe+ { binding = if kind == "heap" && majorGcMs > 0 then SeriesBinding "rts-major.csv" "t_mono_ns" "live_bytes" Map.empty else probe.binding,+ floorPerHour = targetRate * 1800,+ minGrowth = targetRate * fromIntegral (duration - warmup) * 0.25,+ minRelativeGrowth = 0+ }+ | otherwise = probe {floorPerHour = 1.0e30, minGrowth = 1.0e30, minRelativeGrowth = 0}++stableSpec :: Double -> Int -> Int -> LeakSpec+stableSpec majorGcMs duration warmup =+ defaultLeakSpec+ { probes = fmap tune defaultLeakSpec.probes,+ warmupCutSeconds = if majorGcMs > 0 then 0 else fromIntegral warmup,+ minPoints = max 5 (min 20 (duration - warmup - 2)),+ minDurationSeconds = max 5 (fromIntegral (duration - warmup) * 0.7),+ envelopeWindowSeconds = 1,+ resamples = 300+ }+ where+ tune :: ProbeSpec -> ProbeSpec+ tune probe+ | probe.name == "heap.live-bytes" = probe {binding = if majorGcMs > 0 then SeriesBinding "rts-major.csv" "t_mono_ns" "live_bytes" Map.empty else probe.binding}+ | otherwise = ProbeSpec probe.name probe.unit probe.binding probe.aggregation Informational probe.floorPerHour probe.minGrowth probe.minRelativeGrowth probe.limit++assessLeak :: Text -> Double -> LeakReport -> ScenarioReport+assessLeak kind expectedRate report = case findProbe (targetProbe kind) report of+ Nothing -> inconclusiveBecause ("target probe was not reported: " <> targetProbe kind)+ Just probe+ | probe.verdict == InsufficientData -> inconclusiveBecause ("target probe evidence is insufficient: " <> probe.reason)+ | otherwise ->+ let actualRate = (/ 3600) <$> probe.slopePerHour+ tolerance = if kind == "heap" then 0.15 else 0.20+ rateOk = maybe False (\actual -> abs (actual - expectedRate) <= expectedRate * tolerance) actualRate+ intervalOk = case (probe.slopePerHour, probe.intervalPerHour) of+ (Just slope, Just (low, high)) -> low <= slope && slope <= high+ _ -> False+ unrelated = [candidate.probe | candidate <- report.probes, candidate.probe /= probe.probe, candidate.expectation == Bounded, candidate.verdict == LeakSuspected]+ failures = ["target-verdict" | probe.verdict /= LeakSuspected] <> ["target-rate" | not rateOk] <> ["target-interval" | not intervalOk] <> ["unrelated-probe" | not (null unrelated)]+ in if null failures+ then passed+ else failedWith failures ("expected " <> targetProbe kind <> " at " <> showText expectedRate <> "/s; actual=" <> showText actualRate <> "; unrelated=" <> showText unrelated)++findProbe :: Text -> LeakReport -> Maybe ProbeReport+findProbe wanted report = case filter ((== wanted) . (.probe)) report.probes of+ value : _ -> Just value+ [] -> Nothing++targetProbe :: Text -> Text+targetProbe "heap" = "heap.live-bytes"+targetProbe "threads" = "haskell.threads"+targetProbe "fds" = "os.fds"+targetProbe other = error ("unknown leak kind " <> Text.unpack other)++leakKnobs :: [KnobSpec]+leakKnobs =+ [ textKnob "leak.kind" "Resource retained by the worker" "heap" ["heap", "threads", "fds"],+ intKnob "leak.bytes-per-second" "Heap bytes retained each second" 1_048_576 65_536 67_108_864,+ intKnob "leak.units-per-second" "Threads or descriptors retained each second" 5 1 1_000,+ intKnob "leak.chunk-bytes" "Size of each retained heap chunk" 4_096 256 1_048_576,+ intKnob "run.duration-s" "Run duration" 45 20 3_600,+ intKnob "run.warmup-s" "Warm-up duration" 5 0 600,+ intKnob "diagnose.major-gc-interval-ms" "Forced major-GC interval, or zero to disable" 1_000 0 60_000+ ]++stableKnobs :: [KnobSpec]+stableKnobs =+ [ intKnob "alloc.bytes-per-second" "Bytes allocated each second" 8_388_608 65_536 67_108_864,+ intKnob "retain.ring-bytes" "Maximum bytes retained in the ring" 33_554_432 65_536 536_870_912,+ intKnob "run.duration-s" "Run duration" 45 20 3_600,+ intKnob "run.warmup-s" "Warm-up duration" 10 0 600,+ intKnob "diagnose.major-gc-interval-ms" "Forced major-GC interval, or zero to disable" 1_000 0 60_000+ ]++samplingKnobs :: [KnobSpec]+samplingKnobs = fmap setInterval (measureKnobs Soak)+ where+ setInterval :: KnobSpec -> KnobSpec+ setInterval spec+ | renderKnobName spec.name == "measure.sample-interval-ms" = KnobSpec spec.name spec.summary spec.knobType (VInt 500) spec.allowed spec.variants+ | otherwise = spec++telemetryOff :: DimensionSupport+telemetryOff =+ DimensionSupport+ (Supported (Support (TracingOff :| []) TracingOff))+ (Supported (Support (MetricsOff :| []) MetricsOff))+ NotApplicable+ NotApplicable++intKnob :: Text -> Text -> Int -> Int -> Int -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt (fromIntegral def)) (IntRange (fromIntegral low) (fromIntegral high)) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []+textKnob knobName _ _ [] = error ("no values for " <> Text.unpack knobName)++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++scenarioId :: Text -> ScenarioId+scenarioId = either (error . show) id . parseScenarioId++showText :: (Show value) => value -> Text+showText = Text.pack . show
+ src/Kenshou/Diagnose/SelfTest/Stall.hs view
@@ -0,0 +1,285 @@+module Kenshou.Diagnose.SelfTest.Stall+ ( deadlockedWorkersScenario,+ poolStarvedScenario,+ lockWaiterScenario,+ idleSpinnerScenario,+ healthyProgressScenario,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch, withAsync)+import Control.Exception (bracket, finally)+import Control.Monad (forM, forever, replicateM_, void)+import Data.Int (Int32)+import Data.List (sort)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Hasql.Connection qualified as Connection+import Hasql.Connection.Settings qualified as Settings+import Hasql.Decoders qualified as Decoders+import Hasql.Encoders qualified as Encoders+import Hasql.Pool qualified as Pool+import Hasql.Pool.Config qualified as PoolConfig+import Hasql.Session qualified as Session+import Hasql.Statement qualified as Statement+import Kenshou.Core.Context (ArtifactDir (DiagnosisDir), RunContext (..), artifactPath, requirePostgres)+import Kenshou.Core.Dimension+import Kenshou.Core.Env+import Kenshou.Core.Env.Postgres (PostgresEnv (..))+import Kenshou.Core.Id (ScenarioId, parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase (zeroPhases)+import Kenshou.Core.Scenario+import Kenshou.Diagnose.LockGraph (LockGraph (..))+import Kenshou.Diagnose.Pool (PoolStats (..), newPoolObserver)+import Kenshou.Diagnose.Postgres (ActivityRow (..), PostgresSnapshot (..))+import Kenshou.Diagnose.Progress (tick)+import Kenshou.Diagnose.Stall qualified as Stall+import Kenshou.Diagnose.Stall.Types+import Kenshou.Diagnose.Threads (ThreadDump (..), ThreadEntry (..), labelMe)+import System.Directory (doesFileExist)++deadlockedWorkersScenario :: Scenario+deadlockedWorkersScenario = postgresScenario "deadlocked-workers" "Proves that a two-session PostgreSQL deadlock is captured as a cycle." (stallKnobs <> [intKnob "lock.deadlock-timeout-s" "PostgreSQL deadlock timeout" 120 10 600]) runDeadlockedWorkers++poolStarvedScenario :: Scenario+poolStarvedScenario = postgresScenario "pool-starved" "Proves that a saturated pool plus an STM waiter is classified as starvation." (stallKnobs <> [intKnob "pool.size" "Connection-pool size" 2 1 8, intKnob "pool.acquisition-timeout-s" "Pool acquisition timeout" 600 10 3_600]) runPoolStarved++lockWaiterScenario :: Scenario+lockWaiterScenario = postgresScenario "lock-waiter" "Proves that an idle transaction blocking a row waiter is identified." stallKnobs runLockWaiter++idleSpinnerScenario :: Scenario+idleSpinnerScenario = postgresScenario "idle-spinner" "Proves that high query-call rate without progress is classified as idle spin." stallKnobs runIdleSpinner++healthyProgressScenario :: Scenario+healthyProgressScenario = postgresScenario "healthy-progress" "Proves that regular heartbeat progress does not produce a stall capture." stallKnobs runHealthyProgress++postgresScenario :: Text -> Text -> [KnobSpec] -> (RunContext -> IO ScenarioReport) -> Scenario+postgresScenario scenarioName summary knobs run =+ Scenario+ { id = scenarioId ("selftest/diagnose/concurrency/" <> scenarioName),+ revision = 1,+ summary,+ tier = TierSmoke,+ placement = PlaceEither,+ knobs,+ dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg17 :| [Pg18]) telemetryOff,+ phases = zeroPhases,+ requires = postgresRequirement,+ knownDefect = Nothing,+ run+ }++runDeadlockedWorkers :: RunContext -> IO ScenarioReport+runDeadlockedWorkers context = do+ let environment = requirePostgres context+ timeoutSeconds = knobInt context.knobs (name "lock.deadlock-timeout-s")+ config = captureConfig context+ withConnection environment.connectionString "kenshou-selftest-deadlock-a" \left ->+ withConnection environment.connectionString "kenshou-selftest-deadlock-b" \right -> do+ expectSession left (sql "CREATE TABLE IF NOT EXISTS diagnose_selftest (id int PRIMARY KEY, v int NOT NULL)")+ expectSession left (sql "TRUNCATE diagnose_selftest")+ expectSession left (sql "INSERT INTO diagnose_selftest VALUES (1,0),(2,0)")+ expectSession left (sql "BEGIN")+ expectSession left (sql ("SET LOCAL deadlock_timeout='" <> showText timeoutSeconds <> "s'"))+ expectSession left (sql "UPDATE diagnose_selftest SET v=v+1 WHERE id=1")+ expectSession right (sql "BEGIN")+ expectSession right (sql ("SET LOCAL deadlock_timeout='" <> showText timeoutSeconds <> "s'"))+ expectSession right (sql "UPDATE diagnose_selftest SET v=v+1 WHERE id=2")+ leftPid <- connectionPid left+ rightPid <- connectionPid right+ Stall.withWatchdog context config \watchdog -> do+ _ <- Stall.newProgress watchdog "deadlock-work" True+ withAsync (labelMe "kenshou:selftest:deadlock-a" >> expectSession left (sql "UPDATE diagnose_selftest SET v=v+1 WHERE id=2")) \leftWorker ->+ withAsync (labelMe "kenshou:selftest:deadlock-b" >> expectSession right (sql "UPDATE diagnose_selftest SET v=v+1 WHERE id=1")) \rightWorker -> do+ report <- finally (threadDelay 750_000 >> Stall.captureNow watchdog "self-test capture") (terminateBackend environment rightPid)+ void (waitCatch leftWorker)+ void (waitCatch rightWorker)+ ignoreSession left (sql "ROLLBACK")+ ignoreSession right (sql "ROLLBACK")+ let exactCycle = any ((== sort [leftPid, rightPid]) . sort) report.snapshot.graph.cycles+ workersPresent = all (unfinishedLabel report) ["kenshou:selftest:deadlock-a", "kenshou:selftest:deadlock-b"]+ failures = ["classification" | report.classification /= Deadlock] <> ["cycle" | not exactCycle] <> ["thread-dump" | not workersPresent]+ pure $ if null failures then passed else failedWith failures (reportText report)++runLockWaiter :: RunContext -> IO ScenarioReport+runLockWaiter context = do+ let environment = requirePostgres context+ config = captureConfig context+ withConnection environment.connectionString "kenshou-selftest-lock-holder" \holder ->+ withConnection environment.connectionString "kenshou-selftest-lock-waiter" \waiter -> do+ expectSession holder (sql "CREATE TABLE IF NOT EXISTS diagnose_selftest (id int PRIMARY KEY, v int NOT NULL)")+ expectSession holder (sql "TRUNCATE diagnose_selftest")+ expectSession holder (sql "INSERT INTO diagnose_selftest VALUES (1,0)")+ expectSession holder (sql "BEGIN")+ expectSession holder (sql "UPDATE diagnose_selftest SET v=v+1 WHERE id=1")+ expectSession waiter (sql "BEGIN")+ holderPid <- connectionPid holder+ Stall.withWatchdog context config \watchdog -> do+ _ <- Stall.newProgress watchdog "lock-work" True+ withAsync (labelMe "kenshou:selftest:lock-waiter" >> expectSession waiter (sql "UPDATE diagnose_selftest SET v=v+1 WHERE id=1")) \waiterThread -> do+ report <- finally (threadDelay (classificationDelayMicros context) >> Stall.captureNow watchdog "self-test capture") (ignoreSession holder (sql "ROLLBACK"))+ void (waitCatch waiterThread)+ ignoreSession waiter (sql "ROLLBACK")+ let activity = maybe [] (\(snapshot :: PostgresSnapshot) -> snapshot.activity) report.snapshot.postgres+ holderIdle = any (\(row :: ActivityRow) -> row.pid == holderPid && row.state == "idle in transaction") activity+ failures = ["classification" | report.classification /= LockWait] <> ["root-blocker" | report.snapshot.graph.roots /= [holderPid]] <> ["holder-state" | not holderIdle]+ pure $ if null failures then passed else failedWith failures (reportText report)++runPoolStarved :: RunContext -> IO ScenarioReport+runPoolStarved context = do+ let environment = requirePostgres context+ poolSize = fromIntegral (knobInt context.knobs (name "pool.size"))+ acquisitionTimeout = fromIntegral (knobInt context.knobs (name "pool.acquisition-timeout-s"))+ connectionSettings = Settings.connectionString environment.connectionString <> Settings.applicationName "kenshou-selftest-pool"+ (observePool, readPool) <- newPoolObserver "selftest" poolSize+ pool <- Pool.acquire $ PoolConfig.settings [PoolConfig.size poolSize, PoolConfig.acquisitionTimeout acquisitionTimeout, PoolConfig.staticConnectionSettings connectionSettings, PoolConfig.observationHandler observePool]+ holders <- forM [1 .. poolSize] \index -> async (labelMe ("kenshou:selftest:pool-holder-" <> show index) >> void (Pool.use pool (sql "DO $$ BEGIN PERFORM pg_sleep(60); END $$")))+ finally+ ( do+ saturated <- awaitSaturation readPool poolSize 50+ if not saturated+ then pure (failedWith ["pool-setup"] "pool did not reach the configured in-use count")+ else do+ waiter <- async (labelMe "kenshou:selftest:pool-waiter" >> void (Pool.use pool (pure ())))+ finally+ ( Stall.withWatchdog context (captureConfig context) \watchdog -> do+ _ <- Stall.newProgress watchdog "pool-work" True+ Stall.registerPool watchdog "selftest" readPool+ threadDelay (classificationDelayMicros context)+ report <- Stall.captureNow watchdog "self-test capture"+ stats <- readPool+ let failures = ["classification" | report.classification /= PoolStarvation] <> ["occupancy" | stats.inUse /= poolSize] <> ["wait-graph" | not (null report.snapshot.graph.cycles)]+ pure $ if null failures then passed else failedWith failures (reportText report)+ )+ (cancel waiter >> void (waitCatch waiter))+ )+ (terminateApplications environment "kenshou-selftest-pool" >> mapM_ cancel holders >> mapM_ (void . waitCatch) holders >> Pool.release pool)++runIdleSpinner :: RunContext -> IO ScenarioReport+runIdleSpinner context = do+ let environment = requirePostgres context+ withConnection environment.connectionString "kenshou-selftest-idle-spinner" \connection -> do+ expectSession connection (sql "CREATE EXTENSION IF NOT EXISTS pg_stat_statements")+ withAsync (labelMe "kenshou:selftest:idle-spinner" >> forever (void (expectSession connection selectOne))) \spinner ->+ Stall.withWatchdog context (captureConfig context) \watchdog -> do+ _ <- Stall.newProgress watchdog "spin-work" True+ threadDelay 500_000+ report <- Stall.captureNow watchdog "self-test capture"+ cancel spinner+ let rate = report.snapshot.idleSpin.statementCallsPerSecond+ failures = ["classification" | report.classification /= IdleSpin] <> ["statement-rate" | rate < 100 && report.snapshot.idleSpin.cpuCores < 0.5]+ pure $ if null failures then passed else failedWith failures (reportText report)++runHealthyProgress :: RunContext -> IO ScenarioReport+runHealthyProgress context = do+ let deadline = fromIntegral (knobInt context.knobs (name "stall.deadline-s")) :: Double+ config = (captureConfig context) {Stall.pollIntervalSeconds = 0.1, Stall.maxCaptures = 1}+ path <- artifactPath context DiagnosisDir "stall-1.json"+ Stall.withWatchdog context config \watchdog -> do+ progress <- Stall.newProgress watchdog "healthy-work" True+ replicateM_ (ceiling (deadline * 30) :: Int) (tick progress >> threadDelay 100_000)+ captured <- doesFileExist path+ pure $ if captured then failedWith ["false-positive"] "watchdog captured despite continuous progress" else passed++captureConfig :: RunContext -> Stall.WatchdogConfig+captureConfig context =+ Stall.defaultWatchdogConfig+ { Stall.deadlineSeconds = fromIntegral (knobInt context.knobs (name "stall.deadline-s")),+ Stall.pollIntervalSeconds = 100,+ Stall.maxCaptures = 1,+ Stall.onStall = CaptureAndContinue,+ Stall.postgres = Just (requirePostgres context).connectionString,+ Stall.captureStacks = False,+ Stall.spinProbeSeconds = 0.25+ }++classificationDelayMicros :: RunContext -> Int+classificationDelayMicros context =+ (fromIntegral (knobInt context.knobs (name "stall.deadline-s")) * 1_000_000 `div` 2) + 250_000++awaitSaturation :: IO PoolStats -> Int -> Int -> IO Bool+awaitSaturation readPool expected attempts = do+ stats <- readPool+ if stats.inUse == expected+ then pure True+ else+ if attempts <= 0+ then pure False+ else threadDelay 100_000 >> awaitSaturation readPool expected (attempts - 1)++unfinishedLabel :: StallReport -> Text -> Bool+unfinishedLabel report wanted = any matches dump.entries+ where+ dump :: ThreadDump+ dump = report.snapshot.haskellThreads+ matches :: ThreadEntry -> Bool+ matches entry = entry.label == Just wanted && entry.status `notElem` ["finished", "died"]++withConnection :: Text -> Text -> (Connection.Connection -> IO value) -> IO value+withConnection connectionString applicationName = bracket acquire Connection.release+ where+ acquire = Connection.acquire (Settings.connectionString connectionString <> Settings.applicationName applicationName) >>= either (ioError . userError . show) pure++expectSession :: Connection.Connection -> Session.Session value -> IO value+expectSession connection session = Connection.use connection session >>= either (ioError . userError . show) pure++ignoreSession :: Connection.Connection -> Session.Session value -> IO ()+ignoreSession connection session = void (Connection.use connection session)++connectionPid :: Connection.Connection -> IO Int+connectionPid connection = do+ value <- expectSession connection (Session.statement () textPid)+ case reads (Text.unpack value) of+ [(pid, "")] -> pure pid+ _ -> ioError (userError ("invalid PostgreSQL backend pid " <> Text.unpack value))++terminateBackend :: PostgresEnv -> Int -> IO ()+terminateBackend environment pid =+ withConnection environment.connectionString "kenshou-selftest-cleanup" \connection ->+ ignoreSession connection (sql ("DO $$ BEGIN PERFORM pg_terminate_backend(" <> showText pid <> "); END $$"))++terminateApplications :: PostgresEnv -> Text -> IO ()+terminateApplications environment applicationName =+ withConnection environment.connectionString "kenshou-selftest-cleanup" \connection ->+ ignoreSession connection (sql ("DO $$ DECLARE backend record; BEGIN FOR backend IN SELECT pid FROM pg_stat_activity WHERE datname=current_database() AND application_name='" <> applicationName <> "' LOOP PERFORM pg_terminate_backend(backend.pid); END LOOP; END $$"))++textPid :: Statement.Statement () Text+textPid = Statement.unpreparable "SELECT pg_backend_pid()::text" Encoders.noParams (Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.text)))++selectOne :: Session.Session Int32+selectOne = Session.statement () (Statement.unpreparable "SELECT 1" Encoders.noParams (Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.int4))))++sql :: Text -> Session.Session ()+sql statement = Session.statement () (Statement.unpreparable statement Encoders.noParams Decoders.noResult)++reportText :: StallReport -> Text+reportText report = "classification=" <> stallClassText report.classification <> ", reasons=" <> Text.intercalate "; " report.reasons++stallKnobs :: [KnobSpec]+stallKnobs = [intKnob "stall.deadline-s" "Seconds without progress before capture" 5 2 60]++postgresRequirement :: EnvRequirements+postgresRequirement = EnvRequirements (Just (PostgresRequirement [] [("shared_preload_libraries", "'pg_stat_statements'")] False)) [] []++telemetryOff :: DimensionSupport+telemetryOff =+ DimensionSupport+ (Supported (Support (TracingOff :| []) TracingOff))+ (Supported (Support (MetricsOff :| []) MetricsOff))+ NotApplicable+ NotApplicable++intKnob :: Text -> Text -> Int -> Int -> Int -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt (fromIntegral def)) (IntRange (fromIntegral low) (fromIntegral high)) []++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++scenarioId :: Text -> ScenarioId+scenarioId = either (error . show) id . parseScenarioId++showText :: (Show value) => value -> Text+showText = Text.pack . show
+ src/Kenshou/Diagnose/Series.hs view
@@ -0,0 +1,113 @@+module Kenshou.Diagnose.Series+ ( DiagnoseError (..),+ SeriesBinding (..),+ readWide,+ readLong,+ readBinding,+ )+where++import Control.Exception (IOException, try)+import Data.List (elemIndex)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Data.Vector (Vector)+import Data.Vector qualified as Vector+import System.IO (Handle, IOMode (ReadMode), hIsEOF, withFile)+import Text.Read (readMaybe)++data DiagnoseError+ = MissingSeries FilePath+ | MissingColumn FilePath Text+ | InvalidSeriesValue FilePath Int Text+ | CannotReadSeries FilePath Text+ | InvalidRunDirectory FilePath Text+ deriving stock (Eq, Show)++data SeriesBinding = SeriesBinding+ { file :: FilePath,+ timeColumn :: Text,+ valueColumn :: Text,+ filters :: Map Text Text+ }+ deriving stock (Eq, Show)++readWide :: FilePath -> Text -> Text -> IO (Either DiagnoseError (Vector (Double, Double)))+readWide path timeColumn valueColumn = readBinding path (SeriesBinding path timeColumn valueColumn Map.empty)++readLong :: FilePath -> Text -> Text -> Map Text Text -> IO (Either DiagnoseError (Vector (Double, Double)))+readLong path timeColumn valueColumn filters = readBinding path (SeriesBinding path timeColumn valueColumn filters)++readBinding :: FilePath -> SeriesBinding -> IO (Either DiagnoseError (Vector (Double, Double)))+readBinding path binding = do+ result <- try @IOException (withFile path ReadMode (readRows path binding))+ pure case result of+ Left err -> Left (CannotReadSeries path (Text.pack (show err)))+ Right value -> value++readRows :: FilePath -> SeriesBinding -> Handle -> IO (Either DiagnoseError (Vector (Double, Double)))+readRows path binding handle = do+ exhausted <- hIsEOF handle+ if exhausted+ then pure (Left (MissingSeries path))+ else do+ header <- splitCsv <$> Text.hGetLine handle+ case resolveColumns path binding header of+ Left err -> pure (Left err)+ Right columns -> go columns 2 []+ where+ go columns lineNumber accumulated = do+ exhausted <- hIsEOF handle+ if exhausted+ then pure (Right (Vector.fromList (reverse accumulated)))+ else do+ fields <- splitCsv <$> Text.hGetLine handle+ case parseRow path binding columns lineNumber fields of+ Left err -> pure (Left err)+ Right Nothing -> go columns (lineNumber + 1) accumulated+ Right (Just point) -> go columns (lineNumber + 1) (point : accumulated)++data Columns = Columns Int Int [(Int, Text)]++resolveColumns :: FilePath -> SeriesBinding -> [Text] -> Either DiagnoseError Columns+resolveColumns path binding header = do+ timeIndex <- index binding.timeColumn+ valueIndex <- index binding.valueColumn+ filterColumns <- traverse (\(name, expected) -> (,expected) <$> index name) (Map.toList binding.filters)+ pure (Columns timeIndex valueIndex filterColumns)+ where+ index name = maybe (Left (MissingColumn path name)) Right (elemIndex name header)++parseRow :: FilePath -> SeriesBinding -> Columns -> Int -> [Text] -> Either DiagnoseError (Maybe (Double, Double))+parseRow path binding (Columns timeIndex valueIndex filterColumns) lineNumber fields+ | not (all matches filterColumns) = Right Nothing+ | Text.null valueText = Right Nothing+ | otherwise = case (readMaybe (Text.unpack timeText), readMaybe (Text.unpack valueText)) of+ (Just timeValue, Just value) -> Right (Just (normaliseTime binding.timeColumn timeValue, value))+ _ -> Left (InvalidSeriesValue path lineNumber (timeText <> "," <> valueText))+ where+ at index = if index < length fields then fields !! index else ""+ timeText = at timeIndex+ valueText = at valueIndex+ matches (index, expected) = at index == expected++normaliseTime :: Text -> Double -> Double+normaliseTime column value+ | Text.isSuffixOf "_ns" column = value / 1_000_000_000+ | Text.isSuffixOf "_ms" column = value / 1_000+ | otherwise = value++-- Kenshou-owned sampler files contain numeric cells and optional RFC 4180+-- quoting for labels. This small parser handles escaped quotes without loading+-- a 24-hour series into memory.+splitCsv :: Text -> [Text]+splitCsv = fmap Text.pack . go False [] [] . Text.unpack+ where+ go _ field fields [] = reverse (reverse field : fields)+ go True field fields ('"' : '"' : rest) = go True ('"' : field) fields rest+ go quoted field fields ('"' : rest) = go (not quoted) field fields rest+ go False field fields (',' : rest) = go False [] (reverse field : fields) rest+ go quoted field fields (character : rest) = go quoted (character : field) fields rest
+ src/Kenshou/Diagnose/Series/Catalog.hs view
@@ -0,0 +1,43 @@+module Kenshou.Diagnose.Series.Catalog+ ( defaultCatalog,+ discoverProcesses,+ )+where++import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Diagnose.Series+import System.Directory (doesDirectoryExist, listDirectory)+import System.FilePath (dropExtension, takeExtension, takeFileName, (</>))++defaultCatalog :: Map Text SeriesBinding+defaultCatalog =+ Map.fromList+ [ ("heap.live-bytes", wide "rts.csv" "live_bytes_last_gc"),+ ("runtime.mem-in-use-bytes", wide "rts.csv" "mem_in_use_bytes"),+ ("process.rss-bytes", wide "proc.csv" "rss_bytes"),+ ("haskell.threads", wide "rts.csv" "haskell_threads"),+ ("os.threads", wide "proc.csv" "os_threads"),+ ("os.fds", wide "proc.csv" "open_fds"),+ ("pg.connections", long "pg-activity.csv" "connections"),+ ("pg.relation-bytes", long "pg-relations.csv" "total_bytes"),+ ("pg.dead-tuples", long "pg-relations.csv" "n_dead_tup")+ ]+ where+ wide file value = SeriesBinding file "t_mono_ns" value (Map.singleton "phase" "steady")+ long file value = SeriesBinding file "t_mono_ns" value Map.empty++discoverProcesses :: FilePath -> IO [(Text, FilePath)]+discoverProcesses runDirectory = do+ let series = runDirectory </> "series"+ exists <- doesDirectoryExist series+ if not exists+ then pure []+ else do+ names <- listDirectory series+ pure (("main", series </> "rts.csv") : [(suffix name, series </> name) | name <- names, isWorkerRts name])+ where+ isWorkerRts name = takeExtension name == ".csv" && "rts-" `Text.isPrefixOf` Text.pack (takeFileName name)+ suffix name = Text.drop 4 (Text.pack (dropExtension (takeFileName name)))
+ src/Kenshou/Diagnose/Stall.hs view
@@ -0,0 +1,201 @@+module Kenshou.Diagnose.Stall+ ( WatchdogConfig (..),+ defaultWatchdogConfig,+ watchdogConfigFor,+ Watchdog,+ StallDetected (..),+ withWatchdog,+ newProgress,+ registerPool,+ registerStateProbe,+ registerChildProcess,+ suspendDeadline,+ captureNow,+ module Kenshou.Diagnose.Progress,+ module Kenshou.Diagnose.Stall.Types,+ )+where++import Control.Concurrent (ThreadId, myThreadId, threadDelay, throwTo)+import Control.Concurrent.Async (withAsync)+import Control.Exception (Exception, bracket, bracket_)+import Data.Aeson (Value (Null), object, toJSON, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (traverse_)+import Data.IORef+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (diffUTCTime, getCurrentTime)+import Hasql.Connection qualified as Connection+import Hasql.Connection.Settings qualified as Settings+import Kenshou.Core.Bundle qualified as Bundle+import Kenshou.Core.Context qualified as Core+import Kenshou.Diagnose.Context qualified as Context+import Kenshou.Diagnose.Document+import Kenshou.Diagnose.LockGraph (buildGraph)+import Kenshou.Diagnose.Pool (PoolStats)+import Kenshou.Diagnose.Pool qualified as Pool+import Kenshou.Diagnose.Postgres+import Kenshou.Diagnose.Progress+import Kenshou.Diagnose.Stall.Classify (classify)+import Kenshou.Diagnose.Stall.Types+import Kenshou.Diagnose.Threads (dumpThreads, labelMe)+import System.CPUTime (getCPUTime)+import System.Posix.Signals (sigUSR2, signalProcess)+import System.Posix.Types (CPid)++data WatchdogConfig = WatchdogConfig+ { deadlineSeconds :: !Double,+ pollIntervalSeconds :: !Double,+ maxCaptures :: !Int,+ onStall :: !OnStall,+ postgres :: !(Maybe Text),+ advisoryLabels :: ![AdvisoryLabel],+ captureStacks :: !Bool,+ spinProbeSeconds :: !Double+ }+ deriving stock (Eq, Show)++defaultWatchdogConfig :: WatchdogConfig+defaultWatchdogConfig = WatchdogConfig 60 1 3 CaptureAndAbort Nothing [] True 2++-- | The default watchdog naming a bundle's advisory locks and the migration ledger lock.+watchdogConfigFor :: Bundle.LayerBundle -> WatchdogConfig+watchdogConfigFor bundle = defaultWatchdogConfig {advisoryLabels = bundle.advisoryLabels <> [pgMigrateLedgerLock]}++data Watchdog = Watchdog+ { context :: !Core.RunContext,+ config :: !WatchdogConfig,+ scenarioThread :: !ThreadId,+ counters :: !(IORef [ProgressCounter]),+ poolProbes :: !(IORef [(Text, IO PoolStats)]),+ stateProbes :: !(IORef [(Text, IO Value)]),+ children :: !(IORef [(Text, CPid)]),+ suspensions :: !(IORef Int),+ captures :: !(IORef Int),+ database :: !(Maybe (Either Text Connection.Connection))+ }++newtype StallDetected = StallDetected StallReport deriving stock (Show)++instance Exception StallDetected++withWatchdog :: Core.RunContext -> WatchdogConfig -> (Watchdog -> IO value) -> IO value+withWatchdog context config action = do+ scenarioThread <- myThreadId+ bracket (openDatabase config) closeDatabase \database -> do+ watchdog <-+ Watchdog context config scenarioThread+ <$> newIORef []+ <*> newIORef []+ <*> newIORef []+ <*> newIORef []+ <*> newIORef 0+ <*> newIORef 0+ <*> pure database+ withAsync (monitor watchdog) \_ -> action watchdog++openDatabase :: WatchdogConfig -> IO (Maybe (Either Text Connection.Connection))+openDatabase config = traverse acquire config.postgres+ where+ acquire connectionString = do+ result <- Connection.acquire (Settings.connectionString connectionString <> Settings.applicationName "kenshou-diagnose-watchdog")+ pure case result of+ Left err -> Left (Text.pack (show err))+ Right connection -> Right connection++closeDatabase :: Maybe (Either Text Connection.Connection) -> IO ()+closeDatabase = maybe (pure ()) (either (const (pure ())) Connection.release)++newProgress :: Watchdog -> Text -> Bool -> IO ProgressCounter+newProgress watchdog name required = do+ counter <- newProgressCounter name required+ atomicModifyIORef' watchdog.counters (\values -> (counter : values, ()))+ pure counter++registerPool :: Watchdog -> Text -> IO PoolStats -> IO ()+registerPool watchdog name probe = atomicModifyIORef' watchdog.poolProbes (\values -> ((name, namedProbe) : values, ()))+ where+ namedProbe = do+ stats <- probe+ pure stats {Pool.name = name}++registerStateProbe :: Watchdog -> Text -> IO Value -> IO ()+registerStateProbe watchdog name probe = atomicModifyIORef' watchdog.stateProbes (\values -> ((name, probe) : values, ()))++registerChildProcess :: Watchdog -> Text -> CPid -> IO ()+registerChildProcess watchdog role pid = atomicModifyIORef' watchdog.children (\values -> ((role, pid) : values, ()))++suspendDeadline :: Watchdog -> Text -> IO value -> IO value+suspendDeadline watchdog _reason = bracket_ increment decrement+ where+ increment = atomicModifyIORef' watchdog.suspensions (\value -> (value + 1, ()))+ decrement = atomicModifyIORef' watchdog.suspensions (\value -> (max 0 (value - 1), ()))++monitor :: Watchdog -> IO ()+monitor watchdog = do+ labelMe "kenshou:diagnose:watchdog"+ loop+ where+ loop = do+ threadDelay (secondsToMicros watchdog.config.pollIntervalSeconds)+ suspended <- (> 0) <$> readIORef watchdog.suspensions+ count <- readIORef watchdog.captures+ snapshots <- readIORef watchdog.counters >>= traverse snapshotProgress+ now <- getCurrentTime+ let required = filter (.required) snapshots+ stalled = not (null required) && all (\counter -> realToFrac (diffUTCTime now counter.lastAdvancedAt) >= watchdog.config.deadlineSeconds) required+ if suspended || not stalled || count >= watchdog.config.maxCaptures+ then loop+ else do+ report <- captureNow watchdog "progress deadline exceeded"+ case watchdog.config.onStall of+ CaptureAndContinue -> threadDelay (secondsToMicros watchdog.config.deadlineSeconds) >> loop+ CaptureAndAbort -> throwTo watchdog.scenarioThread (StallDetected report)++captureNow :: Watchdog -> Text -> IO StallReport+captureNow watchdog reason = do+ captureNumber <- atomicModifyIORef' watchdog.captures (\value -> let next = value + 1 in (next, next))+ progress <- readIORef watchdog.counters >>= traverse snapshotProgress+ haskellThreads <- dumpThreads watchdog.config.captureStacks+ signalChildren watchdog+ pools <- readIORef watchdog.poolProbes >>= traverse (\(_, probe) -> probe)+ probes <- readIORef watchdog.stateProbes >>= traverse (\(name, probe) -> (name,) <$> probe)+ cpuStart <- getCPUTime+ postgres <- captureDatabase watchdog+ cpuEnd <- getCPUTime+ let elapsed = max 0.001 watchdog.config.spinProbeSeconds+ cpuCores = fromIntegral (cpuEnd - cpuStart) / 1.0e12 / elapsed+ statementCalls = maybe 0 (maybe 0 (.callsPerSecond) . (.statementRate)) postgres+ graph = maybe (buildGraph [] []) (\databaseSnapshot -> buildGraph databaseSnapshot.activity databaseSnapshot.locks) postgres+ snapshot = StallSnapshot watchdog.config.deadlineSeconds progress haskellThreads postgres graph pools (toJSON (Map.fromList probes)) (SpinEvidence cpuCores statementCalls)+ (classification, secondary, reasons) = classify snapshot+ detectedAt <- getCurrentTime+ let report = StallReport detectedAt captureNumber classification secondary (reason : reasons) snapshot+ writeReport watchdog report+ pure report++captureDatabase :: Watchdog -> IO (Maybe PostgresSnapshot)+captureDatabase watchdog = case watchdog.database of+ Nothing -> pure Nothing+ Just (Left err) -> pure (Just (PostgresSnapshot False (Just err) [] [] Nothing Null))+ Just (Right connection) -> Just <$> capturePostgres connection watchdog.config.spinProbeSeconds++signalChildren :: Watchdog -> IO ()+signalChildren watchdog = do+ registered <- readIORef watchdog.children+ traverse_ (signalProcess sigUSR2 . snd) registered+ if null registered then pure () else threadDelay 2_000_000++writeReport :: Watchdog -> StallReport -> IO ()+writeReport watchdog report = do+ let (runId, scenario) = Context.runIdentity watchdog.context+ diagnosis = Diagnosis StallDiagnosis runId scenario report.detectedAt (Generator "kenshou-diagnose" "0.1.0.0" "stall-watchdog-v1") (toJSON report)+ path <- Core.artifactPath watchdog.context Core.DiagnosisDir ("stall-" <> show report.captureNumber <> ".json")+ LazyByteString.writeFile path (encodeDiagnosis diagnosis)+ Core.declareMediaType watchdog.context path "application/json"+ Context.publishDiagnosis watchdog.context "stall" (object ["classification" .= report.classification, "path" .= path])++secondsToMicros :: Double -> Int+secondsToMicros = max 1 . floor . (* 1_000_000)
+ src/Kenshou/Diagnose/Stall/Classify.hs view
@@ -0,0 +1,42 @@+module Kenshou.Diagnose.Stall.Classify+ ( classify,+ )+where++import Data.List (find)+import Data.Maybe (fromMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Diagnose.LockGraph (LockGraph (..))+import Kenshou.Diagnose.Pool (PoolStats (..))+import Kenshou.Diagnose.Postgres (ActivityRow (..), PostgresSnapshot (..))+import Kenshou.Diagnose.Stall.Types+import Kenshou.Diagnose.Threads (ThreadDump (..), ThreadEntry (..))++classify :: StallSnapshot -> (StallClass, [StallClass], [Text])+classify snapshot = (fromMaybe Unknown (fst <$> find snd checks), [class_ | (class_, True) <- checks], reasons)+ where+ checks =+ [ (Deadlock, not (null snapshot.graph.cycles)),+ (LockWait, lockWait),+ (PoolStarvation, poolStarvation),+ (IdleSpin, idleSpin),+ (BlockedIndefinitely, blockedIndefinitely)+ ]+ activity = maybe [] (.activity) snapshot.postgres+ lockWait = null snapshot.graph.cycles && any longLock activity+ longLock row = row.waitEventType == Just "Lock" && fromMaybe 0 row.queryAgeSeconds >= snapshot.deadlineSeconds / 2+ saturated = [pool | pool <- snapshot.pools, pool.saturatedSeconds >= snapshot.deadlineSeconds / 2 && pool.inUse >= pool.size]+ blockedOnStm = any ((== Just "BlockedOnSTM") . (.blockReason)) (nonHarness snapshot.haskellThreads.entries)+ poolStarvation = not lockWait && not (null saturated) && blockedOnStm+ idleSpin = snapshot.idleSpin.cpuCores >= 0.5 || snapshot.idleSpin.statementCallsPerSecond >= 100+ nonHarness = filter (maybe True (not . Text.isPrefixOf "kenshou:diagnose") . (.label))+ quietThread entry = entry.status `elem` ["finished", "died"] || entry.blockReason `elem` [Just "BlockedOnMVar", Just "BlockedOnSTM"]+ allDatabaseIdle = maybe False (\database -> database.available && not (null database.activity) && all ((== "idle") . (.state)) database.activity) snapshot.postgres+ blockedIndefinitely = not idleSpin && all quietThread (nonHarness snapshot.haskellThreads.entries) && allDatabaseIdle+ reasons =+ ["wait-for graph contains a cycle" | not (null snapshot.graph.cycles)]+ <> ["sessions have waited on locks for at least half the watchdog deadline" | lockWait]+ <> ["pool " <> pool.name <> " is saturated" | pool <- saturated]+ <> ["CPU or database statement rate is high while progress is zero" | idleSpin]+ <> ["all non-harness threads are finished or indefinitely blocked" | blockedIndefinitely]
+ src/Kenshou/Diagnose/Stall/Types.hs view
@@ -0,0 +1,77 @@+module Kenshou.Diagnose.Stall.Types+ ( StallClass (..),+ OnStall (..),+ SpinEvidence (..),+ StallSnapshot (..),+ StallReport (..),+ stallClassText,+ )+where++import Data.Aeson+import Data.Aeson.Types (Parser)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime)+import Kenshou.Diagnose.LockGraph (LockGraph)+import Kenshou.Diagnose.Pool (PoolStats)+import Kenshou.Diagnose.Postgres (PostgresSnapshot)+import Kenshou.Diagnose.Progress (ProgressSnapshot)+import Kenshou.Diagnose.Threads (ThreadDump)++data StallClass = Deadlock | LockWait | PoolStarvation | BlockedIndefinitely | IdleSpin | Unknown+ deriving stock (Eq, Ord, Show)++data OnStall = CaptureAndContinue | CaptureAndAbort deriving stock (Eq, Show)++data SpinEvidence = SpinEvidence {cpuCores :: !Double, statementCallsPerSecond :: !Double}+ deriving stock (Eq, Show)++data StallSnapshot = StallSnapshot+ { deadlineSeconds :: !Double,+ progress :: ![ProgressSnapshot],+ haskellThreads :: !ThreadDump,+ postgres :: !(Maybe PostgresSnapshot),+ graph :: !LockGraph,+ pools :: ![PoolStats],+ stateProbes :: !Value,+ idleSpin :: !SpinEvidence+ }+ deriving stock (Eq, Show)++data StallReport = StallReport+ { detectedAt :: !UTCTime,+ captureNumber :: !Int,+ classification :: !StallClass,+ secondary :: ![StallClass],+ reasons :: ![Text],+ snapshot :: !StallSnapshot+ }+ deriving stock (Eq, Show)++stallClassText :: StallClass -> Text+stallClassText Deadlock = "deadlock"+stallClassText LockWait = "lock-wait"+stallClassText PoolStarvation = "pool-starvation"+stallClassText BlockedIndefinitely = "blocked-indefinitely"+stallClassText IdleSpin = "idle-spin"+stallClassText Unknown = "unknown"++parseStallClass :: Text -> Parser StallClass+parseStallClass value = maybe (fail ("unknown stall class " <> Text.unpack value)) pure (lookup value [(stallClassText class_, class_) | class_ <- [Deadlock, LockWait, PoolStarvation, BlockedIndefinitely, IdleSpin, Unknown]])++instance ToJSON StallClass where toJSON = String . stallClassText++instance FromJSON StallClass where parseJSON = withText "StallClass" parseStallClass++instance ToJSON SpinEvidence where toJSON value = object ["cpuCores" .= value.cpuCores, "statementCallsPerSecond" .= value.statementCallsPerSecond]++instance FromJSON SpinEvidence where parseJSON = withObject "SpinEvidence" \value -> SpinEvidence <$> value .:? "cpuCores" .!= 0 <*> value .:? "statementCallsPerSecond" .!= 0++instance ToJSON StallSnapshot where toJSON value = object ["deadlineSeconds" .= value.deadlineSeconds, "progress" .= value.progress, "haskellThreads" .= value.haskellThreads, "postgres" .= value.postgres, "graph" .= value.graph, "pools" .= value.pools, "stateProbes" .= value.stateProbes, "idleSpin" .= value.idleSpin]++instance FromJSON StallSnapshot where parseJSON = withObject "StallSnapshot" \value -> StallSnapshot <$> value .: "deadlineSeconds" <*> value .:? "progress" .!= [] <*> value .: "haskellThreads" <*> value .:? "postgres" <*> value .: "graph" <*> value .:? "pools" .!= [] <*> value .:? "stateProbes" .!= Null <*> value .:? "idleSpin" .!= SpinEvidence 0 0++instance ToJSON StallReport where toJSON value = object ["detectedAt" .= value.detectedAt, "captureNumber" .= value.captureNumber, "classification" .= value.classification, "secondary" .= value.secondary, "reasons" .= value.reasons, "snapshot" .= value.snapshot]++instance FromJSON StallReport where parseJSON = withObject "StallReport" \value -> StallReport <$> value .: "detectedAt" <*> value .: "captureNumber" <*> value .: "classification" <*> value .:? "secondary" .!= [] <*> value .:? "reasons" .!= [] <*> value .: "snapshot"
+ src/Kenshou/Diagnose/Stats.hs view
@@ -0,0 +1,151 @@+module Kenshou.Diagnose.Stats+ ( SlopeEstimate (..),+ theilSen,+ slopeWithInterval,+ windowMinima,+ windowMedians,+ )+where++import Data.List (sort)+import Data.Vector (Vector)+import Data.Vector qualified as Vector+import Data.Word (Word64)+import System.Random.SplitMix (SMGen, mkSMGen, nextWord64)++data SlopeEstimate = SlopeEstimate+ { slope :: !Double,+ intercept :: !Double,+ low :: !Double,+ high :: !Double,+ points :: !Int,+ resamples :: !Int,+ blockLength :: !Int+ }+ deriving stock (Eq, Show)++theilSen :: Vector (Double, Double) -> Maybe (Double, Double)+theilSen original+ | Vector.length samples < 2 = Nothing+ | null slopes = Nothing+ | otherwise = Just (estimate, median [y - estimate * x | (x, y) <- Vector.toList samples])+ where+ samples = capPoints original+ indexed = Vector.toList (Vector.indexed samples)+ slopes =+ [ (y2 - y1) / (x2 - x1)+ | (index, (x1, y1)) <- indexed,+ (_, (x2, y2)) <- drop (index + 1) indexed,+ x2 /= x1+ ]+ estimate = median slopes++slopeWithInterval :: Word64 -> Int -> Double -> Vector (Double, Double) -> Maybe SlopeEstimate+slopeWithInterval seed requested confidence values = do+ (estimate, intercept) <- theilSen values+ let count = Vector.length values+ block = max 1 (ceiling (sqrt (fromIntegral count :: Double)))+ iterations = max 1 requested+ residuals = Vector.map (\(x, y) -> (x, y - (estimate * x + intercept))) values+ estimates = take iterations (bootstrapSlopes block estimate intercept residuals (mkSMGen seed))+ alpha = max 0 (min 0.5 ((1 - confidence) / 2))+ pure+ SlopeEstimate+ { slope = estimate,+ intercept,+ low = quantile alpha estimates,+ high = quantile (1 - alpha) estimates,+ points = count,+ resamples = iterations,+ blockLength = block+ }++windowMinima :: Double -> Vector (Double, Double) -> Vector (Double, Double)+windowMinima width = windowBy width minimumPoint+ where+ minimumPoint values = minimumByValue values++windowMedians :: Double -> Vector (Double, Double) -> Vector (Double, Double)+windowMedians width = windowBy width medianPoint+ where+ medianPoint values =+ let ordered = sortOnValue values+ in ordered !! ((length ordered - 1) `div` 2)++windowBy :: Double -> ([(Double, Double)] -> (Double, Double)) -> Vector (Double, Double) -> Vector (Double, Double)+windowBy width choose values+ | width <= 0 = values+ | Vector.null values = Vector.empty+ | otherwise = Vector.fromList (fmap choose (groups (Vector.toList values)))+ where+ origin = fst (Vector.head values)+ bucket (time, _) = floor ((time - origin) / width) :: Int+ groups [] = []+ groups (first : rest) =+ let key = bucket first+ (same, remaining) = span ((== key) . bucket) rest+ in (first : same) : groups remaining++capPoints :: Vector (Double, Double) -> Vector (Double, Double)+capPoints values+ | Vector.length values <= 2000 = values+ | otherwise =+ let width = fromIntegral (Vector.length values) / 2000 :: Double+ groups = chunk (ceiling width) (Vector.toList values)+ in Vector.fromList (fmap medianPoint groups)+ where+ medianPoint group = sortOnValue group !! ((length group - 1) `div` 2)++bootstrapSlopes :: Int -> Double -> Double -> Vector (Double, Double) -> SMGen -> [Double]+bootstrapSlopes _ _ _ values _ | Vector.length values < 2 = []+bootstrapSlopes block baseEstimate intercept values generator =+ let (sampledResiduals, next) = movingBlockSample block values generator+ sample = Vector.map (\(x, residual) -> (x, baseEstimate * x + intercept + residual)) sampledResiduals+ bootstrapEstimate = maybe 0 fst (theilSen sample)+ in bootstrapEstimate : bootstrapSlopes block baseEstimate intercept values next++movingBlockSample :: Int -> Vector (Double, Double) -> SMGen -> (Vector (Double, Double), SMGen)+movingBlockSample block values initial =+ let count = Vector.length values+ blockCount = ceiling (fromIntegral count / fromIntegral block :: Double)+ (starts, final) = drawStarts blockCount count initial []+ sampledY = take count [snd (values Vector.! ((start + offset) `mod` count)) | start <- starts, offset <- [0 .. block - 1]]+ sampledX = fmap fst (Vector.toList values)+ in (Vector.fromList (zip sampledX sampledY), final)++drawStarts :: Int -> Int -> SMGen -> [Int] -> ([Int], SMGen)+drawStarts 0 _ generator accumulated = (reverse accumulated, generator)+drawStarts remaining count generator accumulated =+ let (word, next) = nextWord64 generator+ start = fromIntegral (word `mod` fromIntegral count)+ in drawStarts (remaining - 1) count next (start : accumulated)++quantile :: Double -> [Double] -> Double+quantile _ [] = 0+quantile fraction values =+ let ordered = sort values+ position = max 0 (min 1 fraction) * fromIntegral (length ordered - 1)+ lowerIndex = floor position+ upperIndex = ceiling position+ weight = position - fromIntegral lowerIndex+ in ordered !! lowerIndex * (1 - weight) + ordered !! upperIndex * weight++median :: [Double] -> Double+median = quantile 0.5++minimumByValue :: [(Double, Double)] -> (Double, Double)+minimumByValue = foldl1 \left@(_, leftValue) right@(_, rightValue) -> if rightValue < leftValue then right else left++sortOnValue :: [(Double, Double)] -> [(Double, Double)]+sortOnValue = sortByValue++sortByValue :: [(Double, Double)] -> [(Double, Double)]+sortByValue [] = []+sortByValue (pivot@(_, pivotValue) : rest) =+ sortByValue [value | value@(_, item) <- rest, item <= pivotValue]+ <> [pivot]+ <> sortByValue [value | value@(_, item) <- rest, item > pivotValue]++chunk :: Int -> [value] -> [[value]]+chunk _ [] = []+chunk size values = let (prefix, suffix) = splitAt (max 1 size) values in prefix : chunk size suffix
+ src/Kenshou/Diagnose/Threads.hs view
@@ -0,0 +1,114 @@+module Kenshou.Diagnose.Threads+ ( ThreadEntry (..),+ ThreadDump (..),+ labelMe,+ forkLabelled,+ dumpThreads,+ installThreadDumpSignal,+ )+where++import Control.Concurrent (ThreadId, forkIO, myThreadId)+import Control.Exception (SomeException, try)+import Data.Aeson (FromJSON (..), ToJSON (..), object, toJSON, withObject, (.!=), (.:), (.:?), (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import GHC.Conc (labelThread, listThreads, threadCapability, threadStatus)+import GHC.Conc.Sync (ThreadStatus (..), fromThreadId, threadLabel)+import GHC.Stack.CloneStack (StackEntry (..), cloneThreadStack, decode)+import Kenshou.Diagnose.Document (Diagnosis (..), DiagnosisKind (ThreadDumpDiagnosis), Generator (..), encodeDiagnosis)+import System.Directory (createDirectoryIfMissing)+import System.FilePath (takeFileName, (</>))+import System.Posix.Process (getProcessID)+import System.Posix.Signals (Handler (Catch), installHandler, sigUSR2)+import System.Timeout (timeout)++data ThreadEntry = ThreadEntry+ { id :: !Text,+ label :: !(Maybe Text),+ status :: !Text,+ blockReason :: !(Maybe Text),+ capability :: !Int,+ locked :: !Bool,+ stack :: ![Text]+ }+ deriving stock (Eq, Show)++data ThreadDump = ThreadDump+ { generatedAt :: !UTCTime,+ processId :: !Int,+ entries :: ![ThreadEntry],+ truncated :: !Bool+ }+ deriving stock (Eq, Show)++labelMe :: String -> IO ()+labelMe name = myThreadId >>= \thread -> labelThread thread name++forkLabelled :: String -> IO () -> IO ThreadId+forkLabelled name action = forkIO (labelMe name >> action)++dumpThreads :: Bool -> IO ThreadDump+dumpThreads captureStacks = do+ now <- getCurrentTime+ pid <- fromIntegral <$> getProcessID+ threads <- listThreads+ let selected = take 200 threads+ entries <- traverse (dumpOne captureStacks) selected+ pure (ThreadDump now pid entries (length threads > length selected))++dumpOne :: Bool -> ThreadId -> IO ThreadEntry+dumpOne captureStacks thread = do+ label <- fmap Text.pack <$> threadLabel thread+ status <- threadStatus thread+ (capability, locked) <- threadCapability thread+ stack <- if captureStacks then decodeStack thread else pure []+ let (statusText, block) = statusParts status+ pure (ThreadEntry (Text.pack (show (fromThreadId thread))) label statusText block capability locked stack)++statusParts :: ThreadStatus -> (Text, Maybe Text)+statusParts ThreadRunning = ("running", Nothing)+statusParts ThreadFinished = ("finished", Nothing)+statusParts ThreadDied = ("died", Nothing)+statusParts (ThreadBlocked reason) = ("blocked", Just (Text.pack (show reason)))++decodeStack :: ThreadId -> IO [Text]+decodeStack thread = do+ decoded <- timeout 250_000 (try @SomeException (cloneThreadStack thread >>= decode))+ pure case decoded of+ Just (Right entries) -> fmap renderStackEntry entries+ _ -> []++renderStackEntry :: StackEntry -> Text+renderStackEntry entry = Text.pack entry.moduleName <> "." <> Text.pack entry.functionName <> " (" <> Text.pack entry.srcLoc <> ", " <> Text.pack (show entry.closureType) <> ")"++installThreadDumpSignal :: FilePath -> Text -> IO ()+installThreadDumpSignal root role = do+ counter <- newIORef (0 :: Int)+ _ <- installHandler sigUSR2 (Catch (capture counter)) Nothing+ pure ()+ where+ capture counter = do+ _ <- forkIO do+ dump <- dumpThreads True+ number <- atomicModifyIORef' counter (\value -> let next = value + 1 in (next, next))+ createDirectoryIfMissing True (root </> "diagnosis")+ let path = root </> "diagnosis" </> ("threads-" <> Text.unpack role <> "-" <> show dump.processId <> "-" <> show number <> ".json")+ document = Diagnosis ThreadDumpDiagnosis (Text.pack (takeFileName root)) ("worker/" <> role) dump.generatedAt (Generator "kenshou-diagnose" "0.1.0.0" "thread-dump-v1") (toJSON dump)+ LazyByteString.writeFile path (encodeDiagnosis document)+ pure ()++instance ToJSON ThreadEntry where+ toJSON entry = object ["id" .= entry.id, "label" .= entry.label, "status" .= entry.status, "blockReason" .= entry.blockReason, "capability" .= entry.capability, "locked" .= entry.locked, "stack" .= entry.stack]++instance FromJSON ThreadEntry where+ parseJSON = withObject "ThreadEntry" \value -> ThreadEntry <$> value .: "id" <*> value .:? "label" <*> value .: "status" <*> value .:? "blockReason" <*> value .: "capability" <*> value .: "locked" <*> value .:? "stack" .!= []++instance ToJSON ThreadDump where+ toJSON dump = object ["generatedAt" .= dump.generatedAt, "processId" .= dump.processId, "entries" .= dump.entries, "truncated" .= dump.truncated]++instance FromJSON ThreadDump where+ parseJSON = withObject "ThreadDump" \value -> ThreadDump <$> value .: "generatedAt" <*> value .: "processId" <*> value .: "entries" <*> value .:? "truncated" .!= False
+ test/Kenshou/Diagnose/LeakSpec.hs view
@@ -0,0 +1,97 @@+module Kenshou.Diagnose.LeakSpec (spec) where++import Data.ByteString.Lazy.Char8 qualified as LazyByteString+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text.IO qualified as Text+import Data.Vector qualified as Vector+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Id (parseScenarioId)+import Kenshou.Diagnose.Document+import Kenshou.Diagnose.Leak+import Kenshou.Diagnose.Leak.MajorGcProbe (withMajorGcProbe)+import Kenshou.Diagnose.Series+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Diagnose.Leak" do+ it "round-trips the versioned diagnosis envelope" do+ let encoded = "{\"schema\":\"kenshou.diagnosis/v1\",\"kind\":\"leak\",\"runId\":\"run\",\"scenario\":\"scenario\",\"generatedAt\":\"2026-09-21T00:00:00Z\",\"generator\":{\"package\":\"kenshou-diagnose\",\"version\":\"0.1.0.0\",\"algorithm\":\"test\"},\"body\":{}}"+ fmap encodeDiagnosis (decodeDiagnosis (LazyByteString.pack encoded)) `shouldSatisfy` either (const False) (not . LazyByteString.null)++ it "loads the default policy shape" do+ result <- loadLeakPolicy "../policies/leak-default.json"+ result `shouldSatisfy` either (const False) (not . null . (.probes))++ it "reads series by header name and reports missing columns" do+ result <- readWide "test/fixtures/series/leak.csv" "t_mono_ns" "live_bytes_last_gc"+ fmap Vector.length result `shouldBe` Right 6+ missing <- readWide "test/fixtures/series/leak.csv" "t_mono_ns" "not_present"+ missing `shouldBe` Left (MissingColumn "test/fixtures/series/leak.csv" "not_present")++ it "maps verdicts onto the shared outcome vocabulary" do+ leakOutcome (LeakReport LeakSuspected Nothing 1 "test" []) `shouldNotBe` leakOutcome (LeakReport Stable Nothing 1 "test" [])++ it "distinguishes a flat sawtooth envelope from a rising one" do+ let stable = Vector.fromList [(fromIntegral index, if even index then 100 else 1000) | index <- [0 .. 79 :: Int]]+ rising = Vector.imap (\index (time, value) -> (time, value + fromIntegral (index `div` 2) * 20)) stable+ (judgeSeries testSpec 42 testProbe stable).verdict `shouldBe` Stable+ (judgeSeries testSpec 42 testProbe rising).verdict `shouldBe` LeakSuspected++ it "uses only samples following major collections for heap evidence" do+ let live = Vector.fromList [(0, 100), (1, 1000), (2, 101), (3, 1000), (4, 102), (5, 1000), (6, 103)]+ majors = Vector.fromList [(0, 0), (1, 0), (2, 1), (3, 1), (4, 2), (5, 2), (6, 3)]+ majorGcSamples live majors `shouldBe` Vector.fromList [(2, 101), (4, 102), (6, 103)]++ it "recognises a plateau after initial growth" do+ let points = Vector.fromList [(fromIntegral index, fromIntegral (min index 30) * 100) | index <- [0 .. 79 :: Int]]+ report = judgeSeries testSpec {envelopeWindowSeconds = 1} 42 testProbe {floorPerHour = 1000} points+ report.verdict `shouldBe` Stable+ report.reason `shouldBe` "plateau-after-growth"++ it "returns insufficient data for a short or threshold-straddling series" do+ let short = Vector.fromList [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)]+ exactFloor = Vector.fromList [(fromIntegral index, fromIntegral index) | index <- [0 .. 79 :: Int]]+ (judgeSeries testSpec 42 testProbe short).verdict `shouldBe` InsufficientData+ (judgeSeries testSpec {envelopeWindowSeconds = 1} 42 testProbe {floorPerHour = 3600} exactFloor).reason `shouldBe` "interval-straddles-floor"++ it "refuses forced major collections for benchmark scenarios" do+ case parseScenarioId "selftest/diagnose/benchmark/refuses-major-gc" of+ Left err -> expectationFailure (show err)+ Right scenario -> do+ let runContext = RunContext undefined scenario undefined undefined undefined undefined undefined undefined undefined undefined undefined undefined+ withMajorGcProbe runContext 1 (pure ()) `shouldThrow` anyIOException++ it "keeps the actual header contract in the default catalog" do+ case [probe | probe <- defaultLeakSpec.probes, probe.name == ("heap.live-bytes" :: Text)] of+ [heap] -> do+ heap.binding.valueColumn `shouldBe` "live_bytes_last_gc"+ heap.binding.filters `shouldBe` Map.singleton "phase" "steady"+ other -> expectationFailure ("expected one heap probe, got " <> show other)++ it "sums connection states for one application at each sample time" do+ withSystemTempDirectory "kenshou-connections" \directory -> do+ createDirectoryIfMissing True (directory </> "series")+ Text.writeFile+ (directory </> "series/pg-activity.csv")+ "t_mono_ns,phase,application_name,connections\n0,steady,worker,1\n0,steady,worker,2\n0,steady,other,9\n2000000000,steady,worker,1\n2000000000,steady,worker,2\n4000000000,steady,worker,1\n4000000000,steady,worker,2\n"+ case [probe | probe <- defaultLeakSpec.probes, probe.name == "pg.connections"] of+ [connectionProbe] -> do+ let selected = connectionProbe {binding = connectionProbe.binding {filters = Map.insert "application_name" "worker" connectionProbe.binding.filters}}+ policy = LeakSpec [selected] 0 2 0 1 200 0.95+ report <- analyseSeriesDirectory directory policy 42+ case report.probes of+ [result] -> do+ result.first `shouldBe` Just 3+ result.last `shouldBe` Just 3+ other -> expectationFailure ("expected one connection report, got " <> show other)+ other -> expectationFailure ("expected one connection probe, got " <> show other)++testProbe :: ProbeSpec+testProbe = ProbeSpec "test" "count" (SeriesBinding "synthetic.csv" "seconds" "value" Map.empty) WindowMin Bounded 3600 1 0 Nothing++testSpec :: LeakSpec+testSpec = LeakSpec [testProbe] 0 20 10 2 200 0.95
+ test/Kenshou/Diagnose/ProfileSpec.hs view
@@ -0,0 +1,18 @@+module Kenshou.Diagnose.ProfileSpec (spec) where++import Data.List (isPrefixOf)+import Kenshou.Diagnose.Profile+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Diagnose.Profile" do+ it "assembles closure-type flags with a bounded event log" do+ let flags = rtsFlags ClosureType "session/kenshou.eventlog" 10+ flags `shouldContain` ["-hT", "-i10.0", "-l-agu", "-olsession/kenshou.eventlog", "--eventlog-flush-interval=5"]++ it "keeps scheduler events opt-in" do+ rtsFlags (Eventlog "gu") "events" 10 `shouldSatisfy` not . any (== "-l-agus")+ rtsFlags (Eventlog "gus") "events" 10 `shouldContain` ["-l-agus"]++ it "selects source and retainer breakdowns for profiled libraries" do+ rtsFlags (Profiled 'r') "profiled" 5 `shouldSatisfy` any ("-hr" `isPrefixOf`)
+ test/Kenshou/Diagnose/RenderSpec.hs view
@@ -0,0 +1,33 @@+module Kenshou.Diagnose.RenderSpec (spec) where++import Control.Monad ((>=>))+import Data.Aeson (Result (..), eitherDecodeFileStrict', fromJSON)+import Data.Text.IO qualified as Text+import Kenshou.Diagnose.Document (Diagnosis (..))+import Kenshou.Diagnose.Leak (analyseRunDirectory, defaultLeakSpec)+import Kenshou.Diagnose.Profile+import Kenshou.Diagnose.Render+import Kenshou.Diagnose.Stall.Types (StallReport)+import Test.Hspec++spec :: Spec+spec = describe "diagnosis rendering goldens" do+ it "renders the leaking run" do+ result <- analyseRunDirectory "test/fixtures/run-leaking" defaultLeakSpec 0+ report <- either (expectationFailure . show >=> const (fail "leak analysis failed")) pure result+ golden <- Text.readFile "test/golden/diagnose-leak.txt"+ renderLeakReport report `shouldBe` golden++ it "renders the stalled run" do+ decoded <- eitherDecodeFileStrict' "test/fixtures/run-stalled/diagnosis/stall-1.json" :: IO (Either String Diagnosis)+ document <- either (expectationFailure >=> const (fail "stall document failed")) pure decoded+ report <- case fromJSON document.body of+ Error err -> expectationFailure err >> fail "stall report failed"+ Success value -> pure (value :: StallReport)+ golden <- Text.readFile "test/golden/diagnose-stall.txt"+ renderStallReport report `shouldBe` golden++ it "renders a profile session" do+ let report = ProfileReport "session" ClosureType "ordinary" ["-hT"] "kenshou" (Just "run") (Just "sha256") 10582 False 0+ golden <- Text.readFile "test/golden/diagnose-profile.txt"+ renderProfileReport report `shouldBe` golden
+ test/Kenshou/Diagnose/StallSpec.hs view
@@ -0,0 +1,152 @@+module Kenshou.Diagnose.StallSpec (spec) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (wait, withAsync)+import Control.Exception (bracket, finally, try)+import Control.Monad ((>=>))+import Data.Aeson (eitherDecodeFileStrict')+import Data.Int (Int64)+import Data.List (sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word32, Word64)+import Hasql.Connection qualified as Connection+import Hasql.Connection.Settings qualified as Settings+import Hasql.Decoders qualified as Decoders+import Hasql.Encoders qualified as Encoders+import Hasql.Session qualified as Session+import Hasql.Statement qualified as Statement+import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Context (RunContext (..), newRunState)+import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..))+import Kenshou.Core.Env (PostgresRequirement (..))+import Kenshou.Core.Env.Postgres (PostgresEnv (..), withPostgresEnv)+import Kenshou.Core.Id (parseRunId, parseScenarioId, selftestLayer)+import Kenshou.Core.Log (nullLogger)+import Kenshou.Core.RunSpec (PostgresSpec (..))+import Kenshou.Diagnose.LockGraph (GraphEdge (..), LockGraph (..), buildGraph)+import Kenshou.Diagnose.Postgres+import Kenshou.Diagnose.Stall qualified as Stall+import Kenshou.Diagnose.Stall.Classify (classify)+import Kenshou.Diagnose.Stall.Types+import System.Directory (doesFileExist)+import System.Environment (lookupEnv)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Diagnose.Stall" do+ it "names a bundle's advisory locks and the migration ledger lock" do+ let label = AdvisoryLabel "queue creation" (HashText "synthetic.queue_jobs")+ config = Stall.watchdogConfigFor (mkBundle selftestLayer [] []) {advisoryLabels = [label]}+ config.advisoryLabels `shouldBe` [label, pgMigrateLedgerLock]+ mapM_+ fixtureCase+ [ ("deadlock", Deadlock),+ ("lock-wait", LockWait),+ ("pool-starvation", PoolStarvation),+ ("blocked-indefinitely", BlockedIndefinitely),+ ("idle-spin", IdleSpin),+ ("unknown", Unknown),+ ("postgres-unavailable", Unknown)+ ]++ it "finds a three-session wait cycle as one cycle" do+ let row pid blockers = ActivityRow pid "test" "active" (Just "Lock") (Just "transactionid") (Just 1) (Just 1) blockers "UPDATE test"+ graph = buildGraph [row 10 [11], row 11 [12], row 12 [10]] []+ graph.cycles `shouldBe` [[10, 11, 12]]++ it "reconstructs signed advisory lock keys" do+ let original = -2 :: Int64+ bits = fromIntegral original :: Word64+ high = fromIntegral (bits `div` 0x100000000) :: Word32+ low = fromIntegral bits :: Word32+ reconstructAdvisoryKey high low `shouldBe` original++ describe "PostgreSQL capture" do+ it "finds a real row-lock edge on PostgreSQL 18" (lockCaptureCase Pg18)+ it "finds a real row-lock edge on PostgreSQL 17" do+ available <- lookupEnv "KENSHOU_PG17_BIN"+ maybe (pendingWith "KENSHOU_PG17_BIN is not set") (const (lockCaptureCase Pg17)) available++ it "captures and aborts a silent scenario at the deadline" $ withSystemTempDirectory "kenshou-watchdog" \directory -> do+ runId <- either (expectationFailure . show >=> const (fail "invalid run id")) pure (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+ scenario <- either (expectationFailure . show >=> const (fail "invalid scenario id")) pure (parseScenarioId "selftest/diagnose/concurrency/watchdog")+ state <- newRunState+ let runContext = RunContext runId scenario undefined undefined undefined undefined undefined undefined undefined directory undefined state+ config = Stall.defaultWatchdogConfig {Stall.deadlineSeconds = 0.2, Stall.pollIntervalSeconds = 0.05, Stall.captureStacks = False}+ result <- try @Stall.StallDetected $ Stall.withWatchdog runContext config \watchdog -> do+ _ <- Stall.newProgress watchdog "work" True+ threadDelay 2_000_000+ result `shouldSatisfy` either (const True) (const False)+ doesFileExist (directory </> "diagnosis" </> "stall-1.json") `shouldReturn` True++fixtureCase :: (FilePath, StallClass) -> Spec+fixtureCase (name, expected) = it ("classifies the " <> name <> " snapshot") do+ decoded <- eitherDecodeFileStrict' ("test/fixtures/snapshots/" <> name <> ".json")+ case decoded of+ Left err -> expectationFailure err+ Right snapshot -> do+ let (actual, secondary, _) = classify snapshot+ actual `shouldBe` expected+ sort secondary `shouldSatisfy` (if expected == Unknown then null else elem expected)++lockCaptureCase :: PgVersion -> IO ()+lockCaptureCase version = withSystemTempDirectory "kenshou-diagnose-lock" \directory -> do+ runId <- either (expectationFailure . show >=> const (fail "invalid run id")) pure (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")+ let requirement = PostgresRequirement [] [] False+ dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just version)+ result <- withPostgresEnv mempty nullLogger directory runId requirement (PostgresEphemeral []) dimensions exerciseLockCapture+ case result of+ Left err -> expectationFailure (show err)+ Right () -> pure ()++exerciseLockCapture :: PostgresEnv -> IO ()+exerciseLockCapture environment =+ withConnection environment.connectionString "kenshou-diagnose-holder" \holder ->+ withConnection environment.connectionString "kenshou-diagnose-waiter" \waiter ->+ withConnection environment.connectionString "kenshou-diagnose-capture" \capture -> do+ expectSession holder (sql "CREATE TABLE diagnose_capture (id int primary key)")+ expectSession holder (sql "INSERT INTO diagnose_capture VALUES (1)")+ holderPid <- connectionPid holder+ waiterPid <- connectionPid waiter+ expectSession holder (sql "BEGIN")+ expectSession holder (sql "UPDATE diagnose_capture SET id=id WHERE id=1")+ expectSession waiter (sql "BEGIN")+ withAsync (expectSession waiter (sql "UPDATE diagnose_capture SET id=id WHERE id=1")) \waiterUpdate -> do+ finally+ ( do+ threadDelay 500_000+ activity <- captureActivity capture >>= either (expectationFailure . Text.unpack >=> const (fail "activity capture failed")) pure+ locks <- captureLocks capture >>= either (expectationFailure . Text.unpack >=> const (fail "lock capture failed")) pure+ let graph = buildGraph activity locks+ graph.edges `shouldSatisfy` any (\edge -> edge.waiter == waiterPid && edge.blocker == holderPid)+ )+ (expectSession holder (sql "ROLLBACK"))+ wait waiterUpdate+ _ <- Connection.use waiter (sql "ROLLBACK")+ pure ()++withConnection :: Text -> Text -> (Connection.Connection -> IO value) -> IO value+withConnection connectionString applicationName = bracket acquire Connection.release+ where+ acquire = do+ result <- Connection.acquire (Settings.connectionString connectionString <> Settings.applicationName applicationName)+ either (ioError . userError . show) pure result++expectSession :: Connection.Connection -> Session.Session value -> IO value+expectSession connection session = Connection.use connection session >>= either (ioError . userError . show) pure++connectionPid :: Connection.Connection -> IO Int+connectionPid connection = do+ raw <- expectSession connection (Session.statement () pidStatement)+ case reads (Text.unpack raw) of+ [(pid, "")] -> pure pid+ _ -> expectationFailure ("invalid backend pid " <> Text.unpack raw) >> fail "invalid backend pid"++pidStatement :: Statement.Statement () Text+pidStatement = Statement.unpreparable "SELECT pg_backend_pid()::text" Encoders.noParams (Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.text)))++sql :: Text -> Session.Session ()+sql statement = Session.statement () (Statement.unpreparable statement Encoders.noParams Decoders.noResult)
+ test/Kenshou/Diagnose/StatsSpec.hs view
@@ -0,0 +1,33 @@+module Kenshou.Diagnose.StatsSpec (spec) where++import Data.Vector qualified as Vector+import Kenshou.Diagnose.Stats+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Diagnose.Stats" do+ it "recovers an exact line" do+ fmap fst (theilSen (line 2 10)) `shouldBe` Just 2++ it "resists twenty percent large outliers" do+ let points = Vector.imap (\index point -> if index `mod` 5 == 0 then (fst point, snd point + 10_000) else point) (line 3 100)+ fmap fst (theilSen points) `shouldSatisfy` maybe False (\slope -> abs (slope - 3) < 0.01)++ it "returns a deterministic moving-block interval containing the exact slope" do+ let first = slopeWithInterval 42 200 0.95 (line 4 80)+ second = slopeWithInterval 42 200 0.95 (line 4 80)+ first `shouldBe` second+ first `shouldSatisfy` maybe False (\estimate -> estimate.low <= 4 && estimate.high >= 4)++ it "covers the known slope for at least ninety of one hundred bootstrap seeds" do+ let points = Vector.fromList [(fromIntegral index, 5 * fromIntegral index + correlatedNoise index) | index <- [0 .. 39 :: Int]]+ contains seed = maybe False (\estimate -> estimate.low <= 5 && estimate.high >= 5) (slopeWithInterval seed 80 0.95 points)+ length (filter contains [1 .. 100]) `shouldSatisfy` (>= 90)++ it "reduces sawtooth windows by their minima" do+ let points = Vector.fromList [(0, 10), (1, 20), (2, 11), (3, 30)]+ windowMinima 2 points `shouldBe` Vector.fromList [(0, 10), (2, 11)]+ where+ line :: Double -> Int -> Vector.Vector (Double, Double)+ line slope count = Vector.fromList [(fromIntegral index, slope * fromIntegral index + 7) | index <- [0 .. count - 1]]+ correlatedNoise index = if index >= 10 && index < 18 then 10 else 0
+ test/Main.hs view
@@ -0,0 +1,16 @@+module Main (main) where++import Kenshou.Diagnose.LeakSpec qualified+import Kenshou.Diagnose.ProfileSpec qualified+import Kenshou.Diagnose.RenderSpec qualified+import Kenshou.Diagnose.StallSpec qualified+import Kenshou.Diagnose.StatsSpec qualified+import Test.Hspec (hspec)++main :: IO ()+main = hspec do+ Kenshou.Diagnose.StatsSpec.spec+ Kenshou.Diagnose.LeakSpec.spec+ Kenshou.Diagnose.StallSpec.spec+ Kenshou.Diagnose.ProfileSpec.spec+ Kenshou.Diagnose.RenderSpec.spec
+ test/fixtures/run-leaking/manifest.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.artifact-manifest/v1","files":[]}
+ test/fixtures/run-leaking/run-result.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-result/v1","outcome":"failed"}
+ test/fixtures/run-leaking/series/rts.csv view
@@ -0,0 +1,32 @@+t_mono_ns,phase,live_bytes_last_gc+300000000000,steady,10000000+361000000000,steady,10100000+422000000000,steady,10200000+483000000000,steady,10300000+544000000000,steady,10400000+605000000000,steady,10500000+666000000000,steady,10600000+727000000000,steady,10700000+788000000000,steady,10800000+849000000000,steady,10900000+910000000000,steady,11000000+971000000000,steady,11100000+1032000000000,steady,11200000+1093000000000,steady,11300000+1154000000000,steady,11400000+1215000000000,steady,11500000+1276000000000,steady,11600000+1337000000000,steady,11700000+1398000000000,steady,11800000+1459000000000,steady,11900000+1520000000000,steady,12000000+1581000000000,steady,12100000+1642000000000,steady,12200000+1703000000000,steady,12300000+1764000000000,steady,12400000+1825000000000,steady,12500000+1886000000000,steady,12600000+1947000000000,steady,12700000+2008000000000,steady,12800000+2069000000000,steady,12900000+2130000000000,steady,13000000
+ test/fixtures/run-stalled/diagnosis/stall-1.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.diagnosis/v1","kind":"stall","runId":"fixture-stalled","scenario":"selftest/diagnose/concurrency/deadlocked-workers","generatedAt":"2026-09-21T00:00:00Z","generator":{"package":"kenshou-diagnose","version":"0.1.0.0","algorithm":"fixture"},"body":{"detectedAt":"2026-09-21T00:00:00Z","captureNumber":1,"classification":"deadlock","secondary":["deadlock"],"reasons":["wait-for graph contains a cycle"],"snapshot":{"deadlineSeconds":5,"progress":[],"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"kenshou:selftest:worker-a","status":"blocked","blockReason":"BlockedOnMVar","capability":0,"locked":false}],"truncated":false},"postgres":{"available":true,"activity":[{"pid":101,"state":"active","waitEventType":"Lock","waitEvent":"transactionid","queryAgeSeconds":4,"blockedBy":[102]},{"pid":102,"state":"active","waitEventType":"Lock","waitEvent":"transactionid","queryAgeSeconds":4,"blockedBy":[101]}],"locks":[],"settings":{}},"graph":{"nodes":[{"pid":101,"applicationName":"worker-a","state":"active","waitEvent":"transactionid","query":"UPDATE a"},{"pid":102,"applicationName":"worker-b","state":"active","waitEvent":"transactionid","query":"UPDATE b"}],"edges":[{"waiter":101,"blocker":102,"label":"transactionid"},{"waiter":102,"blocker":101,"label":"transactionid"}],"cycles":[[101,102]],"roots":[]},"pools":[],"stateProbes":{},"idleSpin":{"cpuCores":0,"statementCallsPerSecond":0}}}}
+ test/fixtures/run-stalled/run-result.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.run-result/v1","outcome":"failed"}
+ test/fixtures/series/leak.csv view
@@ -0,0 +1,7 @@+t_mono_ns,phase,live_bytes_last_gc+0,steady,100+1000000000,steady,200+2000000000,steady,300+3000000000,steady,400+4000000000,steady,500+5000000000,steady,600
+ test/fixtures/snapshots/blocked-indefinitely.json view
@@ -0,0 +1,1 @@+{"deadlineSeconds":10,"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"worker","status":"blocked","blockReason":"BlockedOnMVar","capability":0,"locked":false}],"truncated":false},"postgres":{"available":true,"activity":[{"pid":40,"state":"idle","blockedBy":[]}],"locks":[],"settings":{}},"graph":{"cycles":[],"roots":[]},"idleSpin":{"cpuCores":0,"statementCallsPerSecond":0}}
+ test/fixtures/snapshots/deadlock.json view
@@ -0,0 +1,1 @@+{"deadlineSeconds":10,"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"worker","status":"blocked","blockReason":"BlockedOnMVar","capability":0,"locked":false}],"truncated":false},"postgres":{"available":true,"activity":[{"pid":10,"state":"active","waitEventType":"Lock","waitEvent":"transactionid","queryAgeSeconds":8,"blockedBy":[11]},{"pid":11,"state":"active","waitEventType":"Lock","waitEvent":"transactionid","queryAgeSeconds":8,"blockedBy":[10]}],"locks":[],"settings":{}},"graph":{"cycles":[[10,11]],"roots":[]},"idleSpin":{"cpuCores":0,"statementCallsPerSecond":0}}
+ test/fixtures/snapshots/idle-spin.json view
@@ -0,0 +1,1 @@+{"deadlineSeconds":10,"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"worker","status":"running","capability":0,"locked":false}],"truncated":false},"postgres":{"available":true,"activity":[{"pid":50,"state":"active","blockedBy":[]}],"locks":[],"settings":{}},"graph":{"cycles":[],"roots":[]},"idleSpin":{"cpuCores":0.8,"statementCallsPerSecond":0}}
+ test/fixtures/snapshots/lock-wait.json view
@@ -0,0 +1,1 @@+{"deadlineSeconds":10,"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"worker","status":"blocked","blockReason":"BlockedOnMVar","capability":0,"locked":false}],"truncated":false},"postgres":{"available":true,"activity":[{"pid":20,"state":"active","waitEventType":"Lock","waitEvent":"transactionid","queryAgeSeconds":8,"blockedBy":[21]},{"pid":21,"state":"idle in transaction","blockedBy":[]}],"locks":[],"settings":{}},"graph":{"edges":[{"waiter":20,"blocker":21}],"cycles":[],"roots":[21]},"idleSpin":{"cpuCores":0,"statementCallsPerSecond":0}}
+ test/fixtures/snapshots/pool-starvation.json view
@@ -0,0 +1,1 @@+{"deadlineSeconds":10,"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"worker","status":"blocked","blockReason":"BlockedOnSTM","capability":0,"locked":false}],"truncated":false},"postgres":{"available":true,"activity":[{"pid":30,"state":"idle","blockedBy":[]}],"locks":[],"settings":{}},"graph":{"cycles":[],"roots":[]},"pools":[{"name":"main","size":1,"inUse":1,"saturatedSeconds":8}],"idleSpin":{"cpuCores":0,"statementCallsPerSecond":0}}
@@ -0,0 +1,1 @@+{"deadlineSeconds":10,"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"worker","status":"running","capability":0,"locked":false}],"truncated":false},"postgres":{"available":false,"error":"connection refused","activity":[],"locks":[],"settings":null},"graph":{"cycles":[],"roots":[]},"idleSpin":{"cpuCores":0,"statementCallsPerSecond":0}}
+ test/fixtures/snapshots/unknown.json view
@@ -0,0 +1,1 @@+{"deadlineSeconds":10,"haskellThreads":{"generatedAt":"2026-09-21T00:00:00Z","processId":1,"entries":[{"id":"1","label":"worker","status":"running","capability":0,"locked":false}],"truncated":false},"postgres":{"available":true,"activity":[{"pid":60,"state":"active","blockedBy":[]}],"locks":[],"settings":{}},"graph":{"cycles":[],"roots":[]},"idleSpin":{"cpuCores":0,"statementCallsPerSecond":0}}
+ test/golden/diagnose-leak.txt view
@@ -0,0 +1,9 @@+leak verdict: leak-suspected+heap.live-bytes[main] leak-suspected slope=5901639.344262295 bytes/hour reason=interval-above-floor+process.native-bytes[main] insufficient-data slope=n/a reason=CannotReadSeries "test/fixtures/run-leaking/series/proc.csv" "test/fixtures/run-leaking/series/proc.csv: withFile: does not exist (No such file or directory)"+haskell.threads[main] insufficient-data slope=n/a reason=MissingColumn "test/fixtures/run-leaking/series/rts.csv" "haskell_threads"+os.threads[main] insufficient-data slope=n/a reason=CannotReadSeries "test/fixtures/run-leaking/series/proc.csv" "test/fixtures/run-leaking/series/proc.csv: withFile: does not exist (No such file or directory)"+os.fds[main] insufficient-data slope=n/a reason=CannotReadSeries "test/fixtures/run-leaking/series/proc.csv" "test/fixtures/run-leaking/series/proc.csv: withFile: does not exist (No such file or directory)"+pg.connections[main] insufficient-data slope=n/a reason=CannotReadSeries "test/fixtures/run-leaking/series/pg-activity.csv" "test/fixtures/run-leaking/series/pg-activity.csv: withFile: does not exist (No such file or directory)"+pg.relation-bytes[main] insufficient-data slope=n/a reason=CannotReadSeries "test/fixtures/run-leaking/series/pg-relations.csv" "test/fixtures/run-leaking/series/pg-relations.csv: withFile: does not exist (No such file or directory)"+pg.dead-tuples[main] insufficient-data slope=n/a reason=CannotReadSeries "test/fixtures/run-leaking/series/pg-relations.csv" "test/fixtures/run-leaking/series/pg-relations.csv: withFile: does not exist (No such file or directory)"
+ test/golden/diagnose-profile.txt view
@@ -0,0 +1,4 @@+profile session: session+mode: ClosureType+event log: 10582 bytes+wrapped exit: 0
+ test/golden/diagnose-stall.txt view
@@ -0,0 +1,7 @@+stall 1: deadlock+reason: wait-for graph contains a cycle+wait graph:+cycle: 101 -> 102 -> 101+101 waits for 102 (transactionid)+102 waits for 101 (transactionid)+thread kenshou:selftest:worker-a: blocked BlockedOnMVar
+ test/golden/leak-diagnosis.json view
@@ -0,0 +1,1 @@+{"schema":"kenshou.diagnosis/v1","kind":"leak","runId":"01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55","scenario":"selftest/diagnose/soak/leaking-worker","generatedAt":"2026-09-21T00:00:00Z","generator":{"package":"kenshou-diagnose","version":"0.1.0.0","algorithm":"theil-sen+moving-block-bootstrap/1"},"body":{"verdict":"leak-suspected","window":{"startSeconds":5,"endSeconds":45},"parameters":{"seed":42,"policy":"policies/leak-default.json"},"probes":[]}}