packages feed

kenshou-diagnose-0.1.0.0: src/Kenshou/Diagnose/Stall/Classify.hs

module Kenshou.Diagnose.Stall.Classify
  ( classify,
  )
where

import Data.List (find)
import Data.Maybe (fromMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Diagnose.LockGraph (LockGraph (..))
import Kenshou.Diagnose.Pool (PoolStats (..))
import Kenshou.Diagnose.Postgres (ActivityRow (..), PostgresSnapshot (..))
import Kenshou.Diagnose.Stall.Types
import Kenshou.Diagnose.Threads (ThreadDump (..), ThreadEntry (..))

classify :: StallSnapshot -> (StallClass, [StallClass], [Text])
classify snapshot = (fromMaybe Unknown (fst <$> find snd checks), [class_ | (class_, True) <- checks], reasons)
  where
    checks =
      [ (Deadlock, not (null snapshot.graph.cycles)),
        (LockWait, lockWait),
        (PoolStarvation, poolStarvation),
        (IdleSpin, idleSpin),
        (BlockedIndefinitely, blockedIndefinitely)
      ]
    activity = maybe [] (.activity) snapshot.postgres
    lockWait = null snapshot.graph.cycles && any longLock activity
    longLock row = row.waitEventType == Just "Lock" && fromMaybe 0 row.queryAgeSeconds >= snapshot.deadlineSeconds / 2
    saturated = [pool | pool <- snapshot.pools, pool.saturatedSeconds >= snapshot.deadlineSeconds / 2 && pool.inUse >= pool.size]
    blockedOnStm = any ((== Just "BlockedOnSTM") . (.blockReason)) (nonHarness snapshot.haskellThreads.entries)
    poolStarvation = not lockWait && not (null saturated) && blockedOnStm
    idleSpin = snapshot.idleSpin.cpuCores >= 0.5 || snapshot.idleSpin.statementCallsPerSecond >= 100
    nonHarness = filter (maybe True (not . Text.isPrefixOf "kenshou:diagnose") . (.label))
    quietThread entry = entry.status `elem` ["finished", "died"] || entry.blockReason `elem` [Just "BlockedOnMVar", Just "BlockedOnSTM"]
    allDatabaseIdle = maybe False (\database -> database.available && not (null database.activity) && all ((== "idle") . (.state)) database.activity) snapshot.postgres
    blockedIndefinitely = not idleSpin && all quietThread (nonHarness snapshot.haskellThreads.entries) && allDatabaseIdle
    reasons =
      ["wait-for graph contains a cycle" | not (null snapshot.graph.cycles)]
        <> ["sessions have waited on locks for at least half the watchdog deadline" | lockWait]
        <> ["pool " <> pool.name <> " is saturated" | pool <- saturated]
        <> ["CPU or database statement rate is high while progress is zero" | idleSpin]
        <> ["all non-harness threads are finished or indefinitely blocked" | blockedIndefinitely]