packages feed

kenshou-diagnose-0.1.0.0: src/Kenshou/Diagnose/Leak.hs

module Kenshou.Diagnose.Leak
  ( LeakVerdict (..),
    Expectation (..),
    Aggregation (..),
    ProbeSpec (..),
    LeakSpec (..),
    ProbeReport (..),
    LeakReport (..),
    defaultLeakSpec,
    loadLeakPolicy,
    judgeLeaks,
    judgeLeaksWithWindow,
    analyseRunDirectory,
    analyseSeriesDirectory,
    majorGcSamples,
    judgeSeries,
    leakOutcome,
  )
where

import Data.Aeson
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Map.Strict qualified as Map
import Data.Maybe (catMaybes, fromMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import Data.Vector (Vector)
import Data.Vector qualified as Vector
import Data.Word (Word64)
import Kenshou.Core.Context qualified as Core
import Kenshou.Core.Outcome (Outcome (..))
import Kenshou.Diagnose.Context qualified as Context
import Kenshou.Diagnose.Document
import Kenshou.Diagnose.Series
import Kenshou.Diagnose.Series.Catalog
import Kenshou.Diagnose.Stats
import System.Directory (doesFileExist)
import System.FilePath (takeDirectory, (</>))

data LeakVerdict = LeakSuspected | Stable | InsufficientData
  deriving stock (Eq, Ord, Show)

data Expectation = Bounded | Informational deriving stock (Eq, Show)

data Aggregation = WindowMin | WindowMedian deriving stock (Eq, Show)

data ProbeSpec = ProbeSpec
  { name :: !Text,
    unit :: !Text,
    binding :: !SeriesBinding,
    aggregation :: !Aggregation,
    expectation :: !Expectation,
    floorPerHour :: !Double,
    minGrowth :: !Double,
    minRelativeGrowth :: !Double,
    limit :: !(Maybe Double)
  }
  deriving stock (Eq, Show)

data LeakSpec = LeakSpec
  { probes :: ![ProbeSpec],
    warmupCutSeconds :: !Double,
    minPoints :: !Int,
    minDurationSeconds :: !Double,
    envelopeWindowSeconds :: !Double,
    resamples :: !Int,
    confidence :: !Double
  }
  deriving stock (Eq, Show)

data ProbeReport = ProbeReport
  { probe :: !Text,
    process :: !Text,
    unit :: !Text,
    expectation :: !Expectation,
    basis :: !Text,
    points :: !Int,
    durationSeconds :: !Double,
    first :: !(Maybe Double),
    last :: !(Maybe Double),
    medianLevel :: !(Maybe Double),
    slopePerHour :: !(Maybe Double),
    intervalPerHour :: !(Maybe (Double, Double)),
    secondHalfSlopePerHour :: !(Maybe Double),
    growthOverWindow :: !(Maybe Double),
    projectedHoursToLimit :: !(Maybe Double),
    verdict :: !LeakVerdict,
    reason :: !Text,
    sourceFile :: !FilePath,
    sourceColumn :: !Text
  }
  deriving stock (Eq, Show)

data LeakReport = LeakReport
  { verdict :: !LeakVerdict,
    window :: !(Maybe (Double, Double)),
    seed :: !Word64,
    policy :: !Text,
    probes :: ![ProbeReport]
  }
  deriving stock (Eq, Show)

defaultLeakSpec :: LeakSpec
defaultLeakSpec =
  LeakSpec
    { probes =
        [ bounded "heap.live-bytes" "bytes" WindowMin 1_048_576 2_097_152 0.01,
          bounded "process.native-bytes" "bytes" WindowMedian 8_388_608 8_388_608 0.05,
          bounded "haskell.threads" "count" WindowMedian 1 5 0,
          bounded "os.threads" "count" WindowMedian 1 5 0,
          bounded "os.fds" "count" WindowMedian 1 5 0,
          bounded "pg.connections" "count" WindowMedian 1 5 0,
          informational "pg.relation-bytes" "bytes" WindowMin,
          informational "pg.dead-tuples" "count" WindowMin
        ],
      warmupCutSeconds = 300,
      minPoints = 30,
      minDurationSeconds = 1200,
      envelopeWindowSeconds = 60,
      resamples = 1000,
      confidence = 0.95
    }
  where
    catalog name = fromMaybe (error ("missing default catalog binding " <> Text.unpack name)) (Map.lookup name defaultCatalog)
    bounded name unit aggregation slopeFloor growth relative = ProbeSpec name unit (catalog (if name == "process.native-bytes" then "process.rss-bytes" else name)) aggregation Bounded slopeFloor growth relative Nothing
    informational name unit aggregation = ProbeSpec name unit (catalog name) aggregation Informational 0 0 0 Nothing

loadLeakPolicy :: FilePath -> IO (Either Text LeakSpec)
loadLeakPolicy path = do
  decoded <- eitherDecodeFileStrict' path
  pure case decoded of
    Left err -> Left (Text.pack err)
    Right spec -> Right spec

judgeLeaks :: Core.RunContext -> LeakSpec -> IO LeakReport
judgeLeaks context spec = do
  window <- Context.steadyWindow context
  judgeLeaksWithWindow context window spec

-- | Use the effective measurement window when a scenario overrides its
-- registered phase duration at run time.
judgeLeaksWithWindow :: Core.RunContext -> Maybe (Double, Double) -> LeakSpec -> IO LeakReport
judgeLeaksWithWindow context window spec = do
  report <- analyse (Context.runDirectory context) spec (Context.runSeed context) (pure window)
  generatedAt <- getCurrentTime
  let (runId, scenario) = Context.runIdentity context
      document = Diagnosis LeakDiagnosis runId scenario generatedAt generator (toJSON report)
      relative = "diagnosis/leak.json"
  path <- Core.artifactPath context Core.DiagnosisDir "leak.json"
  LazyByteString.writeFile path (encodeDiagnosis document)
  Core.declareMediaType context relative "application/json"
  Context.publishDiagnosis context "leak" (toJSON report)
  pure report
  where
    generator = Generator "kenshou-diagnose" "0.1.0.0" "theil-sen+moving-block-bootstrap/1"

analyseRunDirectory :: FilePath -> LeakSpec -> Word64 -> IO (Either DiagnoseError LeakReport)
analyseRunDirectory runDirectory spec seed = do
  exists <- doesFileExist (runDirectory </> "run-result.json")
  if not exists
    then pure (Left (InvalidRunDirectory runDirectory "run-result.json is missing"))
    else Right <$> analyse runDirectory spec seed (pure Nothing)

analyseSeriesDirectory :: FilePath -> LeakSpec -> Word64 -> IO LeakReport
analyseSeriesDirectory runDirectory spec seed = analyse runDirectory spec seed (pure Nothing)

analyse :: FilePath -> LeakSpec -> Word64 -> IO (Maybe (Double, Double)) -> IO LeakReport
analyse runDirectory spec seed getWindow = do
  reports <- catMaybes <$> traverse (analyseProbe runDirectory spec seed) spec.probes
  window <- getWindow
  let bounded = [report.verdict | report <- reports, report.expectation == Bounded]
      overall
        | LeakSuspected `elem` bounded = LeakSuspected
        | null bounded || InsufficientData `elem` bounded = InsufficientData
        | otherwise = Stable
  pure (LeakReport overall window seed "policy" reports)

analyseProbe :: FilePath -> LeakSpec -> Word64 -> ProbeSpec -> IO (Maybe ProbeReport)
analyseProbe runDirectory spec seed probe = do
  (series, postMajor) <-
    if probe.name == "heap.live-bytes"
      then heapSeries runDirectory probe
      else do
        result <- if probe.name == "process.native-bytes" then nativeSeries runDirectory probe else if probe.name == "pg.connections" then connectionSeries runDirectory probe else ordinarySeries runDirectory probe
        pure (result, False)
  pure $ Just case series of
    Left err -> emptyReport probe (Text.pack (show err))
    Right raw ->
      let report = judgeSeries (if postMajor then spec {minPoints = min 10 spec.minPoints} else spec) seed probe raw
       in if postMajor then report {basis = "post-major-collection"} else report

heapSeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)), Bool)
heapSeries runDirectory probe = do
  let path = runDirectory </> "series" </> probe.binding.file
      unfiltered = probe.binding {filters = Map.empty}
  live <- readBinding path unfiltered
  majors <- readBinding path (unfiltered {valueColumn = "major_gcs"})
  pure case (live, majors) of
    (Right values, Right counts) -> (Right (majorGcSamples values counts), True)
    (Right _, Left (MissingColumn _ _)) -> (live, False)
    (Left err, _) -> (Left err, False)
    (_, Left err) -> (Left err, False)

majorGcSamples :: Vector (Double, Double) -> Vector (Double, Double) -> Vector (Double, Double)
majorGcSamples live majors = Vector.fromList (go Nothing (Vector.toList (Vector.zip live majors)))
  where
    go _ [] = []
    go previous (((time, value), (_, count)) : rest) =
      let selected = maybe False (count >) previous
       in (if selected then [(time, value)] else []) <> go (Just count) rest

ordinarySeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)))
ordinarySeries runDirectory probe =
  let binding = probe.binding
      path = runDirectory </> "series" </> binding.file
   in readBinding path binding

connectionSeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)))
connectionSeries runDirectory probe =
  fmap (Vector.fromList . Map.toAscList . Map.fromListWith (+) . Vector.toList) <$> ordinarySeries runDirectory probe

nativeSeries :: FilePath -> ProbeSpec -> IO (Either DiagnoseError (Vector (Double, Double)))
nativeSeries runDirectory probe = do
  rss <- ordinarySeries runDirectory probe
  let memBinding = fromMaybe (error "runtime mem binding missing") (Map.lookup "runtime.mem-in-use-bytes" defaultCatalog)
  let childBinding = memBinding {file = takeDirectory probe.binding.file </> memBinding.file}
  mem <- readBinding (runDirectory </> "series" </> childBinding.file) childBinding
  pure ((Vector.zipWith difference) <$> rss <*> mem)
  where
    difference (time, resident) (_, inUse) = (time, max 0 (resident - inUse))

judgeSeries :: LeakSpec -> Word64 -> ProbeSpec -> Vector (Double, Double) -> ProbeReport
judgeSeries spec seed probe raw
  | Vector.length reduced < spec.minPoints = emptyReport probe "too-few-points"
  | duration < spec.minDurationSeconds = emptyReport probe "too-short"
  | otherwise = case slopeWithInterval seed spec.resamples spec.confidence reduced of
      Nothing -> emptyReport probe "slope-unavailable"
      Just estimate -> finish estimate
  where
    cut = Vector.dropWhile (\(time, _) -> time < spec.warmupCutSeconds) raw
    reduced = case probe.aggregation of WindowMin -> windowMinima spec.envelopeWindowSeconds cut; WindowMedian -> windowMedians spec.envelopeWindowSeconds cut
    duration = if Vector.length reduced < 2 then 0 else fst (Vector.last reduced) - fst (Vector.head reduced)
    values = fmap snd (Vector.toList reduced)
    firstValue = snd <$> reduced Vector.!? 0
    lastValue = snd <$> reduced Vector.!? (Vector.length reduced - 1)
    medianValue = if null values then Nothing else Just (median values)
    growth = (-) <$> lastValue <*> firstValue
    finish estimate =
      let slopeHour = estimate.slope * 3600
          lowHour = estimate.low * 3600
          highHour = estimate.high * 3600
          half = Vector.drop (Vector.length reduced `div` 2) reduced
          second = (* 3600) . fst <$> theilSen half
          enoughGrowth = fromMaybe False ((\value level -> value >= probe.minGrowth && value >= probe.minRelativeGrowth * level) <$> growth <*> medianValue)
          plateau = maybe True (< probe.floorPerHour / 2) second
          (verdict, reason)
            | lowHour > probe.floorPerHour && enoughGrowth && not plateau = (LeakSuspected, "interval-above-floor")
            | lowHour > probe.floorPerHour && plateau = (Stable, "plateau-after-growth")
            | highHour < probe.floorPerHour || not enoughGrowth = (Stable, "below-growth-floor")
            | otherwise = (InsufficientData, "interval-straddles-floor")
          projected = do
            limit <- probe.limit
            level <- lastValue
            if slopeHour > 0 then Just ((limit - level) / slopeHour) else Nothing
       in ProbeReport probe.name "main" probe.unit probe.expectation (basisFor probe) (Vector.length reduced) duration firstValue lastValue medianValue (Just slopeHour) (Just (lowHour, highHour)) second growth projected verdict reason probe.binding.file probe.binding.valueColumn

emptyReport :: ProbeSpec -> Text -> ProbeReport
emptyReport probe reason = ProbeReport probe.name "main" probe.unit probe.expectation (basisFor probe) 0 0 Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing InsufficientData reason probe.binding.file probe.binding.valueColumn

basisFor :: ProbeSpec -> Text
basisFor probe = case probe.aggregation of WindowMin -> "window-minimum"; WindowMedian -> "window-median"

leakOutcome :: LeakReport -> Outcome
leakOutcome report = case report.verdict of LeakSuspected -> Failed; Stable -> Passed; InsufficientData -> Inconclusive

verdictText :: LeakVerdict -> Text
verdictText LeakSuspected = "leak-suspected"
verdictText Stable = "stable"
verdictText InsufficientData = "insufficient-data"

instance ToJSON LeakVerdict where toJSON = toJSON . verdictText

instance ToJSON Expectation where toJSON Bounded = String "bounded"; toJSON Informational = String "informational"

instance ToJSON Aggregation where toJSON WindowMin = String "window-min"; toJSON WindowMedian = String "window-median"

instance ToJSON ProbeReport where
  toJSON report = object ["probe" .= report.probe, "process" .= report.process, "unit" .= report.unit, "expectation" .= report.expectation, "basis" .= report.basis, "points" .= report.points, "durationSeconds" .= report.durationSeconds, "level" .= object ["first" .= report.first, "last" .= report.last, "median" .= report.medianLevel], "slopePerHour" .= report.slopePerHour, "intervalPerHour" .= fmap (\(low, high) -> [low, high]) report.intervalPerHour, "secondHalfSlopePerHour" .= report.secondHalfSlopePerHour, "growthOverWindow" .= report.growthOverWindow, "projectedHoursToLimit" .= report.projectedHoursToLimit, "verdict" .= report.verdict, "reason" .= report.reason, "source" .= object ["file" .= report.sourceFile, "column" .= report.sourceColumn]]

instance ToJSON LeakReport where
  toJSON report = object ["verdict" .= report.verdict, "window" .= fmap (\(start, end) -> object ["startSeconds" .= start, "endSeconds" .= end]) report.window, "parameters" .= object ["seed" .= report.seed, "policy" .= report.policy], "probes" .= report.probes]

instance FromJSON LeakSpec where
  parseJSON = withObject "LeakSpec" \value -> do
    schema <- value .:? "schema" .!= ("kenshou.leak-policy/v1" :: Text)
    if schema /= ("kenshou.leak-policy/v1" :: Text)
      then fail "unsupported leak policy schema"
      else LeakSpec <$> value .: "probes" <*> value .: "warmupCutSeconds" <*> value .: "minPoints" <*> value .: "minDurationSeconds" <*> value .: "envelopeWindowSeconds" <*> value .: "resamples" <*> value .: "confidence"

instance FromJSON ProbeSpec where
  parseJSON = withObject "ProbeSpec" \value -> do
    name <- value .: "name"
    unit <- value .: "unit"
    file <- value .: "file"
    timeColumn <- value .:? "timeColumn" .!= "t_mono_ns"
    valueColumn <- value .: "valueColumn"
    aggregationText <- value .: "aggregation"
    expectationText <- value .: "expectation"
    aggregation <- case (aggregationText :: Text) of "window-min" -> pure WindowMin; "window-median" -> pure WindowMedian; other -> fail ("unknown aggregation " <> Text.unpack other)
    expectation <- case (expectationText :: Text) of "bounded" -> pure Bounded; "informational" -> pure Informational; other -> fail ("unknown expectation " <> Text.unpack other)
    let filters = if file == ("rts.csv" :: FilePath) || file == "proc.csv" then Map.singleton "phase" "steady" else Map.empty
    ProbeSpec name unit (SeriesBinding file timeColumn valueColumn filters) aggregation expectation <$> value .: "floorPerHour" <*> value .: "minGrowth" <*> value .: "minRelativeGrowth" <*> value .:? "limit"

median :: [Double] -> Double
median [] = 0
median values = let ordered = quicksort values in ordered !! ((length ordered - 1) `div` 2)
  where
    quicksort [] = []
    quicksort (item : rest) = quicksort (filter (<= item) rest) <> [item] <> quicksort (filter (> item) rest)