packages feed

kenshou-measure-0.1.0.0: src/Kenshou/Measure/Health.hs

module Kenshou.Measure.Health
  ( Severity (..),
    HealthObservation (..),
    HealthConfig (..),
    defaultHealthConfig,
    captureHealthNotices,
    evaluateHealth,
    healthOutcome,
  )
where

import Control.Applicative ((<|>))
import Data.Aeson
import Data.Aeson.Key qualified as Key
import Data.Aeson.KeyMap qualified as KeyMap
import Data.List (isSuffixOf)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe, mapMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.Encoding qualified as Encoding
import Data.Text.IO qualified as Text
import Data.Time (UTCTime)
import Data.Time.Clock.POSIX (posixSecondsToUTCTime)
import Data.Word (Word64)
import Kenshou.Core.Outcome (Outcome (..))
import System.Directory (copyFile, doesFileExist, listDirectory)
import System.Environment (lookupEnv)
import System.FilePath ((</>))
import Text.Read (readMaybe)

data Severity = Info | Soft | Hard deriving stock (Eq, Ord, Show)

data HealthObservation = HealthObservation
  { gate :: Text,
    severity :: Severity,
    fromMonoNs :: Word64,
    toMonoNs :: Word64,
    detail :: Text,
    evidence :: Value
  }
  deriving stock (Eq, Show)

data HealthConfig = HealthConfig
  { cpuSoft :: Double,
    cpuHard :: Double,
    stealSoft :: Double,
    clockStepNs :: Word64,
    pauseNs :: Word64,
    minSteadySamples :: Word64
  }
  deriving stock (Eq, Show)

defaultHealthConfig :: HealthConfig
defaultHealthConfig = HealthConfig 0.70 0.90 0.02 50_000_000 2_000_000_000 1_000

instance ToJSON Severity where
  toJSON Info = String "info"
  toJSON Soft = String "soft"
  toJSON Hard = String "hard"

instance FromJSON Severity where
  parseJSON = withText "Severity" \case
    "info" -> pure Info
    "soft" -> pure Soft
    "hard" -> pure Hard
    _ -> fail "severity must be info, soft, or hard"

instance ToJSON HealthObservation where
  toJSON value =
    object
      [ "gate" .= value.gate,
        "severity" .= value.severity,
        "fromMonoNs" .= value.fromMonoNs,
        "toMonoNs" .= value.toMonoNs,
        "detail" .= value.detail,
        "evidence" .= value.evidence
      ]

instance FromJSON HealthObservation where
  parseJSON = withObject "HealthObservation" \value -> HealthObservation <$> value .: "gate" <*> value .: "severity" <*> value .: "fromMonoNs" <*> value .: "toMonoNs" <*> value .: "detail" <*> value .: "evidence"

data HealthNotice = HealthNotice Text Severity UTCTime Text

instance FromJSON HealthNotice where
  parseJSON = withObject "HealthNotice" \value -> do
    schema <- value .: "schema"
    if schema /= ("kenshou.health-notice/v1" :: Text) then fail "unsupported health notice" else pure ()
    HealthNotice <$> value .: "source" <*> value .: "severity" <*> value .: "at" <*> value .: "detail"

type CsvRow = Map Text Text

captureHealthNotices :: FilePath -> IO (Maybe FilePath)
captureHealthNotices runDir = do
  let captured = runDir </> "health-notices.jsonl"
  source <- lookupEnv "KENSHOU_HEALTH_NOTICES"
  case source of
    Just path | path /= captured -> do
      exists <- doesFileExist path
      if exists then copyFile path captured >> pure (Just captured) else existing captured
    _ -> existing captured
  where
    existing path = do
      exists <- doesFileExist path
      pure (if exists then Just path else Nothing)

evaluateHealth :: HealthConfig -> FilePath -> IO [HealthObservation]
evaluateHealth config runDir = do
  samplerRows <- steadyRows (runDir </> "series" </> "sampler.csv")
  loadRows <- readCsvRows (runDir </> "series" </> "load.csv")
  hostRows <- steadyRows (runDir </> "series" </> "host.csv")
  processRows <- steadyRows (runDir </> "series" </> "proc.csv")
  rtsRows <- steadyRows (runDir </> "series" </> "rts.csv")
  checkpointRows <- steadyRows (runDir </> "series" </> "pg-checkpointer.csv")
  spec <- decodeValue (runDir </> "run-spec.json")
  sampleObservations <- sampleFileObservations config runDir
  noticeObservations <- noticeFileObservations runDir samplerRows
  let bounds = steadyWindow loadRows samplerRows
      intervalNs = round (numberAt ["knobs", "measure.sample-interval-ms"] spec * 1_000_000)
      effectivePauseNs = max config.pauseNs (5 * intervalNs)
      cpu = driverCpu config bounds hostRows processRows rtsRows
      driverHard = any (\item -> item.gate == "driver-cpu-saturation" && item.severity == Hard) cpu
      overload = backlogObservations bounds driverHard spec loadRows
      clocks = samplerObservations config effectivePauseNs bounds samplerRows
      checkpoint = checkpointObservations bounds checkpointRows
      rts = [observation bounds "rts-stats-unavailable" Info "GHC RTS statistics were unavailable" Null | null rtsRows]
  pure (cpu <> overload <> clocks <> checkpoint <> sampleObservations bounds <> rts <> noticeObservations bounds)

healthOutcome :: [HealthObservation] -> Maybe Outcome
healthOutcome observations
  | any ((== Hard) . (.severity)) observations = Just InfrastructureFailure
  | any ((== Soft) . (.severity)) observations = Just Inconclusive
  | otherwise = Nothing

driverCpu :: HealthConfig -> (Word64, Word64) -> [CsvRow] -> [CsvRow] -> [CsvRow] -> [HealthObservation]
driverCpu config bounds hostRows processRows rtsRows = saturation <> steal
  where
    hostDeltas = fmap (\fieldName -> max 0 (delta fieldName hostRows)) ["cpu_user", "cpu_nice", "cpu_system", "cpu_idle", "cpu_iowait", "cpu_irq", "cpu_softirq", "cpu_steal"]
    hostTotal = sum hostDeltas
    hostBusy = if hostTotal <= 0 then Nothing else Just ((hostTotal - hostDeltas !! 3 - hostDeltas !! 4) / hostTotal)
    capabilities = max 1 (fromMaybe 1 (lastNumber "capabilities" rtsRows))
    elapsed = max 1 (delta "t_mono_ns" processRows)
    processBusy = if length processRows < 2 then Nothing else Just (delta "cpu_total_ns" processRows / elapsed / capabilities)
    busy = if length hostRows >= 2 then hostBusy else processBusy
    saturation = case busy of
      Just value | value > config.cpuHard -> [cpuObservation Hard value]
      Just value | value > config.cpuSoft -> [cpuObservation Soft value]
      _ -> []
    stealFraction = if hostTotal <= 0 then 0 else hostDeltas !! 7 / hostTotal
    steal = [observation bounds "cpu-steal" Soft "host CPU steal exceeded the health threshold" (object ["fraction" .= stealFraction, "threshold" .= config.stealSoft]) | stealFraction > config.stealSoft]
    cpuObservation level value = observation bounds "driver-cpu-saturation" level "load driver CPU use exceeded the health threshold" (object ["utilisation" .= value, "softThreshold" .= config.cpuSoft, "hardThreshold" .= config.cpuHard])

backlogObservations :: (Word64, Word64) -> Bool -> Value -> [CsvRow] -> [HealthObservation]
backlogObservations bounds driverHard spec rows =
  case maximumMaybe (mapMaybe (fieldNumber "max_lag_ns") rows) of
    Just maximumLag
      | modelName `elem` [Just "open-constant", Just "open-poisson"] && maximumLag > maximumThreshold ->
          let aborted = maximumLag > abortThreshold
              level = if driverHard || aborted then Hard else Soft
           in [observation bounds "open-loop-backlog" level "open-loop arrivals accumulated sustained lag" (object ["maximumLagNs" .= maximumLag, "thresholdNs" .= maximumThreshold, "abortedEarly" .= aborted])]
    _ -> []
  where
    modelName = textAt ["knobs", "load.model"] spec
    maximumThreshold = numberAt ["knobs", "load.max-lag-ms"] spec * 1_000_000
    abortThreshold = numberAt ["knobs", "load.abort-lag-ms"] spec * 1_000_000

samplerObservations :: HealthConfig -> Word64 -> (Word64, Word64) -> [CsvRow] -> [HealthObservation]
samplerObservations config effectivePauseNs bounds rows = clock <> late <> overrun
  where
    maximumClockStep = maximumOrZero (fmap abs (mapMaybe (fieldNumber "wall_minus_mono_ns") rows))
    maximumLate = maximumOrZero (mapMaybe (fieldNumber "late_ns") rows)
    maximumCost = maximumOrZero (mapMaybe (fieldNumber "cost_ns") rows)
    interval = if length rows < 2 then 0 else delta "scheduled_mono_ns" rows / fromIntegral (length rows - 1)
    clock = [observation bounds "clock-anomaly" Soft "wall and monotonic clocks changed by more than 50 ms" (object ["maximumStepNs" .= maximumClockStep, "thresholdNs" .= config.clockStepNs]) | maximumClockStep > fromIntegral config.clockStepNs]
    late = [observation bounds "clock-anomaly" Hard "the sampler observed a process or virtual-machine pause" (object ["maximumLateNs" .= maximumLate, "thresholdNs" .= effectivePauseNs]) | maximumLate > fromIntegral effectivePauseNs]
    overrun = [observation bounds "sampler-overrun" Soft "sampling cost exceeded the configured interval" (object ["maximumCostNs" .= maximumCost, "intervalNs" .= interval]) | interval > 0 && maximumCost > interval]

checkpointObservations :: (Word64, Word64) -> [CsvRow] -> [HealthObservation]
checkpointObservations _ [] = []
checkpointObservations bounds rows =
  [ observation
      bounds
      "checkpoint-in-window"
      Info
      "PostgreSQL checkpoint activity in the steady window"
      (object ["started" .= started, "completed" .= completed, "overlapFraction" .= overlapFraction])
  ]
  where
    started = max 0 (delta "num_timed" rows) + max 0 (delta "num_requested" rows)
    lsnChanges = length [() | (left, right) <- adjacent rows, Map.lookup "checkpoint_lsn" left /= Map.lookup "checkpoint_lsn" right]
    completed = case lastNumber "num_done" rows of Just _ -> max 0 (delta "num_done" rows); Nothing -> fromIntegral lsnChanges
    intervals = adjacent rows
    active (left, right) = Map.lookup "checkpoint_lsn" left /= Map.lookup "checkpoint_lsn" right || fromMaybe 0 (fieldNumber "buffers_written" right) > fromMaybe 0 (fieldNumber "buffers_written" left)
    overlapFraction :: Double
    overlapFraction = if null intervals then 0 else fromIntegral (length (filter active intervals)) / fromIntegral (length intervals)

sampleFileObservations :: HealthConfig -> FilePath -> IO ((Word64, Word64) -> [HealthObservation])
sampleFileObservations config runDir = do
  let directory = runDir </> "samples"
  files <- listDirectory directory
  decoded <- traverse (decodeValue . (directory </>)) [file | file <- files, ".meta.json" `isSuffixOf` file]
  pure \bounds -> concatMap (fromMeta bounds) decoded
  where
    fromMeta bounds value =
      let operation = fromMaybe "unknown" (textAt ["operation"] value)
          successes = numberAt ["phaseCounts", "steady", "successes"] value
          failures = numberAt ["phaseCounts", "steady", "failures"] value
          samples = successes + failures
          backpressure = numberAt ["backpressureCount"] value
       in [observation bounds "insufficient-samples" Soft ("operation " <> operation <> " has too few steady samples") (object ["operation" .= operation, "samples" .= samples, "minimum" .= config.minSteadySamples]) | samples < fromIntegral config.minSteadySamples]
            <> [observation bounds "recorder-backpressure" Soft ("operation " <> operation <> " experienced recorder back-pressure") (object ["operation" .= operation, "count" .= backpressure]) | backpressure > 0]

noticeFileObservations :: FilePath -> [CsvRow] -> IO ((Word64, Word64) -> [HealthObservation])
noticeFileObservations runDir samplerRows = do
  let path = runDir </> "health-notices.jsonl"
  exists <- doesFileExist path
  if not exists
    then pure (const [])
    else do
      notices <- fmap (mapMaybe decodeNotice . Text.lines) (Text.readFile path)
      let wallBounds = wallTimeBounds samplerRows
      pure \bounds -> [noticeObservation bounds notice | notice@(HealthNotice _ _ noticeAt _) <- notices, within wallBounds noticeAt]
  where
    decodeNotice :: Text -> Maybe HealthNotice
    decodeNotice = decodeStrict' . Encoding.encodeUtf8
    noticeObservation bounds (HealthNotice noticeSource noticeSeverity noticeAt noticeDetail) = observation bounds "host-notice" noticeSeverity noticeDetail (object ["source" .= noticeSource, "at" .= noticeAt])
    within Nothing _ = True
    within (Just (start, end)) instant = instant >= start && instant <= end

observation :: (Word64, Word64) -> Text -> Severity -> Text -> Value -> HealthObservation
observation (start, end) gate severity detail evidence = HealthObservation gate severity start end detail evidence

steadyWindow :: [CsvRow] -> [CsvRow] -> (Word64, Word64)
steadyWindow loadRows samplerRows =
  let steadyStart = listToMaybeNumber "t_mono_ns" [row | row <- loadRows, Map.lookup "phase" row == Just "steady"]
      steadyEnd = listToMaybeNumber "t_mono_ns" [row | row <- loadRows, Map.lookup "phase" row `elem` [Just "drain", Just "done"]]
      fallback = boundsOf samplerRows
   in (floor (fromMaybe (fromIntegral (fst fallback)) steadyStart), floor (fromMaybe (fromIntegral (snd fallback)) steadyEnd))

wallTimeBounds :: [CsvRow] -> Maybe (UTCTime, UTCTime)
wallTimeBounds rows = do
  start <- listToMaybeNumber "t_wall_ms" rows
  end <- lastNumber "t_wall_ms" rows
  pure (toUtc start, toUtc end)
  where
    toUtc value = posixSecondsToUTCTime (realToFrac (value / 1_000))

readCsvRows :: FilePath -> IO [CsvRow]
readCsvRows path = do
  exists <- doesFileExist path
  if not exists
    then pure []
    else do
      rows <- Text.lines <$> Text.readFile path
      pure case rows of
        [] -> []
        header : body -> fmap (Map.fromList . zip (Text.splitOn "," header) . Text.splitOn ",") body

steadyRows :: FilePath -> IO [CsvRow]
steadyRows path = filter ((== Just "steady") . Map.lookup "phase") <$> readCsvRows path

decodeValue :: FilePath -> IO Value
decodeValue path = do
  decoded <- eitherDecodeFileStrict' path
  pure (either (const Null) id decoded)

textAt :: [Text] -> Value -> Maybe Text
textAt [] (String value) = Just value
textAt (name : rest) (Object value) = KeyMap.lookup (Key.fromText name) value >>= textAt rest
textAt _ _ = Nothing

numberAt :: [Text] -> Value -> Double
numberAt [] (Number value) = realToFrac value
numberAt (name : rest) (Object value) = maybe 0 (numberAt rest) (KeyMap.lookup (Key.fromText name) value)
numberAt _ _ = 0

fieldNumber :: Text -> CsvRow -> Maybe Double
fieldNumber name row = Map.lookup name row >>= readMaybe . Text.unpack

delta :: Text -> [CsvRow] -> Double
delta name rows = fromMaybe 0 ((-) <$> (lastNumber name rows) <*> (listToMaybeNumber name rows))

listToMaybeNumber :: Text -> [CsvRow] -> Maybe Double
listToMaybeNumber _ [] = Nothing
listToMaybeNumber name (row : rest) = fieldNumber name row <|> listToMaybeNumber name rest

lastNumber :: Text -> [CsvRow] -> Maybe Double
lastNumber name = listToMaybeNumber name . reverse

boundsOf :: [CsvRow] -> (Word64, Word64)
boundsOf rows = (floor (fromMaybe 0 (listToMaybeNumber "t_mono_ns" rows)), floor (fromMaybe 0 (lastNumber "t_mono_ns" rows)))

adjacent :: [value] -> [(value, value)]
adjacent values = zip values (drop 1 values)

maximumMaybe :: (Ord value) => [value] -> Maybe value
maximumMaybe [] = Nothing
maximumMaybe values = Just (maximum values)

maximumOrZero :: (Ord value, Num value) => [value] -> value
maximumOrZero = fromMaybe 0 . maximumMaybe