packages feed

kenshou-telemetry-0.1.0.0: src/Kenshou/Telemetry/Detect.hs

module Kenshou.Telemetry.Detect
  ( FindingStatus (..),
    FindingSeverity (..),
    Finding (..),
    pipelineFindings,
    queueGrowthFinding,
    endpointFinding,
    handlerFinding,
    continuityFindings,
  )
where

import Data.Aeson (ToJSON (..), Value, object, (.=))
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Telemetry.Compose (HandlerStatsSnapshot (..))
import Kenshou.Telemetry.Continuity (ContinuityResult (..), IsolationResult (..))
import Kenshou.Telemetry.Scrape (EndpointSummary (..))
import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..))

data FindingStatus = Fired | Quiet | NotApplicable deriving stock (Eq, Show)

data FindingSeverity = Info | Degraded | Failure deriving stock (Eq, Ord, Show)

data Finding = Finding
  { detector :: Text,
    status :: FindingStatus,
    severity :: FindingSeverity,
    evidence :: Value
  }
  deriving stock (Eq, Show)

instance ToJSON FindingStatus where
  toJSON Fired = toJSON ("fired" :: Text)
  toJSON Quiet = toJSON ("quiet" :: Text)
  toJSON NotApplicable = toJSON ("not-applicable" :: Text)

instance ToJSON FindingSeverity where
  toJSON Info = toJSON ("info" :: Text)
  toJSON Degraded = toJSON ("degraded" :: Text)
  toJSON Failure = toJSON ("failure" :: Text)

instance ToJSON Finding where
  toJSON value = object ["detector" .= value.detector, "status" .= value.status, "severity" .= value.severity, "evidence" .= value.evidence]

pipelineFindings :: Double -> Maybe PipelineSnapshot -> [(Text, Text, Double, Double)] -> [Finding]
pipelineFindings _ Nothing _ =
  [ Finding "span-drop" NotApplicable Info (object []),
    Finding "export-failure" NotApplicable Info (object []),
    Finding "shutdown-blocked" NotApplicable Info (object [])
  ]
pipelineFindings degradedFraction (Just snapshot) providerCalls =
  [ Finding "span-drop" (fired snapshot.spansDropped) dropSeverity (object ["spansEnded" .= snapshot.spansEnded, "spansDropped" .= snapshot.spansDropped, "fraction" .= dropFraction]),
    Finding "export-failure" (fired snapshot.spansExportFailed) (if snapshot.spansExportFailed > 0 then Degraded else Info) (object ["spansExportFailed" .= snapshot.spansExportFailed, "lastError" .= snapshot.lastExportError]),
    Finding "shutdown-blocked" (if null blocked then Quiet else Fired) (if null blocked then Info else Degraded) (object ["calls" .= [object ["call" .= name, "result" .= result, "durationMs" .= duration, "limitMs" .= limit] | (name, result, duration, limit) <- providerCalls], "blocked" .= fmap (\(name, _, _, _) -> name) blocked])
  ]
  where
    dropFraction = fromIntegral snapshot.spansDropped / fromIntegral (max 1 snapshot.spansEnded)
    dropSeverity = if dropFraction > degradedFraction then Degraded else Info
    blocked = filter (\(_, result, duration, limit) -> duration > limit || "timeout" `Text.isInfixOf` Text.toCaseFold result) providerCalls
    fired count = if count > 0 then Fired else Quiet

queueGrowthFinding :: Maybe Int -> [Int] -> Finding
queueGrowthFinding Nothing _ = Finding "queue-growth" NotApplicable Info (object [])
queueGrowthFinding (Just queueLimit) samples =
  Finding
    "queue-growth"
    (if longest > 10 then Fired else Quiet)
    (if longest > 10 then Degraded else Info)
    (object ["queueLimit" .= queueLimit, "threshold" .= threshold, "longestConsecutiveSamples" .= longest, "samples" .= length samples])
  where
    threshold = fromIntegral queueLimit * (0.9 :: Double)
    longest = longestRun ((> threshold) . fromIntegral) samples

longestRun :: (value -> Bool) -> [value] -> Int
longestRun predicate = snd . foldl step (0, 0)
  where
    step (current, best) value =
      let next = if predicate value then current + 1 else 0
       in (next, max best next)

endpointFinding :: [EndpointSummary] -> Finding
endpointFinding [] = Finding "endpoint-failure" NotApplicable Info (object [])
endpointFinding summaries = Finding "endpoint-failure" status severity (object ["failures" .= failures, "attempted" .= attempted, "fraction" .= fraction])
  where
    attempted = sum [summary.scrapes + summary.skippedTicks | summary <- summaries]
    failures = sum [summary.failures + summary.skippedTicks | summary <- summaries]
    fraction :: Double
    fraction = fromIntegral failures / fromIntegral (max 1 attempted)
    status = if fraction > 0.01 then Fired else Quiet
    severity = if status == Fired then Degraded else Info

handlerFinding :: HandlerStatsSnapshot -> Finding
handlerFinding snapshot = Finding "handler-stall" status severity (object ["handler" .= snapshot.name, "calls" .= snapshot.calls, "durationP99Ns" .= snapshot.durationP99Ns, "limitNs" .= (5_000_000 :: Integer)])
  where
    status = if snapshot.durationP99Ns > 5_000_000 then Fired else Quiet
    severity = if status == Fired then Degraded else Info

continuityFindings :: ContinuityResult -> IsolationResult -> [Finding]
continuityFindings continuity isolation =
  [ Finding "trace-discontinuity" (if continuity.violations > 0 then Fired else Quiet) (if continuity.violations > 0 then Failure else Info) (toJSON continuity),
    Finding "context-leak" (if isolation.violations > 0 then Fired else Quiet) (if isolation.violations > 0 then Failure else Info) (toJSON isolation)
  ]