packages feed

kenshou-telemetry-0.1.0.0: test/Main.hs

module Main (main) where

import Control.Concurrent (threadDelay)
import Control.Concurrent.Async (async, cancel, waitCatch)
import Control.Concurrent.MVar (newEmptyMVar, putMVar, readMVar, tryPutMVar)
import Control.Exception (bracket)
import Control.Monad (forM_, replicateM_, void)
import Data.IORef
import Data.List (find)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (fromJust)
import Data.Text qualified as Text
import Kenshou.Core.Knob (Allowed (..), KnobSpec (..), renderKnobName)
import Kenshou.Core.Scenario (Scenario (..))
import Kenshou.Telemetry.Compose
import Kenshou.Telemetry.Continuity
import Kenshou.Telemetry.Detect (Finding (..), FindingSeverity (Degraded), FindingStatus (Fired), handlerFinding, pipelineFindings, queueGrowthFinding)
import Kenshou.Telemetry.Endpoint
import Kenshou.Telemetry.Metrics
import Kenshou.Telemetry.Overhead
import Kenshou.Telemetry.Scrape
import Kenshou.Telemetry.SelfTest.Arms (armsScenario)
import Kenshou.Telemetry.SelfTest.Service
import Kenshou.Telemetry.Sink qualified as Sink
import Kenshou.Telemetry.Spec
import Kenshou.Telemetry.Tracing
import Kenshou.Telemetry.Tracing.Pipeline
import Kenshou.Telemetry.Tracing.Probe
import Network.HTTP.Types (status200, status404)
import Network.Socket (close)
import Network.Wai (Application, responseLBS)
import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)
import Network.Wai.Handler.WebSockets (websocketsOr)
import Network.WebSockets qualified as WebSockets
import OpenTelemetry.Attributes (lookupAttribute)
import OpenTelemetry.Context.ThreadLocal (getContext)
import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))
import OpenTelemetry.Processor.Batch.Span qualified as Batch
import OpenTelemetry.Processor.Simple.Span qualified as Simple
import OpenTelemetry.Processor.Span (SpanProcessor (..))
import OpenTelemetry.Propagator (emptyTextMap, inject, textMapToList)
import OpenTelemetry.Trace.Core
import System.Directory (createDirectoryIfMissing)
import System.Mem (performMajorGC)
import System.Mem.Weak (Weak, deRefWeak, mkWeakPtr)
import System.Timeout (timeout)
import Test.Hspec

main :: IO ()
main = hspec do
  describe "overhead planning" do
    it "expands the headline one-factor matrix into four arms and twelve slots" do
      let request = overheadRequest (Map.fromList [("telemetry.tracing", ["off", "noop", "sdk-otlp"]), ("telemetry.metrics", ["off", "serve-scraped"])])
      case planOverhead request armsScenario of
        Left err -> expectationFailure (show err)
        Right plan -> do
          length plan.arms `shouldBe` 4
          length plan.slots `shouldBe` 12
          map (length . unique . fmap (.arm.armId)) (blocks plan.slots) `shouldBe` [4, 4, 4]

    it "is deterministic and gives two arms ABBA order across the first two blocks" do
      let request = overheadRequest (Map.singleton "telemetry.tracing" ["off", "noop"])
      case (planOverhead request armsScenario, planOverhead request armsScenario) of
        (Right first, Right second) -> do
          first `shouldBe` second
          let order = fmap (.arm.armId) first.slots
          take 2 order `shouldBe` reverse (take 2 (drop 2 order))
        result -> expectationFailure (show result)

    it "rejects repeated arm values" do
      planOverhead (overheadRequest (Map.singleton "telemetry.tracing" ["off", "off"])) armsScenario `shouldSatisfy` isLeft

  describe "telemetry problem detectors" do
    it "detects synchronous handler stalls while a bounded async wrapper drops instead of blocking" do
      synchronousStats <- newHandlerStats
      timedHandler synchronousStats "sync" (slowHandler 6_000) ()
      synchronous <- snapshotHandlerStats "sync" synchronousStats
      (submit, snapshotAsync) <- asyncHandler 1 (slowHandler 20_000)
      replicateM_ 100 (submit ())
      queued <- snapshotAsync
      handlerFinding synchronous `shouldSatisfy` ((== Fired) . (.status))
      queued.dropped `shouldSatisfy` (> 0)

    it "marks dropped spans as degraded above the policy fraction" do
      let snapshot = PipelineSnapshot 100 100 80 0 20 20 1 (LatencySummary 1 1 1) Nothing
      pipelineFindings 0.01 (Just snapshot) [] `shouldSatisfy` any (\finding -> finding.detector == "span-drop" && finding.status == Fired && finding.severity == Degraded)

    it "fires queue growth only after more than ten consecutive samples above ninety percent" do
      queueGrowthFinding (Just 100) (replicate 10 91) `shouldSatisfy` ((/= Fired) . (.status))
      queueGrowthFinding (Just 100) (replicate 11 91) `shouldSatisfy` ((== Fired) . (.status))

  describe "telemetryKnobs" do
    it "declares each shared knob once" do
      let names = fmap (renderKnobName . (.name)) telemetryKnobs
      length names `shouldBe` length (unique names)
    it "bounds the trace-id ratio to zero through one" do
      fmap (.allowed) (find ((== "otel.sampler-arg") . renderKnobName . (.name)) telemetryKnobs)
        `shouldBe` Just (DoubleRange 0 1)

  describe "tracing arms" do
    it "does not construct a provider under off" do
      runtime <- startTracing (testSpec TracingOff)
      isNothing runtime.tracer `shouldBe` True
      isNothing runtime.provider `shouldBe` True

    it "constructs a disabled tracer under noop" do
      runtime <- startTracing (testSpec TracingNoop)
      let tracer = fromJust runtime.tracer
      tracerIsEnabled tracer `shouldBe` False
      inSpan tracer "ignored" defaultSpanArguments (pure ())
      _ <- stopTracing 1000 runtime
      pure ()

    it "retains a bounded, parented trace and installs W3C propagation" do
      runtime <- startTracing ((testSpec TracingSdkInMemory) {probeRetain = 4})
      let tracer = fromJust runtime.tracer
          provider = fromJust runtime.provider
      replicateM_ 5 (inSpan tracer "discarded" defaultSpanArguments (pure ()))
      headers <- inSpan' tracer "parent" defaultSpanArguments \parentSpan -> do
        addAttribute parentSpan "message.id" ("one" :: Text.Text)
        nested <- inSpan' tracer "child" defaultSpanArguments \childSpan -> addAttribute childSpan "message.id" ("one" :: Text.Text)
        activeContext <- getContext
        carrier <- inject (getTracerProviderPropagators provider) activeContext emptyTextMap
        pure (nested, textMapToList carrier)
      _ <- flushTracing 1000 runtime
      let probe = fromJust runtime.probe
      views <- readSpans probe
      seen <- spansSeen probe
      seen `shouldBe` 7
      length views `shouldBe` 4
      let child = fromJust (find ((== "child") . (.name)) views)
          parent = fromJust (find ((== "parent") . (.name)) views)
      child.traceId `shouldBe` parent.traceId
      child.parentSpanId `shouldBe` Just parent.spanId
      let correlation spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing
          continuity = checkContinuity (SpanSelector (Just "parent") Nothing []) (SpanSelector (Just "child") Nothing []) correlation views
      continuity.violations `shouldBe` 0
      snd headers `shouldSatisfy` any ((== "traceparent") . fst)
      snapshot <- snapshotPipeline (fromJust runtime.pipeline)
      snapshot.spansEnded `shouldBe` 7
      snapshot.spansExportedOk `shouldBe` 7
      snapshot.spansDropped `shouldBe` 0
      _ <- stopTracing 1000 runtime
      pure ()

    it "releases evicted spans before anyone reads the bounded probe" do
      forM_ [0, 1] \capacity -> do
        (probe, provider, weak) <- probeWithEvictedSpan capacity
        performMajorGC
        (isNothing <$> deRefWeak weak) `shouldReturn` True
        spansSeen probe `shouldReturn` 10
        (length <$> readSpans probe) `shouldReturn` capacity
        _ <- shutdownTracerProvider provider (Just 1000000)
        pure ()

    it "exports plain and gzip OTLP requests to the built-in sink" do
      forCompression CompressionNone
      forCompression CompressionGzip

    it "accounts for alternating exporter failures exactly" do
      stats <- newPipelineStats
      calls <- newIORef (0 :: Int)
      let exporter =
            SpanExporter
              { spanExporterExport = \_ -> do
                  call <- atomicModifyIORef' calls (\value -> let next = value + 1 in (next, next))
                  pure (if even call then Failure Nothing else Success),
                spanExporterShutdown = pure ShutdownSuccess,
                spanExporterForceFlush = pure FlushSuccess
              }
      processor <- Simple.simpleProcessor (Simple.SimpleProcessorConfig (instrumentExporter stats exporter) 1_000_000)
      provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions
      let tracer = makeTracer provider (instrumentationLibrary "pipeline-test" "1") tracerOptions
      replicateM_ 10 (inSpan tracer "alternating" defaultSpanArguments (pure ()))
      _ <- shutdownTracerProvider provider (Just 1_000_000)
      snapshot <- snapshotPipeline stats
      snapshot.spansEnded `shouldBe` snapshot.spansExportedOk + snapshot.spansExportFailed + snapshot.spansDropped
      snapshot.spansExportedOk `shouldBe` 5
      snapshot.spansExportFailed `shouldBe` 5

    it "drops instead of blocking when a batch queue is full" do
      stats <- newPipelineStats
      releaseExporter <- newEmptyMVar
      let exporter =
            SpanExporter
              { spanExporterExport = \_ -> readMVar releaseExporter >> pure Success,
                spanExporterShutdown = pure ShutdownSuccess,
                spanExporterForceFlush = pure FlushSuccess
              }
      processor <-
        Batch.batchProcessor
          Batch.batchTimeoutConfig
            { Batch.maxQueueSize = 8,
              Batch.scheduledDelayMillis = 10,
              Batch.maxExportBatchSize = 4,
              Batch.exportTimeoutMillis = 5_000
            }
          (instrumentExporter stats exporter)
      provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions
      let tracer = makeTracer provider (instrumentationLibrary "backpressure-test" "1") tracerOptions
      completed <- timeout 1_000_000 (replicateM_ 10_000 (inSpan tracer "queued" defaultSpanArguments (pure ())))
      completed `shouldBe` Just ()
      beforeRelease <- snapshotPipeline stats
      beforeRelease.spansDropped `shouldSatisfy` (> 0)
      putMVar releaseExporter ()
      void (shutdownTracerProvider provider (Just 1_000_000))

  describe "metrics arms" do
    it "constructs no provider under off" do
      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsOff})
      isNothing runtime.meter `shouldBe` True
      isNothing runtime.provider `shouldBe` True
      isNothing runtime.endpoint `shouldBe` True

    it "collects instruments without opening an endpoint" do
      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsCollect})
      let meter = fromJust runtime.meter
      metrics <- newSyntheticMetrics meter
      void (runSyntheticOperation Nothing (Just metrics) 0 0 1)
      (snapshot, _) <- stopMetrics 1000 runtime
      runtime.endpoint `shouldBe` Nothing
      fmap (.instruments) snapshot `shouldBe` Just 2

    it "serves its Prometheus reader only for a serving arm" do
      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsServe, otelReader = ReaderPrometheus})
      runtime.endpoint `shouldSatisfy` maybe False ((== PrometheusText) . (.kind))
      traverse_ (\endpoint -> awaitHttpReady endpoint.url 1000 >>= (`shouldBe` True)) runtime.endpoint
      void (stopMetrics 1000 runtime)

  describe "metrics scraper" do
    it "skips fixed-schedule ticks when an endpoint is slower than its interval" do
      withSlowHttpServer 75_000 \endpoint -> do
        let output = ".tmp/telemetry-test/slow-scrape"
        createDirectoryIfMissing True output
        scraper <- startScraperInProcess output 20 0
        scraper.register endpoint
        threadDelay 260_000
        summaries <- scraper.finish
        fmap (.skippedTicks) summaries `shouldSatisfy` any (> 0)

    it "detects exhausted WebSocket slots and accepts a correct server" do
      withWebSocketServer (Just 2) \leaky -> do
        result <- wsSlotLeakProbe leaky 2
        result.freshAccepted `shouldBe` False
      withWebSocketServer Nothing \correct -> do
        result <- wsSlotLeakProbe correct 2
        result.freshAccepted `shouldBe` True

testSpec :: TracingArm -> TelemetrySpec
testSpec tracing =
  TelemetrySpec
    { tracing,
      metrics = MetricsOff,
      serviceName = "kenshou.test",
      sampler = AlwaysOn,
      processor = SimpleProcessor 1000,
      exporter = HttpProtobuf,
      endpoint = BuiltinSink SinkHealthy,
      compression = CompressionNone,
      probeRetain = 16,
      shutdownMs = 1000,
      otelReader = ReaderNone,
      otelExportMs = 1000,
      scrapeMs = 1000,
      wsSubscribers = 0,
      helpers = HelperInProcess,
      workerContext = Nothing,
      outDir = ".tmp/telemetry-test",
      report = const (pure ())
    }

forCompression :: OtlpCompression -> IO ()
forCompression compression = Sink.withSink SinkHealthy \sink -> do
  let spec = (testSpec TracingSdkOtlp) {endpoint = ExternalEndpoint sink.endpoint, compression}
  runtime <- startTracing spec
  let tracer = fromJust runtime.tracer
  inSpan tracer "exported" defaultSpanArguments (pure ())
  _ <- flushTracing 2000 runtime
  _ <- stopTracing 2000 runtime
  stats <- sink.snapshot
  stats.spansReceived `shouldBe` 1

isNothing :: Maybe value -> Bool
isNothing Nothing = True
isNothing (Just _) = False

unique :: (Eq value) => [value] -> [value]
unique [] = []
unique (value : values) = value : unique (filter (/= value) values)

withSlowHttpServer :: Int -> (Endpoint -> IO value) -> IO value
withSlowHttpServer delayMicros action = withWarp application \port ->
  action (Endpoint "slow-json" JsonDocument (Text.pack ("http://127.0.0.1:" <> show port <> "/metrics")) Nothing)
  where
    application _ respond = threadDelay delayMicros >> respond (responseLBS status200 [] "delayed")

withWebSocketServer :: Maybe Int -> (Endpoint -> IO value) -> IO value
withWebSocketServer capacity action = do
  accepted <- newIORef (0 :: Int)
  withWarp (websocketsOr WebSockets.defaultConnectionOptions (server accepted) fallback) \port ->
    action (Endpoint "slot-probe" WebSocketPush (Text.pack ("ws://127.0.0.1:" <> show port <> "/ws")) Nothing)
  where
    server accepted pendingConnection = do
      count <- atomicModifyIORef' accepted (\value -> let next = value + 1 in (next, next))
      case capacity of
        Just limit | count > limit -> WebSockets.rejectRequest pendingConnection "slots exhausted"
        _ -> void (WebSockets.acceptRequest pendingConnection)
    fallback _ respond = respond (responseLBS status404 [] "not found")

withWarp :: Application -> (Int -> IO value) -> IO value
withWarp application action = bracket acquire release (\(port, _, _) -> action port)
  where
    acquire = do
      (port, socket) <- openFreePort
      server <- async (runSettingsSocket defaultSettings socket application)
      pure (port, server, socket)
    release (_, server, socket) = cancel server >> void (waitCatch server) >> close socket

traverse_ :: (a -> IO b) -> Maybe a -> IO ()
traverse_ _ Nothing = pure ()
traverse_ action (Just value) = void (action value)

overheadRequest :: Map Text.Text [Text.Text] -> OverheadRequest
overheadRequest factors =
  OverheadRequest
    { requestId = "test-overhead",
      scenarioId = armsScenario.id,
      factors,
      mode = OneFactor,
      includeControl = False,
      trials = 3,
      fixedKnobs = [],
      fixedDimensions = Map.empty,
      seed = 7,
      settleSeconds = 0,
      retries = 1
    }

blocks :: [Slot] -> [[Slot]]
blocks slots = [[slot | slot <- slots, slot.block == block] | block <- unique (fmap (.block) slots)]

isLeft :: Either left right -> Bool
isLeft (Left _) = True
isLeft (Right _) = False

-- Keeping the provider and probe alive must not keep a span outside the
-- retention window alive. Do not force readSpans/spansSeen before the GC.
{-# NOINLINE probeWithEvictedSpan #-}
probeWithEvictedSpan :: Int -> IO (SpanProbe, TracerProvider, Weak SpanHot)
probeWithEvictedSpan capacity = do
  (probe, processor) <- newSpanProbe capacity
  first <- newEmptyMVar
  let observing =
        processor
          { spanProcessorOnEnd = \spanValue -> do
              hot <- readIORef spanValue.spanHot
              weak <- mkWeakPtr hot Nothing
              void (tryPutMVar first weak)
              processor.spanProcessorOnEnd spanValue
          }
  provider <- createTracerProvider [observing] emptyTracerProviderOptions
  let tracer = makeTracer provider (instrumentationLibrary "probe-retention-test" "0.1.0.0") tracerOptions
  replicateM_ 10 (inSpan tracer "evicted" defaultSpanArguments (pure ()))
  weak <- readMVar first
  pure (probe, provider, weak)