kenshou-telemetry-0.1.0.0: src/Kenshou/Telemetry/SelfTest/Problems.hs
module Kenshou.Telemetry.SelfTest.Problems
( traceContinuityScenario,
slowExporterScenario,
)
where
import Control.Concurrent (threadDelay)
import Control.Monad (forM_, replicateM_, void)
import Data.Aeson (object, (.=))
import Data.Int (Int64)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Core.Context (RunContext (..), SummarySection (Verdicts), putSummary)
import Kenshou.Core.Dimension
import Kenshou.Core.Dimension qualified as Dimension
import Kenshou.Core.Env (noEnvironment)
import Kenshou.Core.Id (ScenarioId, parseScenarioId)
import Kenshou.Core.Knob
import Kenshou.Core.Phase (PhasePlan (..))
import Kenshou.Core.Scenario
import Kenshou.Telemetry
import Kenshou.Telemetry.Compose
import Kenshou.Telemetry.Continuity
import Kenshou.Telemetry.Detect
import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..), snapshotPipeline)
import Kenshou.Telemetry.Tracing.Probe (SpanView (..), readSpans)
import OpenTelemetry.Attributes (Attribute (AttributeValue), PrimitiveAttribute (TextAttribute), lookupAttribute)
import OpenTelemetry.Trace.Core (Span, addAttribute, defaultSpanArguments, inSpan, inSpan')
import System.Timeout (timeout)
traceContinuityScenario :: Scenario
traceContinuityScenario =
Scenario
{ id = scenarioId "selftest/telemetry/correctness/trace-continuity",
revision = 1,
summary = "Proves trace discontinuity and context-leak checks are non-vacuous.",
tier = TierSmoke,
placement = PlaceEither,
knobs = [intKnob "continuity.messages" "Synthetic messages in each continuity case" 1000 10 100000] <> telemetryKnobs,
dimensions = onlyArms TracingSdkInMemory MetricsOff,
phases = PhasePlan 0 0 0,
requires = noEnvironment,
knownDefect = Nothing,
run = runTraceContinuity
}
slowExporterScenario :: Scenario
slowExporterScenario =
Scenario
{ id = scenarioId "selftest/telemetry/concurrency/slow-exporter-backpressure",
revision = 1,
summary = "Proves slow exporters and handlers are detected without blocking the emitter.",
tier = TierSmoke,
placement = PlaceEither,
knobs =
[ intKnob "backpressure.phase-seconds" "Maximum synthetic phase duration" 8 2 600,
intKnob "handler.delay-micros" "Injected handler delay" 5000 0 1000000
]
<> telemetryKnobs,
dimensions = onlyArms TracingSdkOtlp MetricsOff,
phases = PhasePlan 0 0 0,
requires = noEnvironment,
knownDefect = Nothing,
run = runSlowExporter
}
runTraceContinuity :: RunContext -> IO ScenarioReport
runTraceContinuity context = case telemetrySpecFromContext context of
Left message -> pure (failedWith ["invalid-telemetry-config"] message)
Right spec -> withTelemetry spec \telemetry -> case telemetry.tracer of
Nothing -> pure (failedWith ["missing-tracer"] "sdk-inmemory did not provide a tracer")
Just tracer -> do
let messages = fromIntegral (knobInt context.knobs (name "continuity.messages"))
forM_ [1 .. messages] \index ->
inSpan' tracer "intact.producer" defaultSpanArguments \producer -> do
addMessageId producer ("intact-" <> tshow index)
inSpan' tracer "intact.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("intact-" <> tshow index)
forM_ [10, 20 .. messages] \index -> do
inSpan' tracer "broken.producer" defaultSpanArguments \producer -> addMessageId producer ("broken-" <> tshow index)
inSpan' tracer "broken.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("broken-" <> tshow index)
inSpan tracer "leak.carrier" defaultSpanArguments do
forM_ [10, 20 .. messages] \index ->
inSpan' tracer "leaky.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("leaky-" <> tshow index)
void telemetry.flushTelemetry
spans <- maybe (pure []) readSpans telemetry.spans
let correlate spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing
intact = checkContinuity (named "intact.producer") (named "intact.consumer") correlate spans
broken = checkContinuity (named "broken.producer") (named "broken.consumer") correlate spans
isolated = checkIsolation (named "leaky.consumer") (const True) spans
expected = messages `div` 10
ok = intact.violations == 0 && intact.consumers == messages && broken.violations == expected && isolated.violations == expected
telemetry.recordContinuity intact (IsolationResult 0 0 [])
telemetry.recordContinuity broken isolated
putSummary context Verdicts "trace-continuity" (object ["intact" .= intact, "broken" .= broken, "leaky" .= isolated, "expectedInjected" .= expected])
pure (if ok then passed else failedWith ["trace-detector-non-vacuity"] "continuity detectors did not identify exactly the injected defects")
runSlowExporter :: RunContext -> IO ScenarioReport
runSlowExporter context = case telemetrySpecFromContext context of
Left message -> pure (failedWith ["invalid-telemetry-config"] message)
Right spec -> withTelemetry (spec {processor = BatchProcessor 256 200 128 2000, shutdownMs = 1000}) \telemetry -> case (telemetry.tracer, telemetry.pipeline) of
(Just tracer, Just pipeline) -> do
let delayMicros = fromIntegral (knobInt context.knobs (name "handler.delay-micros"))
phaseMicros = fromIntegral (knobInt context.knobs (name "backpressure.phase-seconds")) * 1_000_000
synchronousStats <- newHandlerStats
replicateM_ 20 (timedHandler synchronousStats "synchronous" (slowHandler delayMicros) ())
synchronous <- snapshotHandlerStats "synchronous" synchronousStats
asyncStats <- newHandlerStats
(submit, asyncSnapshot) <- asyncHandler 64 (slowHandler delayMicros)
completed <- timeout phaseMicros (replicateM_ 10000 (timedHandler asyncStats "async-submit" submit ()))
threadDelay (min 250_000 phaseMicros)
asynchronous <- snapshotHandlerStats "async-submit" asyncStats
asyncQueue <- asyncSnapshot
telemetry.recordHandlerStats synchronous
telemetry.recordHandlerStats asynchronous
telemetry.setSinkFault SinkHang
emitted <- timeout phaseMicros (replicateM_ 10000 (inSpan tracer "backpressure" defaultSpanArguments (pure ())))
snapshot <- snapshotPipeline pipeline
let syncFired = (handlerFinding synchronous).status == Fired
asyncQuiet = (handlerFinding asynchronous).status == Quiet
ok = completed /= Nothing && emitted /= Nothing && syncFired && asyncQuiet && asyncQueue.dropped > 0 && snapshot.spansDropped > 0
putSummary context Verdicts "slow-exporter-backpressure" (object ["synchronous" .= synchronous, "asynchronous" .= asynchronous, "asyncQueue" .= asyncQueue, "pipeline" .= snapshot, "emitterCompleted" .= (emitted /= Nothing)])
pure (if ok then passed else failedWith ["backpressure-detector-non-vacuity"] "slow exporter or handler detector did not fire as expected")
_ -> pure (failedWith ["missing-sdk"] "sdk-otlp did not provide a tracer and pipeline")
onlyArms :: TracingArm -> MetricsArm -> DimensionSupport
onlyArms tracing metrics =
DimensionSupport (Supported (Support (tracing :| []) tracing)) (Supported (Support (metrics :| []) metrics)) Dimension.NotApplicable Dimension.NotApplicable
named :: Text -> SpanSelector
named spanName = SpanSelector (Just spanName) Nothing []
addMessageId :: Span -> Text -> IO ()
addMessageId spanValue value = addAttribute spanValue "message.id" value
scenarioId :: Text -> ScenarioId
scenarioId text = either (error . Text.unpack) id (parseScenarioId text)
intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec
intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []
name :: Text -> KnobName
name = either (error . Text.unpack) id . mkKnobName
tshow :: (Show value) => value -> Text
tshow = Text.pack . show