diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,11 @@
+# Changelog
+
+## 0.1.0.0 - 2026-10-28
+
+### New Features
+
+- First public release of tracing and metrics arms, endpoint scraping, and observability-overhead reports.
+
+### Other Changes
+
+- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
diff --git a/LICENSE b/LICENSE
new file mode 100644
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Nadeem Bitar
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/kenshou-telemetry.cabal b/kenshou-telemetry.cabal
new file mode 100644
--- /dev/null
+++ b/kenshou-telemetry.cabal
@@ -0,0 +1,135 @@
+cabal-version: 3.0
+name: kenshou-telemetry
+version: 0.1.0.0
+synopsis: Telemetry arms and overhead analysis for Kenshou scenarios
+description: Explicit tracing and metrics providers, endpoint scrapers, pipeline diagnostics, and paired observability-overhead reports.
+license: BSD-3-Clause
+license-file: LICENSE
+author: Nadeem Bitar
+maintainer: nadeem@gmail.com
+homepage: https://github.com/shinzui/kenshou
+bug-reports: https://github.com/shinzui/kenshou/issues
+category: Testing
+build-type: Simple
+extra-doc-files: CHANGELOG.md
+tested-with: ghc >=9.12 && <9.13
+extra-source-files:
+  test/golden/*.json
+
+flag otlp-grpc
+  description: Enable the optional OTLP gRPC exporter
+  default: False
+  manual: True
+
+common warnings
+  ghc-options:
+    -Wall
+    -Wcompat
+    -Widentities
+    -Wincomplete-record-updates
+    -Wincomplete-uni-patterns
+    -Wpartial-fields
+    -Wredundant-constraints
+
+  default-language: GHC2024
+  default-extensions:
+    BlockArguments
+    CPP
+    DuplicateRecordFields
+    ImportQualifiedPost
+    NoFieldSelectors
+    OverloadedLabels
+    OverloadedRecordDot
+    OverloadedStrings
+
+library
+  import: warnings
+  hs-source-dirs: src
+  exposed-modules:
+    Kenshou.Telemetry
+    Kenshou.Telemetry.Compose
+    Kenshou.Telemetry.Continuity
+    Kenshou.Telemetry.Detect
+    Kenshou.Telemetry.Endpoint
+    Kenshou.Telemetry.Metrics
+    Kenshou.Telemetry.Overhead
+    Kenshou.Telemetry.Overhead.Policy
+    Kenshou.Telemetry.Scrape
+    Kenshou.Telemetry.SelfTest
+    Kenshou.Telemetry.SelfTest.Arms
+    Kenshou.Telemetry.SelfTest.Endpoints
+    Kenshou.Telemetry.SelfTest.Problems
+    Kenshou.Telemetry.SelfTest.Service
+    Kenshou.Telemetry.Sink
+    Kenshou.Telemetry.Spec
+    Kenshou.Telemetry.Tracing
+    Kenshou.Telemetry.Tracing.Pipeline
+    Kenshou.Telemetry.Tracing.Probe
+
+  build-depends:
+    aeson >=2.2 && <2.3,
+    async >=2.2 && <2.3,
+    base >=4.21 && <4.22,
+    bytestring >=0.12 && <0.13,
+    containers >=0.7 && <0.8,
+    directory >=1.3 && <1.4,
+    filepath >=1.5 && <1.6,
+    hs-opentelemetry-api ==1.0.0.0,
+    hs-opentelemetry-exporter-otlp ==1.0.0.0,
+    hs-opentelemetry-exporter-prometheus ==1.0.0.0,
+    hs-opentelemetry-otlp ==1.0.0.0,
+    hs-opentelemetry-propagator-w3c ==1.0.0.0,
+    hs-opentelemetry-sdk ==1.0.0.0,
+    http-client >=0.7 && <0.8,
+    http-types >=0.12 && <0.13,
+    kenshou-core ^>=0.1.0.0,
+    kenshou-measure ^>=0.1.0.0,
+    microlens >=0.4 && <0.5,
+    network >=3.2 && <3.3,
+    proto-lens >=0.7 && <0.8,
+    proto-lens-runtime >=0.7 && <0.8,
+    splitmix >=0.1 && <0.2,
+    stm >=2.5 && <2.6,
+    text >=2.1 && <2.2,
+    time >=1.14 && <1.15,
+    unordered-containers >=0.2 && <0.3,
+    vector >=0.13 && <0.14,
+    wai >=3.2 && <3.3,
+    wai-websockets >=3.0 && <3.1,
+    warp >=3.4 && <3.5,
+    websockets >=0.13 && <0.14,
+    zlib >=0.7 && <0.8,
+
+  if flag(otlp-grpc)
+    cpp-options: -DWITH_OTLP_GRPC
+
+test-suite kenshou-telemetry-test
+  import: warnings
+  type: exitcode-stdio-1.0
+  hs-source-dirs: test
+  main-is: Main.hs
+  ghc-options:
+    -threaded
+    -rtsopts
+
+  build-depends:
+    async,
+    base >=4.21 && <4.22,
+    containers,
+    directory >=1.3 && <1.4,
+    hs-opentelemetry-api,
+    hs-opentelemetry-sdk,
+    hspec >=2.11 && <2.12,
+    http-types >=0.12 && <0.13,
+    kenshou-core ^>=0.1.0.0,
+    kenshou-telemetry,
+    network >=3.2 && <3.3,
+    text,
+    wai >=3.2 && <3.3,
+    wai-websockets >=3.0 && <3.1,
+    warp >=3.4 && <3.5,
+    websockets >=0.13 && <0.14,
+
+source-repository head
+  type: git
+  location: https://github.com/shinzui/kenshou.git
diff --git a/src/Kenshou/Telemetry.hs b/src/Kenshou/Telemetry.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry.hs
@@ -0,0 +1,287 @@
+module Kenshou.Telemetry
+  ( TelemetryHandles (..),
+    FlushReport (..),
+    ProviderCallReport (..),
+    withTelemetry,
+    module Kenshou.Telemetry.Spec,
+  )
+where
+
+import Control.Exception (SomeException, displayException, mask, throwIO, try)
+import Control.Monad (void)
+import Data.Aeson (FromJSON (parseJSON), ToJSON (..), Value (..), object, (.=))
+import Data.Aeson.Key qualified as Key
+import Data.Aeson.KeyMap qualified as KeyMap
+import Data.Aeson.Types (parseMaybe, withObject, (.:))
+import Data.IORef
+import Data.Maybe (catMaybes)
+import Data.Text (Text)
+import Data.Text qualified as Text
+import GHC.Clock (getMonotonicTimeNSec)
+import Kenshou.Core.Dimension (renderMetrics, renderTracing)
+import Kenshou.Core.Role (ControlMessage (..), WorkerMessage (..), mkRoleName)
+import Kenshou.Core.Role.Spawn (WorkerHandle (..), withWorker)
+import Kenshou.Telemetry.Compose (HandlerStatsSnapshot)
+import Kenshou.Telemetry.Continuity (ContinuityResult, IsolationResult)
+import Kenshou.Telemetry.Detect
+import Kenshou.Telemetry.Endpoint
+import Kenshou.Telemetry.Metrics
+import Kenshou.Telemetry.Metrics qualified as Metrics
+import Kenshou.Telemetry.Scrape
+import Kenshou.Telemetry.Sink
+import Kenshou.Telemetry.Spec
+import Kenshou.Telemetry.Spec qualified as Spec
+import Kenshou.Telemetry.Tracing
+import Kenshou.Telemetry.Tracing.Pipeline
+import Kenshou.Telemetry.Tracing.Probe
+import OpenTelemetry.Metric.Core (Meter, MeterProvider)
+import OpenTelemetry.Trace.Core (Tracer, TracerProvider)
+import System.Environment (getEnvironment)
+
+data ProviderCallReport = ProviderCallReport
+  { result :: Text,
+    durationMs :: Double
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON ProviderCallReport where
+  toJSON report = object ["result" .= report.result, "durationMs" .= report.durationMs]
+
+data FlushReport = FlushReport
+  { tracing :: Maybe ProviderCallReport,
+    metrics :: Maybe ProviderCallReport
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON FlushReport where
+  toJSON report = object ["tracing" .= report.tracing, "metrics" .= report.metrics]
+
+data TelemetryHandles = TelemetryHandles
+  { tracer :: Maybe Tracer,
+    tracerProvider :: Maybe TracerProvider,
+    meter :: Maybe Meter,
+    meterProvider :: Maybe MeterProvider,
+    spans :: Maybe SpanProbe,
+    pipeline :: Maybe PipelineStats,
+    metricsLive :: Bool,
+    readMetricSums :: IO [(Text, Double)],
+    readMetricGauges :: IO [(Text, Double)],
+    servesEndpoints :: Bool,
+    metricEndpoint :: Maybe Endpoint,
+    registerEndpoint :: Endpoint -> IO (),
+    setSinkFault :: SinkFault -> IO (),
+    recordContinuity :: ContinuityResult -> IsolationResult -> IO (),
+    recordHandlerStats :: HandlerStatsSnapshot -> IO (),
+    flushTelemetry :: IO FlushReport
+  }
+
+withTelemetry :: TelemetrySpec -> (TelemetryHandles -> IO value) -> IO value
+withTelemetry spec action = withConfiguredScraper spec \scraper -> case spec.endpoint of
+  BuiltinSink fault | requiresSink spec -> withConfiguredSink spec fault (\sink -> runWith scraper (spec {Spec.endpoint = ExternalEndpoint sink.endpoint}) (Just sink))
+  _ -> runWith scraper spec Nothing
+  where
+    runWith scraper effectiveSpec sink = mask \restore -> do
+      runtime <- startTracing effectiveSpec
+      metricsRuntime <- startMetrics effectiveSpec
+      stopPipelineSampler <- maybe (pure (pure [])) (startPipelineSampler spec.outDir) runtime.pipeline
+      endpointsRef <- newIORef (maybe [] pure metricsRuntime.endpoint)
+      recordedFindingsRef <- newIORef []
+      handlerStatsRef <- newIORef []
+      traverse_ (\endpoint -> traverse_ (\active -> active.register endpoint) scraper) metricsRuntime.endpoint
+      let register endpoint
+            | spec.metrics `notElem` [MetricsServe, MetricsServeScraped] = ioError (userError "metrics endpoint registered while telemetry.metrics does not serve endpoints")
+            | otherwise = do
+                ready <- if endpoint.kind == WebSocketPush then pure True else awaitHttpReady endpoint.url 10_000
+                if ready
+                  then do
+                    modifyIORef' endpointsRef (<> [endpoint])
+                    traverse_ (\active -> active.register endpoint) scraper
+                  else ioError (userError ("metrics endpoint did not become ready: " <> Text.unpack endpoint.url))
+      let handles =
+            TelemetryHandles
+              { tracer = runtime.tracer,
+                tracerProvider = runtime.provider,
+                meter = metricsRuntime.meter,
+                meterProvider = metricsRuntime.provider,
+                spans = runtime.probe,
+                pipeline = runtime.pipeline,
+                metricsLive = spec.metrics /= MetricsOff,
+                readMetricSums = Metrics.readMetricSums metricsRuntime,
+                readMetricGauges = Metrics.readMetricGauges metricsRuntime,
+                servesEndpoints = spec.metrics `elem` [MetricsServe, MetricsServeScraped],
+                metricEndpoint = metricsRuntime.endpoint,
+                registerEndpoint = register,
+                setSinkFault = maybe (const (pure ())) (.setFault) sink,
+                recordContinuity = \continuity isolation -> modifyIORef' recordedFindingsRef (<> continuityFindings continuity isolation),
+                recordHandlerStats = \snapshot -> modifyIORef' handlerStatsRef (<> [snapshot]),
+                flushTelemetry = FlushReport <$> timedCall (flushTracing spec.shutdownMs runtime) <*> timedCall (flushMetrics spec.shutdownMs metricsRuntime)
+              }
+      bodyResult <- tryAny (restore (action handles))
+      flushResult <- timedCall (flushTracing spec.shutdownMs runtime)
+      metricsFlushResult <- timedCall (flushMetrics spec.shutdownMs metricsRuntime)
+      endpointSummaries <- maybe (pure Nothing) (fmap Just . (.finish)) scraper
+      (metricsSnapshot, metricsShutdownResult) <- timedStopMetrics spec.shutdownMs metricsRuntime
+      shutdownResult <- timedCall (stopTracing spec.shutdownMs runtime)
+      queueSamples <- stopPipelineSampler
+      rawPipelineSnapshot <- traverse snapshotPipeline runtime.pipeline
+      let pipelineSnapshot = fmap (normalizeQueueDepth spec) rawPipelineSnapshot
+      sinkSnapshot <- traverse (.snapshot) sink
+      ambient <- ambientOtelEnvironment
+      endpoints <- readIORef endpointsRef
+      recordedFindings <- readIORef recordedFindingsRef
+      handlerStats <- readIORef handlerStatsRef
+      let endpointValues = maybe (fmap toJSON endpoints) (fmap toJSON) endpointSummaries
+          calls = catMaybes [providerCall "trace-flush" spec.shutdownMs flushResult, providerCall "metrics-flush" spec.shutdownMs metricsFlushResult, providerCall "trace-shutdown" spec.shutdownMs shutdownResult, providerCall "metrics-shutdown" spec.shutdownMs metricsShutdownResult]
+          findings = pipelineFindings 0.01 pipelineSnapshot calls <> [queueGrowthFinding (pipelineQueueLimit spec runtime.pipeline) queueSamples, endpointFinding (maybe [] id endpointSummaries)] <> fmap handlerFinding handlerStats <> recordedFindings
+          summary = telemetrySummary spec ambient pipelineSnapshot sinkSnapshot metricsSnapshot endpointValues (fmap toJSON handlerStats) findings flushResult metricsFlushResult shutdownResult metricsShutdownResult
+      _ <- tryAny (spec.report summary)
+      either throwIO pure bodyResult
+
+requiresSink :: TelemetrySpec -> Bool
+requiresSink spec = spec.tracing == TracingSdkOtlp || (spec.metrics /= MetricsOff && spec.otelReader == ReaderOtlpPeriodic)
+
+withConfiguredSink :: TelemetrySpec -> SinkFault -> (SinkHandle -> IO value) -> IO value
+withConfiguredSink spec fault action = case (spec.helpers, spec.workerContext) of
+  (HelperProcess _, Just context) -> do
+    roleName <- either (ioError . userError . Text.unpack) pure (mkRoleName "selftest/telemetry-otlp-sink")
+    withWorker context roleName "telemetry-otlp-sink" (object ["fault" .= renderSinkFault fault]) \worker -> do
+      ready <- worker.receive 10_000
+      endpoint <- case ready of
+        Just (WrkCustom "ready" payload) -> maybe (ioError (userError "telemetry sink sent an invalid ready message")) pure (parseMaybe (withObject "sink ready" (.: "endpoint")) payload)
+        Just (WrkError message) -> ioError (userError (Text.unpack message))
+        _ -> ioError (userError "telemetry sink did not become ready within 10 seconds")
+      let setFault updated = worker.send (CtlCustom "fault" (String (renderSinkFault updated)))
+          snapshot = do
+            worker.send (CtlCustom "snapshot" Null)
+            response <- worker.receive 10_000
+            case response of
+              Just (WrkCustom "sink-stats" payload) -> maybe (ioError (userError "telemetry sink sent invalid statistics")) pure (parseMaybe parseJSON payload)
+              _ -> ioError (userError "telemetry sink did not return statistics")
+      action (SinkHandle endpoint setFault snapshot)
+  _ -> withSink fault action
+
+withConfiguredScraper :: TelemetrySpec -> (Maybe ScraperHandle -> IO value) -> IO value
+withConfiguredScraper spec action
+  | spec.metrics /= MetricsServeScraped = action Nothing
+  | otherwise = case (spec.helpers, spec.workerContext) of
+      (HelperProcess _, Just context) -> do
+        roleName <- either (ioError . userError . Text.unpack) pure (mkRoleName "selftest/telemetry-scraper")
+        withWorker context roleName "telemetry-scraper" (object ["intervalMs" .= spec.scrapeMs, "wsSubscribers" .= spec.wsSubscribers]) \worker -> do
+          ready <- worker.receive 10_000
+          case ready of
+            Just WrkReady -> pure ()
+            Just (WrkError message) -> ioError (userError (Text.unpack message))
+            _ -> ioError (userError "telemetry scraper did not become ready within 10 seconds")
+          let register endpoint = worker.send (CtlCustom "endpoint" (toJSON endpoint))
+              finish = do
+                worker.send (CtlCustom "finish" Null)
+                response <- worker.receive 10_000
+                case response of
+                  Just (WrkCustom "scrape-summary" payload) -> maybe (ioError (userError "telemetry scraper sent an invalid summary")) pure (parseMaybe parseJSON payload)
+                  _ -> ioError (userError "telemetry scraper did not return its summary")
+          action (Just (ScraperHandle register finish))
+      _ -> do
+        scraper <- startScraperInProcess spec.outDir spec.scrapeMs spec.wsSubscribers
+        action (Just scraper)
+
+telemetrySummary :: TelemetrySpec -> [(Text, Text)] -> Maybe PipelineSnapshot -> Maybe SinkStats -> Maybe MetricsSnapshot -> [Value] -> [Value] -> [Finding] -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Value
+telemetrySummary spec ambient pipelineSnapshot sinkSnapshot metricsSnapshot endpoints handlers findings flushResult metricsFlushResult shutdownResult metricsShutdownResult =
+  object
+    [ "schema" .= ("kenshou.telemetry-summary/v1" :: Text),
+      "arms" .= object ["tracing" .= renderTracing spec.tracing, "metrics" .= renderMetrics spec.metrics],
+      "settings" .= settingsValue spec,
+      "ambientEnv" .= object [Key.fromText key .= value | (key, value) <- ambient],
+      "pipeline" .= pipelineValue pipelineSnapshot flushResult shutdownResult,
+      "sink" .= sinkSnapshot,
+      "metrics" .= object ["snapshot" .= metricsSnapshot, "flush" .= metricsFlushResult, "shutdown" .= metricsShutdownResult],
+      "endpoints" .= endpoints,
+      "handlers" .= handlers,
+      "findings" .= findings
+    ]
+
+providerCall :: Text -> Int -> Maybe ProviderCallReport -> Maybe (Text, Text, Double, Double)
+providerCall _ _ Nothing = Nothing
+providerCall name limit (Just report) = Just (name, report.result, report.durationMs, fromIntegral limit)
+
+normalizeQueueDepth :: TelemetrySpec -> PipelineSnapshot -> PipelineSnapshot
+normalizeQueueDepth spec snapshot = case spec.processor of
+  BatchProcessor queue _ batch _ ->
+    -- The public SDK does not expose its private queue count. The accounting
+    -- backlog includes SDK-dropped spans, so cap its high-water mark at the
+    -- configured queue plus the one batch that may already be exporting.
+    PipelineSnapshot snapshot.spansStarted snapshot.spansEnded snapshot.spansExportedOk snapshot.spansExportFailed snapshot.spansDropped (min snapshot.maxQueueDepth (queue + batch)) snapshot.exportCalls snapshot.exportLatencyNs snapshot.lastExportError
+  SimpleProcessor _ -> snapshot
+
+pipelineQueueLimit :: TelemetrySpec -> Maybe PipelineStats -> Maybe Int
+pipelineQueueLimit spec pipeline = case (pipeline, spec.processor) of
+  (Just _, BatchProcessor queue _ _ _) -> Just queue
+  _ -> Nothing
+
+pipelineValue :: Maybe PipelineSnapshot -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Value
+pipelineValue Nothing _ _ = Null
+pipelineValue (Just snapshot) flushResult shutdownResult = case toJSON snapshot of
+  Object fields -> Object (KeyMap.insert "shutdown" (toJSON shutdownResult) (KeyMap.insert "flush" (toJSON flushResult) fields))
+  value -> value
+
+settingsValue :: TelemetrySpec -> Value
+settingsValue spec =
+  object
+    [ "sampler" .= renderSampler spec.sampler,
+      "processor" .= renderProcessor spec.processor,
+      "exporter" .= (case spec.exporter of HttpProtobuf -> "http-protobuf" :: Text; Grpc -> "grpc"),
+      "endpoint" .= (case spec.endpoint of BuiltinSink _ -> "builtin-sink" :: Text; ExternalEndpoint value -> value),
+      "sinkFault" .= (case spec.endpoint of BuiltinSink fault -> renderSinkFault fault; ExternalEndpoint _ -> "none"),
+      "compression" .= (case spec.compression of CompressionNone -> "none" :: Text; CompressionGzip -> "gzip")
+    ]
+
+renderSampler :: SamplerSpec -> Text
+renderSampler AlwaysOn = "always-on"
+renderSampler AlwaysOff = "always-off"
+renderSampler ParentBasedAlwaysOn = "parentbased-always-on"
+renderSampler (TraceIdRatio ratio) = "traceidratio:" <> Text.pack (show ratio)
+renderSampler (ParentBasedTraceIdRatio ratio) = "parentbased-traceidratio:" <> Text.pack (show ratio)
+
+renderProcessor :: ProcessorSpec -> Value
+renderProcessor (SimpleProcessor timeoutMs) = object ["kind" .= ("simple" :: Text), "exportTimeoutMs" .= timeoutMs]
+renderProcessor (BatchProcessor queue delay batch timeoutMs) =
+  object
+    [ "kind" .= ("batch" :: Text),
+      "maxQueue" .= queue,
+      "scheduleDelayMs" .= delay,
+      "maxExportBatch" .= batch,
+      "exportTimeoutMs" .= timeoutMs
+    ]
+
+timedCall :: (Show result) => IO (Maybe result) -> IO (Maybe ProviderCallReport)
+timedCall operation = do
+  started <- getMonotonicTimeNSec
+  outcome <- tryAny operation
+  finished <- getMonotonicTimeNSec
+  let elapsed = fromIntegral (finished - started) / 1_000_000
+  pure . Just $ case outcome of
+    Left exception -> ProviderCallReport ("exception: " <> Text.pack (displayException exception)) elapsed
+    Right Nothing -> ProviderCallReport "not-applicable" elapsed
+    Right (Just result) -> ProviderCallReport (Text.pack (show result)) elapsed
+
+timedStopMetrics :: Int -> MetricsRuntime -> IO (Maybe MetricsSnapshot, Maybe ProviderCallReport)
+timedStopMetrics timeoutMs runtime = do
+  started <- getMonotonicTimeNSec
+  outcome <- tryAny (stopMetrics timeoutMs runtime)
+  finished <- getMonotonicTimeNSec
+  let elapsed = fromIntegral (finished - started) / 1_000_000
+  pure $ case outcome of
+    Left exception -> (Nothing, Just (ProviderCallReport ("exception: " <> Text.pack (displayException exception)) elapsed))
+    Right (snapshot, Nothing) -> (snapshot, Just (ProviderCallReport "not-applicable" elapsed))
+    Right (snapshot, Just result) -> (snapshot, Just (ProviderCallReport (Text.pack (show result)) elapsed))
+
+ambientOtelEnvironment :: IO [(Text, Text)]
+ambientOtelEnvironment = do
+  environment <- getEnvironment
+  pure [(Text.pack key, Text.pack value) | (key, value) <- environment, "OTEL_" `Text.isPrefixOf` Text.pack key]
+
+tryAny :: IO value -> IO (Either SomeException value)
+tryAny = try
+
+traverse_ :: (a -> IO b) -> Maybe a -> IO ()
+traverse_ _ Nothing = pure ()
+traverse_ action (Just value) = void (action value)
diff --git a/src/Kenshou/Telemetry/Compose.hs b/src/Kenshou/Telemetry/Compose.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Compose.hs
@@ -0,0 +1,96 @@
+module Kenshou.Telemetry.Compose
+  ( HandlerStats,
+    HandlerStatsSnapshot (..),
+    AsyncHandlerStats (..),
+    newHandlerStats,
+    snapshotHandlerStats,
+    composeHandlers,
+    timedHandler,
+    slowHandler,
+    asyncHandler,
+  )
+where
+
+import Control.Concurrent (threadDelay)
+import Control.Concurrent.Async (async)
+import Control.Concurrent.STM
+import Control.Monad (forever)
+import Data.Aeson (ToJSON (..), object, (.=))
+import Data.IORef
+import Data.List (sort)
+import Data.Text (Text)
+import GHC.Clock (getMonotonicTimeNSec)
+
+newtype HandlerStats = HandlerStats (IORef [Integer])
+
+data HandlerStatsSnapshot = HandlerStatsSnapshot
+  { name :: Text,
+    calls :: Int,
+    durationP50Ns :: Integer,
+    durationP99Ns :: Integer,
+    durationMaxNs :: Integer
+  }
+  deriving stock (Eq, Show)
+
+data AsyncHandlerStats = AsyncHandlerStats
+  { accepted :: Int,
+    dropped :: Int,
+    handled :: Int,
+    queueDepth :: Int
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON HandlerStatsSnapshot where
+  toJSON value = object ["name" .= value.name, "calls" .= value.calls, "durationNs" .= object ["p50" .= value.durationP50Ns, "p99" .= value.durationP99Ns, "max" .= value.durationMaxNs]]
+
+instance ToJSON AsyncHandlerStats where
+  toJSON value = object ["accepted" .= value.accepted, "dropped" .= value.dropped, "handled" .= value.handled, "queueDepth" .= value.queueDepth]
+
+newHandlerStats :: IO HandlerStats
+newHandlerStats = HandlerStats <$> newIORef []
+
+snapshotHandlerStats :: Text -> HandlerStats -> IO HandlerStatsSnapshot
+snapshotHandlerStats name (HandlerStats ref) = do
+  durations <- sort <$> readIORef ref
+  pure (HandlerStatsSnapshot name (length durations) (quantile 0.50 durations) (quantile 0.99 durations) (maybe 0 last (nonEmpty durations)))
+
+composeHandlers :: [a -> IO ()] -> a -> IO ()
+composeHandlers [] = const (pure ())
+composeHandlers handlers = \value -> mapM_ ($ value) handlers
+
+timedHandler :: HandlerStats -> Text -> (a -> IO ()) -> a -> IO ()
+timedHandler (HandlerStats ref) _name handler value = do
+  started <- getMonotonicTimeNSec
+  handler value
+  ended <- getMonotonicTimeNSec
+  modifyIORef' ref (fromIntegral (ended - started) :)
+
+slowHandler :: Int -> a -> IO ()
+slowHandler micros _ = threadDelay (max 0 micros)
+
+asyncHandler :: Int -> (a -> IO ()) -> IO (a -> IO (), IO AsyncHandlerStats)
+asyncHandler requestedCapacity handler = do
+  let capacity = max 1 requestedCapacity
+  queue <- newTBQueueIO (fromIntegral capacity)
+  acceptedRef <- newIORef 0
+  droppedRef <- newIORef 0
+  handledRef <- newIORef 0
+  _ <- async . forever $ do
+    value <- atomically (readTBQueue queue)
+    handler value
+    modifyIORef' handledRef (+ 1)
+  let submit value = do
+        accepted <- atomically do
+          full <- isFullTBQueue queue
+          if full then pure False else writeTBQueue queue value >> pure True
+        modifyIORef' (if accepted then acceptedRef else droppedRef) (+ 1)
+      snapshot = AsyncHandlerStats <$> readIORef acceptedRef <*> readIORef droppedRef <*> readIORef handledRef <*> (fromIntegral <$> atomically (lengthTBQueue queue))
+  pure (submit, snapshot)
+
+quantile :: Double -> [Integer] -> Integer
+quantile _ [] = 0
+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))
+
+nonEmpty :: [a] -> Maybe [a]
+nonEmpty [] = Nothing
+nonEmpty values = Just values
diff --git a/src/Kenshou/Telemetry/Continuity.hs b/src/Kenshou/Telemetry/Continuity.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Continuity.hs
@@ -0,0 +1,74 @@
+module Kenshou.Telemetry.Continuity
+  ( SpanSelector (..),
+    ContinuityResult (..),
+    IsolationResult (..),
+    checkContinuity,
+    checkIsolation,
+  )
+where
+
+import Data.Aeson (ToJSON (..), object, (.=))
+import Data.List (find)
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Kenshou.Telemetry.Tracing.Probe (SpanView (..))
+import OpenTelemetry.Attributes (Attribute, lookupAttribute)
+import OpenTelemetry.Trace.Core (SpanKind)
+
+data SpanSelector = SpanSelector
+  { nameIs :: Maybe Text,
+    kindIs :: Maybe SpanKind,
+    hasAttr :: [(Text, Attribute)]
+  }
+  deriving stock (Eq, Show)
+
+data ContinuityResult = ContinuityResult
+  { producers :: Int,
+    consumers :: Int,
+    matched :: Int,
+    violations :: Int,
+    counterExamples :: [Text]
+  }
+  deriving stock (Eq, Show)
+
+data IsolationResult = IsolationResult
+  { checked :: Int,
+    violations :: Int,
+    counterExamples :: [Text]
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON ContinuityResult where
+  toJSON value = object ["producers" .= value.producers, "consumers" .= value.consumers, "matched" .= value.matched, "violations" .= value.violations, "counterExamples" .= value.counterExamples]
+
+instance ToJSON IsolationResult where
+  toJSON value = object ["checked" .= value.checked, "violations" .= value.violations, "counterExamples" .= value.counterExamples]
+
+checkContinuity :: SpanSelector -> SpanSelector -> (SpanView -> Maybe Text) -> [SpanView] -> ContinuityResult
+checkContinuity producerSelector consumerSelector correlation spans =
+  let producerSpans = filter (matches producerSelector) spans
+      consumerSpans = filter (matches consumerSelector) spans
+      outcomes = fmap (checkConsumer producerSpans) consumerSpans
+      failures = take 10 [message | Left message <- outcomes]
+   in ContinuityResult (length producerSpans) (length consumerSpans) (length [() | Right () <- outcomes]) (length [() | Left _ <- outcomes]) failures
+  where
+    checkConsumer producerSpans consumer = case correlation consumer of
+      Nothing -> Left (consumer.name <> ": missing correlation key")
+      Just key -> case find ((== Just key) . correlation) producerSpans of
+        Nothing -> Left (consumer.name <> ": no producer for " <> key)
+        Just producer
+          | consumer.traceId /= producer.traceId -> Left (consumer.name <> ": trace differs for " <> key)
+          | consumer.parentSpanId /= Just producer.spanId -> Left (consumer.name <> ": parent differs for " <> key)
+          | otherwise -> Right ()
+
+checkIsolation :: SpanSelector -> (SpanView -> Bool) -> [SpanView] -> IsolationResult
+checkIsolation selector expectedRoot spans =
+  let checkedSpans = filter (\span -> matches selector span && expectedRoot span) spans
+      failures = [span.name <> ": inherited parent " <> Text.pack (show parent) | span <- checkedSpans, Just parent <- [span.parentSpanId]]
+   in IsolationResult (length checkedSpans) (length failures) (take 10 failures)
+
+matches :: SpanSelector -> SpanView -> Bool
+matches selector span =
+  maybe True (== span.name) selector.nameIs
+    && maybe True (== span.kind) selector.kindIs
+    && all (\(key, expected) -> lookupAttribute span.attributes key == Just expected) selector.hasAttr
diff --git a/src/Kenshou/Telemetry/Detect.hs b/src/Kenshou/Telemetry/Detect.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Detect.hs
@@ -0,0 +1,103 @@
+module Kenshou.Telemetry.Detect
+  ( FindingStatus (..),
+    FindingSeverity (..),
+    Finding (..),
+    pipelineFindings,
+    queueGrowthFinding,
+    endpointFinding,
+    handlerFinding,
+    continuityFindings,
+  )
+where
+
+import Data.Aeson (ToJSON (..), Value, object, (.=))
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Kenshou.Telemetry.Compose (HandlerStatsSnapshot (..))
+import Kenshou.Telemetry.Continuity (ContinuityResult (..), IsolationResult (..))
+import Kenshou.Telemetry.Scrape (EndpointSummary (..))
+import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..))
+
+data FindingStatus = Fired | Quiet | NotApplicable deriving stock (Eq, Show)
+
+data FindingSeverity = Info | Degraded | Failure deriving stock (Eq, Ord, Show)
+
+data Finding = Finding
+  { detector :: Text,
+    status :: FindingStatus,
+    severity :: FindingSeverity,
+    evidence :: Value
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON FindingStatus where
+  toJSON Fired = toJSON ("fired" :: Text)
+  toJSON Quiet = toJSON ("quiet" :: Text)
+  toJSON NotApplicable = toJSON ("not-applicable" :: Text)
+
+instance ToJSON FindingSeverity where
+  toJSON Info = toJSON ("info" :: Text)
+  toJSON Degraded = toJSON ("degraded" :: Text)
+  toJSON Failure = toJSON ("failure" :: Text)
+
+instance ToJSON Finding where
+  toJSON value = object ["detector" .= value.detector, "status" .= value.status, "severity" .= value.severity, "evidence" .= value.evidence]
+
+pipelineFindings :: Double -> Maybe PipelineSnapshot -> [(Text, Text, Double, Double)] -> [Finding]
+pipelineFindings _ Nothing _ =
+  [ Finding "span-drop" NotApplicable Info (object []),
+    Finding "export-failure" NotApplicable Info (object []),
+    Finding "shutdown-blocked" NotApplicable Info (object [])
+  ]
+pipelineFindings degradedFraction (Just snapshot) providerCalls =
+  [ Finding "span-drop" (fired snapshot.spansDropped) dropSeverity (object ["spansEnded" .= snapshot.spansEnded, "spansDropped" .= snapshot.spansDropped, "fraction" .= dropFraction]),
+    Finding "export-failure" (fired snapshot.spansExportFailed) (if snapshot.spansExportFailed > 0 then Degraded else Info) (object ["spansExportFailed" .= snapshot.spansExportFailed, "lastError" .= snapshot.lastExportError]),
+    Finding "shutdown-blocked" (if null blocked then Quiet else Fired) (if null blocked then Info else Degraded) (object ["calls" .= [object ["call" .= name, "result" .= result, "durationMs" .= duration, "limitMs" .= limit] | (name, result, duration, limit) <- providerCalls], "blocked" .= fmap (\(name, _, _, _) -> name) blocked])
+  ]
+  where
+    dropFraction = fromIntegral snapshot.spansDropped / fromIntegral (max 1 snapshot.spansEnded)
+    dropSeverity = if dropFraction > degradedFraction then Degraded else Info
+    blocked = filter (\(_, result, duration, limit) -> duration > limit || "timeout" `Text.isInfixOf` Text.toCaseFold result) providerCalls
+    fired count = if count > 0 then Fired else Quiet
+
+queueGrowthFinding :: Maybe Int -> [Int] -> Finding
+queueGrowthFinding Nothing _ = Finding "queue-growth" NotApplicable Info (object [])
+queueGrowthFinding (Just queueLimit) samples =
+  Finding
+    "queue-growth"
+    (if longest > 10 then Fired else Quiet)
+    (if longest > 10 then Degraded else Info)
+    (object ["queueLimit" .= queueLimit, "threshold" .= threshold, "longestConsecutiveSamples" .= longest, "samples" .= length samples])
+  where
+    threshold = fromIntegral queueLimit * (0.9 :: Double)
+    longest = longestRun ((> threshold) . fromIntegral) samples
+
+longestRun :: (value -> Bool) -> [value] -> Int
+longestRun predicate = snd . foldl step (0, 0)
+  where
+    step (current, best) value =
+      let next = if predicate value then current + 1 else 0
+       in (next, max best next)
+
+endpointFinding :: [EndpointSummary] -> Finding
+endpointFinding [] = Finding "endpoint-failure" NotApplicable Info (object [])
+endpointFinding summaries = Finding "endpoint-failure" status severity (object ["failures" .= failures, "attempted" .= attempted, "fraction" .= fraction])
+  where
+    attempted = sum [summary.scrapes + summary.skippedTicks | summary <- summaries]
+    failures = sum [summary.failures + summary.skippedTicks | summary <- summaries]
+    fraction :: Double
+    fraction = fromIntegral failures / fromIntegral (max 1 attempted)
+    status = if fraction > 0.01 then Fired else Quiet
+    severity = if status == Fired then Degraded else Info
+
+handlerFinding :: HandlerStatsSnapshot -> Finding
+handlerFinding snapshot = Finding "handler-stall" status severity (object ["handler" .= snapshot.name, "calls" .= snapshot.calls, "durationP99Ns" .= snapshot.durationP99Ns, "limitNs" .= (5_000_000 :: Integer)])
+  where
+    status = if snapshot.durationP99Ns > 5_000_000 then Fired else Quiet
+    severity = if status == Fired then Degraded else Info
+
+continuityFindings :: ContinuityResult -> IsolationResult -> [Finding]
+continuityFindings continuity isolation =
+  [ Finding "trace-discontinuity" (if continuity.violations > 0 then Fired else Quiet) (if continuity.violations > 0 then Failure else Info) (toJSON continuity),
+    Finding "context-leak" (if isolation.violations > 0 then Fired else Quiet) (if isolation.violations > 0 then Failure else Info) (toJSON isolation)
+  ]
diff --git a/src/Kenshou/Telemetry/Endpoint.hs b/src/Kenshou/Telemetry/Endpoint.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Endpoint.hs
@@ -0,0 +1,71 @@
+module Kenshou.Telemetry.Endpoint
+  ( EndpointKind (..),
+    Endpoint (..),
+    reserveFreePort,
+    awaitHttpReady,
+  )
+where
+
+import Control.Concurrent (threadDelay)
+import Control.Exception (SomeException, try)
+import Data.Aeson (FromJSON (..), ToJSON (..), Value, object, withObject, withText, (.:), (.:?), (.=))
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Network.HTTP.Client (defaultManagerSettings, httpLbs, newManager, parseRequest, responseStatus)
+import Network.HTTP.Types.Status (statusCode)
+import Network.Socket (close)
+import Network.Wai.Handler.Warp (openFreePort)
+
+data EndpointKind = PrometheusText | JsonDocument | HealthProbe | WebSocketPush
+  deriving stock (Eq, Ord, Show)
+
+data Endpoint = Endpoint
+  { name :: Text,
+    kind :: EndpointKind,
+    url :: Text,
+    wsHello :: Maybe Value
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON EndpointKind where
+  toJSON = toJSON . renderKind
+
+instance FromJSON EndpointKind where
+  parseJSON = withText "EndpointKind" (maybe (fail "unknown endpoint kind") pure . parseKind)
+
+instance ToJSON Endpoint where
+  toJSON endpoint = object ["name" .= endpoint.name, "kind" .= endpoint.kind, "url" .= endpoint.url, "wsHello" .= endpoint.wsHello]
+
+instance FromJSON Endpoint where
+  parseJSON = withObject "Endpoint" (\value -> Endpoint <$> value .: "name" <*> value .: "kind" <*> value .: "url" <*> value .:? "wsHello")
+
+reserveFreePort :: IO Int
+reserveFreePort = do
+  (port, socket) <- openFreePort
+  close socket
+  pure port
+
+awaitHttpReady :: Text -> Int -> IO Bool
+awaitHttpReady url timeoutMs = do
+  manager <- newManager defaultManagerSettings
+  request <- parseRequest (Text.unpack url)
+  let attempts = max 1 (timeoutMs `div` 50)
+      loop 0 = pure False
+      loop remaining = do
+        response <- tryAny (httpLbs request manager)
+        case response of
+          Right value | statusCode (responseStatus value) < 500 -> pure True
+          _ -> threadDelay 50_000 >> loop (remaining - 1)
+  loop attempts
+
+renderKind :: EndpointKind -> Text
+renderKind PrometheusText = "prometheus-text"
+renderKind JsonDocument = "json-document"
+renderKind HealthProbe = "health-probe"
+renderKind WebSocketPush = "websocket-push"
+
+parseKind :: Text -> Maybe EndpointKind
+parseKind value = lookup value [(renderKind kind, kind) | kind <- [PrometheusText, JsonDocument, HealthProbe, WebSocketPush]]
+
+tryAny :: IO value -> IO (Either SomeException value)
+tryAny = try
diff --git a/src/Kenshou/Telemetry/Metrics.hs b/src/Kenshou/Telemetry/Metrics.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Metrics.hs
@@ -0,0 +1,116 @@
+module Kenshou.Telemetry.Metrics
+  ( MetricsRuntime (..),
+    MetricsSnapshot (..),
+    startMetrics,
+    flushMetrics,
+    readMetricSums,
+    readMetricGauges,
+    stopMetrics,
+  )
+where
+
+import Control.Concurrent.Async (Async, async, cancel, waitCatch)
+import Control.Monad (void)
+import Data.Aeson (ToJSON (..), object, (.=))
+import Data.Text qualified as Text
+import Data.Vector qualified as Vector
+import Kenshou.Telemetry.Endpoint
+import Kenshou.Telemetry.Spec
+import Kenshou.Telemetry.Tracing (otlpExporterConfig)
+import Network.Socket (Socket, close)
+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)
+import OpenTelemetry.Exporter.Metric (GaugeDataPoint (..), MetricExport (..), NumberValue (..), ResourceMetricsExport (..), ScopeMetricsExport (..), SumDataPoint (..))
+import OpenTelemetry.Exporter.OTLP.Metric qualified as OtlpMetric
+import OpenTelemetry.Exporter.Prometheus.WAI (prometheusApplication)
+import OpenTelemetry.MeterProvider (SdkMeterEnv, collectResourceMetrics, createMeterProvider, defaultSdkMeterProviderOptions)
+import OpenTelemetry.Metric.Core (Meter, MeterProvider, forceFlushMeterProvider, getMeter, shutdownMeterProvider)
+import OpenTelemetry.MetricReader (PeriodicMetricReaderHandle (..), PeriodicMetricReaderOptions (..), forkPeriodicMetricReader)
+import OpenTelemetry.Resource (emptyMaterializedResources)
+import OpenTelemetry.Trace.Core (FlushResult, ShutdownResult, instrumentationLibrary)
+
+data MetricsRuntime = MetricsRuntime
+  { meter :: Maybe Meter,
+    provider :: Maybe MeterProvider,
+    env :: Maybe SdkMeterEnv,
+    endpoint :: Maybe Endpoint,
+    server :: Maybe (Async (), Socket),
+    reader :: Maybe PeriodicMetricReaderHandle
+  }
+
+data MetricsSnapshot = MetricsSnapshot
+  { resourceBatches :: Int,
+    instruments :: Int
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON MetricsSnapshot where
+  toJSON snapshot = object ["resourceBatches" .= snapshot.resourceBatches, "instruments" .= snapshot.instruments]
+
+startMetrics :: TelemetrySpec -> IO MetricsRuntime
+startMetrics spec = case spec.metrics of
+  MetricsOff -> pure (MetricsRuntime Nothing Nothing Nothing Nothing Nothing Nothing)
+  _ -> do
+    (provider, env) <- createMeterProvider emptyMaterializedResources defaultSdkMeterProviderOptions
+    meter <- getMeter provider (instrumentationLibrary "kenshou" "0.1.0.0")
+    (endpoint, server) <- case (spec.metrics, spec.otelReader) of
+      (arm, ReaderPrometheus) | arm `elem` [MetricsServe, MetricsServeScraped] -> do
+        (port, socket) <- openFreePort
+        running <- async (runSettingsSocket defaultSettings socket (prometheusApplication (Vector.fromList <$> collectResourceMetrics env)))
+        let url = "http://127.0.0.1:" <> Text.pack (show port) <> "/metrics"
+        pure (Just (Endpoint "otel-prometheus" PrometheusText url Nothing), Just (running, socket))
+      _ -> pure (Nothing, Nothing)
+    reader <- case spec.otelReader of
+      ReaderOtlpPeriodic -> do
+        exporter <- OtlpMetric.otlpMetricExporter (otlpExporterConfig spec)
+        Just <$> forkPeriodicMetricReader env exporter (PeriodicMetricReaderOptions (spec.otelExportMs * 1000))
+      _ -> pure Nothing
+    pure (MetricsRuntime (Just meter) (Just provider) (Just env) endpoint server reader)
+
+flushMetrics :: Int -> MetricsRuntime -> IO (Maybe FlushResult)
+flushMetrics timeoutMs runtime = traverse (\provider -> forceFlushMeterProvider provider (Just (timeoutMs * 1000))) runtime.provider
+
+readMetricSums :: MetricsRuntime -> IO [(Text.Text, Double)]
+readMetricSums runtime = do
+  batches <- maybe (pure []) collectResourceMetrics runtime.env
+  pure
+    [ (metric.mesName, sum (fmap pointValue metric.mesSumPoints))
+    | resource <- batches,
+      scope <- Vector.toList resource.resourceMetricsScopes,
+      metric@MetricExportSum {} <- Vector.toList scope.scopeMetricsExports
+    ]
+  where
+    pointValue point = case point.sumDataPointValue of
+      IntNumber value -> fromIntegral value
+      DoubleNumber value -> value
+
+readMetricGauges :: MetricsRuntime -> IO [(Text.Text, Double)]
+readMetricGauges runtime = do
+  batches <- maybe (pure []) collectResourceMetrics runtime.env
+  pure
+    [ (metric.megName, pointValue point.gaugeDataPointValue)
+    | resource <- batches,
+      scope <- Vector.toList resource.resourceMetricsScopes,
+      metric@MetricExportGauge {} <- Vector.toList scope.scopeMetricsExports,
+      point <- Vector.toList metric.megGaugePoints
+    ]
+  where
+    pointValue = \case
+      IntNumber value -> fromIntegral value
+      DoubleNumber value -> value
+
+stopMetrics :: Int -> MetricsRuntime -> IO (Maybe MetricsSnapshot, Maybe ShutdownResult)
+stopMetrics timeoutMs runtime = do
+  traverse_ (.stopPeriodicMetricReader) runtime.reader
+  batches <- maybe (pure []) collectResourceMetrics runtime.env
+  let snapshot = case runtime.env of
+        Nothing -> Nothing
+        Just _ -> Just (MetricsSnapshot (length batches) (sum [Vector.length scope.scopeMetricsExports | resource <- batches, scope <- Vector.toList resource.resourceMetricsScopes]))
+  shutdown <- traverse (\provider -> shutdownMeterProvider provider (Just (timeoutMs * 1000))) runtime.provider
+  case runtime.server of
+    Nothing -> pure ()
+    Just (running, socket) -> cancel running >> void (waitCatch running) >> close socket
+  pure (snapshot, shutdown)
+
+traverse_ :: (a -> IO b) -> Maybe a -> IO ()
+traverse_ _ Nothing = pure ()
+traverse_ action (Just value) = void (action value)
diff --git a/src/Kenshou/Telemetry/Overhead.hs b/src/Kenshou/Telemetry/Overhead.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Overhead.hs
@@ -0,0 +1,648 @@
+module Kenshou.Telemetry.Overhead
+  ( Arm (..),
+    Slot (..),
+    OverheadMode (..),
+    OverheadRequest (..),
+    OverheadPlan (..),
+    SlotRun (..),
+    OverheadState (..),
+    OverheadHooks (..),
+    OverheadComparison (..),
+    OverheadReport (..),
+    UsageError (..),
+    planOverhead,
+    executeOverhead,
+    analyseOverhead,
+    loadOverheadState,
+    overheadVerdictExitCode,
+  )
+where
+
+import Control.Concurrent (threadDelay)
+import Control.Exception (SomeException, try)
+import Control.Monad (foldM, when)
+import Data.Aeson
+import Data.Aeson.Key qualified as Key
+import Data.Aeson.KeyMap qualified as KeyMap
+import Data.ByteString.Lazy qualified as LazyByteString
+import Data.List (nub, sort, sortOn)
+import Data.List.NonEmpty (NonEmpty (..))
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
+import Data.Maybe (catMaybes, fromMaybe, listToMaybe)
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Data.Word (Word64)
+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)
+import Kenshou.Core.Dimension
+import Kenshou.Core.Id
+import Kenshou.Core.Knob
+import Kenshou.Core.Manifest (verifyManifest)
+import Kenshou.Core.RunSpec
+import Kenshou.Core.Scenario (Scenario (..))
+import Kenshou.Measure.Compare
+import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..))
+import Kenshou.Measure.Compare.Policy (Policy (..))
+import Kenshou.Measure.Stats (Interval (..))
+import Kenshou.Telemetry.Overhead.Policy
+import System.Directory (createDirectoryIfMissing, doesFileExist, renameFile)
+import System.Exit (ExitCode (..))
+import System.FilePath ((</>))
+import System.Random.SplitMix (mkSMGen, nextWord64)
+
+data Arm = Arm
+  { armId :: Text,
+    dimensions :: Map Text Text,
+    knobs :: Map Text KnobValue,
+    control :: Bool
+  }
+  deriving stock (Eq, Show)
+
+data Slot = Slot
+  { block :: Int,
+    position :: Int,
+    arm :: Arm,
+    spec :: RunSpec
+  }
+  deriving stock (Eq, Show)
+
+data OverheadMode = OneFactor | FullFactorial deriving stock (Eq, Show)
+
+data OverheadRequest = OverheadRequest
+  { requestId :: Text,
+    scenarioId :: ScenarioId,
+    factors :: Map Text [Text],
+    mode :: OverheadMode,
+    includeControl :: Bool,
+    trials :: Int,
+    fixedKnobs :: [(KnobName, RawKnob)],
+    fixedDimensions :: Map Text Text,
+    seed :: Word64,
+    settleSeconds :: Int,
+    retries :: Int
+  }
+  deriving stock (Eq, Show)
+
+data OverheadPlan = OverheadPlan
+  { planId :: Text,
+    scenarioId :: ScenarioId,
+    scenarioRevision :: Int,
+    mode :: OverheadMode,
+    factorValues :: Map Text [Text],
+    baselineArm :: Text,
+    arms :: [Arm],
+    trials :: Int,
+    seed :: Word64,
+    settleSeconds :: Int,
+    retries :: Int,
+    fixedDimensions :: Map Text Text,
+    fixedKnobs :: Map Text KnobValue,
+    slots :: [Slot]
+  }
+  deriving stock (Eq, Show)
+
+data SlotRun = SlotRun
+  { block :: Int,
+    position :: Int,
+    armId :: Text,
+    runIds :: [RunId],
+    exitCode :: Maybe Int,
+    complete :: Bool
+  }
+  deriving stock (Eq, Show)
+
+data OverheadState = OverheadState
+  { plan :: OverheadPlan,
+    slots :: [SlotRun]
+  }
+  deriving stock (Eq, Show)
+
+data OverheadHooks = OverheadHooks
+  { runChild :: RunSpec -> FilePath -> IO ExitCode,
+    compare :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison),
+    leakCheck :: Maybe (FilePath -> IO (Maybe Value))
+  }
+
+data OverheadComparison = OverheadComparison
+  { candidate :: Text,
+    baseline :: Text,
+    factor :: Text,
+    fromValue :: Text,
+    toValue :: Text,
+    comparisonPath :: Maybe FilePath,
+    verdict :: Verdict,
+    degraded :: Bool,
+    deltas :: Value,
+    reasons :: [Text]
+  }
+  deriving stock (Eq, Show)
+
+data OverheadReport = OverheadReport
+  { reportId :: Text,
+    scenarioId :: ScenarioId,
+    mode :: OverheadMode,
+    baselineDimensions :: Map Text Text,
+    fixedDimensions :: Map Text Text,
+    fixedKnobs :: Map Text KnobValue,
+    trials :: Int,
+    seed :: Word64,
+    validBlocks :: Int,
+    order :: [[Text]],
+    armRuns :: [ArmEvidence],
+    comparisons :: [OverheadComparison],
+    reportFindings :: [Value],
+    policyIdentity :: Value,
+    verdict :: Verdict
+  }
+  deriving stock (Eq, Show)
+
+data ArmEvidence = ArmEvidence
+  { arm :: Arm,
+    runs :: [RunId],
+    telemetry :: Value,
+    leak :: Maybe Value,
+    findings :: [Value],
+    degraded :: Bool
+  }
+  deriving stock (Eq, Show)
+
+newtype UsageError = UsageError Text deriving stock (Eq, Show)
+
+instance ToJSON Arm where
+  toJSON value = object ["arm" .= value.armId, "dimensions" .= value.dimensions, "knobs" .= value.knobs, "control" .= value.control]
+
+instance FromJSON Arm where
+  parseJSON = withObject "Arm" \value -> Arm <$> value .: "arm" <*> value .: "dimensions" <*> value .: "knobs" <*> value .:? "control" .!= False
+
+instance ToJSON OverheadMode where
+  toJSON OneFactor = String "one-factor"
+  toJSON FullFactorial = String "full"
+
+instance FromJSON OverheadMode where
+  parseJSON = withText "OverheadMode" \case
+    "one-factor" -> pure OneFactor
+    "full" -> pure FullFactorial
+    _ -> fail "mode must be one-factor or full"
+
+instance ToJSON Slot where
+  toJSON value = object ["block" .= value.block, "position" .= value.position, "arm" .= value.arm, "spec" .= value.spec]
+
+instance FromJSON Slot where
+  parseJSON = withObject "Slot" \value -> Slot <$> value .: "block" <*> value .: "position" <*> value .: "arm" <*> value .: "spec"
+
+instance ToJSON OverheadPlan where
+  toJSON value =
+    object
+      [ "schema" .= ("kenshou.overhead-plan/v1" :: Text),
+        "id" .= value.planId,
+        "scenario" .= value.scenarioId,
+        "scenarioRevision" .= value.scenarioRevision,
+        "mode" .= value.mode,
+        "factorValues" .= value.factorValues,
+        "baselineArm" .= value.baselineArm,
+        "arms" .= value.arms,
+        "trials" .= value.trials,
+        "seed" .= value.seed,
+        "settleSeconds" .= value.settleSeconds,
+        "retries" .= value.retries,
+        "fixedDimensions" .= value.fixedDimensions,
+        "fixedKnobs" .= value.fixedKnobs,
+        "slots" .= value.slots
+      ]
+
+instance FromJSON OverheadPlan where
+  parseJSON = withObject "OverheadPlan" \value -> do
+    schema <- value .: "schema"
+    if schema /= ("kenshou.overhead-plan/v1" :: Text) then fail "unsupported overhead plan" else pure ()
+    OverheadPlan <$> value .: "id" <*> value .: "scenario" <*> value .: "scenarioRevision" <*> value .: "mode" <*> value .: "factorValues" <*> value .: "baselineArm" <*> value .: "arms" <*> value .: "trials" <*> value .: "seed" <*> value .: "settleSeconds" <*> value .: "retries" <*> value .: "fixedDimensions" <*> value .: "fixedKnobs" <*> value .: "slots"
+
+instance ToJSON SlotRun where
+  toJSON value = object ["block" .= value.block, "position" .= value.position, "arm" .= value.armId, "runIds" .= value.runIds, "exitCode" .= value.exitCode, "complete" .= value.complete]
+
+instance FromJSON SlotRun where
+  parseJSON = withObject "SlotRun" \value -> SlotRun <$> value .: "block" <*> value .: "position" <*> value .: "arm" <*> value .:? "runIds" .!= [] <*> value .:? "exitCode" <*> value .:? "complete" .!= False
+
+instance ToJSON OverheadState where
+  toJSON value = object ["schema" .= ("kenshou.overhead-state/v1" :: Text), "plan" .= value.plan, "slots" .= value.slots]
+
+instance FromJSON OverheadState where
+  parseJSON = withObject "OverheadState" \value -> do
+    schema <- value .: "schema"
+    if schema /= ("kenshou.overhead-state/v1" :: Text) then fail "unsupported overhead state" else pure ()
+    OverheadState <$> value .: "plan" <*> value .:? "slots" .!= []
+
+instance ToJSON OverheadComparison where
+  toJSON value =
+    object
+      [ "candidate" .= value.candidate,
+        "baseline" .= value.baseline,
+        "factor" .= value.factor,
+        "from" .= value.fromValue,
+        "to" .= value.toValue,
+        "comparison" .= value.comparisonPath,
+        "verdict" .= value.verdict,
+        "degraded" .= value.degraded,
+        "deltas" .= value.deltas,
+        "reasons" .= value.reasons
+      ]
+
+instance ToJSON OverheadReport where
+  toJSON value =
+    object
+      [ "schema" .= ("kenshou.overhead-report/v1" :: Text),
+        "id" .= value.reportId,
+        "scenario" .= value.scenarioId,
+        "mode" .= value.mode,
+        "baseline" .= value.baselineDimensions,
+        "fixed" .= object ["knobs" .= value.fixedKnobs, "dimensions" .= value.fixedDimensions],
+        "trials" .= value.trials,
+        "seed" .= value.seed,
+        "validBlocks" .= value.validBlocks,
+        "order" .= value.order,
+        "arms" .= [object ["arm" .= row.arm.armId, "dimensions" .= row.arm.dimensions, "runs" .= row.runs, "telemetry" .= row.telemetry, "leak" .= row.leak, "findings" .= row.findings] | row <- value.armRuns],
+        "comparisons" .= value.comparisons,
+        "findings" .= value.reportFindings,
+        "policy" .= value.policyIdentity,
+        "cohort" .= object [],
+        "verdict" .= value.verdict,
+        "algorithm" .= object ["name" .= ("kenshou-overhead" :: Text), "version" .= (1 :: Int)]
+      ]
+
+planOverhead :: OverheadRequest -> Scenario -> Either UsageError OverheadPlan
+planOverhead request scenario = do
+  whenEither (request.scenarioId /= scenario.id) "scenario request does not match the selected scenario"
+  whenEither (request.trials < 3) "trials must be at least 3"
+  whenEither (request.settleSeconds < 0) "settle-seconds must be non-negative"
+  whenEither (request.retries < 0) "retries must be non-negative"
+  whenEither (Map.null request.factors) "at least one --arms factor is required"
+  traverse_ validateFactor (Map.toList request.factors)
+  whenEither (any (`Map.member` request.fixedDimensions) (Map.keys request.factors)) "a varied factor cannot also be fixed with --dim"
+  resolvedKnobs <- firstUsage (resolveKnobs scenario.knobs request.fixedKnobs)
+  let knobMap = Map.fromList [(renderKnobName name, value) | (name, value) <- Map.toList (resolvedKnobsMap resolvedKnobs)]
+      baselineCell = Map.map (fromMaybe "" . listToMaybe) request.factors
+      cells = case request.mode of
+        OneFactor -> baselineCell : [Map.insert factor armValue baselineCell | (factor, values) <- Map.toAscList request.factors, armValue <- drop 1 values]
+        FullFactorial -> cartesian (Map.toAscList request.factors)
+  plannedArms <- traverse (makeArm scenario knobMap) (zip [0 :: Int ..] (nub cells))
+  baselinePlanned <- maybe (Left (UsageError "overhead planning produced no baseline arm")) Right (listToMaybe plannedArms)
+  let withControl = plannedArms <> [Arm "control" baselinePlanned.dimensions knobMap True | request.includeControl]
+      shuffled = shuffle request.seed withControl
+      template =
+        OverheadPlan
+          request.requestId
+          scenario.id
+          scenario.revision
+          request.mode
+          request.factors
+          baselinePlanned.armId
+          withControl
+          request.trials
+          request.seed
+          request.settleSeconds
+          request.retries
+          request.fixedDimensions
+          knobMap
+          []
+      initialSlots = concatMap (blockSlots template shuffled) [0 .. request.trials - 1]
+  pure (setPlanSlots initialSlots template)
+  where
+    validateFactor (factor, values) = do
+      whenEither (factor `notElem` ["telemetry.tracing", "telemetry.metrics"]) ("unknown overhead factor " <> factor)
+      whenEither (null values) (factor <> " has no values")
+      whenEither (length values /= length (nub values)) (factor <> " repeats a value")
+    makeArm selected knobMap (index, cell) = do
+      let assignments = Map.toList (request.fixedDimensions <> cell)
+      dimensions <- firstUsage (resolveDimensions selected.dimensions assignments)
+      let armName = "a" <> Text.pack (show index)
+      pure (Arm armName (Map.fromList (renderDimensions dimensions)) knobMap False)
+
+blockSlots :: OverheadPlan -> [Arm] -> Int -> [Slot]
+blockSlots plan shuffled blockNumber = zipWith make [0 ..] ordered
+  where
+    count = length shuffled
+    rotated = rotate ((blockNumber `div` 2) `mod` max 1 count) shuffled
+    ordered = if even blockNumber then rotated else reverse rotated
+    make position arm =
+      let rawKnobs = [(name, RawJson (toJSON value)) | (nameText, value) <- Map.toList arm.knobs, Right name <- [mkKnobName nameText]]
+          blockSeed = (plan.seed + fromIntegral blockNumber) `mod` 9007199254740992
+          seedValue = either (const Nothing) Just (mkSeed blockSeed)
+          base :: RunSpec
+          base = minimalRunSpec plan.scenarioId
+          runSpec =
+            RunSpec
+              base.runId
+              base.scenario
+              (Just plan.scenarioRevision)
+              rawKnobs
+              (Map.toList arm.dimensions)
+              seedValue
+              base.phases
+              base.timeoutSeconds
+              base.environment
+              base.cohortExpectation
+              (Just (ComparisonMembership plan.planId arm.armId blockNumber (blockNumber * count + position)))
+              (Map.fromList [("overhead", plan.planId), ("overhead-arm", arm.armId)])
+       in Slot blockNumber position arm runSpec
+
+executeOverhead :: OverheadHooks -> OverheadPlan -> FilePath -> IO OverheadState
+executeOverhead hooks plan output = do
+  createDirectoryIfMissing True (output </> "runs")
+  createDirectoryIfMissing True (output </> "comparisons")
+  loaded <- loadOverheadState output
+  initial <- case loaded of
+    Right state | state.plan.planId == plan.planId -> pure state
+    Right _ -> fail "state.json belongs to a different overhead plan"
+    Left _ -> pure (OverheadState plan [])
+  writeState output initial
+  runBlocks initial 0
+  where
+    shuffled = shuffle plan.seed plan.arms
+    maximumBlocks = plan.trials * 2
+    runBlocks state blockNumber
+      | validBlockCount state >= plan.trials = pure state
+      | blockNumber >= maximumBlocks = pure state
+      | otherwise = do
+          let slotsForBlock = blockSlots plan shuffled blockNumber
+          stateWithSlots <- ensureSlots output state slotsForBlock
+          finished <- foldM (runSlot hooks output plan) stateWithSlots slotsForBlock
+          runBlocks finished (blockNumber + 1)
+
+analyseOverhead :: OverheadHooks -> OverheadPolicy -> OverheadPlan -> OverheadState -> FilePath -> IO OverheadReport
+analyseOverhead hooks policy plan state output = do
+  createDirectoryIfMissing True (output </> "comparisons")
+  baseline <- maybe (fail "overhead plan has no baseline arm") pure (findArm plan.baselineArm plan.arms)
+  let valid = validBlockNumbers state
+      candidates = filter ((/= plan.baselineArm) . (.armId)) plan.arms
+  armRows <- traverse (armRow valid) plan.arms
+  let degradedArms = Map.fromList [(row.arm.armId, row.degraded) | row <- armRows]
+  comparisons <- traverse (compareArm degradedArms valid baseline) candidates
+  let overall = if length valid < 3 then VerdictInfrastructureFailure else worstVerdict (fmap (.verdict) comparisons)
+      report =
+        OverheadReport
+          plan.planId
+          plan.scenarioId
+          plan.mode
+          baseline.dimensions
+          plan.fixedDimensions
+          plan.fixedKnobs
+          plan.trials
+          plan.seed
+          (length valid)
+          [[slot.armId | slot <- sortOn (.position) state.slots, slot.block == blockNumber] | blockNumber <- sort valid]
+          armRows
+          comparisons
+          (concatMap (.findings) armRows)
+          (object ["name" .= policy.defaultPolicy.name, "sha256" .= sha256Hex (canonicalEncode (toJSON policy))])
+          overall
+  LazyByteString.writeFile (output </> "overhead-report.json") (encode report)
+  pure report
+  where
+    compareArm degradedArms valid baseline candidate
+      | length valid < 3 = pure (failedComparison candidate baseline VerdictInfrastructureFailure ["fewer than three valid blocks"])
+      | otherwise = do
+          let baselineDirs = mapMaybeRunDirs output state valid baseline.armId
+              candidateDirs = mapMaybeRunDirs output state valid candidate.armId
+              changes = [(name, from, to) | (name, from) <- Map.toList baseline.dimensions, Just to <- [Map.lookup name candidate.dimensions], from /= to]
+              transition = case changes of [change] -> Just change; _ -> Nothing
+              axes =
+                if candidate.control
+                  then VaryControl :| []
+                  else case changes of
+                    [] -> VaryControl :| []
+                    (name, _, _) : rest -> VaryDimension name :| [VaryDimension other | (other, _, _) <- rest]
+              selectedPolicy = if candidate.control then policy.controlPolicy else policyForTransition policy transition
+          compared <- hooks.compare selectedPolicy axes baselineDirs candidateDirs
+          case compared of
+            Left err -> pure (failedComparison candidate baseline VerdictInfrastructureFailure [Text.pack (show err)])
+            Right comparison -> do
+              let relative = "comparisons" </> Text.unpack candidate.armId <> "-vs-" <> Text.unpack baseline.armId <> ".json"
+                  candidateDegraded = Map.findWithDefault False candidate.armId degradedArms
+                  reportedVerdict
+                    | candidateDegraded && comparison.verdict `elem` [VerdictPass, VerdictRegression] = VerdictInconclusive
+                    | candidate.control && comparison.verdict /= VerdictPass = VerdictInconclusive
+                    | otherwise = comparison.verdict
+              LazyByteString.writeFile (output </> relative) (encode comparison)
+              pure
+                OverheadComparison
+                  { candidate = candidate.armId,
+                    baseline = baseline.armId,
+                    factor = maybe (if candidate.control then "control" else "multiple") (\(name, _, _) -> name) transition,
+                    fromValue = maybe "baseline" (\(_, value, _) -> value) transition,
+                    toValue = maybe (if candidate.control then "control" else "candidate") (\(_, _, value) -> value) transition,
+                    comparisonPath = Just relative,
+                    verdict = reportedVerdict,
+                    degraded = candidateDegraded,
+                    deltas = metricDeltas comparison,
+                    reasons = comparison.reasons
+                  }
+    armRow valid arm = do
+      let runIds = mapMaybeRunIds state valid arm.armId
+          directories = fmap (\runId -> output </> "runs" </> Text.unpack (renderRunId runId)) runIds
+      leak <- case (hooks.leakCheck, reverse directories) of (Just check, directory : _) -> check directory; _ -> pure Nothing
+      telemetryValues <- catMaybes <$> traverse readTelemetry directories
+      let findings = concatMap findingsFromTelemetry telemetryValues
+      pure (ArmEvidence arm runIds (aggregateTelemetry telemetryValues) leak findings (any degradedFinding findings))
+
+loadOverheadState :: FilePath -> IO (Either String OverheadState)
+loadOverheadState output = do
+  let path = output </> "state.json"
+  exists <- doesFileExist path
+  if exists then eitherDecodeFileStrict' path else pure (Left "state.json does not exist")
+
+overheadVerdictExitCode :: Verdict -> Int
+overheadVerdictExitCode = verdictExitCode
+
+ensureSlots :: FilePath -> OverheadState -> [Slot] -> IO OverheadState
+ensureSlots output state wanted = do
+  let existingKeys = [(slot.block, slot.position) | slot <- state.slots]
+      additions = [SlotRun slot.block slot.position slot.arm.armId [] Nothing False | slot <- wanted, (slot.block, slot.position) `notElem` existingKeys]
+      updated = setStateSlots (state.slots <> additions) state
+  writeState output updated
+  pure updated
+
+runSlot :: OverheadHooks -> FilePath -> OverheadPlan -> OverheadState -> Slot -> IO OverheadState
+runSlot hooks output plan state slot = case findSlotRun slot state.slots of
+  Just recorded | recorded.complete -> do
+    valid <- completedRunIsValid output recorded
+    if valid then pure state else attempt state 0
+  _ -> attempt state 0
+  where
+    attempt current retryNumber = do
+      runId <- newRunId
+      let spec = setRunId runId slot.spec
+          prepared = updateSlot slot (\old -> old {runIds = old.runIds <> [runId], exitCode = Nothing, complete = False}) current
+      writeState output prepared
+      result <- try (hooks.runChild spec (output </> "runs")) :: IO (Either SomeException ExitCode)
+      let code = either (const 4) exitNumber result
+          runDirectory = output </> "runs" </> Text.unpack (renderRunId runId)
+      complete <- if code == 0 then either (const False) (const True) <$> verifyManifest runDirectory else pure False
+      let finished = updateSlot slot (\old -> old {exitCode = Just code, complete}) prepared
+      writeState output finished
+      when (plan.settleSeconds > 0) (threadDelay (plan.settleSeconds * 1_000_000))
+      if code == 4 && retryNumber < plan.retries then attempt finished (retryNumber + 1) else pure finished
+
+writeState :: FilePath -> OverheadState -> IO ()
+writeState output state = do
+  let temporary = output </> "state.json.tmp"
+  LazyByteString.writeFile temporary (encode state)
+  renameFile temporary (output </> "state.json")
+
+validBlockCount :: OverheadState -> Int
+validBlockCount = length . validBlockNumbers
+
+validBlockNumbers :: OverheadState -> [Int]
+validBlockNumbers state =
+  [ blockNumber
+  | blockNumber <- nub (fmap (.block) state.slots),
+    let blockRuns = filter ((== blockNumber) . (.block)) state.slots,
+    length blockRuns == length state.plan.arms,
+    all (.complete) blockRuns
+  ]
+
+completedRunIsValid :: FilePath -> SlotRun -> IO Bool
+completedRunIsValid output recorded = case reverse recorded.runIds of
+  [] -> pure False
+  runId : _ -> either (const False) (const True) <$> verifyManifest (output </> "runs" </> Text.unpack (renderRunId runId))
+
+findSlotRun :: Slot -> [SlotRun] -> Maybe SlotRun
+findSlotRun slot = listToMaybe . filter (\recorded -> recorded.block == slot.block && recorded.position == slot.position)
+
+updateSlot :: Slot -> (SlotRun -> SlotRun) -> OverheadState -> OverheadState
+updateSlot slot change state = setStateSlots (fmap update state.slots) state
+  where
+    update current | current.block == slot.block && current.position == slot.position = change current
+    update current = current
+
+mapMaybeRunIds :: OverheadState -> [Int] -> Text -> [RunId]
+mapMaybeRunIds state blocks armName =
+  catMaybes
+    [ listToMaybe (reverse slot.runIds)
+    | blockNumber <- sort blocks,
+      slot <- state.slots,
+      slot.block == blockNumber,
+      slot.armId == armName,
+      slot.complete
+    ]
+
+mapMaybeRunDirs :: FilePath -> OverheadState -> [Int] -> Text -> [FilePath]
+mapMaybeRunDirs output state blocks armName = fmap (\runId -> output </> "runs" </> Text.unpack (renderRunId runId)) (mapMaybeRunIds state blocks armName)
+
+readTelemetry :: FilePath -> IO (Maybe Value)
+readTelemetry directory = do
+  decoded <- eitherDecodeFileStrict' (directory </> "run-result.json") :: IO (Either String Value)
+  pure (either (const Nothing) (valueAt ["summaries", "telemetry", "telemetry"]) decoded)
+
+aggregateTelemetry :: [Value] -> Value
+aggregateTelemetry [] = Null
+aggregateTelemetry values =
+  object
+    [ "runs" .= length values,
+      "spansEnded" .= sumNumbers ["pipeline", "spansEnded"],
+      "spansDropped" .= sumNumbers ["pipeline", "spansDropped"],
+      "exportFailures" .= sumNumbers ["pipeline", "spansExportFailed"],
+      "shutdownMsMax" .= maximumOrZero (mapMaybeNumber ["pipeline", "shutdown", "durationMs"] values),
+      "sinkSpansReceived" .= sumNumbers ["sink", "spansReceived"]
+    ]
+  where
+    sumNumbers path = sum (mapMaybeNumber path values)
+
+findingsFromTelemetry :: Value -> [Value]
+findingsFromTelemetry value = case valueAt ["findings"] value of Just (Array findings) -> toList findings; _ -> []
+
+degradedFinding :: Value -> Bool
+degradedFinding value = textAt ["status"] value == Just "fired" && textAt ["severity"] value == Just "degraded"
+
+valueAt :: [Text] -> Value -> Maybe Value
+valueAt [] value = Just value
+valueAt (key : rest) (Object values) = KeyMap.lookup (Key.fromText key) values >>= valueAt rest
+valueAt _ _ = Nothing
+
+textAt :: [Text] -> Value -> Maybe Text
+textAt path value = case valueAt path value of Just (String textValue) -> Just textValue; _ -> Nothing
+
+mapMaybeNumber :: [Text] -> [Value] -> [Double]
+mapMaybeNumber path = foldr (\value numbers -> maybe numbers (: numbers) (numberAt path value)) []
+
+numberAt :: [Text] -> Value -> Maybe Double
+numberAt path value = case valueAt path value of Just (Number number) -> Just (realToFrac number); _ -> Nothing
+
+maximumOrZero :: [Double] -> Double
+maximumOrZero [] = 0
+maximumOrZero values = maximum values
+
+toList :: (Foldable collection) => collection value -> [value]
+toList = foldr (:) []
+
+metricDeltas :: Comparison -> Value
+metricDeltas comparison =
+  toJSON
+    ( Map.map
+        (\metric -> object ["relative" .= (metric.ratio.estimate - 1), "ci95" .= [metric.ratio.low - 1, metric.ratio.high - 1], "absolute" .= metric.delta.estimate, "unit" .= metric.unit])
+        comparison.metrics
+    )
+
+setPlanSlots :: [Slot] -> OverheadPlan -> OverheadPlan
+setPlanSlots value plan = OverheadPlan plan.planId plan.scenarioId plan.scenarioRevision plan.mode plan.factorValues plan.baselineArm plan.arms plan.trials plan.seed plan.settleSeconds plan.retries plan.fixedDimensions plan.fixedKnobs value
+
+setStateSlots :: [SlotRun] -> OverheadState -> OverheadState
+setStateSlots value state = OverheadState state.plan value
+
+setRunId :: RunId -> RunSpec -> RunSpec
+setRunId value spec =
+  RunSpec
+    (Just value)
+    spec.scenario
+    spec.scenarioRevision
+    spec.knobs
+    spec.dimensions
+    spec.seed
+    spec.phases
+    spec.timeoutSeconds
+    spec.environment
+    spec.cohortExpectation
+    spec.comparison
+    spec.labels
+
+failedComparison :: Arm -> Arm -> Verdict -> [Text] -> OverheadComparison
+failedComparison candidate baseline verdict reasons =
+  OverheadComparison candidate.armId baseline.armId (if candidate.control then "control" else "unknown") "baseline" "candidate" Nothing verdict False (object []) reasons
+
+worstVerdict :: [Verdict] -> Verdict
+worstVerdict [] = VerdictInfrastructureFailure
+worstVerdict values = maximumByRank values
+  where
+    rank VerdictPass = 0 :: Int
+    rank VerdictInconclusive = 1
+    rank VerdictRegression = 2
+    rank VerdictInfrastructureFailure = 3
+    maximumByRank (first : rest) = foldl (\current candidate -> if rank candidate > rank current then candidate else current) first rest
+    maximumByRank [] = VerdictInfrastructureFailure
+
+findArm :: Text -> [Arm] -> Maybe Arm
+findArm name = listToMaybe . filter ((== name) . (.armId))
+
+shuffle :: Word64 -> [value] -> [value]
+shuffle seed values = fmap snd (sortOn fst (zip (randomWords (mkSMGen seed)) values))
+  where
+    randomWords generator = let (word, next) = nextWord64 generator in word : randomWords next
+
+rotate :: Int -> [value] -> [value]
+rotate _ [] = []
+rotate count values = drop offset values <> take offset values where offset = count `mod` length values
+
+cartesian :: [(Text, [Text])] -> [Map Text Text]
+cartesian [] = [Map.empty]
+cartesian ((name, values) : rest) = [Map.insert name value suffix | value <- values, suffix <- cartesian rest]
+
+whenEither :: Bool -> Text -> Either UsageError ()
+whenEither condition message = if condition then Left (UsageError message) else Right ()
+
+firstUsage :: (Show error) => Either error value -> Either UsageError value
+firstUsage = either (Left . UsageError . Text.pack . show) Right
+
+exitNumber :: ExitCode -> Int
+exitNumber ExitSuccess = 0
+exitNumber (ExitFailure code) = code
+
+traverse_ :: (a -> Either error b) -> [a] -> Either error ()
+traverse_ action = foldM (\() value -> action value >> pure ()) ()
diff --git a/src/Kenshou/Telemetry/Overhead/Policy.hs b/src/Kenshou/Telemetry/Overhead/Policy.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Overhead/Policy.hs
@@ -0,0 +1,50 @@
+module Kenshou.Telemetry.Overhead.Policy
+  ( OverheadPolicy (..),
+    decodeOverheadPolicy,
+    policyForTransition,
+  )
+where
+
+import Data.Aeson
+import Data.ByteString (ByteString)
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
+import Data.Text (Text)
+import Kenshou.Measure.Compare.Policy (MetricRule (..), Policy (..))
+
+data OverheadPolicy = OverheadPolicy
+  { maxDegradedSpanFraction :: Double,
+    defaultPolicy :: Policy,
+    controlPolicy :: Policy,
+    transitions :: Map Text Policy
+  }
+  deriving stock (Eq, Show)
+
+instance FromJSON OverheadPolicy where
+  parseJSON = withObject "OverheadPolicy" \value -> do
+    schema <- value .: "schema"
+    if schema /= ("kenshou.overhead-policy/v1" :: Text)
+      then fail "unsupported overhead policy"
+      else OverheadPolicy <$> value .: "maxDegradedSpanFraction" <*> value .: "default" <*> value .: "control" <*> value .:? "transitions" .!= Map.empty
+
+instance ToJSON OverheadPolicy where
+  toJSON value =
+    object
+      [ "schema" .= ("kenshou.overhead-policy/v1" :: Text),
+        "maxDegradedSpanFraction" .= value.maxDegradedSpanFraction,
+        "default" .= value.defaultPolicy,
+        "control" .= value.controlPolicy,
+        "transitions" .= value.transitions
+      ]
+
+decodeOverheadPolicy :: ByteString -> Either String OverheadPolicy
+decodeOverheadPolicy = eitherDecodeStrict'
+
+policyForTransition :: OverheadPolicy -> Maybe (Text, Text, Text) -> Policy
+policyForTransition policy transition = case transition of
+  Nothing -> policy.defaultPolicy
+  Just (factor, fromValue, toValue) -> case Map.lookup (factor <> ":" <> fromValue <> "->" <> toValue) policy.transitions of
+    Nothing -> policy.defaultPolicy
+    Just selected ->
+      let selectedMatches = fmap (.match) selected.metricRules
+       in selected {metricRules = selected.metricRules <> filter ((`notElem` selectedMatches) . (.match)) policy.defaultPolicy.metricRules}
diff --git a/src/Kenshou/Telemetry/Scrape.hs b/src/Kenshou/Telemetry/Scrape.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Scrape.hs
@@ -0,0 +1,301 @@
+module Kenshou.Telemetry.Scrape
+  ( EndpointSummary (..),
+    ScraperHandle (..),
+    startScraperInProcess,
+    runScraperInProcess,
+    runScraperRole,
+    SlotLeakResult (..),
+    wsSlotLeakProbe,
+  )
+where
+
+import Control.Concurrent.Async (Async, async, cancel, mapConcurrently_, waitCatch)
+import Control.Concurrent.MVar (MVar, newMVar, withMVar)
+import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)
+import Control.Monad (forM_, replicateM_, void)
+import Data.Aeson (FromJSON (..), ToJSON (..), encode, object, withObject, (.:), (.=))
+import Data.Aeson.Types (parseMaybe, (.!=), (.:?))
+import Data.ByteString.Lazy qualified as LazyByteString
+import Data.Foldable (traverse_)
+import Data.IORef
+import Data.List (sort)
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
+import Data.Text (Text)
+import Data.Text qualified as Text
+import GHC.Clock (getMonotonicTimeNSec)
+import Kenshou.Core.Role (ControlMessage (..), RoleContext (..), WorkerInit (..), WorkerMessage (..))
+import Kenshou.Measure.Clock (Origin (..), captureOrigin, sleepUntilNs)
+import Kenshou.Telemetry.Endpoint
+import Network.HTTP.Client (Manager, defaultManagerSettings, httpLbs, newManager, parseRequest, responseBody, responseStatus)
+import Network.HTTP.Types.Status (statusCode)
+import Network.WebSockets qualified as WebSockets
+import System.Directory (createDirectoryIfMissing)
+import System.FilePath ((</>))
+import System.IO (BufferMode (LineBuffering), Handle, IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)
+import Text.Read (readMaybe)
+
+data ScrapeSample = ScrapeSample
+  { status :: Int,
+    latencyNs :: Integer,
+    bodyBytes :: Integer,
+    skipped :: Int,
+    errorMessage :: Maybe Text
+  }
+
+data EndpointSummary = EndpointSummary
+  { endpoint :: Text,
+    kind :: EndpointKind,
+    scrapes :: Int,
+    failures :: Int,
+    skippedTicks :: Int,
+    latencyP50Ns :: Integer,
+    latencyP99Ns :: Integer,
+    latencyMaxNs :: Integer,
+    meanBodyBytes :: Double,
+    maxBodyBytes :: Integer
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON EndpointSummary where
+  toJSON summary =
+    object
+      [ "endpoint" .= summary.endpoint,
+        "kind" .= summary.kind,
+        "scrapes" .= summary.scrapes,
+        "failures" .= summary.failures,
+        "skippedTicks" .= summary.skippedTicks,
+        "latencyNs" .= object ["p50" .= summary.latencyP50Ns, "p99" .= summary.latencyP99Ns, "max" .= summary.latencyMaxNs],
+        "bodyBytes" .= object ["mean" .= summary.meanBodyBytes, "max" .= summary.maxBodyBytes]
+      ]
+
+instance FromJSON EndpointSummary where
+  parseJSON = withObject "EndpointSummary" \value -> do
+    latency <- value .: "latencyNs"
+    body <- value .: "bodyBytes"
+    EndpointSummary
+      <$> value .: "endpoint"
+      <*> value .: "kind"
+      <*> value .: "scrapes"
+      <*> value .: "failures"
+      <*> value .: "skippedTicks"
+      <*> latency .: "p50"
+      <*> latency .: "p99"
+      <*> latency .: "max"
+      <*> body .: "mean"
+      <*> body .: "max"
+
+data ScraperHandle = ScraperHandle
+  { register :: Endpoint -> IO (),
+    finish :: IO [EndpointSummary]
+  }
+
+data ScraperState = ScraperState
+  { manager :: Manager,
+    outputDir :: FilePath,
+    intervalNs :: Integer,
+    wsSubscribers :: Int,
+    workers :: IORef [(Endpoint, Async ())],
+    endpoints :: IORef [Endpoint],
+    samples :: IORef (Map Text [ScrapeSample])
+  }
+
+startScraperInProcess :: FilePath -> Int -> Int -> IO ScraperHandle
+startScraperInProcess outputDir intervalMs wsSubscribers = do
+  createDirectoryIfMissing True (outputDir </> "series")
+  manager <- newManager defaultManagerSettings
+  workers <- newIORef []
+  endpoints <- newIORef []
+  samples <- newIORef Map.empty
+  let state = ScraperState manager outputDir (fromIntegral intervalMs * 1_000_000) wsSubscribers workers endpoints samples
+  pure (ScraperHandle (registerEndpoint state) (finishScraper state))
+
+runScraperInProcess :: FilePath -> Int -> Int -> [Endpoint] -> IO (IO [EndpointSummary])
+runScraperInProcess outputDir intervalMs wsSubscribers endpoints = do
+  scraper <- startScraperInProcess outputDir intervalMs wsSubscribers
+  traverse_ scraper.register endpoints
+  pure scraper.finish
+
+runScraperRole :: RoleContext -> IO ()
+runScraperRole context = do
+  let (intervalMs, subscribers) = maybe (15_000, 0) id (parseMaybe parseConfig context.init.args)
+  scraper <- startScraperInProcess context.init.outDir intervalMs subscribers
+  context.send WrkReady
+  loop scraper
+  where
+    loop scraper =
+      context.receive >>= \case
+        Just (CtlCustom "endpoint" payload) -> case parseMaybe parseJSON payload of
+          Nothing -> context.send (WrkError "invalid scraper endpoint") >> loop scraper
+          Just endpoint -> scraper.register endpoint >> loop scraper
+        Just (CtlCustom "finish" _) -> scraper.finish >>= context.send . WrkCustom "scrape-summary" . toJSON >> loop scraper
+        Just (CtlStop _) -> void scraper.finish
+        Just _ -> loop scraper
+        Nothing -> void scraper.finish
+    parseConfig = withObject "scraper config" (\value -> (,) <$> value .: "intervalMs" <*> value .:? "wsSubscribers" .!= 0)
+
+registerEndpoint :: ScraperState -> Endpoint -> IO ()
+registerEndpoint state endpoint = do
+  modifyIORef' state.endpoints (<> [endpoint])
+  if endpoint.kind == WebSocketPush
+    then do
+      modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [])
+      if state.wsSubscribers <= 0
+        then pure ()
+        else do
+          worker <- async (websocketSubscribers state endpoint)
+          modifyIORef' state.workers (<> [(endpoint, worker)])
+    else do
+      worker <- async (scrapeLoop state endpoint)
+      modifyIORef' state.workers (<> [(endpoint, worker)])
+
+scrapeLoop :: ScraperState -> Endpoint -> IO ()
+scrapeLoop state endpoint = do
+  request <- parseRequest (Text.unpack endpoint.url)
+  handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> ".csv")) WriteMode
+  hSetBuffering handle LineBuffering
+  hPutStrLn handle "t_wall_ns,endpoint,kind,status,latency_ns,body_bytes,skipped,error"
+  started <- getMonotonicTimeNSec
+  let loop tick = do
+        sleepUntilNs (started + fromIntegral tick * fromIntegral state.intervalNs)
+        origin <- captureOrigin
+        callStarted <- getMonotonicTimeNSec
+        result <- trySync (httpLbs request state.manager)
+        callEnded <- getMonotonicTimeNSec
+        let elapsed = fromIntegral (callEnded - callStarted)
+            behind = max 0 (fromIntegral (callEnded - started) `div` state.intervalNs - tick)
+            sample = case result of
+              Left exception -> ScrapeSample 0 elapsed 0 (fromIntegral behind) (Just (Text.pack (displayException exception)))
+              Right response -> ScrapeSample (statusCode (responseStatus response)) elapsed (fromIntegral (LazyByteString.length (responseBody response))) (fromIntegral behind) Nothing
+        modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [sample])
+        hPutStrLn handle (csvRow origin endpoint sample)
+        loop (tick + 1 + behind)
+  loop 1 `finally` hClose handle
+
+websocketSubscribers :: ScraperState -> Endpoint -> IO ()
+websocketSubscribers state endpoint = do
+  (host, port, path) <- either (ioError . userError . Text.unpack) pure (parseWebSocketUrl endpoint.url)
+  handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> "-ws.csv")) WriteMode
+  hSetBuffering handle LineBuffering
+  hPutStrLn handle "t_wall_ns,subscriber,event,frames,bytes,gap_ns,error"
+  lock <- newMVar ()
+  mapConcurrently_ (subscriber host port path handle lock) [1 .. state.wsSubscribers] `finally` hClose handle
+  where
+    subscriber host port path handle lock subscriberId = do
+      outcome <- trySync $ WebSockets.runClient (Text.unpack host) port (Text.unpack path) \connection -> do
+        traverse_ (WebSockets.sendTextData connection . encode) endpoint.wsHello
+        writeWsRow handle lock subscriberId "connect" 0 0 0 Nothing
+        previous <- newIORef Nothing
+        let loop frames = do
+              payload <- WebSockets.receiveData connection :: IO LazyByteString.ByteString
+              now <- getMonotonicTimeNSec
+              prior <- atomicModifyIORef' previous (\value -> (Just now, value))
+              writeWsRow handle lock subscriberId "frame" frames (fromIntegral (LazyByteString.length payload)) (maybe 0 (fromIntegral . (now -)) prior) Nothing
+              loop (frames + 1)
+        loop 1
+      case outcome of
+        Left exception -> writeWsRow handle lock subscriberId "error" 0 0 0 (Just (Text.pack (displayException exception)))
+        Right () -> writeWsRow handle lock subscriberId "close" 0 0 0 Nothing
+
+writeWsRow :: Handle -> MVar () -> Int -> Text -> Int -> Integer -> Integer -> Maybe Text -> IO ()
+writeWsRow handle lock subscriberId event frames bytes gap errorMessage = withMVar lock \_ -> do
+  origin <- captureOrigin
+  hPutStrLn handle . Text.unpack . Text.intercalate "," $
+    [ Text.pack (show origin.wallUnixNs),
+      Text.pack (show subscriberId),
+      event,
+      Text.pack (show frames),
+      Text.pack (show bytes),
+      Text.pack (show gap),
+      maybe "" (Text.replace "," ";" . Text.replace "\n" " ") errorMessage
+    ]
+
+finishScraper :: ScraperState -> IO [EndpointSummary]
+finishScraper state = do
+  workers <- readIORef state.workers
+  forM_ workers (cancel . snd)
+  forM_ workers (void . waitCatch . snd)
+  captured <- readIORef state.samples
+  endpoints <- readIORef state.endpoints
+  pure [summarize endpoint (Map.findWithDefault [] endpoint.name captured) | endpoint <- endpoints]
+
+summarize :: Endpoint -> [ScrapeSample] -> EndpointSummary
+summarize endpoint samples =
+  EndpointSummary
+    { endpoint = endpoint.name,
+      kind = endpoint.kind,
+      scrapes = length samples,
+      failures = length [() | sample <- samples, sample.status < 200 || sample.status >= 300 || sample.errorMessage /= Nothing],
+      skippedTicks = sum (fmap (.skipped) samples),
+      latencyP50Ns = quantile 0.50 latencies,
+      latencyP99Ns = quantile 0.99 latencies,
+      latencyMaxNs = maybe 0 last nonEmptyLatencies,
+      meanBodyBytes = if null samples then 0 else fromIntegral (sum (fmap (.bodyBytes) samples)) / fromIntegral (length samples),
+      maxBodyBytes = maybe 0 last nonEmptyBodies
+    }
+  where
+    latencies = sort (fmap (.latencyNs) samples)
+    bodies = sort (fmap (.bodyBytes) samples)
+    nonEmptyLatencies = if null latencies then Nothing else Just latencies
+    nonEmptyBodies = if null bodies then Nothing else Just bodies
+
+quantile :: Double -> [Integer] -> Integer
+quantile _ [] = 0
+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))
+
+data SlotLeakResult = SlotLeakResult
+  { attempted :: Int,
+    freshAccepted :: Bool
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON SlotLeakResult where
+  toJSON result = object ["attempted" .= result.attempted, "freshAccepted" .= result.freshAccepted]
+
+wsSlotLeakProbe :: Endpoint -> Int -> IO SlotLeakResult
+wsSlotLeakProbe endpoint attempts = case parseWebSocketUrl endpoint.url of
+  Left _ -> pure (SlotLeakResult attempts False)
+  Right (host, port, path) -> do
+    replicateM_ attempts (void (trySync (connect host port path)))
+    fresh <- trySync (connect host port path)
+    pure (SlotLeakResult attempts (either (const False) (const True) fresh))
+  where
+    connect host port path = WebSockets.runClient (Text.unpack host) port (Text.unpack path) (const (pure ()))
+
+parseWebSocketUrl :: Text -> Either Text (Text, Int, Text)
+parseWebSocketUrl url = do
+  remainder <- maybe (Left "WebSocket URL must start with ws://") Right (Text.stripPrefix "ws://" url)
+  let (authority, rawPath) = Text.breakOn "/" remainder
+      path = if Text.null rawPath then "/" else rawPath
+      pieces = Text.splitOn ":" authority
+  case pieces of
+    [host] -> Right (host, 80, path)
+    [host, rawPort] -> maybe (Left "WebSocket URL has an invalid port") (\port -> Right (host, port, path)) (readMaybe (Text.unpack rawPort))
+    _ -> Left "WebSocket URL has an invalid authority"
+
+csvRow :: Origin -> Endpoint -> ScrapeSample -> String
+csvRow origin endpoint sample =
+  Text.unpack . Text.intercalate "," $
+    [ Text.pack (show origin.wallUnixNs),
+      endpoint.name,
+      Text.pack (show endpoint.kind),
+      Text.pack (show sample.status),
+      Text.pack (show sample.latencyNs),
+      Text.pack (show sample.bodyBytes),
+      Text.pack (show sample.skipped),
+      maybe "" sanitize sample.errorMessage
+    ]
+  where
+    sanitize = Text.replace "," ";" . Text.replace "\n" " "
+
+tryAny :: IO value -> IO (Either SomeException value)
+tryAny = try
+
+trySync :: IO value -> IO (Either SomeException value)
+trySync action = do
+  outcome <- tryAny action
+  case outcome of
+    Left exception -> case fromException exception :: Maybe SomeAsyncException of
+      Just _ -> throwIO exception
+      Nothing -> pure outcome
+    Right _ -> pure outcome
diff --git a/src/Kenshou/Telemetry/SelfTest.hs b/src/Kenshou/Telemetry/SelfTest.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/SelfTest.hs
@@ -0,0 +1,28 @@
+module Kenshou.Telemetry.SelfTest (selfTestBundle) where
+
+import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)
+import Kenshou.Core.Id (selftestLayer)
+import Kenshou.Core.Role (WorkerRole (..), mkRoleName)
+import Kenshou.Telemetry.Scrape (runScraperRole)
+import Kenshou.Telemetry.SelfTest.Arms (armsScenario)
+import Kenshou.Telemetry.SelfTest.Problems (slowExporterScenario, traceContinuityScenario)
+import Kenshou.Telemetry.Sink (runSinkRole)
+
+selfTestBundle :: LayerBundle
+selfTestBundle = mkBundle selftestLayer [armsScenario, traceContinuityScenario, slowExporterScenario] [sinkRole, scraperRole]
+
+sinkRole :: WorkerRole
+sinkRole =
+  WorkerRole
+    { name = either (error . show) id (mkRoleName "selftest/telemetry-otlp-sink"),
+      summary = "Receives and counts OTLP payloads for telemetry scenarios.",
+      run = runSinkRole
+    }
+
+scraperRole :: WorkerRole
+scraperRole =
+  WorkerRole
+    { name = either (error . show) id (mkRoleName "selftest/telemetry-scraper"),
+      summary = "Scrapes telemetry endpoints on a fixed schedule.",
+      run = runScraperRole
+    }
diff --git a/src/Kenshou/Telemetry/SelfTest/Arms.hs b/src/Kenshou/Telemetry/SelfTest/Arms.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/SelfTest/Arms.hs
@@ -0,0 +1,78 @@
+module Kenshou.Telemetry.SelfTest.Arms (armsScenario) where
+
+import Data.Int (Int64)
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Kenshou.Core.Context (RunContext (..))
+import Kenshou.Core.Dimension (allTelemetryArms, noDimensions)
+import Kenshou.Core.Env (noEnvironment)
+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)
+import Kenshou.Core.Knob
+import Kenshou.Core.Phase qualified as Core
+import Kenshou.Core.Scenario
+import Kenshou.Measure.Clock (Nanos (..))
+import Kenshou.Measure.Health (HealthConfig (..))
+import Kenshou.Measure.Knobs
+import Kenshou.Measure.Load
+import Kenshou.Measure.Phase qualified as Measure
+import Kenshou.Measure.Recorder (OpName (..))
+import Kenshou.Measure.Session
+import Kenshou.Telemetry
+import Kenshou.Telemetry.SelfTest.Endpoints (withSyntheticEndpoints)
+import Kenshou.Telemetry.SelfTest.Service
+
+armsScenario :: Scenario
+armsScenario =
+  Scenario
+    { id = either (error . Text.unpack) id (parseScenarioId "selftest/telemetry/benchmark/arms-on-synthetic-service"),
+      revision = 1,
+      summary = "Measures one synthetic service under every tracing and metrics arm.",
+      tier = TierSmoke,
+      placement = PlaceEither,
+      knobs = loadKnobs defaults <> measureKnobs Benchmark <> workKnobs <> telemetryKnobs,
+      dimensions = allTelemetryArms noDimensions,
+      phases = Core.PhasePlan 0.1 10 0.1,
+      requires = noEnvironment,
+      knownDefect = Nothing,
+      run = runArms
+    }
+  where
+    defaults = defaultLoadDefaults {model = "closed", workers = 4, thinkTimeUs = 0}
+
+runArms :: RunContext -> IO ScenarioReport
+runArms context = case (telemetrySpecFromContext context, loadModelFromKnobs context.knobs, measureConfigFromKnobs context phases) of
+  (Left message, _, _) -> pure (failedWith ["invalid-telemetry-config"] message)
+  (_, Left message, _) -> pure (failedWith ["invalid-load-config"] message)
+  (_, _, Left message) -> pure (failedWith ["invalid-measure-config"] message)
+  (Right telemetrySpec, Right loadModel, Right measureConfig) ->
+    withTelemetry telemetrySpec \telemetry -> do
+      withSyntheticEndpoints telemetry.servesEndpoints telemetry.registerEndpoint do
+        syntheticMetrics <- traverse newSyntheticMetrics telemetry.meter
+        let spanCount = integer "work.spans-per-op"
+            attributeCount = integer "work.attributes-per-span"
+            cpuMicros = integer "work.cpu-micros"
+            operation = Operation (OpName "synthetic") (\_ _ -> runSyntheticOperation telemetry.tracer syntheticMetrics spanCount attributeCount cpuMicros)
+        -- This closed-loop benchmark intentionally saturates the measured
+        -- process when work.cpu-micros is zero. That CPU is the subject under
+        -- test, rather than an overloaded open-loop driver.
+        let closedLoopConfig = measureConfig {healthConfig = measureConfig.healthConfig {cpuSoft = 1.1, cpuHard = 1.2}}
+        (_, measurement) <- withMeasurement context closedLoopConfig (\session -> runLoad session loadModel operation)
+        pure (passed {outcome = measuredOutcome measurement passed.outcome})
+  where
+    duration = fromIntegral (knobInt context.knobs (name "load.duration-seconds")) * 1_000_000_000
+    phases = Measure.PhasePlan (Nanos 100_000_000) (Measure.SteadyFor (Nanos duration)) (Nanos 100_000_000)
+    integer key = fromIntegral (knobInt context.knobs (name key))
+
+workKnobs :: [KnobSpec]
+workKnobs =
+  [ intKnob "work.cpu-micros" "Synthetic CPU work per operation" 1000 0 100000,
+    intKnob "work.spans-per-op" "Spans emitted per operation" 2 1 16,
+    intKnob "work.attributes-per-span" "Attributes attached to every span" 6 0 64,
+    intKnob "load.duration-seconds" "Steady measurement duration" 10 1 3600
+  ]
+
+intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec
+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []
+
+name :: Text -> KnobName
+name = either (error . Text.unpack) id . mkKnobName
diff --git a/src/Kenshou/Telemetry/SelfTest/Endpoints.hs b/src/Kenshou/Telemetry/SelfTest/Endpoints.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/SelfTest/Endpoints.hs
@@ -0,0 +1,41 @@
+module Kenshou.Telemetry.SelfTest.Endpoints (withSyntheticEndpoints) where
+
+import Control.Concurrent (threadDelay)
+import Control.Concurrent.Async (async, cancel, waitCatch)
+import Control.Exception (bracket)
+import Control.Monad (forever, void)
+import Data.Aeson (encode, object, (.=))
+import Data.Text qualified as Text
+import Kenshou.Telemetry.Endpoint
+import Network.HTTP.Types (status200, status404)
+import Network.Socket (close)
+import Network.Wai (Application, pathInfo, responseLBS)
+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)
+import Network.Wai.Handler.WebSockets (websocketsOr)
+import Network.WebSockets qualified as WebSockets
+
+withSyntheticEndpoints :: Bool -> (Endpoint -> IO ()) -> IO value -> IO value
+withSyntheticEndpoints False _ action = action
+withSyntheticEndpoints True register action = bracket acquire release \(port, _) -> do
+  let base = Text.pack ("127.0.0.1:" <> show port)
+  register (Endpoint "synthetic-json" JsonDocument ("http://" <> base <> "/metrics") Nothing)
+  register (Endpoint "synthetic-push" WebSocketPush ("ws://" <> base <> "/ws/metrics") Nothing)
+  action
+  where
+    acquire = do
+      (port, socket) <- openFreePort
+      server <- async (runSettingsSocket defaultSettings socket application)
+      pure (port, (server, socket))
+    release (_, (server, socket)) = cancel server >> void (waitCatch server) >> close socket
+
+application :: Application
+application = websocketsOr WebSockets.defaultConnectionOptions websocket http
+  where
+    http request respond
+      | pathInfo request == ["metrics"] = respond (responseLBS status200 [("Content-Type", "application/json")] (encode (object ["operations" .= (1 :: Int)])))
+      | otherwise = respond (responseLBS status404 [] "not found")
+    websocket pending = do
+      connection <- WebSockets.acceptRequest pending
+      forever do
+        WebSockets.sendTextData connection (encode (object ["operations" .= (1 :: Int)]))
+        threadDelay 100_000
diff --git a/src/Kenshou/Telemetry/SelfTest/Problems.hs b/src/Kenshou/Telemetry/SelfTest/Problems.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/SelfTest/Problems.hs
@@ -0,0 +1,146 @@
+module Kenshou.Telemetry.SelfTest.Problems
+  ( traceContinuityScenario,
+    slowExporterScenario,
+  )
+where
+
+import Control.Concurrent (threadDelay)
+import Control.Monad (forM_, replicateM_, void)
+import Data.Aeson (object, (.=))
+import Data.Int (Int64)
+import Data.List.NonEmpty (NonEmpty (..))
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Kenshou.Core.Context (RunContext (..), SummarySection (Verdicts), putSummary)
+import Kenshou.Core.Dimension
+import Kenshou.Core.Dimension qualified as Dimension
+import Kenshou.Core.Env (noEnvironment)
+import Kenshou.Core.Id (ScenarioId, parseScenarioId)
+import Kenshou.Core.Knob
+import Kenshou.Core.Phase (PhasePlan (..))
+import Kenshou.Core.Scenario
+import Kenshou.Telemetry
+import Kenshou.Telemetry.Compose
+import Kenshou.Telemetry.Continuity
+import Kenshou.Telemetry.Detect
+import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..), snapshotPipeline)
+import Kenshou.Telemetry.Tracing.Probe (SpanView (..), readSpans)
+import OpenTelemetry.Attributes (Attribute (AttributeValue), PrimitiveAttribute (TextAttribute), lookupAttribute)
+import OpenTelemetry.Trace.Core (Span, addAttribute, defaultSpanArguments, inSpan, inSpan')
+import System.Timeout (timeout)
+
+traceContinuityScenario :: Scenario
+traceContinuityScenario =
+  Scenario
+    { id = scenarioId "selftest/telemetry/correctness/trace-continuity",
+      revision = 1,
+      summary = "Proves trace discontinuity and context-leak checks are non-vacuous.",
+      tier = TierSmoke,
+      placement = PlaceEither,
+      knobs = [intKnob "continuity.messages" "Synthetic messages in each continuity case" 1000 10 100000] <> telemetryKnobs,
+      dimensions = onlyArms TracingSdkInMemory MetricsOff,
+      phases = PhasePlan 0 0 0,
+      requires = noEnvironment,
+      knownDefect = Nothing,
+      run = runTraceContinuity
+    }
+
+slowExporterScenario :: Scenario
+slowExporterScenario =
+  Scenario
+    { id = scenarioId "selftest/telemetry/concurrency/slow-exporter-backpressure",
+      revision = 1,
+      summary = "Proves slow exporters and handlers are detected without blocking the emitter.",
+      tier = TierSmoke,
+      placement = PlaceEither,
+      knobs =
+        [ intKnob "backpressure.phase-seconds" "Maximum synthetic phase duration" 8 2 600,
+          intKnob "handler.delay-micros" "Injected handler delay" 5000 0 1000000
+        ]
+          <> telemetryKnobs,
+      dimensions = onlyArms TracingSdkOtlp MetricsOff,
+      phases = PhasePlan 0 0 0,
+      requires = noEnvironment,
+      knownDefect = Nothing,
+      run = runSlowExporter
+    }
+
+runTraceContinuity :: RunContext -> IO ScenarioReport
+runTraceContinuity context = case telemetrySpecFromContext context of
+  Left message -> pure (failedWith ["invalid-telemetry-config"] message)
+  Right spec -> withTelemetry spec \telemetry -> case telemetry.tracer of
+    Nothing -> pure (failedWith ["missing-tracer"] "sdk-inmemory did not provide a tracer")
+    Just tracer -> do
+      let messages = fromIntegral (knobInt context.knobs (name "continuity.messages"))
+      forM_ [1 .. messages] \index ->
+        inSpan' tracer "intact.producer" defaultSpanArguments \producer -> do
+          addMessageId producer ("intact-" <> tshow index)
+          inSpan' tracer "intact.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("intact-" <> tshow index)
+      forM_ [10, 20 .. messages] \index -> do
+        inSpan' tracer "broken.producer" defaultSpanArguments \producer -> addMessageId producer ("broken-" <> tshow index)
+        inSpan' tracer "broken.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("broken-" <> tshow index)
+      inSpan tracer "leak.carrier" defaultSpanArguments do
+        forM_ [10, 20 .. messages] \index ->
+          inSpan' tracer "leaky.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("leaky-" <> tshow index)
+      void telemetry.flushTelemetry
+      spans <- maybe (pure []) readSpans telemetry.spans
+      let correlate spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing
+          intact = checkContinuity (named "intact.producer") (named "intact.consumer") correlate spans
+          broken = checkContinuity (named "broken.producer") (named "broken.consumer") correlate spans
+          isolated = checkIsolation (named "leaky.consumer") (const True) spans
+          expected = messages `div` 10
+          ok = intact.violations == 0 && intact.consumers == messages && broken.violations == expected && isolated.violations == expected
+      telemetry.recordContinuity intact (IsolationResult 0 0 [])
+      telemetry.recordContinuity broken isolated
+      putSummary context Verdicts "trace-continuity" (object ["intact" .= intact, "broken" .= broken, "leaky" .= isolated, "expectedInjected" .= expected])
+      pure (if ok then passed else failedWith ["trace-detector-non-vacuity"] "continuity detectors did not identify exactly the injected defects")
+
+runSlowExporter :: RunContext -> IO ScenarioReport
+runSlowExporter context = case telemetrySpecFromContext context of
+  Left message -> pure (failedWith ["invalid-telemetry-config"] message)
+  Right spec -> withTelemetry (spec {processor = BatchProcessor 256 200 128 2000, shutdownMs = 1000}) \telemetry -> case (telemetry.tracer, telemetry.pipeline) of
+    (Just tracer, Just pipeline) -> do
+      let delayMicros = fromIntegral (knobInt context.knobs (name "handler.delay-micros"))
+          phaseMicros = fromIntegral (knobInt context.knobs (name "backpressure.phase-seconds")) * 1_000_000
+      synchronousStats <- newHandlerStats
+      replicateM_ 20 (timedHandler synchronousStats "synchronous" (slowHandler delayMicros) ())
+      synchronous <- snapshotHandlerStats "synchronous" synchronousStats
+      asyncStats <- newHandlerStats
+      (submit, asyncSnapshot) <- asyncHandler 64 (slowHandler delayMicros)
+      completed <- timeout phaseMicros (replicateM_ 10000 (timedHandler asyncStats "async-submit" submit ()))
+      threadDelay (min 250_000 phaseMicros)
+      asynchronous <- snapshotHandlerStats "async-submit" asyncStats
+      asyncQueue <- asyncSnapshot
+      telemetry.recordHandlerStats synchronous
+      telemetry.recordHandlerStats asynchronous
+      telemetry.setSinkFault SinkHang
+      emitted <- timeout phaseMicros (replicateM_ 10000 (inSpan tracer "backpressure" defaultSpanArguments (pure ())))
+      snapshot <- snapshotPipeline pipeline
+      let syncFired = (handlerFinding synchronous).status == Fired
+          asyncQuiet = (handlerFinding asynchronous).status == Quiet
+          ok = completed /= Nothing && emitted /= Nothing && syncFired && asyncQuiet && asyncQueue.dropped > 0 && snapshot.spansDropped > 0
+      putSummary context Verdicts "slow-exporter-backpressure" (object ["synchronous" .= synchronous, "asynchronous" .= asynchronous, "asyncQueue" .= asyncQueue, "pipeline" .= snapshot, "emitterCompleted" .= (emitted /= Nothing)])
+      pure (if ok then passed else failedWith ["backpressure-detector-non-vacuity"] "slow exporter or handler detector did not fire as expected")
+    _ -> pure (failedWith ["missing-sdk"] "sdk-otlp did not provide a tracer and pipeline")
+
+onlyArms :: TracingArm -> MetricsArm -> DimensionSupport
+onlyArms tracing metrics =
+  DimensionSupport (Supported (Support (tracing :| []) tracing)) (Supported (Support (metrics :| []) metrics)) Dimension.NotApplicable Dimension.NotApplicable
+
+named :: Text -> SpanSelector
+named spanName = SpanSelector (Just spanName) Nothing []
+
+addMessageId :: Span -> Text -> IO ()
+addMessageId spanValue value = addAttribute spanValue "message.id" value
+
+scenarioId :: Text -> ScenarioId
+scenarioId text = either (error . Text.unpack) id (parseScenarioId text)
+
+intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec
+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []
+
+name :: Text -> KnobName
+name = either (error . Text.unpack) id . mkKnobName
+
+tshow :: (Show value) => value -> Text
+tshow = Text.pack . show
diff --git a/src/Kenshou/Telemetry/SelfTest/Service.hs b/src/Kenshou/Telemetry/SelfTest/Service.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/SelfTest/Service.hs
@@ -0,0 +1,55 @@
+module Kenshou.Telemetry.SelfTest.Service
+  ( SyntheticMetrics,
+    newSyntheticMetrics,
+    runSyntheticOperation,
+  )
+where
+
+import Control.Concurrent (threadDelay)
+import Control.Exception (evaluate)
+import Control.Monad (forM_)
+import Data.Bits (xor)
+import Data.Int (Int64)
+import Data.Text qualified as Text
+import Kenshou.Measure.Recorder (OpResult (OpOk))
+import OpenTelemetry.Attributes (emptyAttributes)
+import OpenTelemetry.Metric.Core (Counter (..), Histogram (..), Meter (..), defaultAdvisoryParameters)
+import OpenTelemetry.Trace.Core (Tracer, addAttribute, defaultSpanArguments, inSpan')
+
+data SyntheticMetrics = SyntheticMetrics (Counter Int64) Histogram
+
+newSyntheticMetrics :: Meter -> IO SyntheticMetrics
+newSyntheticMetrics meter =
+  SyntheticMetrics
+    <$> meter.meterCreateCounterInt64 "synthetic.operations" (Just "operations") Nothing defaultAdvisoryParameters
+    <*> meter.meterCreateHistogram "synthetic.work.duration" (Just "us") Nothing defaultAdvisoryParameters
+
+runSyntheticOperation :: Maybe Tracer -> Maybe SyntheticMetrics -> Int -> Int -> Int -> IO OpResult
+runSyntheticOperation tracer metrics spanCount attributeCount cpuMicros = do
+  withSpans tracer spanCount attributeCount do
+    _ <- evaluate (burnCpu (max 1 (cpuMicros * 40)) 0x9e3779b9)
+    threadDelay cpuMicros
+    pure ()
+  case metrics of
+    Nothing -> pure ()
+    Just (SyntheticMetrics counter histogram) -> do
+      counter.counterAdd 1 emptyAttributes
+      histogram.histogramRecord (fromIntegral cpuMicros) emptyAttributes
+  pure (OpOk 1)
+
+withSpans :: Maybe Tracer -> Int -> Int -> IO value -> IO value
+withSpans Nothing _ _ action = action
+withSpans (Just tracer) count attributeCount action = go count
+  where
+    go remaining
+      | remaining <= 0 = action
+      | otherwise =
+          inSpan' tracer ("synthetic-" <> Text.pack (show remaining)) defaultSpanArguments \spanValue -> do
+            forM_ [1 .. attributeCount] \index -> addAttribute spanValue ("work.attribute." <> Text.pack (show index)) index
+            go (remaining - 1)
+
+burnCpu :: Int -> Int -> Int
+burnCpu 0 accumulator = accumulator
+burnCpu remaining accumulator =
+  let mixed = (accumulator * 1664525 + 1013904223) `xor` remaining
+   in burnCpu (remaining - 1) mixed
diff --git a/src/Kenshou/Telemetry/Sink.hs b/src/Kenshou/Telemetry/Sink.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Sink.hs
@@ -0,0 +1,111 @@
+module Kenshou.Telemetry.Sink
+  ( SinkStats (..),
+    SinkHandle (..),
+    sinkApplication,
+    withSink,
+    runSinkRole,
+  )
+where
+
+import Codec.Compression.GZip qualified as GZip
+import Control.Concurrent (threadDelay)
+import Control.Concurrent.Async (async, cancel, waitCatch)
+import Control.Exception (bracket)
+import Control.Monad (void)
+import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), object, withObject, (.:), (.=))
+import Data.Aeson.Types (parseMaybe)
+import Data.ByteString.Lazy qualified as LazyByteString
+import Data.IORef
+import Data.ProtoLens (decodeMessage)
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Data.Vector qualified as Vector
+import Kenshou.Core.Role (ControlMessage (..), RoleContext (..), WorkerInit (..), WorkerMessage (..))
+import Kenshou.Telemetry.Spec (SinkFault (..), renderSinkFault)
+import Lens.Micro ((^.))
+import Network.HTTP.Types (status200, status404, status503)
+import Network.Socket (close)
+import Network.Wai (Application, Request (..), responseLBS, strictRequestBody)
+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)
+import Proto.Opentelemetry.Proto.Collector.Trace.V1.TraceService (ExportTraceServiceRequest)
+import Proto.Opentelemetry.Proto.Collector.Trace.V1.TraceService_Fields qualified as TraceService
+import Proto.Opentelemetry.Proto.Trace.V1.Trace_Fields qualified as Trace
+
+data SinkStats = SinkStats {requests :: Int, bytes :: Integer, spansReceived :: Int}
+  deriving stock (Eq, Show)
+
+instance ToJSON SinkStats where
+  toJSON stats = object ["requests" .= stats.requests, "bytes" .= stats.bytes, "spansReceived" .= stats.spansReceived]
+
+instance FromJSON SinkStats where
+  parseJSON = withObject "SinkStats" (\value -> SinkStats <$> value .: "requests" <*> value .: "bytes" <*> value .: "spansReceived")
+
+data SinkHandle = SinkHandle
+  { endpoint :: Text,
+    setFault :: SinkFault -> IO (),
+    snapshot :: IO SinkStats
+  }
+
+sinkApplication :: IORef SinkFault -> IORef SinkStats -> Application
+sinkApplication faultRef statsRef request respond
+  | requestMethod request /= "POST" || pathInfo request `notElem` [["v1", "traces"], ["v1", "metrics"]] = respond (responseLBS status404 [] "not found")
+  | otherwise = do
+      fault <- readIORef faultRef
+      case fault of
+        SinkDelay200ms -> threadDelay 200000
+        SinkDelay2000ms -> threadDelay 2000000
+        SinkHang -> threadDelay 120000000
+        _ -> pure ()
+      body <- strictRequestBody request
+      let decodedBody = if lookup "Content-Encoding" (requestHeaders request) == Just "gzip" then GZip.decompress body else body
+          spans = if pathInfo request == ["v1", "traces"] then countSpans decodedBody else 0
+      atomicModifyIORef' statsRef \stats -> (SinkStats (stats.requests + 1) (stats.bytes + fromIntegral (LazyByteString.length body)) (stats.spansReceived + spans), ())
+      respond (responseLBS (if fault == SinkStatus503 then status503 else status200) [("Content-Type", "application/x-protobuf")] "")
+
+withSink :: SinkFault -> (SinkHandle -> IO value) -> IO value
+withSink initialFault action = bracket acquire release use
+  where
+    acquire = do
+      faultRef <- newIORef initialFault
+      statsRef <- newIORef (SinkStats 0 0 0)
+      (port, socket) <- openFreePort
+      if initialFault == SinkRefuse
+        then close socket >> pure (SinkHandle (url port) (writeIORef faultRef) (readIORef statsRef), Nothing)
+        else do
+          server <- async (runSettingsSocket defaultSettings socket (sinkApplication faultRef statsRef))
+          pure (SinkHandle (url port) (writeIORef faultRef) (readIORef statsRef), Just (server, socket))
+    release (_, Nothing) = pure ()
+    release (_, Just (server, socket)) = cancel server >> void (waitCatch server) >> close socket
+    use (handle, _) = action handle
+    url port = "http://127.0.0.1:" <> Text.pack (show port)
+
+runSinkRole :: RoleContext -> IO ()
+runSinkRole context = withSink initialFault \sink -> do
+  context.send (WrkCustom "ready" (object ["endpoint" .= sink.endpoint]))
+  loop sink
+  stats <- sink.snapshot
+  context.send (WrkCustom "sink-stats" (toJSON stats))
+  where
+    loop sink =
+      context.receive >>= \case
+        Just (CtlCustom "fault" (String raw)) -> case parseFault raw of
+          Nothing -> context.send (WrkError ("unknown sink fault: " <> raw))
+          Just fault -> sink.setFault fault >> loop sink
+        Just (CtlCustom "snapshot" _) -> sink.snapshot >>= context.send . WrkCustom "sink-stats" . toJSON >> loop sink
+        Just (CtlStop _) -> pure ()
+        Just _ -> loop sink
+        Nothing -> pure ()
+    initialFault = maybe SinkHealthy id (parseMaybe (withObject "sink arguments" (\value -> value .: "fault" >>= maybe (fail "unknown sink fault") pure . parseFault)) context.init.args)
+
+countSpans :: LazyByteString.ByteString -> Int
+countSpans body = case decodeMessage (LazyByteString.toStrict body) :: Either String ExportTraceServiceRequest of
+  Left _ -> 0
+  Right request ->
+    sum
+      [ Vector.length (scope ^. Trace.vec'spans)
+      | resource <- Vector.toList (request ^. TraceService.vec'resourceSpans),
+        scope <- Vector.toList (resource ^. Trace.vec'scopeSpans)
+      ]
+
+parseFault :: Text -> Maybe SinkFault
+parseFault raw = lookup raw [(renderSinkFault fault, fault) | fault <- [SinkHealthy, SinkDelay200ms, SinkDelay2000ms, SinkStatus503, SinkHang, SinkRefuse]]
diff --git a/src/Kenshou/Telemetry/Spec.hs b/src/Kenshou/Telemetry/Spec.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Spec.hs
@@ -0,0 +1,230 @@
+module Kenshou.Telemetry.Spec
+  ( TracingArm (..),
+    MetricsArm (..),
+    SamplerSpec (..),
+    ProcessorSpec (..),
+    OtlpProtocol (..),
+    OtlpEndpoint (..),
+    OtlpCompression (..),
+    SinkFault (..),
+    OtelReader (..),
+    HelperPlacement (..),
+    TelemetrySpec (..),
+    telemetryKnobs,
+    telemetrySpecFromContext,
+    telemetrySpecFromWorker,
+    renderSinkFault,
+  )
+where
+
+import Data.Aeson (Value, encode)
+import Data.ByteString.Lazy qualified as LBS
+import Data.Int (Int64)
+import Data.List.NonEmpty (NonEmpty (..))
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Kenshou.Core.Context (RunContext (..), SummarySection (Telemetry), putSummary)
+import Kenshou.Core.Dimension (Dimensions (..), MetricsArm (..), TracingArm (..))
+import Kenshou.Core.Id (renderScenarioId)
+import Kenshou.Core.Knob
+import Kenshou.Core.Role (RoleContext (..), WorkerInit (..))
+import System.Directory (createDirectoryIfMissing)
+import System.FilePath ((</>))
+
+data SamplerSpec = AlwaysOn | AlwaysOff | ParentBasedAlwaysOn | TraceIdRatio Double | ParentBasedTraceIdRatio Double
+  deriving stock (Eq, Show)
+
+data ProcessorSpec
+  = BatchProcessor
+      { maxQueue :: Int,
+        scheduleDelayMs :: Int,
+        maxExportBatch :: Int,
+        exportTimeoutMs :: Int
+      }
+  | SimpleProcessor {exportTimeoutMs :: Int}
+  deriving stock (Eq, Show)
+
+data OtlpProtocol = HttpProtobuf | Grpc deriving stock (Eq, Show)
+
+data SinkFault = SinkHealthy | SinkDelay200ms | SinkDelay2000ms | SinkStatus503 | SinkHang | SinkRefuse
+  deriving stock (Eq, Ord, Show)
+
+data OtlpEndpoint = BuiltinSink SinkFault | ExternalEndpoint Text deriving stock (Eq, Show)
+
+data OtlpCompression = CompressionNone | CompressionGzip deriving stock (Eq, Show)
+
+data OtelReader = ReaderPrometheus | ReaderOtlpPeriodic | ReaderNone deriving stock (Eq, Show)
+
+data HelperPlacement = HelperProcess FilePath | HelperInProcess deriving stock (Eq, Show)
+
+data TelemetrySpec = TelemetrySpec
+  { tracing :: TracingArm,
+    metrics :: MetricsArm,
+    serviceName :: Text,
+    sampler :: SamplerSpec,
+    processor :: ProcessorSpec,
+    exporter :: OtlpProtocol,
+    endpoint :: OtlpEndpoint,
+    compression :: OtlpCompression,
+    probeRetain :: Int,
+    shutdownMs :: Int,
+    otelReader :: OtelReader,
+    otelExportMs :: Int,
+    scrapeMs :: Int,
+    wsSubscribers :: Int,
+    helpers :: HelperPlacement,
+    workerContext :: Maybe RunContext,
+    outDir :: FilePath,
+    report :: Value -> IO ()
+  }
+
+telemetryKnobs :: [KnobSpec]
+telemetryKnobs =
+  [ textKnob "otel.sampler" "OpenTelemetry sampler" "parentbased-always-on" ["always-on", "always-off", "parentbased-always-on", "traceidratio", "parentbased-traceidratio"],
+    doubleKnob "otel.sampler-arg" "Trace-id-ratio sampler fraction" 1 0 1,
+    textKnob "otel.processor" "Span processor" "batch" ["batch", "simple"],
+    intKnob "otel.bsp.max-queue" "Batch processor queue bound" 2048 1 1048576,
+    intKnob "otel.bsp.schedule-delay-ms" "Batch export schedule delay" 5000 10 60000,
+    intKnob "otel.bsp.max-export-batch" "Maximum export batch" 512 1 1048576,
+    intKnob "otel.bsp.export-timeout-ms" "Exporter timeout" 30000 100 120000,
+    textKnob "otel.exporter" "OTLP protocol" "http-protobuf" ["http-protobuf", "grpc"],
+    KnobSpec (name "otel.endpoint") "External OTLP base URL, empty for built-in sink" KnobText (VText "") AnyValue [],
+    textKnob "otel.sink-fault" "Built-in OTLP sink fault" "none" ["none", "delay-200ms", "delay-2000ms", "status-503", "hang", "refuse"],
+    textKnob "otel.compression" "OTLP compression" "none" ["none", "gzip"],
+    intKnob "otel.inmemory.retain" "Recent spans retained by the bounded probe" 4096 0 1000000,
+    intKnob "otel.shutdown-timeout-ms" "Provider flush and shutdown timeout" 5000 100 120000,
+    intKnob "metrics.scrape-interval-ms" "Metrics scrape interval" 15000 100 600000,
+    intKnob "metrics.ws-subscribers" "WebSocket subscribers per endpoint" 0 0 64,
+    textKnob "metrics.otel-reader" "OpenTelemetry metric reader" "prometheus" ["prometheus", "otlp-periodic", "none"],
+    intKnob "metrics.otel-export-interval-ms" "Periodic OTLP metric export interval" 60000 1000 600000
+  ]
+
+telemetrySpecFromContext :: RunContext -> Either Text TelemetrySpec
+telemetrySpecFromContext context =
+  makeTelemetrySpec
+    context.knobs
+    context.dimensions
+    (Text.replace "/" "." (renderScenarioId context.scenario))
+    (HelperProcess "kenshou")
+    (Just context)
+    context.outDir
+    (putSummary context Telemetry "telemetry")
+
+-- A child owns its providers, endpoints and files. It must never write into
+-- the parent's series or retain the parent's in-memory summary state.
+telemetrySpecFromWorker :: RoleContext -> FilePath -> Either Text TelemetrySpec
+telemetrySpecFromWorker context directory =
+  makeTelemetrySpec
+    context.init.knobs
+    context.init.dimensions
+    (Text.replace "/" "." context.init.instanceName)
+    HelperInProcess
+    Nothing
+    directory
+    (\value -> createDirectoryIfMissing True (directory </> "logs") >> LBS.writeFile (directory </> "logs" </> "telemetry-summary.json") (encode value))
+
+makeTelemetrySpec :: ResolvedKnobs -> Dimensions -> Text -> HelperPlacement -> Maybe RunContext -> FilePath -> (Value -> IO ()) -> Either Text TelemetrySpec
+makeTelemetrySpec knobs dimensions serviceName helpers workerContext outDir report = do
+  let Dimensions resolvedTracing resolvedMetrics _ _ = dimensions
+  tracing <- maybe (Left "telemetry.tracing is not applicable") Right resolvedTracing
+  metrics <- maybe (Left "telemetry.metrics is not applicable") Right resolvedMetrics
+  sampler <- parseSampler (knobText knobs (name "otel.sampler")) (knobDouble knobs (name "otel.sampler-arg"))
+  processor <- parseProcessor knobs
+  exporter <- parseProtocol (knobText knobs (name "otel.exporter"))
+  endpoint <- parseEndpoint knobs
+  compression <- parseCompression (knobText knobs (name "otel.compression"))
+  reader <- parseReader (knobText knobs (name "metrics.otel-reader"))
+  pure
+    TelemetrySpec
+      { tracing,
+        metrics,
+        serviceName = serviceName,
+        sampler,
+        processor,
+        exporter,
+        endpoint,
+        compression,
+        probeRetain = integer "otel.inmemory.retain",
+        shutdownMs = integer "otel.shutdown-timeout-ms",
+        otelReader = reader,
+        otelExportMs = integer "metrics.otel-export-interval-ms",
+        scrapeMs = integer "metrics.scrape-interval-ms",
+        wsSubscribers = integer "metrics.ws-subscribers",
+        helpers,
+        workerContext,
+        outDir,
+        report
+      }
+  where
+    integer key = fromIntegral (knobInt knobs (name key))
+
+parseSampler :: Text -> Double -> Either Text SamplerSpec
+parseSampler "always-on" _ = Right AlwaysOn
+parseSampler "always-off" _ = Right AlwaysOff
+parseSampler "parentbased-always-on" _ = Right ParentBasedAlwaysOn
+parseSampler "traceidratio" ratio = Right (TraceIdRatio ratio)
+parseSampler "parentbased-traceidratio" ratio = Right (ParentBasedTraceIdRatio ratio)
+parseSampler value _ = Left ("unsupported otel.sampler: " <> value)
+
+parseProcessor :: ResolvedKnobs -> Either Text ProcessorSpec
+parseProcessor knobs = case knobText knobs (name "otel.processor") of
+  "simple" -> Right (SimpleProcessor timeoutMs)
+  "batch"
+    | batch > queue -> Left "otel.bsp.max-export-batch must not exceed otel.bsp.max-queue"
+    | otherwise -> Right (BatchProcessor queue delay batch timeoutMs)
+  value -> Left ("unsupported otel.processor: " <> value)
+  where
+    queue = integer "otel.bsp.max-queue"
+    delay = integer "otel.bsp.schedule-delay-ms"
+    batch = integer "otel.bsp.max-export-batch"
+    timeoutMs = integer "otel.bsp.export-timeout-ms"
+    integer key = fromIntegral (knobInt knobs (name key))
+
+parseProtocol :: Text -> Either Text OtlpProtocol
+parseProtocol "http-protobuf" = Right HttpProtobuf
+parseProtocol "grpc" = Left "otel.exporter=grpc requires building kenshou-telemetry with -fotlp-grpc"
+parseProtocol value = Left ("unsupported otel.exporter: " <> value)
+
+parseEndpoint :: ResolvedKnobs -> Either Text OtlpEndpoint
+parseEndpoint knobs = case (knobText knobs (name "otel.endpoint"), parseSinkFault (knobText knobs (name "otel.sink-fault"))) of
+  (_, Left message) -> Left message
+  ("", Right fault) -> Right (BuiltinSink fault)
+  (url, Right SinkHealthy) -> Right (ExternalEndpoint url)
+  (_, Right _) -> Left "otel.sink-fault is only valid with the built-in sink"
+
+parseCompression :: Text -> Either Text OtlpCompression
+parseCompression "none" = Right CompressionNone
+parseCompression "gzip" = Right CompressionGzip
+parseCompression value = Left ("unsupported otel.compression: " <> value)
+
+parseReader :: Text -> Either Text OtelReader
+parseReader "prometheus" = Right ReaderPrometheus
+parseReader "otlp-periodic" = Right ReaderOtlpPeriodic
+parseReader "none" = Right ReaderNone
+parseReader value = Left ("unsupported metrics.otel-reader: " <> value)
+
+parseSinkFault :: Text -> Either Text SinkFault
+parseSinkFault value = maybe (Left ("unsupported otel.sink-fault: " <> value)) Right (lookup value values)
+  where
+    values = [(renderSinkFault fault, fault) | fault <- [SinkHealthy, SinkDelay200ms, SinkDelay2000ms, SinkStatus503, SinkHang, SinkRefuse]]
+
+renderSinkFault :: SinkFault -> Text
+renderSinkFault SinkHealthy = "none"
+renderSinkFault SinkDelay200ms = "delay-200ms"
+renderSinkFault SinkDelay2000ms = "delay-2000ms"
+renderSinkFault SinkStatus503 = "status-503"
+renderSinkFault SinkHang = "hang"
+renderSinkFault SinkRefuse = "refuse"
+
+intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec
+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []
+
+doubleKnob :: Text -> Text -> Double -> Double -> Double -> KnobSpec
+doubleKnob key summary def low high = KnobSpec (name key) summary KnobDouble (VDouble def) (DoubleRange low high) []
+
+textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec
+textKnob key summary def (first : rest) = KnobSpec (name key) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []
+textKnob key _ _ [] = error ("text knob has no values: " <> Text.unpack key)
+
+name :: Text -> KnobName
+name = either (error . Text.unpack) id . mkKnobName
diff --git a/src/Kenshou/Telemetry/Tracing.hs b/src/Kenshou/Telemetry/Tracing.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Tracing.hs
@@ -0,0 +1,122 @@
+module Kenshou.Telemetry.Tracing
+  ( TracingRuntime (..),
+    startTracing,
+    flushTracing,
+    stopTracing,
+    otlpExporterConfig,
+  )
+where
+
+import Control.Monad (when)
+import Data.Text qualified as Text
+import Kenshou.Telemetry.Spec
+import Kenshou.Telemetry.Tracing.Pipeline
+import Kenshou.Telemetry.Tracing.Probe
+import OpenTelemetry.Exporter.OTLP.Span qualified as OtlpSpan
+import OpenTelemetry.Exporter.Span (SpanExporter)
+import OpenTelemetry.Processor.Batch.Span qualified as Batch
+import OpenTelemetry.Processor.Simple.Span qualified as Simple
+import OpenTelemetry.Processor.Span (SpanProcessor)
+import OpenTelemetry.Propagator (setGlobalTextMapPropagator)
+import OpenTelemetry.Propagator.W3CTraceContext (w3cTraceContextPropagator)
+import OpenTelemetry.Resource (materializeResources, mkResource, (.=))
+import OpenTelemetry.Trace.Core
+import OpenTelemetry.Trace.Id.Generator.Default (defaultIdGenerator)
+import OpenTelemetry.Trace.Sampler qualified as Sampler
+
+data TracingRuntime = TracingRuntime
+  { tracer :: Maybe Tracer,
+    provider :: Maybe TracerProvider,
+    probe :: Maybe SpanProbe,
+    pipeline :: Maybe PipelineStats
+  }
+
+startTracing :: TelemetrySpec -> IO TracingRuntime
+startTracing spec = do
+  when (spec.tracing /= TracingOff) (setGlobalTextMapPropagator w3cTraceContextPropagator)
+  case spec.tracing of
+    TracingOff -> pure (TracingRuntime Nothing Nothing Nothing Nothing)
+    TracingNoop -> do
+      provider <- createTracerProvider [] (providerOptions spec)
+      pure (runtime provider Nothing Nothing)
+    TracingSdkInMemory -> do
+      stats <- newPipelineStats
+      (probe, probeProcessor) <- newSpanProbe spec.probeRetain
+      provider <- createTracerProvider [countingProcessor stats, instrumentProcessorSuccess stats probeProcessor] (providerOptions spec)
+      pure (runtime provider (Just probe) (Just stats))
+    TracingSdkOtlp -> do
+      stats <- newPipelineStats
+      exporter <- instrumentExporter stats <$> OtlpSpan.otlpExporter (otlpExporterConfig spec)
+      processor <- makeProcessor spec.processor exporter
+      provider <- createTracerProvider [countingProcessor stats, processor] (providerOptions spec)
+      pure (runtime provider Nothing (Just stats))
+  where
+    runtime provider probe pipeline = TracingRuntime (Just (makeTracer provider (instrumentationLibrary "kenshou" "0.1.0.0") tracerOptions)) (Just provider) probe pipeline
+
+flushTracing :: Int -> TracingRuntime -> IO (Maybe FlushResult)
+flushTracing timeoutMs runtime = traverse (\provider -> forceFlushTracerProvider provider (Just (timeoutMs * 1000))) runtime.provider
+
+stopTracing :: Int -> TracingRuntime -> IO (Maybe ShutdownResult)
+stopTracing timeoutMs runtime = traverse (\provider -> shutdownTracerProvider provider (Just (timeoutMs * 1000))) runtime.provider
+
+providerOptions :: TelemetrySpec -> TracerProviderOptions
+providerOptions spec =
+  emptyTracerProviderOptions
+    { tracerProviderOptionsIdGenerator = defaultIdGenerator,
+      tracerProviderOptionsSampler = sampler spec.sampler,
+      tracerProviderOptionsResources = materializeResources (mkResource ["service.name" .= spec.serviceName]),
+      tracerProviderOptionsPropagators = w3cTraceContextPropagator
+    }
+
+sampler :: SamplerSpec -> Sampler.Sampler
+sampler AlwaysOn = Sampler.alwaysOn
+sampler AlwaysOff = Sampler.alwaysOff
+sampler ParentBasedAlwaysOn = Sampler.parentBased (Sampler.parentBasedOptions Sampler.alwaysOn)
+sampler (TraceIdRatio ratio) = Sampler.traceIdRatioBased ratio
+sampler (ParentBasedTraceIdRatio ratio) = Sampler.parentBased (Sampler.parentBasedOptions (Sampler.traceIdRatioBased ratio))
+
+makeProcessor :: ProcessorSpec -> SpanExporter -> IO SpanProcessor
+makeProcessor (SimpleProcessor timeoutMs) exporter = Simple.simpleProcessor (Simple.SimpleProcessorConfig exporter (timeoutMs * 1000))
+makeProcessor (BatchProcessor queue delay batch timeoutMs) exporter =
+  Batch.batchProcessor
+    Batch.batchTimeoutConfig
+      { Batch.maxQueueSize = queue,
+        Batch.scheduledDelayMillis = delay,
+        Batch.maxExportBatchSize = batch,
+        Batch.exportTimeoutMillis = timeoutMs
+      }
+    exporter
+
+otlpExporterConfig :: TelemetrySpec -> OtlpSpan.OTLPExporterConfig
+otlpExporterConfig spec =
+  OtlpSpan.OTLPExporterConfig
+    { OtlpSpan.otlpEndpoint = Just (Text.unpack (endpointBase spec.endpoint)),
+      OtlpSpan.otlpTracesEndpoint = Nothing,
+      OtlpSpan.otlpMetricsEndpoint = Nothing,
+      OtlpSpan.otlpInsecure = True,
+      OtlpSpan.otlpSpanInsecure = True,
+      OtlpSpan.otlpMetricInsecure = True,
+      OtlpSpan.otlpCertificate = Nothing,
+      OtlpSpan.otlpTracesCertificate = Nothing,
+      OtlpSpan.otlpMetricCertificate = Nothing,
+      OtlpSpan.otlpHeaders = Nothing,
+      OtlpSpan.otlpTracesHeaders = Nothing,
+      OtlpSpan.otlpMetricsHeaders = Nothing,
+      OtlpSpan.otlpCompression = Just (case spec.compression of CompressionNone -> OtlpSpan.None; CompressionGzip -> OtlpSpan.GZip),
+      OtlpSpan.otlpTracesCompression = Nothing,
+      OtlpSpan.otlpMetricsCompression = Nothing,
+      OtlpSpan.otlpTimeout = Just (processorTimeout spec.processor),
+      OtlpSpan.otlpTracesTimeout = Nothing,
+      OtlpSpan.otlpMetricsTimeout = Nothing,
+      OtlpSpan.otlpProtocol = Just OtlpSpan.HttpProtobuf,
+      OtlpSpan.otlpTracesProtocol = Nothing,
+      OtlpSpan.otlpMetricsProtocol = Nothing
+    }
+
+endpointBase :: OtlpEndpoint -> Text.Text
+endpointBase (ExternalEndpoint url) = url
+endpointBase (BuiltinSink _) = "http://127.0.0.1:4318"
+
+processorTimeout :: ProcessorSpec -> Int
+processorTimeout (SimpleProcessor timeoutMs) = timeoutMs
+processorTimeout (BatchProcessor _ _ _ timeoutMs) = timeoutMs
diff --git a/src/Kenshou/Telemetry/Tracing/Pipeline.hs b/src/Kenshou/Telemetry/Tracing/Pipeline.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Tracing/Pipeline.hs
@@ -0,0 +1,187 @@
+module Kenshou.Telemetry.Tracing.Pipeline
+  ( PipelineStats,
+    PipelineSnapshot (..),
+    LatencySummary (..),
+    newPipelineStats,
+    countingProcessor,
+    instrumentProcessorSuccess,
+    instrumentExporter,
+    snapshotPipeline,
+    startPipelineSampler,
+  )
+where
+
+import Control.Concurrent (threadDelay)
+import Control.Concurrent.Async (async, cancel, waitCatch)
+import Control.Exception (displayException, finally, onException)
+import Control.Monad (void)
+import Data.Aeson (ToJSON (..), object, (.=))
+import Data.HashMap.Strict qualified as HashMap
+import Data.IORef
+import Data.List (sort)
+import Data.Text (Text)
+import Data.Text qualified as Text
+import Data.Vector qualified as Vector
+import GHC.Clock (getMonotonicTimeNSec)
+import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))
+import OpenTelemetry.Internal.Common.Types (FlushResult (FlushSuccess), ShutdownResult (ShutdownSuccess))
+import OpenTelemetry.Processor.Span (SpanProcessor (..))
+import System.Directory (createDirectoryIfMissing)
+import System.FilePath ((</>))
+import System.IO (BufferMode (LineBuffering), IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)
+
+data PipelineStats = PipelineStats
+  { started :: IORef Int,
+    ended :: IORef Int,
+    exportedOk :: IORef Int,
+    exportFailed :: IORef Int,
+    exportCalls :: IORef Int,
+    exportLatencies :: IORef [Integer],
+    maxQueueDepth :: IORef Int,
+    lastExportError :: IORef (Maybe Text)
+  }
+
+data LatencySummary = LatencySummary {p50 :: Integer, p99 :: Integer, maximum :: Integer}
+  deriving stock (Eq, Show)
+
+data PipelineSnapshot = PipelineSnapshot
+  { spansStarted :: Int,
+    spansEnded :: Int,
+    spansExportedOk :: Int,
+    spansExportFailed :: Int,
+    spansDropped :: Int,
+    maxQueueDepth :: Int,
+    exportCalls :: Int,
+    exportLatencyNs :: LatencySummary,
+    lastExportError :: Maybe Text
+  }
+  deriving stock (Eq, Show)
+
+instance ToJSON LatencySummary where
+  toJSON summary = object ["p50" .= summary.p50, "p99" .= summary.p99, "max" .= summary.maximum]
+
+instance ToJSON PipelineSnapshot where
+  toJSON snapshot =
+    object
+      [ "spansStarted" .= snapshot.spansStarted,
+        "spansEnded" .= snapshot.spansEnded,
+        "spansExportedOk" .= snapshot.spansExportedOk,
+        "spansExportFailed" .= snapshot.spansExportFailed,
+        "spansDropped" .= snapshot.spansDropped,
+        "maxQueueDepth" .= snapshot.maxQueueDepth,
+        "exportCalls" .= snapshot.exportCalls,
+        "exportLatencyNs" .= snapshot.exportLatencyNs,
+        "lastExportError" .= snapshot.lastExportError
+      ]
+
+newPipelineStats :: IO PipelineStats
+newPipelineStats = PipelineStats <$> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef [] <*> newIORef 0 <*> newIORef Nothing
+
+countingProcessor :: PipelineStats -> SpanProcessor
+countingProcessor stats =
+  SpanProcessor
+    { spanProcessorOnStart = \_ _ -> increment stats.started 1,
+      spanProcessorOnEnd = \_ -> increment stats.ended 1 >> updateQueueHighWater stats,
+      spanProcessorShutdown = pure ShutdownSuccess,
+      spanProcessorForceFlush = pure FlushSuccess
+    }
+
+instrumentProcessorSuccess :: PipelineStats -> SpanProcessor -> SpanProcessor
+instrumentProcessorSuccess stats processor =
+  processor
+    { spanProcessorOnEnd = \spanValue -> processor.spanProcessorOnEnd spanValue >> increment stats.exportedOk 1
+    }
+
+instrumentExporter :: PipelineStats -> SpanExporter -> SpanExporter
+instrumentExporter stats exporter =
+  exporter
+    { spanExporterExport = \batch -> do
+        let spanCount = sum (fmap Vector.length (HashMap.elems batch))
+        startedAt <- getMonotonicTimeNSec
+        increment stats.exportCalls 1
+        result <-
+          exporter.spanExporterExport batch
+            `onException` do
+              increment stats.exportFailed spanCount
+              writeIORef stats.lastExportError (Just "exporter raised an exception")
+        endedAt <- getMonotonicTimeNSec
+        modifyIORef' stats.exportLatencies (fromIntegral (endedAt - startedAt) :)
+        case result of
+          Success -> increment stats.exportedOk spanCount
+          Failure exception -> do
+            increment stats.exportFailed spanCount
+            writeIORef stats.lastExportError (fmap (Text.pack . displayException) exception)
+        pure result
+    }
+
+snapshotPipeline :: PipelineStats -> IO PipelineSnapshot
+snapshotPipeline stats = do
+  spansStarted <- readIORef stats.started
+  spansEnded <- readIORef stats.ended
+  spansExportedOk <- readIORef stats.exportedOk
+  spansExportFailed <- readIORef stats.exportFailed
+  exportCalls <- readIORef stats.exportCalls
+  latencies <- readIORef stats.exportLatencies
+  maxQueueDepth <- readIORef stats.maxQueueDepth
+  lastExportError <- readIORef stats.lastExportError
+  let spansDropped = max 0 (spansEnded - spansExportedOk - spansExportFailed)
+  pure
+    PipelineSnapshot
+      { spansStarted,
+        spansEnded,
+        spansExportedOk,
+        spansExportFailed,
+        spansDropped,
+        maxQueueDepth,
+        exportCalls,
+        exportLatencyNs = summarize latencies,
+        lastExportError
+      }
+
+startPipelineSampler :: FilePath -> PipelineStats -> IO (IO [Int])
+startPipelineSampler outputDir stats = do
+  createDirectoryIfMissing True (outputDir </> "series")
+  samples <- newIORef []
+  worker <- async do
+    handle <- openFile (outputDir </> "series" </> "otel-pipeline.csv") WriteMode
+    hSetBuffering handle LineBuffering
+    hPutStrLn handle "t_mono_ns,spans_started,spans_ended,exported_ok,export_failed,queue_depth,export_calls"
+    let loop = do
+          threadDelay 1_000_000
+          timestamp <- getMonotonicTimeNSec
+          started <- readIORef stats.started
+          ended <- readIORef stats.ended
+          exportedOk <- readIORef stats.exportedOk
+          exportFailed <- readIORef stats.exportFailed
+          exportCalls <- readIORef stats.exportCalls
+          let queueDepth = max 0 (ended - exportedOk - exportFailed)
+          modifyIORef' samples (queueDepth :)
+          hPutStrLn handle (comma [timestamp, fromIntegral started, fromIntegral ended, fromIntegral exportedOk, fromIntegral exportFailed, fromIntegral queueDepth, fromIntegral exportCalls])
+          loop
+    loop `finally` hClose handle
+  pure do
+    cancel worker
+    void (waitCatch worker)
+    reverse <$> readIORef samples
+  where
+    comma = foldr1 (\left right -> left <> "," <> right) . fmap show
+
+increment :: IORef Int -> Int -> IO ()
+increment ref amount = void (atomicModifyIORef' ref (\value -> let updated = value + amount in (updated, updated)))
+
+updateQueueHighWater :: PipelineStats -> IO ()
+updateQueueHighWater stats = do
+  ended <- readIORef stats.ended
+  exportedOk <- readIORef stats.exportedOk
+  exportFailed <- readIORef stats.exportFailed
+  let queued = max 0 (ended - exportedOk - exportFailed)
+  atomicModifyIORef' stats.maxQueueDepth (\high -> (max high queued, ()))
+
+summarize :: [Integer] -> LatencySummary
+summarize [] = LatencySummary 0 0 0
+summarize values =
+  let ordered = sort values
+   in LatencySummary (quantile 0.50 ordered) (quantile 0.99 ordered) (last ordered)
+
+quantile :: Double -> [Integer] -> Integer
+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))
diff --git a/src/Kenshou/Telemetry/Tracing/Probe.hs b/src/Kenshou/Telemetry/Tracing/Probe.hs
new file mode 100644
--- /dev/null
+++ b/src/Kenshou/Telemetry/Tracing/Probe.hs
@@ -0,0 +1,82 @@
+module Kenshou.Telemetry.Tracing.Probe
+  ( SpanView (..),
+    SpanProbe,
+    newSpanProbe,
+    readSpans,
+    spansSeen,
+  )
+where
+
+import Data.Foldable qualified as Foldable
+import Data.IORef
+import Data.Sequence (Seq (..), (|>))
+import Data.Sequence qualified as Seq
+import Data.Text (Text)
+import Data.Word (Word64)
+import OpenTelemetry.Attributes (Attributes)
+import OpenTelemetry.Common (optionalTimestampToMaybe, timestampToNanoseconds)
+import OpenTelemetry.Internal.Common.Types (FlushResult (FlushSuccess), ShutdownResult (ShutdownSuccess))
+import OpenTelemetry.Processor.Span (SpanProcessor (..))
+import OpenTelemetry.Trace.Core (ImmutableSpan (..), SpanContext (..), SpanHot (..), SpanKind, SpanStatus, getSpanContext)
+import OpenTelemetry.Trace.Id (SpanId, TraceId)
+
+data SpanView = SpanView
+  { name :: Text,
+    kind :: SpanKind,
+    traceId :: TraceId,
+    spanId :: SpanId,
+    parentSpanId :: Maybe SpanId,
+    attributes :: Attributes,
+    startNs :: Word64,
+    endNs :: Word64,
+    status :: SpanStatus
+  }
+  deriving stock (Eq, Show)
+
+-- Evict during capture, even when nobody reads the probe during a long run.
+-- A strict IORef update alone only forces the outer constructor.
+data ProbeState = ProbeState {seen :: !Int, retained :: !(Seq SpanView)}
+
+data SpanProbe = SpanProbe {capacity :: Int, state :: IORef ProbeState}
+
+newSpanProbe :: Int -> IO (SpanProbe, SpanProcessor)
+newSpanProbe capacity = do
+  state <- newIORef (ProbeState 0 Seq.empty)
+  let probe = SpanProbe (max 0 capacity) state
+      processor =
+        SpanProcessor
+          { spanProcessorOnStart = \_ _ -> pure (),
+            spanProcessorOnEnd = capture probe,
+            spanProcessorShutdown = pure ShutdownSuccess,
+            spanProcessorForceFlush = pure FlushSuccess
+          }
+  pure (probe, processor)
+
+readSpans :: SpanProbe -> IO [SpanView]
+readSpans probe = toList . (.retained) <$> readIORef probe.state
+  where
+    toList = Foldable.toList
+
+spansSeen :: SpanProbe -> IO Int
+spansSeen probe = (.seen) <$> readIORef probe.state
+
+capture :: SpanProbe -> ImmutableSpan -> IO ()
+capture probe spanValue = do
+  hot <- readIORef spanValue.spanHot
+  parent <- traverse getSpanContext spanValue.spanParent
+  let view =
+        SpanView
+          { name = hot.hotName,
+            kind = spanValue.spanKind,
+            traceId = spanValue.spanContext.traceId,
+            spanId = spanValue.spanContext.spanId,
+            parentSpanId = (.spanId) <$> parent,
+            attributes = hot.hotAttributes,
+            startNs = timestampToNanoseconds spanValue.spanStart,
+            endNs = maybe (timestampToNanoseconds spanValue.spanStart) timestampToNanoseconds (optionalTimestampToMaybe hot.hotEnd),
+            status = hot.hotStatus
+          }
+  atomicModifyIORef' probe.state \current ->
+    let appended = current.retained |> view
+        trimmed = if Seq.length appended > probe.capacity then Seq.drop 1 appended else appended
+     in (ProbeState (current.seen + 1) trimmed, ())
diff --git a/test/Main.hs b/test/Main.hs
new file mode 100644
--- /dev/null
+++ b/test/Main.hs
@@ -0,0 +1,366 @@
+module Main (main) where
+
+import Control.Concurrent (threadDelay)
+import Control.Concurrent.Async (async, cancel, waitCatch)
+import Control.Concurrent.MVar (newEmptyMVar, putMVar, readMVar, tryPutMVar)
+import Control.Exception (bracket)
+import Control.Monad (forM_, replicateM_, void)
+import Data.IORef
+import Data.List (find)
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
+import Data.Maybe (fromJust)
+import Data.Text qualified as Text
+import Kenshou.Core.Knob (Allowed (..), KnobSpec (..), renderKnobName)
+import Kenshou.Core.Scenario (Scenario (..))
+import Kenshou.Telemetry.Compose
+import Kenshou.Telemetry.Continuity
+import Kenshou.Telemetry.Detect (Finding (..), FindingSeverity (Degraded), FindingStatus (Fired), handlerFinding, pipelineFindings, queueGrowthFinding)
+import Kenshou.Telemetry.Endpoint
+import Kenshou.Telemetry.Metrics
+import Kenshou.Telemetry.Overhead
+import Kenshou.Telemetry.Scrape
+import Kenshou.Telemetry.SelfTest.Arms (armsScenario)
+import Kenshou.Telemetry.SelfTest.Service
+import Kenshou.Telemetry.Sink qualified as Sink
+import Kenshou.Telemetry.Spec
+import Kenshou.Telemetry.Tracing
+import Kenshou.Telemetry.Tracing.Pipeline
+import Kenshou.Telemetry.Tracing.Probe
+import Network.HTTP.Types (status200, status404)
+import Network.Socket (close)
+import Network.Wai (Application, responseLBS)
+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)
+import Network.Wai.Handler.WebSockets (websocketsOr)
+import Network.WebSockets qualified as WebSockets
+import OpenTelemetry.Attributes (lookupAttribute)
+import OpenTelemetry.Context.ThreadLocal (getContext)
+import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))
+import OpenTelemetry.Processor.Batch.Span qualified as Batch
+import OpenTelemetry.Processor.Simple.Span qualified as Simple
+import OpenTelemetry.Processor.Span (SpanProcessor (..))
+import OpenTelemetry.Propagator (emptyTextMap, inject, textMapToList)
+import OpenTelemetry.Trace.Core
+import System.Directory (createDirectoryIfMissing)
+import System.Mem (performMajorGC)
+import System.Mem.Weak (Weak, deRefWeak, mkWeakPtr)
+import System.Timeout (timeout)
+import Test.Hspec
+
+main :: IO ()
+main = hspec do
+  describe "overhead planning" do
+    it "expands the headline one-factor matrix into four arms and twelve slots" do
+      let request = overheadRequest (Map.fromList [("telemetry.tracing", ["off", "noop", "sdk-otlp"]), ("telemetry.metrics", ["off", "serve-scraped"])])
+      case planOverhead request armsScenario of
+        Left err -> expectationFailure (show err)
+        Right plan -> do
+          length plan.arms `shouldBe` 4
+          length plan.slots `shouldBe` 12
+          map (length . unique . fmap (.arm.armId)) (blocks plan.slots) `shouldBe` [4, 4, 4]
+
+    it "is deterministic and gives two arms ABBA order across the first two blocks" do
+      let request = overheadRequest (Map.singleton "telemetry.tracing" ["off", "noop"])
+      case (planOverhead request armsScenario, planOverhead request armsScenario) of
+        (Right first, Right second) -> do
+          first `shouldBe` second
+          let order = fmap (.arm.armId) first.slots
+          take 2 order `shouldBe` reverse (take 2 (drop 2 order))
+        result -> expectationFailure (show result)
+
+    it "rejects repeated arm values" do
+      planOverhead (overheadRequest (Map.singleton "telemetry.tracing" ["off", "off"])) armsScenario `shouldSatisfy` isLeft
+
+  describe "telemetry problem detectors" do
+    it "detects synchronous handler stalls while a bounded async wrapper drops instead of blocking" do
+      synchronousStats <- newHandlerStats
+      timedHandler synchronousStats "sync" (slowHandler 6_000) ()
+      synchronous <- snapshotHandlerStats "sync" synchronousStats
+      (submit, snapshotAsync) <- asyncHandler 1 (slowHandler 20_000)
+      replicateM_ 100 (submit ())
+      queued <- snapshotAsync
+      handlerFinding synchronous `shouldSatisfy` ((== Fired) . (.status))
+      queued.dropped `shouldSatisfy` (> 0)
+
+    it "marks dropped spans as degraded above the policy fraction" do
+      let snapshot = PipelineSnapshot 100 100 80 0 20 20 1 (LatencySummary 1 1 1) Nothing
+      pipelineFindings 0.01 (Just snapshot) [] `shouldSatisfy` any (\finding -> finding.detector == "span-drop" && finding.status == Fired && finding.severity == Degraded)
+
+    it "fires queue growth only after more than ten consecutive samples above ninety percent" do
+      queueGrowthFinding (Just 100) (replicate 10 91) `shouldSatisfy` ((/= Fired) . (.status))
+      queueGrowthFinding (Just 100) (replicate 11 91) `shouldSatisfy` ((== Fired) . (.status))
+
+  describe "telemetryKnobs" do
+    it "declares each shared knob once" do
+      let names = fmap (renderKnobName . (.name)) telemetryKnobs
+      length names `shouldBe` length (unique names)
+    it "bounds the trace-id ratio to zero through one" do
+      fmap (.allowed) (find ((== "otel.sampler-arg") . renderKnobName . (.name)) telemetryKnobs)
+        `shouldBe` Just (DoubleRange 0 1)
+
+  describe "tracing arms" do
+    it "does not construct a provider under off" do
+      runtime <- startTracing (testSpec TracingOff)
+      isNothing runtime.tracer `shouldBe` True
+      isNothing runtime.provider `shouldBe` True
+
+    it "constructs a disabled tracer under noop" do
+      runtime <- startTracing (testSpec TracingNoop)
+      let tracer = fromJust runtime.tracer
+      tracerIsEnabled tracer `shouldBe` False
+      inSpan tracer "ignored" defaultSpanArguments (pure ())
+      _ <- stopTracing 1000 runtime
+      pure ()
+
+    it "retains a bounded, parented trace and installs W3C propagation" do
+      runtime <- startTracing ((testSpec TracingSdkInMemory) {probeRetain = 4})
+      let tracer = fromJust runtime.tracer
+          provider = fromJust runtime.provider
+      replicateM_ 5 (inSpan tracer "discarded" defaultSpanArguments (pure ()))
+      headers <- inSpan' tracer "parent" defaultSpanArguments \parentSpan -> do
+        addAttribute parentSpan "message.id" ("one" :: Text.Text)
+        nested <- inSpan' tracer "child" defaultSpanArguments \childSpan -> addAttribute childSpan "message.id" ("one" :: Text.Text)
+        activeContext <- getContext
+        carrier <- inject (getTracerProviderPropagators provider) activeContext emptyTextMap
+        pure (nested, textMapToList carrier)
+      _ <- flushTracing 1000 runtime
+      let probe = fromJust runtime.probe
+      views <- readSpans probe
+      seen <- spansSeen probe
+      seen `shouldBe` 7
+      length views `shouldBe` 4
+      let child = fromJust (find ((== "child") . (.name)) views)
+          parent = fromJust (find ((== "parent") . (.name)) views)
+      child.traceId `shouldBe` parent.traceId
+      child.parentSpanId `shouldBe` Just parent.spanId
+      let correlation spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing
+          continuity = checkContinuity (SpanSelector (Just "parent") Nothing []) (SpanSelector (Just "child") Nothing []) correlation views
+      continuity.violations `shouldBe` 0
+      snd headers `shouldSatisfy` any ((== "traceparent") . fst)
+      snapshot <- snapshotPipeline (fromJust runtime.pipeline)
+      snapshot.spansEnded `shouldBe` 7
+      snapshot.spansExportedOk `shouldBe` 7
+      snapshot.spansDropped `shouldBe` 0
+      _ <- stopTracing 1000 runtime
+      pure ()
+
+    it "releases evicted spans before anyone reads the bounded probe" do
+      forM_ [0, 1] \capacity -> do
+        (probe, provider, weak) <- probeWithEvictedSpan capacity
+        performMajorGC
+        (isNothing <$> deRefWeak weak) `shouldReturn` True
+        spansSeen probe `shouldReturn` 10
+        (length <$> readSpans probe) `shouldReturn` capacity
+        _ <- shutdownTracerProvider provider (Just 1000000)
+        pure ()
+
+    it "exports plain and gzip OTLP requests to the built-in sink" do
+      forCompression CompressionNone
+      forCompression CompressionGzip
+
+    it "accounts for alternating exporter failures exactly" do
+      stats <- newPipelineStats
+      calls <- newIORef (0 :: Int)
+      let exporter =
+            SpanExporter
+              { spanExporterExport = \_ -> do
+                  call <- atomicModifyIORef' calls (\value -> let next = value + 1 in (next, next))
+                  pure (if even call then Failure Nothing else Success),
+                spanExporterShutdown = pure ShutdownSuccess,
+                spanExporterForceFlush = pure FlushSuccess
+              }
+      processor <- Simple.simpleProcessor (Simple.SimpleProcessorConfig (instrumentExporter stats exporter) 1_000_000)
+      provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions
+      let tracer = makeTracer provider (instrumentationLibrary "pipeline-test" "1") tracerOptions
+      replicateM_ 10 (inSpan tracer "alternating" defaultSpanArguments (pure ()))
+      _ <- shutdownTracerProvider provider (Just 1_000_000)
+      snapshot <- snapshotPipeline stats
+      snapshot.spansEnded `shouldBe` snapshot.spansExportedOk + snapshot.spansExportFailed + snapshot.spansDropped
+      snapshot.spansExportedOk `shouldBe` 5
+      snapshot.spansExportFailed `shouldBe` 5
+
+    it "drops instead of blocking when a batch queue is full" do
+      stats <- newPipelineStats
+      releaseExporter <- newEmptyMVar
+      let exporter =
+            SpanExporter
+              { spanExporterExport = \_ -> readMVar releaseExporter >> pure Success,
+                spanExporterShutdown = pure ShutdownSuccess,
+                spanExporterForceFlush = pure FlushSuccess
+              }
+      processor <-
+        Batch.batchProcessor
+          Batch.batchTimeoutConfig
+            { Batch.maxQueueSize = 8,
+              Batch.scheduledDelayMillis = 10,
+              Batch.maxExportBatchSize = 4,
+              Batch.exportTimeoutMillis = 5_000
+            }
+          (instrumentExporter stats exporter)
+      provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions
+      let tracer = makeTracer provider (instrumentationLibrary "backpressure-test" "1") tracerOptions
+      completed <- timeout 1_000_000 (replicateM_ 10_000 (inSpan tracer "queued" defaultSpanArguments (pure ())))
+      completed `shouldBe` Just ()
+      beforeRelease <- snapshotPipeline stats
+      beforeRelease.spansDropped `shouldSatisfy` (> 0)
+      putMVar releaseExporter ()
+      void (shutdownTracerProvider provider (Just 1_000_000))
+
+  describe "metrics arms" do
+    it "constructs no provider under off" do
+      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsOff})
+      isNothing runtime.meter `shouldBe` True
+      isNothing runtime.provider `shouldBe` True
+      isNothing runtime.endpoint `shouldBe` True
+
+    it "collects instruments without opening an endpoint" do
+      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsCollect})
+      let meter = fromJust runtime.meter
+      metrics <- newSyntheticMetrics meter
+      void (runSyntheticOperation Nothing (Just metrics) 0 0 1)
+      (snapshot, _) <- stopMetrics 1000 runtime
+      runtime.endpoint `shouldBe` Nothing
+      fmap (.instruments) snapshot `shouldBe` Just 2
+
+    it "serves its Prometheus reader only for a serving arm" do
+      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsServe, otelReader = ReaderPrometheus})
+      runtime.endpoint `shouldSatisfy` maybe False ((== PrometheusText) . (.kind))
+      traverse_ (\endpoint -> awaitHttpReady endpoint.url 1000 >>= (`shouldBe` True)) runtime.endpoint
+      void (stopMetrics 1000 runtime)
+
+  describe "metrics scraper" do
+    it "skips fixed-schedule ticks when an endpoint is slower than its interval" do
+      withSlowHttpServer 75_000 \endpoint -> do
+        let output = ".tmp/telemetry-test/slow-scrape"
+        createDirectoryIfMissing True output
+        scraper <- startScraperInProcess output 20 0
+        scraper.register endpoint
+        threadDelay 260_000
+        summaries <- scraper.finish
+        fmap (.skippedTicks) summaries `shouldSatisfy` any (> 0)
+
+    it "detects exhausted WebSocket slots and accepts a correct server" do
+      withWebSocketServer (Just 2) \leaky -> do
+        result <- wsSlotLeakProbe leaky 2
+        result.freshAccepted `shouldBe` False
+      withWebSocketServer Nothing \correct -> do
+        result <- wsSlotLeakProbe correct 2
+        result.freshAccepted `shouldBe` True
+
+testSpec :: TracingArm -> TelemetrySpec
+testSpec tracing =
+  TelemetrySpec
+    { tracing,
+      metrics = MetricsOff,
+      serviceName = "kenshou.test",
+      sampler = AlwaysOn,
+      processor = SimpleProcessor 1000,
+      exporter = HttpProtobuf,
+      endpoint = BuiltinSink SinkHealthy,
+      compression = CompressionNone,
+      probeRetain = 16,
+      shutdownMs = 1000,
+      otelReader = ReaderNone,
+      otelExportMs = 1000,
+      scrapeMs = 1000,
+      wsSubscribers = 0,
+      helpers = HelperInProcess,
+      workerContext = Nothing,
+      outDir = ".tmp/telemetry-test",
+      report = const (pure ())
+    }
+
+forCompression :: OtlpCompression -> IO ()
+forCompression compression = Sink.withSink SinkHealthy \sink -> do
+  let spec = (testSpec TracingSdkOtlp) {endpoint = ExternalEndpoint sink.endpoint, compression}
+  runtime <- startTracing spec
+  let tracer = fromJust runtime.tracer
+  inSpan tracer "exported" defaultSpanArguments (pure ())
+  _ <- flushTracing 2000 runtime
+  _ <- stopTracing 2000 runtime
+  stats <- sink.snapshot
+  stats.spansReceived `shouldBe` 1
+
+isNothing :: Maybe value -> Bool
+isNothing Nothing = True
+isNothing (Just _) = False
+
+unique :: (Eq value) => [value] -> [value]
+unique [] = []
+unique (value : values) = value : unique (filter (/= value) values)
+
+withSlowHttpServer :: Int -> (Endpoint -> IO value) -> IO value
+withSlowHttpServer delayMicros action = withWarp application \port ->
+  action (Endpoint "slow-json" JsonDocument (Text.pack ("http://127.0.0.1:" <> show port <> "/metrics")) Nothing)
+  where
+    application _ respond = threadDelay delayMicros >> respond (responseLBS status200 [] "delayed")
+
+withWebSocketServer :: Maybe Int -> (Endpoint -> IO value) -> IO value
+withWebSocketServer capacity action = do
+  accepted <- newIORef (0 :: Int)
+  withWarp (websocketsOr WebSockets.defaultConnectionOptions (server accepted) fallback) \port ->
+    action (Endpoint "slot-probe" WebSocketPush (Text.pack ("ws://127.0.0.1:" <> show port <> "/ws")) Nothing)
+  where
+    server accepted pendingConnection = do
+      count <- atomicModifyIORef' accepted (\value -> let next = value + 1 in (next, next))
+      case capacity of
+        Just limit | count > limit -> WebSockets.rejectRequest pendingConnection "slots exhausted"
+        _ -> void (WebSockets.acceptRequest pendingConnection)
+    fallback _ respond = respond (responseLBS status404 [] "not found")
+
+withWarp :: Application -> (Int -> IO value) -> IO value
+withWarp application action = bracket acquire release (\(port, _, _) -> action port)
+  where
+    acquire = do
+      (port, socket) <- openFreePort
+      server <- async (runSettingsSocket defaultSettings socket application)
+      pure (port, server, socket)
+    release (_, server, socket) = cancel server >> void (waitCatch server) >> close socket
+
+traverse_ :: (a -> IO b) -> Maybe a -> IO ()
+traverse_ _ Nothing = pure ()
+traverse_ action (Just value) = void (action value)
+
+overheadRequest :: Map Text.Text [Text.Text] -> OverheadRequest
+overheadRequest factors =
+  OverheadRequest
+    { requestId = "test-overhead",
+      scenarioId = armsScenario.id,
+      factors,
+      mode = OneFactor,
+      includeControl = False,
+      trials = 3,
+      fixedKnobs = [],
+      fixedDimensions = Map.empty,
+      seed = 7,
+      settleSeconds = 0,
+      retries = 1
+    }
+
+blocks :: [Slot] -> [[Slot]]
+blocks slots = [[slot | slot <- slots, slot.block == block] | block <- unique (fmap (.block) slots)]
+
+isLeft :: Either left right -> Bool
+isLeft (Left _) = True
+isLeft (Right _) = False
+
+-- Keeping the provider and probe alive must not keep a span outside the
+-- retention window alive. Do not force readSpans/spansSeen before the GC.
+{-# NOINLINE probeWithEvictedSpan #-}
+probeWithEvictedSpan :: Int -> IO (SpanProbe, TracerProvider, Weak SpanHot)
+probeWithEvictedSpan capacity = do
+  (probe, processor) <- newSpanProbe capacity
+  first <- newEmptyMVar
+  let observing =
+        processor
+          { spanProcessorOnEnd = \spanValue -> do
+              hot <- readIORef spanValue.spanHot
+              weak <- mkWeakPtr hot Nothing
+              void (tryPutMVar first weak)
+              processor.spanProcessorOnEnd spanValue
+          }
+  provider <- createTracerProvider [observing] emptyTracerProviderOptions
+  let tracer = makeTracer provider (instrumentationLibrary "probe-retention-test" "0.1.0.0") tracerOptions
+  replicateM_ 10 (inSpan tracer "evicted" defaultSpanArguments (pure ()))
+  weak <- readMVar first
+  pure (probe, provider, weak)
diff --git a/test/golden/overhead-report.minimal.json b/test/golden/overhead-report.minimal.json
new file mode 100644
--- /dev/null
+++ b/test/golden/overhead-report.minimal.json
@@ -0,0 +1,21 @@
+{
+  "schema": "kenshou.overhead-report/v1",
+  "id": "01a0c631-ff36-77c3-8772-216b4ab24eed",
+  "scenario": "selftest/telemetry/benchmark/arms-on-synthetic-service",
+  "mode": "one-factor",
+  "baseline": {"telemetry.tracing": "off", "telemetry.metrics": "off"},
+  "fixed": {"knobs": {}, "dimensions": {}},
+  "trials": 3,
+  "seed": 7,
+  "validBlocks": 3,
+  "order": [["a0", "a1"], ["a1", "a0"], ["a0", "a1"]],
+  "arms": [
+    {"arm": "a0", "dimensions": {"telemetry.tracing": "off"}, "runs": [], "telemetry": null, "leak": null, "findings": []}
+  ],
+  "comparisons": [],
+  "findings": [],
+  "policy": {"name": "telemetry-overhead-default", "sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},
+  "cohort": {},
+  "verdict": "pass",
+  "algorithm": {"name": "kenshou-overhead", "version": 1}
+}
