packages feed

kenshou-telemetry (empty) → 0.1.0.0

raw patch · 24 files changed

+3387/−0 lines, 24 filesdep +aesondep +asyncdep +base

Dependencies added: aeson, async, base, bytestring, containers, directory, filepath, hs-opentelemetry-api, hs-opentelemetry-exporter-otlp, hs-opentelemetry-exporter-prometheus, hs-opentelemetry-otlp, hs-opentelemetry-propagator-w3c, hs-opentelemetry-sdk, hspec, http-client, http-types, kenshou-core, kenshou-measure, kenshou-telemetry, microlens, network, proto-lens, proto-lens-runtime, splitmix, stm, text, time, unordered-containers, vector, wai, wai-websockets, warp, websockets, zlib

Files

+ CHANGELOG.md view
@@ -0,0 +1,11 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of tracing and metrics arms, endpoint scraping, and observability-overhead reports.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+   list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+   this list of conditions and the following disclaimer in the documentation+   and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+   may be used to endorse or promote products derived from this software without+   specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ kenshou-telemetry.cabal view
@@ -0,0 +1,135 @@+cabal-version: 3.0+name: kenshou-telemetry+version: 0.1.0.0+synopsis: Telemetry arms and overhead analysis for Kenshou scenarios+description: Explicit tracing and metrics providers, endpoint scrapers, pipeline diagnostics, and paired observability-overhead reports.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+  test/golden/*.json++flag otlp-grpc+  description: Enable the optional OTLP gRPC exporter+  default: False+  manual: True++common warnings+  ghc-options:+    -Wall+    -Wcompat+    -Widentities+    -Wincomplete-record-updates+    -Wincomplete-uni-patterns+    -Wpartial-fields+    -Wredundant-constraints++  default-language: GHC2024+  default-extensions:+    BlockArguments+    CPP+    DuplicateRecordFields+    ImportQualifiedPost+    NoFieldSelectors+    OverloadedLabels+    OverloadedRecordDot+    OverloadedStrings++library+  import: warnings+  hs-source-dirs: src+  exposed-modules:+    Kenshou.Telemetry+    Kenshou.Telemetry.Compose+    Kenshou.Telemetry.Continuity+    Kenshou.Telemetry.Detect+    Kenshou.Telemetry.Endpoint+    Kenshou.Telemetry.Metrics+    Kenshou.Telemetry.Overhead+    Kenshou.Telemetry.Overhead.Policy+    Kenshou.Telemetry.Scrape+    Kenshou.Telemetry.SelfTest+    Kenshou.Telemetry.SelfTest.Arms+    Kenshou.Telemetry.SelfTest.Endpoints+    Kenshou.Telemetry.SelfTest.Problems+    Kenshou.Telemetry.SelfTest.Service+    Kenshou.Telemetry.Sink+    Kenshou.Telemetry.Spec+    Kenshou.Telemetry.Tracing+    Kenshou.Telemetry.Tracing.Pipeline+    Kenshou.Telemetry.Tracing.Probe++  build-depends:+    aeson >=2.2 && <2.3,+    async >=2.2 && <2.3,+    base >=4.21 && <4.22,+    bytestring >=0.12 && <0.13,+    containers >=0.7 && <0.8,+    directory >=1.3 && <1.4,+    filepath >=1.5 && <1.6,+    hs-opentelemetry-api ==1.0.0.0,+    hs-opentelemetry-exporter-otlp ==1.0.0.0,+    hs-opentelemetry-exporter-prometheus ==1.0.0.0,+    hs-opentelemetry-otlp ==1.0.0.0,+    hs-opentelemetry-propagator-w3c ==1.0.0.0,+    hs-opentelemetry-sdk ==1.0.0.0,+    http-client >=0.7 && <0.8,+    http-types >=0.12 && <0.13,+    kenshou-core ^>=0.1.0.0,+    kenshou-measure ^>=0.1.0.0,+    microlens >=0.4 && <0.5,+    network >=3.2 && <3.3,+    proto-lens >=0.7 && <0.8,+    proto-lens-runtime >=0.7 && <0.8,+    splitmix >=0.1 && <0.2,+    stm >=2.5 && <2.6,+    text >=2.1 && <2.2,+    time >=1.14 && <1.15,+    unordered-containers >=0.2 && <0.3,+    vector >=0.13 && <0.14,+    wai >=3.2 && <3.3,+    wai-websockets >=3.0 && <3.1,+    warp >=3.4 && <3.5,+    websockets >=0.13 && <0.14,+    zlib >=0.7 && <0.8,++  if flag(otlp-grpc)+    cpp-options: -DWITH_OTLP_GRPC++test-suite kenshou-telemetry-test+  import: warnings+  type: exitcode-stdio-1.0+  hs-source-dirs: test+  main-is: Main.hs+  ghc-options:+    -threaded+    -rtsopts++  build-depends:+    async,+    base >=4.21 && <4.22,+    containers,+    directory >=1.3 && <1.4,+    hs-opentelemetry-api,+    hs-opentelemetry-sdk,+    hspec >=2.11 && <2.12,+    http-types >=0.12 && <0.13,+    kenshou-core ^>=0.1.0.0,+    kenshou-telemetry,+    network >=3.2 && <3.3,+    text,+    wai >=3.2 && <3.3,+    wai-websockets >=3.0 && <3.1,+    warp >=3.4 && <3.5,+    websockets >=0.13 && <0.14,++source-repository head+  type: git+  location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Telemetry.hs view
@@ -0,0 +1,287 @@+module Kenshou.Telemetry+  ( TelemetryHandles (..),+    FlushReport (..),+    ProviderCallReport (..),+    withTelemetry,+    module Kenshou.Telemetry.Spec,+  )+where++import Control.Exception (SomeException, displayException, mask, throwIO, try)+import Control.Monad (void)+import Data.Aeson (FromJSON (parseJSON), ToJSON (..), Value (..), object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseMaybe, withObject, (.:))+import Data.IORef+import Data.Maybe (catMaybes)+import Data.Text (Text)+import Data.Text qualified as Text+import GHC.Clock (getMonotonicTimeNSec)+import Kenshou.Core.Dimension (renderMetrics, renderTracing)+import Kenshou.Core.Role (ControlMessage (..), WorkerMessage (..), mkRoleName)+import Kenshou.Core.Role.Spawn (WorkerHandle (..), withWorker)+import Kenshou.Telemetry.Compose (HandlerStatsSnapshot)+import Kenshou.Telemetry.Continuity (ContinuityResult, IsolationResult)+import Kenshou.Telemetry.Detect+import Kenshou.Telemetry.Endpoint+import Kenshou.Telemetry.Metrics+import Kenshou.Telemetry.Metrics qualified as Metrics+import Kenshou.Telemetry.Scrape+import Kenshou.Telemetry.Sink+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Spec qualified as Spec+import Kenshou.Telemetry.Tracing+import Kenshou.Telemetry.Tracing.Pipeline+import Kenshou.Telemetry.Tracing.Probe+import OpenTelemetry.Metric.Core (Meter, MeterProvider)+import OpenTelemetry.Trace.Core (Tracer, TracerProvider)+import System.Environment (getEnvironment)++data ProviderCallReport = ProviderCallReport+  { result :: Text,+    durationMs :: Double+  }+  deriving stock (Eq, Show)++instance ToJSON ProviderCallReport where+  toJSON report = object ["result" .= report.result, "durationMs" .= report.durationMs]++data FlushReport = FlushReport+  { tracing :: Maybe ProviderCallReport,+    metrics :: Maybe ProviderCallReport+  }+  deriving stock (Eq, Show)++instance ToJSON FlushReport where+  toJSON report = object ["tracing" .= report.tracing, "metrics" .= report.metrics]++data TelemetryHandles = TelemetryHandles+  { tracer :: Maybe Tracer,+    tracerProvider :: Maybe TracerProvider,+    meter :: Maybe Meter,+    meterProvider :: Maybe MeterProvider,+    spans :: Maybe SpanProbe,+    pipeline :: Maybe PipelineStats,+    metricsLive :: Bool,+    readMetricSums :: IO [(Text, Double)],+    readMetricGauges :: IO [(Text, Double)],+    servesEndpoints :: Bool,+    metricEndpoint :: Maybe Endpoint,+    registerEndpoint :: Endpoint -> IO (),+    setSinkFault :: SinkFault -> IO (),+    recordContinuity :: ContinuityResult -> IsolationResult -> IO (),+    recordHandlerStats :: HandlerStatsSnapshot -> IO (),+    flushTelemetry :: IO FlushReport+  }++withTelemetry :: TelemetrySpec -> (TelemetryHandles -> IO value) -> IO value+withTelemetry spec action = withConfiguredScraper spec \scraper -> case spec.endpoint of+  BuiltinSink fault | requiresSink spec -> withConfiguredSink spec fault (\sink -> runWith scraper (spec {Spec.endpoint = ExternalEndpoint sink.endpoint}) (Just sink))+  _ -> runWith scraper spec Nothing+  where+    runWith scraper effectiveSpec sink = mask \restore -> do+      runtime <- startTracing effectiveSpec+      metricsRuntime <- startMetrics effectiveSpec+      stopPipelineSampler <- maybe (pure (pure [])) (startPipelineSampler spec.outDir) runtime.pipeline+      endpointsRef <- newIORef (maybe [] pure metricsRuntime.endpoint)+      recordedFindingsRef <- newIORef []+      handlerStatsRef <- newIORef []+      traverse_ (\endpoint -> traverse_ (\active -> active.register endpoint) scraper) metricsRuntime.endpoint+      let register endpoint+            | spec.metrics `notElem` [MetricsServe, MetricsServeScraped] = ioError (userError "metrics endpoint registered while telemetry.metrics does not serve endpoints")+            | otherwise = do+                ready <- if endpoint.kind == WebSocketPush then pure True else awaitHttpReady endpoint.url 10_000+                if ready+                  then do+                    modifyIORef' endpointsRef (<> [endpoint])+                    traverse_ (\active -> active.register endpoint) scraper+                  else ioError (userError ("metrics endpoint did not become ready: " <> Text.unpack endpoint.url))+      let handles =+            TelemetryHandles+              { tracer = runtime.tracer,+                tracerProvider = runtime.provider,+                meter = metricsRuntime.meter,+                meterProvider = metricsRuntime.provider,+                spans = runtime.probe,+                pipeline = runtime.pipeline,+                metricsLive = spec.metrics /= MetricsOff,+                readMetricSums = Metrics.readMetricSums metricsRuntime,+                readMetricGauges = Metrics.readMetricGauges metricsRuntime,+                servesEndpoints = spec.metrics `elem` [MetricsServe, MetricsServeScraped],+                metricEndpoint = metricsRuntime.endpoint,+                registerEndpoint = register,+                setSinkFault = maybe (const (pure ())) (.setFault) sink,+                recordContinuity = \continuity isolation -> modifyIORef' recordedFindingsRef (<> continuityFindings continuity isolation),+                recordHandlerStats = \snapshot -> modifyIORef' handlerStatsRef (<> [snapshot]),+                flushTelemetry = FlushReport <$> timedCall (flushTracing spec.shutdownMs runtime) <*> timedCall (flushMetrics spec.shutdownMs metricsRuntime)+              }+      bodyResult <- tryAny (restore (action handles))+      flushResult <- timedCall (flushTracing spec.shutdownMs runtime)+      metricsFlushResult <- timedCall (flushMetrics spec.shutdownMs metricsRuntime)+      endpointSummaries <- maybe (pure Nothing) (fmap Just . (.finish)) scraper+      (metricsSnapshot, metricsShutdownResult) <- timedStopMetrics spec.shutdownMs metricsRuntime+      shutdownResult <- timedCall (stopTracing spec.shutdownMs runtime)+      queueSamples <- stopPipelineSampler+      rawPipelineSnapshot <- traverse snapshotPipeline runtime.pipeline+      let pipelineSnapshot = fmap (normalizeQueueDepth spec) rawPipelineSnapshot+      sinkSnapshot <- traverse (.snapshot) sink+      ambient <- ambientOtelEnvironment+      endpoints <- readIORef endpointsRef+      recordedFindings <- readIORef recordedFindingsRef+      handlerStats <- readIORef handlerStatsRef+      let endpointValues = maybe (fmap toJSON endpoints) (fmap toJSON) endpointSummaries+          calls = catMaybes [providerCall "trace-flush" spec.shutdownMs flushResult, providerCall "metrics-flush" spec.shutdownMs metricsFlushResult, providerCall "trace-shutdown" spec.shutdownMs shutdownResult, providerCall "metrics-shutdown" spec.shutdownMs metricsShutdownResult]+          findings = pipelineFindings 0.01 pipelineSnapshot calls <> [queueGrowthFinding (pipelineQueueLimit spec runtime.pipeline) queueSamples, endpointFinding (maybe [] id endpointSummaries)] <> fmap handlerFinding handlerStats <> recordedFindings+          summary = telemetrySummary spec ambient pipelineSnapshot sinkSnapshot metricsSnapshot endpointValues (fmap toJSON handlerStats) findings flushResult metricsFlushResult shutdownResult metricsShutdownResult+      _ <- tryAny (spec.report summary)+      either throwIO pure bodyResult++requiresSink :: TelemetrySpec -> Bool+requiresSink spec = spec.tracing == TracingSdkOtlp || (spec.metrics /= MetricsOff && spec.otelReader == ReaderOtlpPeriodic)++withConfiguredSink :: TelemetrySpec -> SinkFault -> (SinkHandle -> IO value) -> IO value+withConfiguredSink spec fault action = case (spec.helpers, spec.workerContext) of+  (HelperProcess _, Just context) -> do+    roleName <- either (ioError . userError . Text.unpack) pure (mkRoleName "selftest/telemetry-otlp-sink")+    withWorker context roleName "telemetry-otlp-sink" (object ["fault" .= renderSinkFault fault]) \worker -> do+      ready <- worker.receive 10_000+      endpoint <- case ready of+        Just (WrkCustom "ready" payload) -> maybe (ioError (userError "telemetry sink sent an invalid ready message")) pure (parseMaybe (withObject "sink ready" (.: "endpoint")) payload)+        Just (WrkError message) -> ioError (userError (Text.unpack message))+        _ -> ioError (userError "telemetry sink did not become ready within 10 seconds")+      let setFault updated = worker.send (CtlCustom "fault" (String (renderSinkFault updated)))+          snapshot = do+            worker.send (CtlCustom "snapshot" Null)+            response <- worker.receive 10_000+            case response of+              Just (WrkCustom "sink-stats" payload) -> maybe (ioError (userError "telemetry sink sent invalid statistics")) pure (parseMaybe parseJSON payload)+              _ -> ioError (userError "telemetry sink did not return statistics")+      action (SinkHandle endpoint setFault snapshot)+  _ -> withSink fault action++withConfiguredScraper :: TelemetrySpec -> (Maybe ScraperHandle -> IO value) -> IO value+withConfiguredScraper spec action+  | spec.metrics /= MetricsServeScraped = action Nothing+  | otherwise = case (spec.helpers, spec.workerContext) of+      (HelperProcess _, Just context) -> do+        roleName <- either (ioError . userError . Text.unpack) pure (mkRoleName "selftest/telemetry-scraper")+        withWorker context roleName "telemetry-scraper" (object ["intervalMs" .= spec.scrapeMs, "wsSubscribers" .= spec.wsSubscribers]) \worker -> do+          ready <- worker.receive 10_000+          case ready of+            Just WrkReady -> pure ()+            Just (WrkError message) -> ioError (userError (Text.unpack message))+            _ -> ioError (userError "telemetry scraper did not become ready within 10 seconds")+          let register endpoint = worker.send (CtlCustom "endpoint" (toJSON endpoint))+              finish = do+                worker.send (CtlCustom "finish" Null)+                response <- worker.receive 10_000+                case response of+                  Just (WrkCustom "scrape-summary" payload) -> maybe (ioError (userError "telemetry scraper sent an invalid summary")) pure (parseMaybe parseJSON payload)+                  _ -> ioError (userError "telemetry scraper did not return its summary")+          action (Just (ScraperHandle register finish))+      _ -> do+        scraper <- startScraperInProcess spec.outDir spec.scrapeMs spec.wsSubscribers+        action (Just scraper)++telemetrySummary :: TelemetrySpec -> [(Text, Text)] -> Maybe PipelineSnapshot -> Maybe SinkStats -> Maybe MetricsSnapshot -> [Value] -> [Value] -> [Finding] -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Value+telemetrySummary spec ambient pipelineSnapshot sinkSnapshot metricsSnapshot endpoints handlers findings flushResult metricsFlushResult shutdownResult metricsShutdownResult =+  object+    [ "schema" .= ("kenshou.telemetry-summary/v1" :: Text),+      "arms" .= object ["tracing" .= renderTracing spec.tracing, "metrics" .= renderMetrics spec.metrics],+      "settings" .= settingsValue spec,+      "ambientEnv" .= object [Key.fromText key .= value | (key, value) <- ambient],+      "pipeline" .= pipelineValue pipelineSnapshot flushResult shutdownResult,+      "sink" .= sinkSnapshot,+      "metrics" .= object ["snapshot" .= metricsSnapshot, "flush" .= metricsFlushResult, "shutdown" .= metricsShutdownResult],+      "endpoints" .= endpoints,+      "handlers" .= handlers,+      "findings" .= findings+    ]++providerCall :: Text -> Int -> Maybe ProviderCallReport -> Maybe (Text, Text, Double, Double)+providerCall _ _ Nothing = Nothing+providerCall name limit (Just report) = Just (name, report.result, report.durationMs, fromIntegral limit)++normalizeQueueDepth :: TelemetrySpec -> PipelineSnapshot -> PipelineSnapshot+normalizeQueueDepth spec snapshot = case spec.processor of+  BatchProcessor queue _ batch _ ->+    -- The public SDK does not expose its private queue count. The accounting+    -- backlog includes SDK-dropped spans, so cap its high-water mark at the+    -- configured queue plus the one batch that may already be exporting.+    PipelineSnapshot snapshot.spansStarted snapshot.spansEnded snapshot.spansExportedOk snapshot.spansExportFailed snapshot.spansDropped (min snapshot.maxQueueDepth (queue + batch)) snapshot.exportCalls snapshot.exportLatencyNs snapshot.lastExportError+  SimpleProcessor _ -> snapshot++pipelineQueueLimit :: TelemetrySpec -> Maybe PipelineStats -> Maybe Int+pipelineQueueLimit spec pipeline = case (pipeline, spec.processor) of+  (Just _, BatchProcessor queue _ _ _) -> Just queue+  _ -> Nothing++pipelineValue :: Maybe PipelineSnapshot -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Value+pipelineValue Nothing _ _ = Null+pipelineValue (Just snapshot) flushResult shutdownResult = case toJSON snapshot of+  Object fields -> Object (KeyMap.insert "shutdown" (toJSON shutdownResult) (KeyMap.insert "flush" (toJSON flushResult) fields))+  value -> value++settingsValue :: TelemetrySpec -> Value+settingsValue spec =+  object+    [ "sampler" .= renderSampler spec.sampler,+      "processor" .= renderProcessor spec.processor,+      "exporter" .= (case spec.exporter of HttpProtobuf -> "http-protobuf" :: Text; Grpc -> "grpc"),+      "endpoint" .= (case spec.endpoint of BuiltinSink _ -> "builtin-sink" :: Text; ExternalEndpoint value -> value),+      "sinkFault" .= (case spec.endpoint of BuiltinSink fault -> renderSinkFault fault; ExternalEndpoint _ -> "none"),+      "compression" .= (case spec.compression of CompressionNone -> "none" :: Text; CompressionGzip -> "gzip")+    ]++renderSampler :: SamplerSpec -> Text+renderSampler AlwaysOn = "always-on"+renderSampler AlwaysOff = "always-off"+renderSampler ParentBasedAlwaysOn = "parentbased-always-on"+renderSampler (TraceIdRatio ratio) = "traceidratio:" <> Text.pack (show ratio)+renderSampler (ParentBasedTraceIdRatio ratio) = "parentbased-traceidratio:" <> Text.pack (show ratio)++renderProcessor :: ProcessorSpec -> Value+renderProcessor (SimpleProcessor timeoutMs) = object ["kind" .= ("simple" :: Text), "exportTimeoutMs" .= timeoutMs]+renderProcessor (BatchProcessor queue delay batch timeoutMs) =+  object+    [ "kind" .= ("batch" :: Text),+      "maxQueue" .= queue,+      "scheduleDelayMs" .= delay,+      "maxExportBatch" .= batch,+      "exportTimeoutMs" .= timeoutMs+    ]++timedCall :: (Show result) => IO (Maybe result) -> IO (Maybe ProviderCallReport)+timedCall operation = do+  started <- getMonotonicTimeNSec+  outcome <- tryAny operation+  finished <- getMonotonicTimeNSec+  let elapsed = fromIntegral (finished - started) / 1_000_000+  pure . Just $ case outcome of+    Left exception -> ProviderCallReport ("exception: " <> Text.pack (displayException exception)) elapsed+    Right Nothing -> ProviderCallReport "not-applicable" elapsed+    Right (Just result) -> ProviderCallReport (Text.pack (show result)) elapsed++timedStopMetrics :: Int -> MetricsRuntime -> IO (Maybe MetricsSnapshot, Maybe ProviderCallReport)+timedStopMetrics timeoutMs runtime = do+  started <- getMonotonicTimeNSec+  outcome <- tryAny (stopMetrics timeoutMs runtime)+  finished <- getMonotonicTimeNSec+  let elapsed = fromIntegral (finished - started) / 1_000_000+  pure $ case outcome of+    Left exception -> (Nothing, Just (ProviderCallReport ("exception: " <> Text.pack (displayException exception)) elapsed))+    Right (snapshot, Nothing) -> (snapshot, Just (ProviderCallReport "not-applicable" elapsed))+    Right (snapshot, Just result) -> (snapshot, Just (ProviderCallReport (Text.pack (show result)) elapsed))++ambientOtelEnvironment :: IO [(Text, Text)]+ambientOtelEnvironment = do+  environment <- getEnvironment+  pure [(Text.pack key, Text.pack value) | (key, value) <- environment, "OTEL_" `Text.isPrefixOf` Text.pack key]++tryAny :: IO value -> IO (Either SomeException value)+tryAny = try++traverse_ :: (a -> IO b) -> Maybe a -> IO ()+traverse_ _ Nothing = pure ()+traverse_ action (Just value) = void (action value)
+ src/Kenshou/Telemetry/Compose.hs view
@@ -0,0 +1,96 @@+module Kenshou.Telemetry.Compose+  ( HandlerStats,+    HandlerStatsSnapshot (..),+    AsyncHandlerStats (..),+    newHandlerStats,+    snapshotHandlerStats,+    composeHandlers,+    timedHandler,+    slowHandler,+    asyncHandler,+  )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async)+import Control.Concurrent.STM+import Control.Monad (forever)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.IORef+import Data.List (sort)+import Data.Text (Text)+import GHC.Clock (getMonotonicTimeNSec)++newtype HandlerStats = HandlerStats (IORef [Integer])++data HandlerStatsSnapshot = HandlerStatsSnapshot+  { name :: Text,+    calls :: Int,+    durationP50Ns :: Integer,+    durationP99Ns :: Integer,+    durationMaxNs :: Integer+  }+  deriving stock (Eq, Show)++data AsyncHandlerStats = AsyncHandlerStats+  { accepted :: Int,+    dropped :: Int,+    handled :: Int,+    queueDepth :: Int+  }+  deriving stock (Eq, Show)++instance ToJSON HandlerStatsSnapshot where+  toJSON value = object ["name" .= value.name, "calls" .= value.calls, "durationNs" .= object ["p50" .= value.durationP50Ns, "p99" .= value.durationP99Ns, "max" .= value.durationMaxNs]]++instance ToJSON AsyncHandlerStats where+  toJSON value = object ["accepted" .= value.accepted, "dropped" .= value.dropped, "handled" .= value.handled, "queueDepth" .= value.queueDepth]++newHandlerStats :: IO HandlerStats+newHandlerStats = HandlerStats <$> newIORef []++snapshotHandlerStats :: Text -> HandlerStats -> IO HandlerStatsSnapshot+snapshotHandlerStats name (HandlerStats ref) = do+  durations <- sort <$> readIORef ref+  pure (HandlerStatsSnapshot name (length durations) (quantile 0.50 durations) (quantile 0.99 durations) (maybe 0 last (nonEmpty durations)))++composeHandlers :: [a -> IO ()] -> a -> IO ()+composeHandlers [] = const (pure ())+composeHandlers handlers = \value -> mapM_ ($ value) handlers++timedHandler :: HandlerStats -> Text -> (a -> IO ()) -> a -> IO ()+timedHandler (HandlerStats ref) _name handler value = do+  started <- getMonotonicTimeNSec+  handler value+  ended <- getMonotonicTimeNSec+  modifyIORef' ref (fromIntegral (ended - started) :)++slowHandler :: Int -> a -> IO ()+slowHandler micros _ = threadDelay (max 0 micros)++asyncHandler :: Int -> (a -> IO ()) -> IO (a -> IO (), IO AsyncHandlerStats)+asyncHandler requestedCapacity handler = do+  let capacity = max 1 requestedCapacity+  queue <- newTBQueueIO (fromIntegral capacity)+  acceptedRef <- newIORef 0+  droppedRef <- newIORef 0+  handledRef <- newIORef 0+  _ <- async . forever $ do+    value <- atomically (readTBQueue queue)+    handler value+    modifyIORef' handledRef (+ 1)+  let submit value = do+        accepted <- atomically do+          full <- isFullTBQueue queue+          if full then pure False else writeTBQueue queue value >> pure True+        modifyIORef' (if accepted then acceptedRef else droppedRef) (+ 1)+      snapshot = AsyncHandlerStats <$> readIORef acceptedRef <*> readIORef droppedRef <*> readIORef handledRef <*> (fromIntegral <$> atomically (lengthTBQueue queue))+  pure (submit, snapshot)++quantile :: Double -> [Integer] -> Integer+quantile _ [] = 0+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))++nonEmpty :: [a] -> Maybe [a]+nonEmpty [] = Nothing+nonEmpty values = Just values
+ src/Kenshou/Telemetry/Continuity.hs view
@@ -0,0 +1,74 @@+module Kenshou.Telemetry.Continuity+  ( SpanSelector (..),+    ContinuityResult (..),+    IsolationResult (..),+    checkContinuity,+    checkIsolation,+  )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.List (find)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Telemetry.Tracing.Probe (SpanView (..))+import OpenTelemetry.Attributes (Attribute, lookupAttribute)+import OpenTelemetry.Trace.Core (SpanKind)++data SpanSelector = SpanSelector+  { nameIs :: Maybe Text,+    kindIs :: Maybe SpanKind,+    hasAttr :: [(Text, Attribute)]+  }+  deriving stock (Eq, Show)++data ContinuityResult = ContinuityResult+  { producers :: Int,+    consumers :: Int,+    matched :: Int,+    violations :: Int,+    counterExamples :: [Text]+  }+  deriving stock (Eq, Show)++data IsolationResult = IsolationResult+  { checked :: Int,+    violations :: Int,+    counterExamples :: [Text]+  }+  deriving stock (Eq, Show)++instance ToJSON ContinuityResult where+  toJSON value = object ["producers" .= value.producers, "consumers" .= value.consumers, "matched" .= value.matched, "violations" .= value.violations, "counterExamples" .= value.counterExamples]++instance ToJSON IsolationResult where+  toJSON value = object ["checked" .= value.checked, "violations" .= value.violations, "counterExamples" .= value.counterExamples]++checkContinuity :: SpanSelector -> SpanSelector -> (SpanView -> Maybe Text) -> [SpanView] -> ContinuityResult+checkContinuity producerSelector consumerSelector correlation spans =+  let producerSpans = filter (matches producerSelector) spans+      consumerSpans = filter (matches consumerSelector) spans+      outcomes = fmap (checkConsumer producerSpans) consumerSpans+      failures = take 10 [message | Left message <- outcomes]+   in ContinuityResult (length producerSpans) (length consumerSpans) (length [() | Right () <- outcomes]) (length [() | Left _ <- outcomes]) failures+  where+    checkConsumer producerSpans consumer = case correlation consumer of+      Nothing -> Left (consumer.name <> ": missing correlation key")+      Just key -> case find ((== Just key) . correlation) producerSpans of+        Nothing -> Left (consumer.name <> ": no producer for " <> key)+        Just producer+          | consumer.traceId /= producer.traceId -> Left (consumer.name <> ": trace differs for " <> key)+          | consumer.parentSpanId /= Just producer.spanId -> Left (consumer.name <> ": parent differs for " <> key)+          | otherwise -> Right ()++checkIsolation :: SpanSelector -> (SpanView -> Bool) -> [SpanView] -> IsolationResult+checkIsolation selector expectedRoot spans =+  let checkedSpans = filter (\span -> matches selector span && expectedRoot span) spans+      failures = [span.name <> ": inherited parent " <> Text.pack (show parent) | span <- checkedSpans, Just parent <- [span.parentSpanId]]+   in IsolationResult (length checkedSpans) (length failures) (take 10 failures)++matches :: SpanSelector -> SpanView -> Bool+matches selector span =+  maybe True (== span.name) selector.nameIs+    && maybe True (== span.kind) selector.kindIs+    && all (\(key, expected) -> lookupAttribute span.attributes key == Just expected) selector.hasAttr
+ src/Kenshou/Telemetry/Detect.hs view
@@ -0,0 +1,103 @@+module Kenshou.Telemetry.Detect+  ( FindingStatus (..),+    FindingSeverity (..),+    Finding (..),+    pipelineFindings,+    queueGrowthFinding,+    endpointFinding,+    handlerFinding,+    continuityFindings,+  )+where++import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Telemetry.Compose (HandlerStatsSnapshot (..))+import Kenshou.Telemetry.Continuity (ContinuityResult (..), IsolationResult (..))+import Kenshou.Telemetry.Scrape (EndpointSummary (..))+import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..))++data FindingStatus = Fired | Quiet | NotApplicable deriving stock (Eq, Show)++data FindingSeverity = Info | Degraded | Failure deriving stock (Eq, Ord, Show)++data Finding = Finding+  { detector :: Text,+    status :: FindingStatus,+    severity :: FindingSeverity,+    evidence :: Value+  }+  deriving stock (Eq, Show)++instance ToJSON FindingStatus where+  toJSON Fired = toJSON ("fired" :: Text)+  toJSON Quiet = toJSON ("quiet" :: Text)+  toJSON NotApplicable = toJSON ("not-applicable" :: Text)++instance ToJSON FindingSeverity where+  toJSON Info = toJSON ("info" :: Text)+  toJSON Degraded = toJSON ("degraded" :: Text)+  toJSON Failure = toJSON ("failure" :: Text)++instance ToJSON Finding where+  toJSON value = object ["detector" .= value.detector, "status" .= value.status, "severity" .= value.severity, "evidence" .= value.evidence]++pipelineFindings :: Double -> Maybe PipelineSnapshot -> [(Text, Text, Double, Double)] -> [Finding]+pipelineFindings _ Nothing _ =+  [ Finding "span-drop" NotApplicable Info (object []),+    Finding "export-failure" NotApplicable Info (object []),+    Finding "shutdown-blocked" NotApplicable Info (object [])+  ]+pipelineFindings degradedFraction (Just snapshot) providerCalls =+  [ Finding "span-drop" (fired snapshot.spansDropped) dropSeverity (object ["spansEnded" .= snapshot.spansEnded, "spansDropped" .= snapshot.spansDropped, "fraction" .= dropFraction]),+    Finding "export-failure" (fired snapshot.spansExportFailed) (if snapshot.spansExportFailed > 0 then Degraded else Info) (object ["spansExportFailed" .= snapshot.spansExportFailed, "lastError" .= snapshot.lastExportError]),+    Finding "shutdown-blocked" (if null blocked then Quiet else Fired) (if null blocked then Info else Degraded) (object ["calls" .= [object ["call" .= name, "result" .= result, "durationMs" .= duration, "limitMs" .= limit] | (name, result, duration, limit) <- providerCalls], "blocked" .= fmap (\(name, _, _, _) -> name) blocked])+  ]+  where+    dropFraction = fromIntegral snapshot.spansDropped / fromIntegral (max 1 snapshot.spansEnded)+    dropSeverity = if dropFraction > degradedFraction then Degraded else Info+    blocked = filter (\(_, result, duration, limit) -> duration > limit || "timeout" `Text.isInfixOf` Text.toCaseFold result) providerCalls+    fired count = if count > 0 then Fired else Quiet++queueGrowthFinding :: Maybe Int -> [Int] -> Finding+queueGrowthFinding Nothing _ = Finding "queue-growth" NotApplicable Info (object [])+queueGrowthFinding (Just queueLimit) samples =+  Finding+    "queue-growth"+    (if longest > 10 then Fired else Quiet)+    (if longest > 10 then Degraded else Info)+    (object ["queueLimit" .= queueLimit, "threshold" .= threshold, "longestConsecutiveSamples" .= longest, "samples" .= length samples])+  where+    threshold = fromIntegral queueLimit * (0.9 :: Double)+    longest = longestRun ((> threshold) . fromIntegral) samples++longestRun :: (value -> Bool) -> [value] -> Int+longestRun predicate = snd . foldl step (0, 0)+  where+    step (current, best) value =+      let next = if predicate value then current + 1 else 0+       in (next, max best next)++endpointFinding :: [EndpointSummary] -> Finding+endpointFinding [] = Finding "endpoint-failure" NotApplicable Info (object [])+endpointFinding summaries = Finding "endpoint-failure" status severity (object ["failures" .= failures, "attempted" .= attempted, "fraction" .= fraction])+  where+    attempted = sum [summary.scrapes + summary.skippedTicks | summary <- summaries]+    failures = sum [summary.failures + summary.skippedTicks | summary <- summaries]+    fraction :: Double+    fraction = fromIntegral failures / fromIntegral (max 1 attempted)+    status = if fraction > 0.01 then Fired else Quiet+    severity = if status == Fired then Degraded else Info++handlerFinding :: HandlerStatsSnapshot -> Finding+handlerFinding snapshot = Finding "handler-stall" status severity (object ["handler" .= snapshot.name, "calls" .= snapshot.calls, "durationP99Ns" .= snapshot.durationP99Ns, "limitNs" .= (5_000_000 :: Integer)])+  where+    status = if snapshot.durationP99Ns > 5_000_000 then Fired else Quiet+    severity = if status == Fired then Degraded else Info++continuityFindings :: ContinuityResult -> IsolationResult -> [Finding]+continuityFindings continuity isolation =+  [ Finding "trace-discontinuity" (if continuity.violations > 0 then Fired else Quiet) (if continuity.violations > 0 then Failure else Info) (toJSON continuity),+    Finding "context-leak" (if isolation.violations > 0 then Fired else Quiet) (if isolation.violations > 0 then Failure else Info) (toJSON isolation)+  ]
+ src/Kenshou/Telemetry/Endpoint.hs view
@@ -0,0 +1,71 @@+module Kenshou.Telemetry.Endpoint+  ( EndpointKind (..),+    Endpoint (..),+    reserveFreePort,+    awaitHttpReady,+  )+where++import Control.Concurrent (threadDelay)+import Control.Exception (SomeException, try)+import Data.Aeson (FromJSON (..), ToJSON (..), Value, object, withObject, withText, (.:), (.:?), (.=))+import Data.Text (Text)+import Data.Text qualified as Text+import Network.HTTP.Client (defaultManagerSettings, httpLbs, newManager, parseRequest, responseStatus)+import Network.HTTP.Types.Status (statusCode)+import Network.Socket (close)+import Network.Wai.Handler.Warp (openFreePort)++data EndpointKind = PrometheusText | JsonDocument | HealthProbe | WebSocketPush+  deriving stock (Eq, Ord, Show)++data Endpoint = Endpoint+  { name :: Text,+    kind :: EndpointKind,+    url :: Text,+    wsHello :: Maybe Value+  }+  deriving stock (Eq, Show)++instance ToJSON EndpointKind where+  toJSON = toJSON . renderKind++instance FromJSON EndpointKind where+  parseJSON = withText "EndpointKind" (maybe (fail "unknown endpoint kind") pure . parseKind)++instance ToJSON Endpoint where+  toJSON endpoint = object ["name" .= endpoint.name, "kind" .= endpoint.kind, "url" .= endpoint.url, "wsHello" .= endpoint.wsHello]++instance FromJSON Endpoint where+  parseJSON = withObject "Endpoint" (\value -> Endpoint <$> value .: "name" <*> value .: "kind" <*> value .: "url" <*> value .:? "wsHello")++reserveFreePort :: IO Int+reserveFreePort = do+  (port, socket) <- openFreePort+  close socket+  pure port++awaitHttpReady :: Text -> Int -> IO Bool+awaitHttpReady url timeoutMs = do+  manager <- newManager defaultManagerSettings+  request <- parseRequest (Text.unpack url)+  let attempts = max 1 (timeoutMs `div` 50)+      loop 0 = pure False+      loop remaining = do+        response <- tryAny (httpLbs request manager)+        case response of+          Right value | statusCode (responseStatus value) < 500 -> pure True+          _ -> threadDelay 50_000 >> loop (remaining - 1)+  loop attempts++renderKind :: EndpointKind -> Text+renderKind PrometheusText = "prometheus-text"+renderKind JsonDocument = "json-document"+renderKind HealthProbe = "health-probe"+renderKind WebSocketPush = "websocket-push"++parseKind :: Text -> Maybe EndpointKind+parseKind value = lookup value [(renderKind kind, kind) | kind <- [PrometheusText, JsonDocument, HealthProbe, WebSocketPush]]++tryAny :: IO value -> IO (Either SomeException value)+tryAny = try
+ src/Kenshou/Telemetry/Metrics.hs view
@@ -0,0 +1,116 @@+module Kenshou.Telemetry.Metrics+  ( MetricsRuntime (..),+    MetricsSnapshot (..),+    startMetrics,+    flushMetrics,+    readMetricSums,+    readMetricGauges,+    stopMetrics,+  )+where++import Control.Concurrent.Async (Async, async, cancel, waitCatch)+import Control.Monad (void)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.Text qualified as Text+import Data.Vector qualified as Vector+import Kenshou.Telemetry.Endpoint+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Tracing (otlpExporterConfig)+import Network.Socket (Socket, close)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import OpenTelemetry.Exporter.Metric (GaugeDataPoint (..), MetricExport (..), NumberValue (..), ResourceMetricsExport (..), ScopeMetricsExport (..), SumDataPoint (..))+import OpenTelemetry.Exporter.OTLP.Metric qualified as OtlpMetric+import OpenTelemetry.Exporter.Prometheus.WAI (prometheusApplication)+import OpenTelemetry.MeterProvider (SdkMeterEnv, collectResourceMetrics, createMeterProvider, defaultSdkMeterProviderOptions)+import OpenTelemetry.Metric.Core (Meter, MeterProvider, forceFlushMeterProvider, getMeter, shutdownMeterProvider)+import OpenTelemetry.MetricReader (PeriodicMetricReaderHandle (..), PeriodicMetricReaderOptions (..), forkPeriodicMetricReader)+import OpenTelemetry.Resource (emptyMaterializedResources)+import OpenTelemetry.Trace.Core (FlushResult, ShutdownResult, instrumentationLibrary)++data MetricsRuntime = MetricsRuntime+  { meter :: Maybe Meter,+    provider :: Maybe MeterProvider,+    env :: Maybe SdkMeterEnv,+    endpoint :: Maybe Endpoint,+    server :: Maybe (Async (), Socket),+    reader :: Maybe PeriodicMetricReaderHandle+  }++data MetricsSnapshot = MetricsSnapshot+  { resourceBatches :: Int,+    instruments :: Int+  }+  deriving stock (Eq, Show)++instance ToJSON MetricsSnapshot where+  toJSON snapshot = object ["resourceBatches" .= snapshot.resourceBatches, "instruments" .= snapshot.instruments]++startMetrics :: TelemetrySpec -> IO MetricsRuntime+startMetrics spec = case spec.metrics of+  MetricsOff -> pure (MetricsRuntime Nothing Nothing Nothing Nothing Nothing Nothing)+  _ -> do+    (provider, env) <- createMeterProvider emptyMaterializedResources defaultSdkMeterProviderOptions+    meter <- getMeter provider (instrumentationLibrary "kenshou" "0.1.0.0")+    (endpoint, server) <- case (spec.metrics, spec.otelReader) of+      (arm, ReaderPrometheus) | arm `elem` [MetricsServe, MetricsServeScraped] -> do+        (port, socket) <- openFreePort+        running <- async (runSettingsSocket defaultSettings socket (prometheusApplication (Vector.fromList <$> collectResourceMetrics env)))+        let url = "http://127.0.0.1:" <> Text.pack (show port) <> "/metrics"+        pure (Just (Endpoint "otel-prometheus" PrometheusText url Nothing), Just (running, socket))+      _ -> pure (Nothing, Nothing)+    reader <- case spec.otelReader of+      ReaderOtlpPeriodic -> do+        exporter <- OtlpMetric.otlpMetricExporter (otlpExporterConfig spec)+        Just <$> forkPeriodicMetricReader env exporter (PeriodicMetricReaderOptions (spec.otelExportMs * 1000))+      _ -> pure Nothing+    pure (MetricsRuntime (Just meter) (Just provider) (Just env) endpoint server reader)++flushMetrics :: Int -> MetricsRuntime -> IO (Maybe FlushResult)+flushMetrics timeoutMs runtime = traverse (\provider -> forceFlushMeterProvider provider (Just (timeoutMs * 1000))) runtime.provider++readMetricSums :: MetricsRuntime -> IO [(Text.Text, Double)]+readMetricSums runtime = do+  batches <- maybe (pure []) collectResourceMetrics runtime.env+  pure+    [ (metric.mesName, sum (fmap pointValue metric.mesSumPoints))+    | resource <- batches,+      scope <- Vector.toList resource.resourceMetricsScopes,+      metric@MetricExportSum {} <- Vector.toList scope.scopeMetricsExports+    ]+  where+    pointValue point = case point.sumDataPointValue of+      IntNumber value -> fromIntegral value+      DoubleNumber value -> value++readMetricGauges :: MetricsRuntime -> IO [(Text.Text, Double)]+readMetricGauges runtime = do+  batches <- maybe (pure []) collectResourceMetrics runtime.env+  pure+    [ (metric.megName, pointValue point.gaugeDataPointValue)+    | resource <- batches,+      scope <- Vector.toList resource.resourceMetricsScopes,+      metric@MetricExportGauge {} <- Vector.toList scope.scopeMetricsExports,+      point <- Vector.toList metric.megGaugePoints+    ]+  where+    pointValue = \case+      IntNumber value -> fromIntegral value+      DoubleNumber value -> value++stopMetrics :: Int -> MetricsRuntime -> IO (Maybe MetricsSnapshot, Maybe ShutdownResult)+stopMetrics timeoutMs runtime = do+  traverse_ (.stopPeriodicMetricReader) runtime.reader+  batches <- maybe (pure []) collectResourceMetrics runtime.env+  let snapshot = case runtime.env of+        Nothing -> Nothing+        Just _ -> Just (MetricsSnapshot (length batches) (sum [Vector.length scope.scopeMetricsExports | resource <- batches, scope <- Vector.toList resource.resourceMetricsScopes]))+  shutdown <- traverse (\provider -> shutdownMeterProvider provider (Just (timeoutMs * 1000))) runtime.provider+  case runtime.server of+    Nothing -> pure ()+    Just (running, socket) -> cancel running >> void (waitCatch running) >> close socket+  pure (snapshot, shutdown)++traverse_ :: (a -> IO b) -> Maybe a -> IO ()+traverse_ _ Nothing = pure ()+traverse_ action (Just value) = void (action value)
+ src/Kenshou/Telemetry/Overhead.hs view
@@ -0,0 +1,648 @@+module Kenshou.Telemetry.Overhead+  ( Arm (..),+    Slot (..),+    OverheadMode (..),+    OverheadRequest (..),+    OverheadPlan (..),+    SlotRun (..),+    OverheadState (..),+    OverheadHooks (..),+    OverheadComparison (..),+    OverheadReport (..),+    UsageError (..),+    planOverhead,+    executeOverhead,+    analyseOverhead,+    loadOverheadState,+    overheadVerdictExitCode,+  )+where++import Control.Concurrent (threadDelay)+import Control.Exception (SomeException, try)+import Control.Monad (foldM, when)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (nub, sort, sortOn)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, fromMaybe, listToMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Dimension+import Kenshou.Core.Id+import Kenshou.Core.Knob+import Kenshou.Core.Manifest (verifyManifest)+import Kenshou.Core.RunSpec+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..))+import Kenshou.Measure.Compare.Policy (Policy (..))+import Kenshou.Measure.Stats (Interval (..))+import Kenshou.Telemetry.Overhead.Policy+import System.Directory (createDirectoryIfMissing, doesFileExist, renameFile)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Random.SplitMix (mkSMGen, nextWord64)++data Arm = Arm+  { armId :: Text,+    dimensions :: Map Text Text,+    knobs :: Map Text KnobValue,+    control :: Bool+  }+  deriving stock (Eq, Show)++data Slot = Slot+  { block :: Int,+    position :: Int,+    arm :: Arm,+    spec :: RunSpec+  }+  deriving stock (Eq, Show)++data OverheadMode = OneFactor | FullFactorial deriving stock (Eq, Show)++data OverheadRequest = OverheadRequest+  { requestId :: Text,+    scenarioId :: ScenarioId,+    factors :: Map Text [Text],+    mode :: OverheadMode,+    includeControl :: Bool,+    trials :: Int,+    fixedKnobs :: [(KnobName, RawKnob)],+    fixedDimensions :: Map Text Text,+    seed :: Word64,+    settleSeconds :: Int,+    retries :: Int+  }+  deriving stock (Eq, Show)++data OverheadPlan = OverheadPlan+  { planId :: Text,+    scenarioId :: ScenarioId,+    scenarioRevision :: Int,+    mode :: OverheadMode,+    factorValues :: Map Text [Text],+    baselineArm :: Text,+    arms :: [Arm],+    trials :: Int,+    seed :: Word64,+    settleSeconds :: Int,+    retries :: Int,+    fixedDimensions :: Map Text Text,+    fixedKnobs :: Map Text KnobValue,+    slots :: [Slot]+  }+  deriving stock (Eq, Show)++data SlotRun = SlotRun+  { block :: Int,+    position :: Int,+    armId :: Text,+    runIds :: [RunId],+    exitCode :: Maybe Int,+    complete :: Bool+  }+  deriving stock (Eq, Show)++data OverheadState = OverheadState+  { plan :: OverheadPlan,+    slots :: [SlotRun]+  }+  deriving stock (Eq, Show)++data OverheadHooks = OverheadHooks+  { runChild :: RunSpec -> FilePath -> IO ExitCode,+    compare :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison),+    leakCheck :: Maybe (FilePath -> IO (Maybe Value))+  }++data OverheadComparison = OverheadComparison+  { candidate :: Text,+    baseline :: Text,+    factor :: Text,+    fromValue :: Text,+    toValue :: Text,+    comparisonPath :: Maybe FilePath,+    verdict :: Verdict,+    degraded :: Bool,+    deltas :: Value,+    reasons :: [Text]+  }+  deriving stock (Eq, Show)++data OverheadReport = OverheadReport+  { reportId :: Text,+    scenarioId :: ScenarioId,+    mode :: OverheadMode,+    baselineDimensions :: Map Text Text,+    fixedDimensions :: Map Text Text,+    fixedKnobs :: Map Text KnobValue,+    trials :: Int,+    seed :: Word64,+    validBlocks :: Int,+    order :: [[Text]],+    armRuns :: [ArmEvidence],+    comparisons :: [OverheadComparison],+    reportFindings :: [Value],+    policyIdentity :: Value,+    verdict :: Verdict+  }+  deriving stock (Eq, Show)++data ArmEvidence = ArmEvidence+  { arm :: Arm,+    runs :: [RunId],+    telemetry :: Value,+    leak :: Maybe Value,+    findings :: [Value],+    degraded :: Bool+  }+  deriving stock (Eq, Show)++newtype UsageError = UsageError Text deriving stock (Eq, Show)++instance ToJSON Arm where+  toJSON value = object ["arm" .= value.armId, "dimensions" .= value.dimensions, "knobs" .= value.knobs, "control" .= value.control]++instance FromJSON Arm where+  parseJSON = withObject "Arm" \value -> Arm <$> value .: "arm" <*> value .: "dimensions" <*> value .: "knobs" <*> value .:? "control" .!= False++instance ToJSON OverheadMode where+  toJSON OneFactor = String "one-factor"+  toJSON FullFactorial = String "full"++instance FromJSON OverheadMode where+  parseJSON = withText "OverheadMode" \case+    "one-factor" -> pure OneFactor+    "full" -> pure FullFactorial+    _ -> fail "mode must be one-factor or full"++instance ToJSON Slot where+  toJSON value = object ["block" .= value.block, "position" .= value.position, "arm" .= value.arm, "spec" .= value.spec]++instance FromJSON Slot where+  parseJSON = withObject "Slot" \value -> Slot <$> value .: "block" <*> value .: "position" <*> value .: "arm" <*> value .: "spec"++instance ToJSON OverheadPlan where+  toJSON value =+    object+      [ "schema" .= ("kenshou.overhead-plan/v1" :: Text),+        "id" .= value.planId,+        "scenario" .= value.scenarioId,+        "scenarioRevision" .= value.scenarioRevision,+        "mode" .= value.mode,+        "factorValues" .= value.factorValues,+        "baselineArm" .= value.baselineArm,+        "arms" .= value.arms,+        "trials" .= value.trials,+        "seed" .= value.seed,+        "settleSeconds" .= value.settleSeconds,+        "retries" .= value.retries,+        "fixedDimensions" .= value.fixedDimensions,+        "fixedKnobs" .= value.fixedKnobs,+        "slots" .= value.slots+      ]++instance FromJSON OverheadPlan where+  parseJSON = withObject "OverheadPlan" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.overhead-plan/v1" :: Text) then fail "unsupported overhead plan" else pure ()+    OverheadPlan <$> value .: "id" <*> value .: "scenario" <*> value .: "scenarioRevision" <*> value .: "mode" <*> value .: "factorValues" <*> value .: "baselineArm" <*> value .: "arms" <*> value .: "trials" <*> value .: "seed" <*> value .: "settleSeconds" <*> value .: "retries" <*> value .: "fixedDimensions" <*> value .: "fixedKnobs" <*> value .: "slots"++instance ToJSON SlotRun where+  toJSON value = object ["block" .= value.block, "position" .= value.position, "arm" .= value.armId, "runIds" .= value.runIds, "exitCode" .= value.exitCode, "complete" .= value.complete]++instance FromJSON SlotRun where+  parseJSON = withObject "SlotRun" \value -> SlotRun <$> value .: "block" <*> value .: "position" <*> value .: "arm" <*> value .:? "runIds" .!= [] <*> value .:? "exitCode" <*> value .:? "complete" .!= False++instance ToJSON OverheadState where+  toJSON value = object ["schema" .= ("kenshou.overhead-state/v1" :: Text), "plan" .= value.plan, "slots" .= value.slots]++instance FromJSON OverheadState where+  parseJSON = withObject "OverheadState" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.overhead-state/v1" :: Text) then fail "unsupported overhead state" else pure ()+    OverheadState <$> value .: "plan" <*> value .:? "slots" .!= []++instance ToJSON OverheadComparison where+  toJSON value =+    object+      [ "candidate" .= value.candidate,+        "baseline" .= value.baseline,+        "factor" .= value.factor,+        "from" .= value.fromValue,+        "to" .= value.toValue,+        "comparison" .= value.comparisonPath,+        "verdict" .= value.verdict,+        "degraded" .= value.degraded,+        "deltas" .= value.deltas,+        "reasons" .= value.reasons+      ]++instance ToJSON OverheadReport where+  toJSON value =+    object+      [ "schema" .= ("kenshou.overhead-report/v1" :: Text),+        "id" .= value.reportId,+        "scenario" .= value.scenarioId,+        "mode" .= value.mode,+        "baseline" .= value.baselineDimensions,+        "fixed" .= object ["knobs" .= value.fixedKnobs, "dimensions" .= value.fixedDimensions],+        "trials" .= value.trials,+        "seed" .= value.seed,+        "validBlocks" .= value.validBlocks,+        "order" .= value.order,+        "arms" .= [object ["arm" .= row.arm.armId, "dimensions" .= row.arm.dimensions, "runs" .= row.runs, "telemetry" .= row.telemetry, "leak" .= row.leak, "findings" .= row.findings] | row <- value.armRuns],+        "comparisons" .= value.comparisons,+        "findings" .= value.reportFindings,+        "policy" .= value.policyIdentity,+        "cohort" .= object [],+        "verdict" .= value.verdict,+        "algorithm" .= object ["name" .= ("kenshou-overhead" :: Text), "version" .= (1 :: Int)]+      ]++planOverhead :: OverheadRequest -> Scenario -> Either UsageError OverheadPlan+planOverhead request scenario = do+  whenEither (request.scenarioId /= scenario.id) "scenario request does not match the selected scenario"+  whenEither (request.trials < 3) "trials must be at least 3"+  whenEither (request.settleSeconds < 0) "settle-seconds must be non-negative"+  whenEither (request.retries < 0) "retries must be non-negative"+  whenEither (Map.null request.factors) "at least one --arms factor is required"+  traverse_ validateFactor (Map.toList request.factors)+  whenEither (any (`Map.member` request.fixedDimensions) (Map.keys request.factors)) "a varied factor cannot also be fixed with --dim"+  resolvedKnobs <- firstUsage (resolveKnobs scenario.knobs request.fixedKnobs)+  let knobMap = Map.fromList [(renderKnobName name, value) | (name, value) <- Map.toList (resolvedKnobsMap resolvedKnobs)]+      baselineCell = Map.map (fromMaybe "" . listToMaybe) request.factors+      cells = case request.mode of+        OneFactor -> baselineCell : [Map.insert factor armValue baselineCell | (factor, values) <- Map.toAscList request.factors, armValue <- drop 1 values]+        FullFactorial -> cartesian (Map.toAscList request.factors)+  plannedArms <- traverse (makeArm scenario knobMap) (zip [0 :: Int ..] (nub cells))+  baselinePlanned <- maybe (Left (UsageError "overhead planning produced no baseline arm")) Right (listToMaybe plannedArms)+  let withControl = plannedArms <> [Arm "control" baselinePlanned.dimensions knobMap True | request.includeControl]+      shuffled = shuffle request.seed withControl+      template =+        OverheadPlan+          request.requestId+          scenario.id+          scenario.revision+          request.mode+          request.factors+          baselinePlanned.armId+          withControl+          request.trials+          request.seed+          request.settleSeconds+          request.retries+          request.fixedDimensions+          knobMap+          []+      initialSlots = concatMap (blockSlots template shuffled) [0 .. request.trials - 1]+  pure (setPlanSlots initialSlots template)+  where+    validateFactor (factor, values) = do+      whenEither (factor `notElem` ["telemetry.tracing", "telemetry.metrics"]) ("unknown overhead factor " <> factor)+      whenEither (null values) (factor <> " has no values")+      whenEither (length values /= length (nub values)) (factor <> " repeats a value")+    makeArm selected knobMap (index, cell) = do+      let assignments = Map.toList (request.fixedDimensions <> cell)+      dimensions <- firstUsage (resolveDimensions selected.dimensions assignments)+      let armName = "a" <> Text.pack (show index)+      pure (Arm armName (Map.fromList (renderDimensions dimensions)) knobMap False)++blockSlots :: OverheadPlan -> [Arm] -> Int -> [Slot]+blockSlots plan shuffled blockNumber = zipWith make [0 ..] ordered+  where+    count = length shuffled+    rotated = rotate ((blockNumber `div` 2) `mod` max 1 count) shuffled+    ordered = if even blockNumber then rotated else reverse rotated+    make position arm =+      let rawKnobs = [(name, RawJson (toJSON value)) | (nameText, value) <- Map.toList arm.knobs, Right name <- [mkKnobName nameText]]+          blockSeed = (plan.seed + fromIntegral blockNumber) `mod` 9007199254740992+          seedValue = either (const Nothing) Just (mkSeed blockSeed)+          base :: RunSpec+          base = minimalRunSpec plan.scenarioId+          runSpec =+            RunSpec+              base.runId+              base.scenario+              (Just plan.scenarioRevision)+              rawKnobs+              (Map.toList arm.dimensions)+              seedValue+              base.phases+              base.timeoutSeconds+              base.environment+              base.cohortExpectation+              (Just (ComparisonMembership plan.planId arm.armId blockNumber (blockNumber * count + position)))+              (Map.fromList [("overhead", plan.planId), ("overhead-arm", arm.armId)])+       in Slot blockNumber position arm runSpec++executeOverhead :: OverheadHooks -> OverheadPlan -> FilePath -> IO OverheadState+executeOverhead hooks plan output = do+  createDirectoryIfMissing True (output </> "runs")+  createDirectoryIfMissing True (output </> "comparisons")+  loaded <- loadOverheadState output+  initial <- case loaded of+    Right state | state.plan.planId == plan.planId -> pure state+    Right _ -> fail "state.json belongs to a different overhead plan"+    Left _ -> pure (OverheadState plan [])+  writeState output initial+  runBlocks initial 0+  where+    shuffled = shuffle plan.seed plan.arms+    maximumBlocks = plan.trials * 2+    runBlocks state blockNumber+      | validBlockCount state >= plan.trials = pure state+      | blockNumber >= maximumBlocks = pure state+      | otherwise = do+          let slotsForBlock = blockSlots plan shuffled blockNumber+          stateWithSlots <- ensureSlots output state slotsForBlock+          finished <- foldM (runSlot hooks output plan) stateWithSlots slotsForBlock+          runBlocks finished (blockNumber + 1)++analyseOverhead :: OverheadHooks -> OverheadPolicy -> OverheadPlan -> OverheadState -> FilePath -> IO OverheadReport+analyseOverhead hooks policy plan state output = do+  createDirectoryIfMissing True (output </> "comparisons")+  baseline <- maybe (fail "overhead plan has no baseline arm") pure (findArm plan.baselineArm plan.arms)+  let valid = validBlockNumbers state+      candidates = filter ((/= plan.baselineArm) . (.armId)) plan.arms+  armRows <- traverse (armRow valid) plan.arms+  let degradedArms = Map.fromList [(row.arm.armId, row.degraded) | row <- armRows]+  comparisons <- traverse (compareArm degradedArms valid baseline) candidates+  let overall = if length valid < 3 then VerdictInfrastructureFailure else worstVerdict (fmap (.verdict) comparisons)+      report =+        OverheadReport+          plan.planId+          plan.scenarioId+          plan.mode+          baseline.dimensions+          plan.fixedDimensions+          plan.fixedKnobs+          plan.trials+          plan.seed+          (length valid)+          [[slot.armId | slot <- sortOn (.position) state.slots, slot.block == blockNumber] | blockNumber <- sort valid]+          armRows+          comparisons+          (concatMap (.findings) armRows)+          (object ["name" .= policy.defaultPolicy.name, "sha256" .= sha256Hex (canonicalEncode (toJSON policy))])+          overall+  LazyByteString.writeFile (output </> "overhead-report.json") (encode report)+  pure report+  where+    compareArm degradedArms valid baseline candidate+      | length valid < 3 = pure (failedComparison candidate baseline VerdictInfrastructureFailure ["fewer than three valid blocks"])+      | otherwise = do+          let baselineDirs = mapMaybeRunDirs output state valid baseline.armId+              candidateDirs = mapMaybeRunDirs output state valid candidate.armId+              changes = [(name, from, to) | (name, from) <- Map.toList baseline.dimensions, Just to <- [Map.lookup name candidate.dimensions], from /= to]+              transition = case changes of [change] -> Just change; _ -> Nothing+              axes =+                if candidate.control+                  then VaryControl :| []+                  else case changes of+                    [] -> VaryControl :| []+                    (name, _, _) : rest -> VaryDimension name :| [VaryDimension other | (other, _, _) <- rest]+              selectedPolicy = if candidate.control then policy.controlPolicy else policyForTransition policy transition+          compared <- hooks.compare selectedPolicy axes baselineDirs candidateDirs+          case compared of+            Left err -> pure (failedComparison candidate baseline VerdictInfrastructureFailure [Text.pack (show err)])+            Right comparison -> do+              let relative = "comparisons" </> Text.unpack candidate.armId <> "-vs-" <> Text.unpack baseline.armId <> ".json"+                  candidateDegraded = Map.findWithDefault False candidate.armId degradedArms+                  reportedVerdict+                    | candidateDegraded && comparison.verdict `elem` [VerdictPass, VerdictRegression] = VerdictInconclusive+                    | candidate.control && comparison.verdict /= VerdictPass = VerdictInconclusive+                    | otherwise = comparison.verdict+              LazyByteString.writeFile (output </> relative) (encode comparison)+              pure+                OverheadComparison+                  { candidate = candidate.armId,+                    baseline = baseline.armId,+                    factor = maybe (if candidate.control then "control" else "multiple") (\(name, _, _) -> name) transition,+                    fromValue = maybe "baseline" (\(_, value, _) -> value) transition,+                    toValue = maybe (if candidate.control then "control" else "candidate") (\(_, _, value) -> value) transition,+                    comparisonPath = Just relative,+                    verdict = reportedVerdict,+                    degraded = candidateDegraded,+                    deltas = metricDeltas comparison,+                    reasons = comparison.reasons+                  }+    armRow valid arm = do+      let runIds = mapMaybeRunIds state valid arm.armId+          directories = fmap (\runId -> output </> "runs" </> Text.unpack (renderRunId runId)) runIds+      leak <- case (hooks.leakCheck, reverse directories) of (Just check, directory : _) -> check directory; _ -> pure Nothing+      telemetryValues <- catMaybes <$> traverse readTelemetry directories+      let findings = concatMap findingsFromTelemetry telemetryValues+      pure (ArmEvidence arm runIds (aggregateTelemetry telemetryValues) leak findings (any degradedFinding findings))++loadOverheadState :: FilePath -> IO (Either String OverheadState)+loadOverheadState output = do+  let path = output </> "state.json"+  exists <- doesFileExist path+  if exists then eitherDecodeFileStrict' path else pure (Left "state.json does not exist")++overheadVerdictExitCode :: Verdict -> Int+overheadVerdictExitCode = verdictExitCode++ensureSlots :: FilePath -> OverheadState -> [Slot] -> IO OverheadState+ensureSlots output state wanted = do+  let existingKeys = [(slot.block, slot.position) | slot <- state.slots]+      additions = [SlotRun slot.block slot.position slot.arm.armId [] Nothing False | slot <- wanted, (slot.block, slot.position) `notElem` existingKeys]+      updated = setStateSlots (state.slots <> additions) state+  writeState output updated+  pure updated++runSlot :: OverheadHooks -> FilePath -> OverheadPlan -> OverheadState -> Slot -> IO OverheadState+runSlot hooks output plan state slot = case findSlotRun slot state.slots of+  Just recorded | recorded.complete -> do+    valid <- completedRunIsValid output recorded+    if valid then pure state else attempt state 0+  _ -> attempt state 0+  where+    attempt current retryNumber = do+      runId <- newRunId+      let spec = setRunId runId slot.spec+          prepared = updateSlot slot (\old -> old {runIds = old.runIds <> [runId], exitCode = Nothing, complete = False}) current+      writeState output prepared+      result <- try (hooks.runChild spec (output </> "runs")) :: IO (Either SomeException ExitCode)+      let code = either (const 4) exitNumber result+          runDirectory = output </> "runs" </> Text.unpack (renderRunId runId)+      complete <- if code == 0 then either (const False) (const True) <$> verifyManifest runDirectory else pure False+      let finished = updateSlot slot (\old -> old {exitCode = Just code, complete}) prepared+      writeState output finished+      when (plan.settleSeconds > 0) (threadDelay (plan.settleSeconds * 1_000_000))+      if code == 4 && retryNumber < plan.retries then attempt finished (retryNumber + 1) else pure finished++writeState :: FilePath -> OverheadState -> IO ()+writeState output state = do+  let temporary = output </> "state.json.tmp"+  LazyByteString.writeFile temporary (encode state)+  renameFile temporary (output </> "state.json")++validBlockCount :: OverheadState -> Int+validBlockCount = length . validBlockNumbers++validBlockNumbers :: OverheadState -> [Int]+validBlockNumbers state =+  [ blockNumber+  | blockNumber <- nub (fmap (.block) state.slots),+    let blockRuns = filter ((== blockNumber) . (.block)) state.slots,+    length blockRuns == length state.plan.arms,+    all (.complete) blockRuns+  ]++completedRunIsValid :: FilePath -> SlotRun -> IO Bool+completedRunIsValid output recorded = case reverse recorded.runIds of+  [] -> pure False+  runId : _ -> either (const False) (const True) <$> verifyManifest (output </> "runs" </> Text.unpack (renderRunId runId))++findSlotRun :: Slot -> [SlotRun] -> Maybe SlotRun+findSlotRun slot = listToMaybe . filter (\recorded -> recorded.block == slot.block && recorded.position == slot.position)++updateSlot :: Slot -> (SlotRun -> SlotRun) -> OverheadState -> OverheadState+updateSlot slot change state = setStateSlots (fmap update state.slots) state+  where+    update current | current.block == slot.block && current.position == slot.position = change current+    update current = current++mapMaybeRunIds :: OverheadState -> [Int] -> Text -> [RunId]+mapMaybeRunIds state blocks armName =+  catMaybes+    [ listToMaybe (reverse slot.runIds)+    | blockNumber <- sort blocks,+      slot <- state.slots,+      slot.block == blockNumber,+      slot.armId == armName,+      slot.complete+    ]++mapMaybeRunDirs :: FilePath -> OverheadState -> [Int] -> Text -> [FilePath]+mapMaybeRunDirs output state blocks armName = fmap (\runId -> output </> "runs" </> Text.unpack (renderRunId runId)) (mapMaybeRunIds state blocks armName)++readTelemetry :: FilePath -> IO (Maybe Value)+readTelemetry directory = do+  decoded <- eitherDecodeFileStrict' (directory </> "run-result.json") :: IO (Either String Value)+  pure (either (const Nothing) (valueAt ["summaries", "telemetry", "telemetry"]) decoded)++aggregateTelemetry :: [Value] -> Value+aggregateTelemetry [] = Null+aggregateTelemetry values =+  object+    [ "runs" .= length values,+      "spansEnded" .= sumNumbers ["pipeline", "spansEnded"],+      "spansDropped" .= sumNumbers ["pipeline", "spansDropped"],+      "exportFailures" .= sumNumbers ["pipeline", "spansExportFailed"],+      "shutdownMsMax" .= maximumOrZero (mapMaybeNumber ["pipeline", "shutdown", "durationMs"] values),+      "sinkSpansReceived" .= sumNumbers ["sink", "spansReceived"]+    ]+  where+    sumNumbers path = sum (mapMaybeNumber path values)++findingsFromTelemetry :: Value -> [Value]+findingsFromTelemetry value = case valueAt ["findings"] value of Just (Array findings) -> toList findings; _ -> []++degradedFinding :: Value -> Bool+degradedFinding value = textAt ["status"] value == Just "fired" && textAt ["severity"] value == Just "degraded"++valueAt :: [Text] -> Value -> Maybe Value+valueAt [] value = Just value+valueAt (key : rest) (Object values) = KeyMap.lookup (Key.fromText key) values >>= valueAt rest+valueAt _ _ = Nothing++textAt :: [Text] -> Value -> Maybe Text+textAt path value = case valueAt path value of Just (String textValue) -> Just textValue; _ -> Nothing++mapMaybeNumber :: [Text] -> [Value] -> [Double]+mapMaybeNumber path = foldr (\value numbers -> maybe numbers (: numbers) (numberAt path value)) []++numberAt :: [Text] -> Value -> Maybe Double+numberAt path value = case valueAt path value of Just (Number number) -> Just (realToFrac number); _ -> Nothing++maximumOrZero :: [Double] -> Double+maximumOrZero [] = 0+maximumOrZero values = maximum values++toList :: (Foldable collection) => collection value -> [value]+toList = foldr (:) []++metricDeltas :: Comparison -> Value+metricDeltas comparison =+  toJSON+    ( Map.map+        (\metric -> object ["relative" .= (metric.ratio.estimate - 1), "ci95" .= [metric.ratio.low - 1, metric.ratio.high - 1], "absolute" .= metric.delta.estimate, "unit" .= metric.unit])+        comparison.metrics+    )++setPlanSlots :: [Slot] -> OverheadPlan -> OverheadPlan+setPlanSlots value plan = OverheadPlan plan.planId plan.scenarioId plan.scenarioRevision plan.mode plan.factorValues plan.baselineArm plan.arms plan.trials plan.seed plan.settleSeconds plan.retries plan.fixedDimensions plan.fixedKnobs value++setStateSlots :: [SlotRun] -> OverheadState -> OverheadState+setStateSlots value state = OverheadState state.plan value++setRunId :: RunId -> RunSpec -> RunSpec+setRunId value spec =+  RunSpec+    (Just value)+    spec.scenario+    spec.scenarioRevision+    spec.knobs+    spec.dimensions+    spec.seed+    spec.phases+    spec.timeoutSeconds+    spec.environment+    spec.cohortExpectation+    spec.comparison+    spec.labels++failedComparison :: Arm -> Arm -> Verdict -> [Text] -> OverheadComparison+failedComparison candidate baseline verdict reasons =+  OverheadComparison candidate.armId baseline.armId (if candidate.control then "control" else "unknown") "baseline" "candidate" Nothing verdict False (object []) reasons++worstVerdict :: [Verdict] -> Verdict+worstVerdict [] = VerdictInfrastructureFailure+worstVerdict values = maximumByRank values+  where+    rank VerdictPass = 0 :: Int+    rank VerdictInconclusive = 1+    rank VerdictRegression = 2+    rank VerdictInfrastructureFailure = 3+    maximumByRank (first : rest) = foldl (\current candidate -> if rank candidate > rank current then candidate else current) first rest+    maximumByRank [] = VerdictInfrastructureFailure++findArm :: Text -> [Arm] -> Maybe Arm+findArm name = listToMaybe . filter ((== name) . (.armId))++shuffle :: Word64 -> [value] -> [value]+shuffle seed values = fmap snd (sortOn fst (zip (randomWords (mkSMGen seed)) values))+  where+    randomWords generator = let (word, next) = nextWord64 generator in word : randomWords next++rotate :: Int -> [value] -> [value]+rotate _ [] = []+rotate count values = drop offset values <> take offset values where offset = count `mod` length values++cartesian :: [(Text, [Text])] -> [Map Text Text]+cartesian [] = [Map.empty]+cartesian ((name, values) : rest) = [Map.insert name value suffix | value <- values, suffix <- cartesian rest]++whenEither :: Bool -> Text -> Either UsageError ()+whenEither condition message = if condition then Left (UsageError message) else Right ()++firstUsage :: (Show error) => Either error value -> Either UsageError value+firstUsage = either (Left . UsageError . Text.pack . show) Right++exitNumber :: ExitCode -> Int+exitNumber ExitSuccess = 0+exitNumber (ExitFailure code) = code++traverse_ :: (a -> Either error b) -> [a] -> Either error ()+traverse_ action = foldM (\() value -> action value >> pure ()) ()
+ src/Kenshou/Telemetry/Overhead/Policy.hs view
@@ -0,0 +1,50 @@+module Kenshou.Telemetry.Overhead.Policy+  ( OverheadPolicy (..),+    decodeOverheadPolicy,+    policyForTransition,+  )+where++import Data.Aeson+import Data.ByteString (ByteString)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Measure.Compare.Policy (MetricRule (..), Policy (..))++data OverheadPolicy = OverheadPolicy+  { maxDegradedSpanFraction :: Double,+    defaultPolicy :: Policy,+    controlPolicy :: Policy,+    transitions :: Map Text Policy+  }+  deriving stock (Eq, Show)++instance FromJSON OverheadPolicy where+  parseJSON = withObject "OverheadPolicy" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.overhead-policy/v1" :: Text)+      then fail "unsupported overhead policy"+      else OverheadPolicy <$> value .: "maxDegradedSpanFraction" <*> value .: "default" <*> value .: "control" <*> value .:? "transitions" .!= Map.empty++instance ToJSON OverheadPolicy where+  toJSON value =+    object+      [ "schema" .= ("kenshou.overhead-policy/v1" :: Text),+        "maxDegradedSpanFraction" .= value.maxDegradedSpanFraction,+        "default" .= value.defaultPolicy,+        "control" .= value.controlPolicy,+        "transitions" .= value.transitions+      ]++decodeOverheadPolicy :: ByteString -> Either String OverheadPolicy+decodeOverheadPolicy = eitherDecodeStrict'++policyForTransition :: OverheadPolicy -> Maybe (Text, Text, Text) -> Policy+policyForTransition policy transition = case transition of+  Nothing -> policy.defaultPolicy+  Just (factor, fromValue, toValue) -> case Map.lookup (factor <> ":" <> fromValue <> "->" <> toValue) policy.transitions of+    Nothing -> policy.defaultPolicy+    Just selected ->+      let selectedMatches = fmap (.match) selected.metricRules+       in selected {metricRules = selected.metricRules <> filter ((`notElem` selectedMatches) . (.match)) policy.defaultPolicy.metricRules}
+ src/Kenshou/Telemetry/Scrape.hs view
@@ -0,0 +1,301 @@+module Kenshou.Telemetry.Scrape+  ( EndpointSummary (..),+    ScraperHandle (..),+    startScraperInProcess,+    runScraperInProcess,+    runScraperRole,+    SlotLeakResult (..),+    wsSlotLeakProbe,+  )+where++import Control.Concurrent.Async (Async, async, cancel, mapConcurrently_, waitCatch)+import Control.Concurrent.MVar (MVar, newMVar, withMVar)+import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)+import Control.Monad (forM_, replicateM_, void)+import Data.Aeson (FromJSON (..), ToJSON (..), encode, object, withObject, (.:), (.=))+import Data.Aeson.Types (parseMaybe, (.!=), (.:?))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (traverse_)+import Data.IORef+import Data.List (sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import GHC.Clock (getMonotonicTimeNSec)+import Kenshou.Core.Role (ControlMessage (..), RoleContext (..), WorkerInit (..), WorkerMessage (..))+import Kenshou.Measure.Clock (Origin (..), captureOrigin, sleepUntilNs)+import Kenshou.Telemetry.Endpoint+import Network.HTTP.Client (Manager, defaultManagerSettings, httpLbs, newManager, parseRequest, responseBody, responseStatus)+import Network.HTTP.Types.Status (statusCode)+import Network.WebSockets qualified as WebSockets+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO (BufferMode (LineBuffering), Handle, IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)+import Text.Read (readMaybe)++data ScrapeSample = ScrapeSample+  { status :: Int,+    latencyNs :: Integer,+    bodyBytes :: Integer,+    skipped :: Int,+    errorMessage :: Maybe Text+  }++data EndpointSummary = EndpointSummary+  { endpoint :: Text,+    kind :: EndpointKind,+    scrapes :: Int,+    failures :: Int,+    skippedTicks :: Int,+    latencyP50Ns :: Integer,+    latencyP99Ns :: Integer,+    latencyMaxNs :: Integer,+    meanBodyBytes :: Double,+    maxBodyBytes :: Integer+  }+  deriving stock (Eq, Show)++instance ToJSON EndpointSummary where+  toJSON summary =+    object+      [ "endpoint" .= summary.endpoint,+        "kind" .= summary.kind,+        "scrapes" .= summary.scrapes,+        "failures" .= summary.failures,+        "skippedTicks" .= summary.skippedTicks,+        "latencyNs" .= object ["p50" .= summary.latencyP50Ns, "p99" .= summary.latencyP99Ns, "max" .= summary.latencyMaxNs],+        "bodyBytes" .= object ["mean" .= summary.meanBodyBytes, "max" .= summary.maxBodyBytes]+      ]++instance FromJSON EndpointSummary where+  parseJSON = withObject "EndpointSummary" \value -> do+    latency <- value .: "latencyNs"+    body <- value .: "bodyBytes"+    EndpointSummary+      <$> value .: "endpoint"+      <*> value .: "kind"+      <*> value .: "scrapes"+      <*> value .: "failures"+      <*> value .: "skippedTicks"+      <*> latency .: "p50"+      <*> latency .: "p99"+      <*> latency .: "max"+      <*> body .: "mean"+      <*> body .: "max"++data ScraperHandle = ScraperHandle+  { register :: Endpoint -> IO (),+    finish :: IO [EndpointSummary]+  }++data ScraperState = ScraperState+  { manager :: Manager,+    outputDir :: FilePath,+    intervalNs :: Integer,+    wsSubscribers :: Int,+    workers :: IORef [(Endpoint, Async ())],+    endpoints :: IORef [Endpoint],+    samples :: IORef (Map Text [ScrapeSample])+  }++startScraperInProcess :: FilePath -> Int -> Int -> IO ScraperHandle+startScraperInProcess outputDir intervalMs wsSubscribers = do+  createDirectoryIfMissing True (outputDir </> "series")+  manager <- newManager defaultManagerSettings+  workers <- newIORef []+  endpoints <- newIORef []+  samples <- newIORef Map.empty+  let state = ScraperState manager outputDir (fromIntegral intervalMs * 1_000_000) wsSubscribers workers endpoints samples+  pure (ScraperHandle (registerEndpoint state) (finishScraper state))++runScraperInProcess :: FilePath -> Int -> Int -> [Endpoint] -> IO (IO [EndpointSummary])+runScraperInProcess outputDir intervalMs wsSubscribers endpoints = do+  scraper <- startScraperInProcess outputDir intervalMs wsSubscribers+  traverse_ scraper.register endpoints+  pure scraper.finish++runScraperRole :: RoleContext -> IO ()+runScraperRole context = do+  let (intervalMs, subscribers) = maybe (15_000, 0) id (parseMaybe parseConfig context.init.args)+  scraper <- startScraperInProcess context.init.outDir intervalMs subscribers+  context.send WrkReady+  loop scraper+  where+    loop scraper =+      context.receive >>= \case+        Just (CtlCustom "endpoint" payload) -> case parseMaybe parseJSON payload of+          Nothing -> context.send (WrkError "invalid scraper endpoint") >> loop scraper+          Just endpoint -> scraper.register endpoint >> loop scraper+        Just (CtlCustom "finish" _) -> scraper.finish >>= context.send . WrkCustom "scrape-summary" . toJSON >> loop scraper+        Just (CtlStop _) -> void scraper.finish+        Just _ -> loop scraper+        Nothing -> void scraper.finish+    parseConfig = withObject "scraper config" (\value -> (,) <$> value .: "intervalMs" <*> value .:? "wsSubscribers" .!= 0)++registerEndpoint :: ScraperState -> Endpoint -> IO ()+registerEndpoint state endpoint = do+  modifyIORef' state.endpoints (<> [endpoint])+  if endpoint.kind == WebSocketPush+    then do+      modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [])+      if state.wsSubscribers <= 0+        then pure ()+        else do+          worker <- async (websocketSubscribers state endpoint)+          modifyIORef' state.workers (<> [(endpoint, worker)])+    else do+      worker <- async (scrapeLoop state endpoint)+      modifyIORef' state.workers (<> [(endpoint, worker)])++scrapeLoop :: ScraperState -> Endpoint -> IO ()+scrapeLoop state endpoint = do+  request <- parseRequest (Text.unpack endpoint.url)+  handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> ".csv")) WriteMode+  hSetBuffering handle LineBuffering+  hPutStrLn handle "t_wall_ns,endpoint,kind,status,latency_ns,body_bytes,skipped,error"+  started <- getMonotonicTimeNSec+  let loop tick = do+        sleepUntilNs (started + fromIntegral tick * fromIntegral state.intervalNs)+        origin <- captureOrigin+        callStarted <- getMonotonicTimeNSec+        result <- trySync (httpLbs request state.manager)+        callEnded <- getMonotonicTimeNSec+        let elapsed = fromIntegral (callEnded - callStarted)+            behind = max 0 (fromIntegral (callEnded - started) `div` state.intervalNs - tick)+            sample = case result of+              Left exception -> ScrapeSample 0 elapsed 0 (fromIntegral behind) (Just (Text.pack (displayException exception)))+              Right response -> ScrapeSample (statusCode (responseStatus response)) elapsed (fromIntegral (LazyByteString.length (responseBody response))) (fromIntegral behind) Nothing+        modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [sample])+        hPutStrLn handle (csvRow origin endpoint sample)+        loop (tick + 1 + behind)+  loop 1 `finally` hClose handle++websocketSubscribers :: ScraperState -> Endpoint -> IO ()+websocketSubscribers state endpoint = do+  (host, port, path) <- either (ioError . userError . Text.unpack) pure (parseWebSocketUrl endpoint.url)+  handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> "-ws.csv")) WriteMode+  hSetBuffering handle LineBuffering+  hPutStrLn handle "t_wall_ns,subscriber,event,frames,bytes,gap_ns,error"+  lock <- newMVar ()+  mapConcurrently_ (subscriber host port path handle lock) [1 .. state.wsSubscribers] `finally` hClose handle+  where+    subscriber host port path handle lock subscriberId = do+      outcome <- trySync $ WebSockets.runClient (Text.unpack host) port (Text.unpack path) \connection -> do+        traverse_ (WebSockets.sendTextData connection . encode) endpoint.wsHello+        writeWsRow handle lock subscriberId "connect" 0 0 0 Nothing+        previous <- newIORef Nothing+        let loop frames = do+              payload <- WebSockets.receiveData connection :: IO LazyByteString.ByteString+              now <- getMonotonicTimeNSec+              prior <- atomicModifyIORef' previous (\value -> (Just now, value))+              writeWsRow handle lock subscriberId "frame" frames (fromIntegral (LazyByteString.length payload)) (maybe 0 (fromIntegral . (now -)) prior) Nothing+              loop (frames + 1)+        loop 1+      case outcome of+        Left exception -> writeWsRow handle lock subscriberId "error" 0 0 0 (Just (Text.pack (displayException exception)))+        Right () -> writeWsRow handle lock subscriberId "close" 0 0 0 Nothing++writeWsRow :: Handle -> MVar () -> Int -> Text -> Int -> Integer -> Integer -> Maybe Text -> IO ()+writeWsRow handle lock subscriberId event frames bytes gap errorMessage = withMVar lock \_ -> do+  origin <- captureOrigin+  hPutStrLn handle . Text.unpack . Text.intercalate "," $+    [ Text.pack (show origin.wallUnixNs),+      Text.pack (show subscriberId),+      event,+      Text.pack (show frames),+      Text.pack (show bytes),+      Text.pack (show gap),+      maybe "" (Text.replace "," ";" . Text.replace "\n" " ") errorMessage+    ]++finishScraper :: ScraperState -> IO [EndpointSummary]+finishScraper state = do+  workers <- readIORef state.workers+  forM_ workers (cancel . snd)+  forM_ workers (void . waitCatch . snd)+  captured <- readIORef state.samples+  endpoints <- readIORef state.endpoints+  pure [summarize endpoint (Map.findWithDefault [] endpoint.name captured) | endpoint <- endpoints]++summarize :: Endpoint -> [ScrapeSample] -> EndpointSummary+summarize endpoint samples =+  EndpointSummary+    { endpoint = endpoint.name,+      kind = endpoint.kind,+      scrapes = length samples,+      failures = length [() | sample <- samples, sample.status < 200 || sample.status >= 300 || sample.errorMessage /= Nothing],+      skippedTicks = sum (fmap (.skipped) samples),+      latencyP50Ns = quantile 0.50 latencies,+      latencyP99Ns = quantile 0.99 latencies,+      latencyMaxNs = maybe 0 last nonEmptyLatencies,+      meanBodyBytes = if null samples then 0 else fromIntegral (sum (fmap (.bodyBytes) samples)) / fromIntegral (length samples),+      maxBodyBytes = maybe 0 last nonEmptyBodies+    }+  where+    latencies = sort (fmap (.latencyNs) samples)+    bodies = sort (fmap (.bodyBytes) samples)+    nonEmptyLatencies = if null latencies then Nothing else Just latencies+    nonEmptyBodies = if null bodies then Nothing else Just bodies++quantile :: Double -> [Integer] -> Integer+quantile _ [] = 0+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))++data SlotLeakResult = SlotLeakResult+  { attempted :: Int,+    freshAccepted :: Bool+  }+  deriving stock (Eq, Show)++instance ToJSON SlotLeakResult where+  toJSON result = object ["attempted" .= result.attempted, "freshAccepted" .= result.freshAccepted]++wsSlotLeakProbe :: Endpoint -> Int -> IO SlotLeakResult+wsSlotLeakProbe endpoint attempts = case parseWebSocketUrl endpoint.url of+  Left _ -> pure (SlotLeakResult attempts False)+  Right (host, port, path) -> do+    replicateM_ attempts (void (trySync (connect host port path)))+    fresh <- trySync (connect host port path)+    pure (SlotLeakResult attempts (either (const False) (const True) fresh))+  where+    connect host port path = WebSockets.runClient (Text.unpack host) port (Text.unpack path) (const (pure ()))++parseWebSocketUrl :: Text -> Either Text (Text, Int, Text)+parseWebSocketUrl url = do+  remainder <- maybe (Left "WebSocket URL must start with ws://") Right (Text.stripPrefix "ws://" url)+  let (authority, rawPath) = Text.breakOn "/" remainder+      path = if Text.null rawPath then "/" else rawPath+      pieces = Text.splitOn ":" authority+  case pieces of+    [host] -> Right (host, 80, path)+    [host, rawPort] -> maybe (Left "WebSocket URL has an invalid port") (\port -> Right (host, port, path)) (readMaybe (Text.unpack rawPort))+    _ -> Left "WebSocket URL has an invalid authority"++csvRow :: Origin -> Endpoint -> ScrapeSample -> String+csvRow origin endpoint sample =+  Text.unpack . Text.intercalate "," $+    [ Text.pack (show origin.wallUnixNs),+      endpoint.name,+      Text.pack (show endpoint.kind),+      Text.pack (show sample.status),+      Text.pack (show sample.latencyNs),+      Text.pack (show sample.bodyBytes),+      Text.pack (show sample.skipped),+      maybe "" sanitize sample.errorMessage+    ]+  where+    sanitize = Text.replace "," ";" . Text.replace "\n" " "++tryAny :: IO value -> IO (Either SomeException value)+tryAny = try++trySync :: IO value -> IO (Either SomeException value)+trySync action = do+  outcome <- tryAny action+  case outcome of+    Left exception -> case fromException exception :: Maybe SomeAsyncException of+      Just _ -> throwIO exception+      Nothing -> pure outcome+    Right _ -> pure outcome
+ src/Kenshou/Telemetry/SelfTest.hs view
@@ -0,0 +1,28 @@+module Kenshou.Telemetry.SelfTest (selfTestBundle) where++import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Id (selftestLayer)+import Kenshou.Core.Role (WorkerRole (..), mkRoleName)+import Kenshou.Telemetry.Scrape (runScraperRole)+import Kenshou.Telemetry.SelfTest.Arms (armsScenario)+import Kenshou.Telemetry.SelfTest.Problems (slowExporterScenario, traceContinuityScenario)+import Kenshou.Telemetry.Sink (runSinkRole)++selfTestBundle :: LayerBundle+selfTestBundle = mkBundle selftestLayer [armsScenario, traceContinuityScenario, slowExporterScenario] [sinkRole, scraperRole]++sinkRole :: WorkerRole+sinkRole =+  WorkerRole+    { name = either (error . show) id (mkRoleName "selftest/telemetry-otlp-sink"),+      summary = "Receives and counts OTLP payloads for telemetry scenarios.",+      run = runSinkRole+    }++scraperRole :: WorkerRole+scraperRole =+  WorkerRole+    { name = either (error . show) id (mkRoleName "selftest/telemetry-scraper"),+      summary = "Scrapes telemetry endpoints on a fixed schedule.",+      run = runScraperRole+    }
+ src/Kenshou/Telemetry/SelfTest/Arms.hs view
@@ -0,0 +1,78 @@+module Kenshou.Telemetry.SelfTest.Arms (armsScenario) where++import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Dimension (allTelemetryArms, noDimensions)+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Clock (Nanos (..))+import Kenshou.Measure.Health (HealthConfig (..))+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Phase qualified as Measure+import Kenshou.Measure.Recorder (OpName (..))+import Kenshou.Measure.Session+import Kenshou.Telemetry+import Kenshou.Telemetry.SelfTest.Endpoints (withSyntheticEndpoints)+import Kenshou.Telemetry.SelfTest.Service++armsScenario :: Scenario+armsScenario =+  Scenario+    { id = either (error . Text.unpack) id (parseScenarioId "selftest/telemetry/benchmark/arms-on-synthetic-service"),+      revision = 1,+      summary = "Measures one synthetic service under every tracing and metrics arm.",+      tier = TierSmoke,+      placement = PlaceEither,+      knobs = loadKnobs defaults <> measureKnobs Benchmark <> workKnobs <> telemetryKnobs,+      dimensions = allTelemetryArms noDimensions,+      phases = Core.PhasePlan 0.1 10 0.1,+      requires = noEnvironment,+      knownDefect = Nothing,+      run = runArms+    }+  where+    defaults = defaultLoadDefaults {model = "closed", workers = 4, thinkTimeUs = 0}++runArms :: RunContext -> IO ScenarioReport+runArms context = case (telemetrySpecFromContext context, loadModelFromKnobs context.knobs, measureConfigFromKnobs context phases) of+  (Left message, _, _) -> pure (failedWith ["invalid-telemetry-config"] message)+  (_, Left message, _) -> pure (failedWith ["invalid-load-config"] message)+  (_, _, Left message) -> pure (failedWith ["invalid-measure-config"] message)+  (Right telemetrySpec, Right loadModel, Right measureConfig) ->+    withTelemetry telemetrySpec \telemetry -> do+      withSyntheticEndpoints telemetry.servesEndpoints telemetry.registerEndpoint do+        syntheticMetrics <- traverse newSyntheticMetrics telemetry.meter+        let spanCount = integer "work.spans-per-op"+            attributeCount = integer "work.attributes-per-span"+            cpuMicros = integer "work.cpu-micros"+            operation = Operation (OpName "synthetic") (\_ _ -> runSyntheticOperation telemetry.tracer syntheticMetrics spanCount attributeCount cpuMicros)+        -- This closed-loop benchmark intentionally saturates the measured+        -- process when work.cpu-micros is zero. That CPU is the subject under+        -- test, rather than an overloaded open-loop driver.+        let closedLoopConfig = measureConfig {healthConfig = measureConfig.healthConfig {cpuSoft = 1.1, cpuHard = 1.2}}+        (_, measurement) <- withMeasurement context closedLoopConfig (\session -> runLoad session loadModel operation)+        pure (passed {outcome = measuredOutcome measurement passed.outcome})+  where+    duration = fromIntegral (knobInt context.knobs (name "load.duration-seconds")) * 1_000_000_000+    phases = Measure.PhasePlan (Nanos 100_000_000) (Measure.SteadyFor (Nanos duration)) (Nanos 100_000_000)+    integer key = fromIntegral (knobInt context.knobs (name key))++workKnobs :: [KnobSpec]+workKnobs =+  [ intKnob "work.cpu-micros" "Synthetic CPU work per operation" 1000 0 100000,+    intKnob "work.spans-per-op" "Spans emitted per operation" 2 1 16,+    intKnob "work.attributes-per-span" "Attributes attached to every span" 6 0 64,+    intKnob "load.duration-seconds" "Steady measurement duration" 10 1 3600+  ]++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []++name :: Text -> KnobName+name = either (error . Text.unpack) id . mkKnobName
+ src/Kenshou/Telemetry/SelfTest/Endpoints.hs view
@@ -0,0 +1,41 @@+module Kenshou.Telemetry.SelfTest.Endpoints (withSyntheticEndpoints) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Exception (bracket)+import Control.Monad (forever, void)+import Data.Aeson (encode, object, (.=))+import Data.Text qualified as Text+import Kenshou.Telemetry.Endpoint+import Network.HTTP.Types (status200, status404)+import Network.Socket (close)+import Network.Wai (Application, pathInfo, responseLBS)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import Network.Wai.Handler.WebSockets (websocketsOr)+import Network.WebSockets qualified as WebSockets++withSyntheticEndpoints :: Bool -> (Endpoint -> IO ()) -> IO value -> IO value+withSyntheticEndpoints False _ action = action+withSyntheticEndpoints True register action = bracket acquire release \(port, _) -> do+  let base = Text.pack ("127.0.0.1:" <> show port)+  register (Endpoint "synthetic-json" JsonDocument ("http://" <> base <> "/metrics") Nothing)+  register (Endpoint "synthetic-push" WebSocketPush ("ws://" <> base <> "/ws/metrics") Nothing)+  action+  where+    acquire = do+      (port, socket) <- openFreePort+      server <- async (runSettingsSocket defaultSettings socket application)+      pure (port, (server, socket))+    release (_, (server, socket)) = cancel server >> void (waitCatch server) >> close socket++application :: Application+application = websocketsOr WebSockets.defaultConnectionOptions websocket http+  where+    http request respond+      | pathInfo request == ["metrics"] = respond (responseLBS status200 [("Content-Type", "application/json")] (encode (object ["operations" .= (1 :: Int)])))+      | otherwise = respond (responseLBS status404 [] "not found")+    websocket pending = do+      connection <- WebSockets.acceptRequest pending+      forever do+        WebSockets.sendTextData connection (encode (object ["operations" .= (1 :: Int)]))+        threadDelay 100_000
+ src/Kenshou/Telemetry/SelfTest/Problems.hs view
@@ -0,0 +1,146 @@+module Kenshou.Telemetry.SelfTest.Problems+  ( traceContinuityScenario,+    slowExporterScenario,+  )+where++import Control.Concurrent (threadDelay)+import Control.Monad (forM_, replicateM_, void)+import Data.Aeson (object, (.=))+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (RunContext (..), SummarySection (Verdicts), putSummary)+import Kenshou.Core.Dimension+import Kenshou.Core.Dimension qualified as Dimension+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (ScenarioId, parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase (PhasePlan (..))+import Kenshou.Core.Scenario+import Kenshou.Telemetry+import Kenshou.Telemetry.Compose+import Kenshou.Telemetry.Continuity+import Kenshou.Telemetry.Detect+import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..), snapshotPipeline)+import Kenshou.Telemetry.Tracing.Probe (SpanView (..), readSpans)+import OpenTelemetry.Attributes (Attribute (AttributeValue), PrimitiveAttribute (TextAttribute), lookupAttribute)+import OpenTelemetry.Trace.Core (Span, addAttribute, defaultSpanArguments, inSpan, inSpan')+import System.Timeout (timeout)++traceContinuityScenario :: Scenario+traceContinuityScenario =+  Scenario+    { id = scenarioId "selftest/telemetry/correctness/trace-continuity",+      revision = 1,+      summary = "Proves trace discontinuity and context-leak checks are non-vacuous.",+      tier = TierSmoke,+      placement = PlaceEither,+      knobs = [intKnob "continuity.messages" "Synthetic messages in each continuity case" 1000 10 100000] <> telemetryKnobs,+      dimensions = onlyArms TracingSdkInMemory MetricsOff,+      phases = PhasePlan 0 0 0,+      requires = noEnvironment,+      knownDefect = Nothing,+      run = runTraceContinuity+    }++slowExporterScenario :: Scenario+slowExporterScenario =+  Scenario+    { id = scenarioId "selftest/telemetry/concurrency/slow-exporter-backpressure",+      revision = 1,+      summary = "Proves slow exporters and handlers are detected without blocking the emitter.",+      tier = TierSmoke,+      placement = PlaceEither,+      knobs =+        [ intKnob "backpressure.phase-seconds" "Maximum synthetic phase duration" 8 2 600,+          intKnob "handler.delay-micros" "Injected handler delay" 5000 0 1000000+        ]+          <> telemetryKnobs,+      dimensions = onlyArms TracingSdkOtlp MetricsOff,+      phases = PhasePlan 0 0 0,+      requires = noEnvironment,+      knownDefect = Nothing,+      run = runSlowExporter+    }++runTraceContinuity :: RunContext -> IO ScenarioReport+runTraceContinuity context = case telemetrySpecFromContext context of+  Left message -> pure (failedWith ["invalid-telemetry-config"] message)+  Right spec -> withTelemetry spec \telemetry -> case telemetry.tracer of+    Nothing -> pure (failedWith ["missing-tracer"] "sdk-inmemory did not provide a tracer")+    Just tracer -> do+      let messages = fromIntegral (knobInt context.knobs (name "continuity.messages"))+      forM_ [1 .. messages] \index ->+        inSpan' tracer "intact.producer" defaultSpanArguments \producer -> do+          addMessageId producer ("intact-" <> tshow index)+          inSpan' tracer "intact.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("intact-" <> tshow index)+      forM_ [10, 20 .. messages] \index -> do+        inSpan' tracer "broken.producer" defaultSpanArguments \producer -> addMessageId producer ("broken-" <> tshow index)+        inSpan' tracer "broken.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("broken-" <> tshow index)+      inSpan tracer "leak.carrier" defaultSpanArguments do+        forM_ [10, 20 .. messages] \index ->+          inSpan' tracer "leaky.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("leaky-" <> tshow index)+      void telemetry.flushTelemetry+      spans <- maybe (pure []) readSpans telemetry.spans+      let correlate spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing+          intact = checkContinuity (named "intact.producer") (named "intact.consumer") correlate spans+          broken = checkContinuity (named "broken.producer") (named "broken.consumer") correlate spans+          isolated = checkIsolation (named "leaky.consumer") (const True) spans+          expected = messages `div` 10+          ok = intact.violations == 0 && intact.consumers == messages && broken.violations == expected && isolated.violations == expected+      telemetry.recordContinuity intact (IsolationResult 0 0 [])+      telemetry.recordContinuity broken isolated+      putSummary context Verdicts "trace-continuity" (object ["intact" .= intact, "broken" .= broken, "leaky" .= isolated, "expectedInjected" .= expected])+      pure (if ok then passed else failedWith ["trace-detector-non-vacuity"] "continuity detectors did not identify exactly the injected defects")++runSlowExporter :: RunContext -> IO ScenarioReport+runSlowExporter context = case telemetrySpecFromContext context of+  Left message -> pure (failedWith ["invalid-telemetry-config"] message)+  Right spec -> withTelemetry (spec {processor = BatchProcessor 256 200 128 2000, shutdownMs = 1000}) \telemetry -> case (telemetry.tracer, telemetry.pipeline) of+    (Just tracer, Just pipeline) -> do+      let delayMicros = fromIntegral (knobInt context.knobs (name "handler.delay-micros"))+          phaseMicros = fromIntegral (knobInt context.knobs (name "backpressure.phase-seconds")) * 1_000_000+      synchronousStats <- newHandlerStats+      replicateM_ 20 (timedHandler synchronousStats "synchronous" (slowHandler delayMicros) ())+      synchronous <- snapshotHandlerStats "synchronous" synchronousStats+      asyncStats <- newHandlerStats+      (submit, asyncSnapshot) <- asyncHandler 64 (slowHandler delayMicros)+      completed <- timeout phaseMicros (replicateM_ 10000 (timedHandler asyncStats "async-submit" submit ()))+      threadDelay (min 250_000 phaseMicros)+      asynchronous <- snapshotHandlerStats "async-submit" asyncStats+      asyncQueue <- asyncSnapshot+      telemetry.recordHandlerStats synchronous+      telemetry.recordHandlerStats asynchronous+      telemetry.setSinkFault SinkHang+      emitted <- timeout phaseMicros (replicateM_ 10000 (inSpan tracer "backpressure" defaultSpanArguments (pure ())))+      snapshot <- snapshotPipeline pipeline+      let syncFired = (handlerFinding synchronous).status == Fired+          asyncQuiet = (handlerFinding asynchronous).status == Quiet+          ok = completed /= Nothing && emitted /= Nothing && syncFired && asyncQuiet && asyncQueue.dropped > 0 && snapshot.spansDropped > 0+      putSummary context Verdicts "slow-exporter-backpressure" (object ["synchronous" .= synchronous, "asynchronous" .= asynchronous, "asyncQueue" .= asyncQueue, "pipeline" .= snapshot, "emitterCompleted" .= (emitted /= Nothing)])+      pure (if ok then passed else failedWith ["backpressure-detector-non-vacuity"] "slow exporter or handler detector did not fire as expected")+    _ -> pure (failedWith ["missing-sdk"] "sdk-otlp did not provide a tracer and pipeline")++onlyArms :: TracingArm -> MetricsArm -> DimensionSupport+onlyArms tracing metrics =+  DimensionSupport (Supported (Support (tracing :| []) tracing)) (Supported (Support (metrics :| []) metrics)) Dimension.NotApplicable Dimension.NotApplicable++named :: Text -> SpanSelector+named spanName = SpanSelector (Just spanName) Nothing []++addMessageId :: Span -> Text -> IO ()+addMessageId spanValue value = addAttribute spanValue "message.id" value++scenarioId :: Text -> ScenarioId+scenarioId text = either (error . Text.unpack) id (parseScenarioId text)++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []++name :: Text -> KnobName+name = either (error . Text.unpack) id . mkKnobName++tshow :: (Show value) => value -> Text+tshow = Text.pack . show
+ src/Kenshou/Telemetry/SelfTest/Service.hs view
@@ -0,0 +1,55 @@+module Kenshou.Telemetry.SelfTest.Service+  ( SyntheticMetrics,+    newSyntheticMetrics,+    runSyntheticOperation,+  )+where++import Control.Concurrent (threadDelay)+import Control.Exception (evaluate)+import Control.Monad (forM_)+import Data.Bits (xor)+import Data.Int (Int64)+import Data.Text qualified as Text+import Kenshou.Measure.Recorder (OpResult (OpOk))+import OpenTelemetry.Attributes (emptyAttributes)+import OpenTelemetry.Metric.Core (Counter (..), Histogram (..), Meter (..), defaultAdvisoryParameters)+import OpenTelemetry.Trace.Core (Tracer, addAttribute, defaultSpanArguments, inSpan')++data SyntheticMetrics = SyntheticMetrics (Counter Int64) Histogram++newSyntheticMetrics :: Meter -> IO SyntheticMetrics+newSyntheticMetrics meter =+  SyntheticMetrics+    <$> meter.meterCreateCounterInt64 "synthetic.operations" (Just "operations") Nothing defaultAdvisoryParameters+    <*> meter.meterCreateHistogram "synthetic.work.duration" (Just "us") Nothing defaultAdvisoryParameters++runSyntheticOperation :: Maybe Tracer -> Maybe SyntheticMetrics -> Int -> Int -> Int -> IO OpResult+runSyntheticOperation tracer metrics spanCount attributeCount cpuMicros = do+  withSpans tracer spanCount attributeCount do+    _ <- evaluate (burnCpu (max 1 (cpuMicros * 40)) 0x9e3779b9)+    threadDelay cpuMicros+    pure ()+  case metrics of+    Nothing -> pure ()+    Just (SyntheticMetrics counter histogram) -> do+      counter.counterAdd 1 emptyAttributes+      histogram.histogramRecord (fromIntegral cpuMicros) emptyAttributes+  pure (OpOk 1)++withSpans :: Maybe Tracer -> Int -> Int -> IO value -> IO value+withSpans Nothing _ _ action = action+withSpans (Just tracer) count attributeCount action = go count+  where+    go remaining+      | remaining <= 0 = action+      | otherwise =+          inSpan' tracer ("synthetic-" <> Text.pack (show remaining)) defaultSpanArguments \spanValue -> do+            forM_ [1 .. attributeCount] \index -> addAttribute spanValue ("work.attribute." <> Text.pack (show index)) index+            go (remaining - 1)++burnCpu :: Int -> Int -> Int+burnCpu 0 accumulator = accumulator+burnCpu remaining accumulator =+  let mixed = (accumulator * 1664525 + 1013904223) `xor` remaining+   in burnCpu (remaining - 1) mixed
+ src/Kenshou/Telemetry/Sink.hs view
@@ -0,0 +1,111 @@+module Kenshou.Telemetry.Sink+  ( SinkStats (..),+    SinkHandle (..),+    sinkApplication,+    withSink,+    runSinkRole,+  )+where++import Codec.Compression.GZip qualified as GZip+import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Exception (bracket)+import Control.Monad (void)+import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), object, withObject, (.:), (.=))+import Data.Aeson.Types (parseMaybe)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef+import Data.ProtoLens (decodeMessage)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Vector qualified as Vector+import Kenshou.Core.Role (ControlMessage (..), RoleContext (..), WorkerInit (..), WorkerMessage (..))+import Kenshou.Telemetry.Spec (SinkFault (..), renderSinkFault)+import Lens.Micro ((^.))+import Network.HTTP.Types (status200, status404, status503)+import Network.Socket (close)+import Network.Wai (Application, Request (..), responseLBS, strictRequestBody)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import Proto.Opentelemetry.Proto.Collector.Trace.V1.TraceService (ExportTraceServiceRequest)+import Proto.Opentelemetry.Proto.Collector.Trace.V1.TraceService_Fields qualified as TraceService+import Proto.Opentelemetry.Proto.Trace.V1.Trace_Fields qualified as Trace++data SinkStats = SinkStats {requests :: Int, bytes :: Integer, spansReceived :: Int}+  deriving stock (Eq, Show)++instance ToJSON SinkStats where+  toJSON stats = object ["requests" .= stats.requests, "bytes" .= stats.bytes, "spansReceived" .= stats.spansReceived]++instance FromJSON SinkStats where+  parseJSON = withObject "SinkStats" (\value -> SinkStats <$> value .: "requests" <*> value .: "bytes" <*> value .: "spansReceived")++data SinkHandle = SinkHandle+  { endpoint :: Text,+    setFault :: SinkFault -> IO (),+    snapshot :: IO SinkStats+  }++sinkApplication :: IORef SinkFault -> IORef SinkStats -> Application+sinkApplication faultRef statsRef request respond+  | requestMethod request /= "POST" || pathInfo request `notElem` [["v1", "traces"], ["v1", "metrics"]] = respond (responseLBS status404 [] "not found")+  | otherwise = do+      fault <- readIORef faultRef+      case fault of+        SinkDelay200ms -> threadDelay 200000+        SinkDelay2000ms -> threadDelay 2000000+        SinkHang -> threadDelay 120000000+        _ -> pure ()+      body <- strictRequestBody request+      let decodedBody = if lookup "Content-Encoding" (requestHeaders request) == Just "gzip" then GZip.decompress body else body+          spans = if pathInfo request == ["v1", "traces"] then countSpans decodedBody else 0+      atomicModifyIORef' statsRef \stats -> (SinkStats (stats.requests + 1) (stats.bytes + fromIntegral (LazyByteString.length body)) (stats.spansReceived + spans), ())+      respond (responseLBS (if fault == SinkStatus503 then status503 else status200) [("Content-Type", "application/x-protobuf")] "")++withSink :: SinkFault -> (SinkHandle -> IO value) -> IO value+withSink initialFault action = bracket acquire release use+  where+    acquire = do+      faultRef <- newIORef initialFault+      statsRef <- newIORef (SinkStats 0 0 0)+      (port, socket) <- openFreePort+      if initialFault == SinkRefuse+        then close socket >> pure (SinkHandle (url port) (writeIORef faultRef) (readIORef statsRef), Nothing)+        else do+          server <- async (runSettingsSocket defaultSettings socket (sinkApplication faultRef statsRef))+          pure (SinkHandle (url port) (writeIORef faultRef) (readIORef statsRef), Just (server, socket))+    release (_, Nothing) = pure ()+    release (_, Just (server, socket)) = cancel server >> void (waitCatch server) >> close socket+    use (handle, _) = action handle+    url port = "http://127.0.0.1:" <> Text.pack (show port)++runSinkRole :: RoleContext -> IO ()+runSinkRole context = withSink initialFault \sink -> do+  context.send (WrkCustom "ready" (object ["endpoint" .= sink.endpoint]))+  loop sink+  stats <- sink.snapshot+  context.send (WrkCustom "sink-stats" (toJSON stats))+  where+    loop sink =+      context.receive >>= \case+        Just (CtlCustom "fault" (String raw)) -> case parseFault raw of+          Nothing -> context.send (WrkError ("unknown sink fault: " <> raw))+          Just fault -> sink.setFault fault >> loop sink+        Just (CtlCustom "snapshot" _) -> sink.snapshot >>= context.send . WrkCustom "sink-stats" . toJSON >> loop sink+        Just (CtlStop _) -> pure ()+        Just _ -> loop sink+        Nothing -> pure ()+    initialFault = maybe SinkHealthy id (parseMaybe (withObject "sink arguments" (\value -> value .: "fault" >>= maybe (fail "unknown sink fault") pure . parseFault)) context.init.args)++countSpans :: LazyByteString.ByteString -> Int+countSpans body = case decodeMessage (LazyByteString.toStrict body) :: Either String ExportTraceServiceRequest of+  Left _ -> 0+  Right request ->+    sum+      [ Vector.length (scope ^. Trace.vec'spans)+      | resource <- Vector.toList (request ^. TraceService.vec'resourceSpans),+        scope <- Vector.toList (resource ^. Trace.vec'scopeSpans)+      ]++parseFault :: Text -> Maybe SinkFault+parseFault raw = lookup raw [(renderSinkFault fault, fault) | fault <- [SinkHealthy, SinkDelay200ms, SinkDelay2000ms, SinkStatus503, SinkHang, SinkRefuse]]
+ src/Kenshou/Telemetry/Spec.hs view
@@ -0,0 +1,230 @@+module Kenshou.Telemetry.Spec+  ( TracingArm (..),+    MetricsArm (..),+    SamplerSpec (..),+    ProcessorSpec (..),+    OtlpProtocol (..),+    OtlpEndpoint (..),+    OtlpCompression (..),+    SinkFault (..),+    OtelReader (..),+    HelperPlacement (..),+    TelemetrySpec (..),+    telemetryKnobs,+    telemetrySpecFromContext,+    telemetrySpecFromWorker,+    renderSinkFault,+  )+where++import Data.Aeson (Value, encode)+import Data.ByteString.Lazy qualified as LBS+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (RunContext (..), SummarySection (Telemetry), putSummary)+import Kenshou.Core.Dimension (Dimensions (..), MetricsArm (..), TracingArm (..))+import Kenshou.Core.Id (renderScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Role (RoleContext (..), WorkerInit (..))+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))++data SamplerSpec = AlwaysOn | AlwaysOff | ParentBasedAlwaysOn | TraceIdRatio Double | ParentBasedTraceIdRatio Double+  deriving stock (Eq, Show)++data ProcessorSpec+  = BatchProcessor+      { maxQueue :: Int,+        scheduleDelayMs :: Int,+        maxExportBatch :: Int,+        exportTimeoutMs :: Int+      }+  | SimpleProcessor {exportTimeoutMs :: Int}+  deriving stock (Eq, Show)++data OtlpProtocol = HttpProtobuf | Grpc deriving stock (Eq, Show)++data SinkFault = SinkHealthy | SinkDelay200ms | SinkDelay2000ms | SinkStatus503 | SinkHang | SinkRefuse+  deriving stock (Eq, Ord, Show)++data OtlpEndpoint = BuiltinSink SinkFault | ExternalEndpoint Text deriving stock (Eq, Show)++data OtlpCompression = CompressionNone | CompressionGzip deriving stock (Eq, Show)++data OtelReader = ReaderPrometheus | ReaderOtlpPeriodic | ReaderNone deriving stock (Eq, Show)++data HelperPlacement = HelperProcess FilePath | HelperInProcess deriving stock (Eq, Show)++data TelemetrySpec = TelemetrySpec+  { tracing :: TracingArm,+    metrics :: MetricsArm,+    serviceName :: Text,+    sampler :: SamplerSpec,+    processor :: ProcessorSpec,+    exporter :: OtlpProtocol,+    endpoint :: OtlpEndpoint,+    compression :: OtlpCompression,+    probeRetain :: Int,+    shutdownMs :: Int,+    otelReader :: OtelReader,+    otelExportMs :: Int,+    scrapeMs :: Int,+    wsSubscribers :: Int,+    helpers :: HelperPlacement,+    workerContext :: Maybe RunContext,+    outDir :: FilePath,+    report :: Value -> IO ()+  }++telemetryKnobs :: [KnobSpec]+telemetryKnobs =+  [ textKnob "otel.sampler" "OpenTelemetry sampler" "parentbased-always-on" ["always-on", "always-off", "parentbased-always-on", "traceidratio", "parentbased-traceidratio"],+    doubleKnob "otel.sampler-arg" "Trace-id-ratio sampler fraction" 1 0 1,+    textKnob "otel.processor" "Span processor" "batch" ["batch", "simple"],+    intKnob "otel.bsp.max-queue" "Batch processor queue bound" 2048 1 1048576,+    intKnob "otel.bsp.schedule-delay-ms" "Batch export schedule delay" 5000 10 60000,+    intKnob "otel.bsp.max-export-batch" "Maximum export batch" 512 1 1048576,+    intKnob "otel.bsp.export-timeout-ms" "Exporter timeout" 30000 100 120000,+    textKnob "otel.exporter" "OTLP protocol" "http-protobuf" ["http-protobuf", "grpc"],+    KnobSpec (name "otel.endpoint") "External OTLP base URL, empty for built-in sink" KnobText (VText "") AnyValue [],+    textKnob "otel.sink-fault" "Built-in OTLP sink fault" "none" ["none", "delay-200ms", "delay-2000ms", "status-503", "hang", "refuse"],+    textKnob "otel.compression" "OTLP compression" "none" ["none", "gzip"],+    intKnob "otel.inmemory.retain" "Recent spans retained by the bounded probe" 4096 0 1000000,+    intKnob "otel.shutdown-timeout-ms" "Provider flush and shutdown timeout" 5000 100 120000,+    intKnob "metrics.scrape-interval-ms" "Metrics scrape interval" 15000 100 600000,+    intKnob "metrics.ws-subscribers" "WebSocket subscribers per endpoint" 0 0 64,+    textKnob "metrics.otel-reader" "OpenTelemetry metric reader" "prometheus" ["prometheus", "otlp-periodic", "none"],+    intKnob "metrics.otel-export-interval-ms" "Periodic OTLP metric export interval" 60000 1000 600000+  ]++telemetrySpecFromContext :: RunContext -> Either Text TelemetrySpec+telemetrySpecFromContext context =+  makeTelemetrySpec+    context.knobs+    context.dimensions+    (Text.replace "/" "." (renderScenarioId context.scenario))+    (HelperProcess "kenshou")+    (Just context)+    context.outDir+    (putSummary context Telemetry "telemetry")++-- A child owns its providers, endpoints and files. It must never write into+-- the parent's series or retain the parent's in-memory summary state.+telemetrySpecFromWorker :: RoleContext -> FilePath -> Either Text TelemetrySpec+telemetrySpecFromWorker context directory =+  makeTelemetrySpec+    context.init.knobs+    context.init.dimensions+    (Text.replace "/" "." context.init.instanceName)+    HelperInProcess+    Nothing+    directory+    (\value -> createDirectoryIfMissing True (directory </> "logs") >> LBS.writeFile (directory </> "logs" </> "telemetry-summary.json") (encode value))++makeTelemetrySpec :: ResolvedKnobs -> Dimensions -> Text -> HelperPlacement -> Maybe RunContext -> FilePath -> (Value -> IO ()) -> Either Text TelemetrySpec+makeTelemetrySpec knobs dimensions serviceName helpers workerContext outDir report = do+  let Dimensions resolvedTracing resolvedMetrics _ _ = dimensions+  tracing <- maybe (Left "telemetry.tracing is not applicable") Right resolvedTracing+  metrics <- maybe (Left "telemetry.metrics is not applicable") Right resolvedMetrics+  sampler <- parseSampler (knobText knobs (name "otel.sampler")) (knobDouble knobs (name "otel.sampler-arg"))+  processor <- parseProcessor knobs+  exporter <- parseProtocol (knobText knobs (name "otel.exporter"))+  endpoint <- parseEndpoint knobs+  compression <- parseCompression (knobText knobs (name "otel.compression"))+  reader <- parseReader (knobText knobs (name "metrics.otel-reader"))+  pure+    TelemetrySpec+      { tracing,+        metrics,+        serviceName = serviceName,+        sampler,+        processor,+        exporter,+        endpoint,+        compression,+        probeRetain = integer "otel.inmemory.retain",+        shutdownMs = integer "otel.shutdown-timeout-ms",+        otelReader = reader,+        otelExportMs = integer "metrics.otel-export-interval-ms",+        scrapeMs = integer "metrics.scrape-interval-ms",+        wsSubscribers = integer "metrics.ws-subscribers",+        helpers,+        workerContext,+        outDir,+        report+      }+  where+    integer key = fromIntegral (knobInt knobs (name key))++parseSampler :: Text -> Double -> Either Text SamplerSpec+parseSampler "always-on" _ = Right AlwaysOn+parseSampler "always-off" _ = Right AlwaysOff+parseSampler "parentbased-always-on" _ = Right ParentBasedAlwaysOn+parseSampler "traceidratio" ratio = Right (TraceIdRatio ratio)+parseSampler "parentbased-traceidratio" ratio = Right (ParentBasedTraceIdRatio ratio)+parseSampler value _ = Left ("unsupported otel.sampler: " <> value)++parseProcessor :: ResolvedKnobs -> Either Text ProcessorSpec+parseProcessor knobs = case knobText knobs (name "otel.processor") of+  "simple" -> Right (SimpleProcessor timeoutMs)+  "batch"+    | batch > queue -> Left "otel.bsp.max-export-batch must not exceed otel.bsp.max-queue"+    | otherwise -> Right (BatchProcessor queue delay batch timeoutMs)+  value -> Left ("unsupported otel.processor: " <> value)+  where+    queue = integer "otel.bsp.max-queue"+    delay = integer "otel.bsp.schedule-delay-ms"+    batch = integer "otel.bsp.max-export-batch"+    timeoutMs = integer "otel.bsp.export-timeout-ms"+    integer key = fromIntegral (knobInt knobs (name key))++parseProtocol :: Text -> Either Text OtlpProtocol+parseProtocol "http-protobuf" = Right HttpProtobuf+parseProtocol "grpc" = Left "otel.exporter=grpc requires building kenshou-telemetry with -fotlp-grpc"+parseProtocol value = Left ("unsupported otel.exporter: " <> value)++parseEndpoint :: ResolvedKnobs -> Either Text OtlpEndpoint+parseEndpoint knobs = case (knobText knobs (name "otel.endpoint"), parseSinkFault (knobText knobs (name "otel.sink-fault"))) of+  (_, Left message) -> Left message+  ("", Right fault) -> Right (BuiltinSink fault)+  (url, Right SinkHealthy) -> Right (ExternalEndpoint url)+  (_, Right _) -> Left "otel.sink-fault is only valid with the built-in sink"++parseCompression :: Text -> Either Text OtlpCompression+parseCompression "none" = Right CompressionNone+parseCompression "gzip" = Right CompressionGzip+parseCompression value = Left ("unsupported otel.compression: " <> value)++parseReader :: Text -> Either Text OtelReader+parseReader "prometheus" = Right ReaderPrometheus+parseReader "otlp-periodic" = Right ReaderOtlpPeriodic+parseReader "none" = Right ReaderNone+parseReader value = Left ("unsupported metrics.otel-reader: " <> value)++parseSinkFault :: Text -> Either Text SinkFault+parseSinkFault value = maybe (Left ("unsupported otel.sink-fault: " <> value)) Right (lookup value values)+  where+    values = [(renderSinkFault fault, fault) | fault <- [SinkHealthy, SinkDelay200ms, SinkDelay2000ms, SinkStatus503, SinkHang, SinkRefuse]]++renderSinkFault :: SinkFault -> Text+renderSinkFault SinkHealthy = "none"+renderSinkFault SinkDelay200ms = "delay-200ms"+renderSinkFault SinkDelay2000ms = "delay-2000ms"+renderSinkFault SinkStatus503 = "status-503"+renderSinkFault SinkHang = "hang"+renderSinkFault SinkRefuse = "refuse"++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []++doubleKnob :: Text -> Text -> Double -> Double -> Double -> KnobSpec+doubleKnob key summary def low high = KnobSpec (name key) summary KnobDouble (VDouble def) (DoubleRange low high) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob key summary def (first : rest) = KnobSpec (name key) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []+textKnob key _ _ [] = error ("text knob has no values: " <> Text.unpack key)++name :: Text -> KnobName+name = either (error . Text.unpack) id . mkKnobName
+ src/Kenshou/Telemetry/Tracing.hs view
@@ -0,0 +1,122 @@+module Kenshou.Telemetry.Tracing+  ( TracingRuntime (..),+    startTracing,+    flushTracing,+    stopTracing,+    otlpExporterConfig,+  )+where++import Control.Monad (when)+import Data.Text qualified as Text+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Tracing.Pipeline+import Kenshou.Telemetry.Tracing.Probe+import OpenTelemetry.Exporter.OTLP.Span qualified as OtlpSpan+import OpenTelemetry.Exporter.Span (SpanExporter)+import OpenTelemetry.Processor.Batch.Span qualified as Batch+import OpenTelemetry.Processor.Simple.Span qualified as Simple+import OpenTelemetry.Processor.Span (SpanProcessor)+import OpenTelemetry.Propagator (setGlobalTextMapPropagator)+import OpenTelemetry.Propagator.W3CTraceContext (w3cTraceContextPropagator)+import OpenTelemetry.Resource (materializeResources, mkResource, (.=))+import OpenTelemetry.Trace.Core+import OpenTelemetry.Trace.Id.Generator.Default (defaultIdGenerator)+import OpenTelemetry.Trace.Sampler qualified as Sampler++data TracingRuntime = TracingRuntime+  { tracer :: Maybe Tracer,+    provider :: Maybe TracerProvider,+    probe :: Maybe SpanProbe,+    pipeline :: Maybe PipelineStats+  }++startTracing :: TelemetrySpec -> IO TracingRuntime+startTracing spec = do+  when (spec.tracing /= TracingOff) (setGlobalTextMapPropagator w3cTraceContextPropagator)+  case spec.tracing of+    TracingOff -> pure (TracingRuntime Nothing Nothing Nothing Nothing)+    TracingNoop -> do+      provider <- createTracerProvider [] (providerOptions spec)+      pure (runtime provider Nothing Nothing)+    TracingSdkInMemory -> do+      stats <- newPipelineStats+      (probe, probeProcessor) <- newSpanProbe spec.probeRetain+      provider <- createTracerProvider [countingProcessor stats, instrumentProcessorSuccess stats probeProcessor] (providerOptions spec)+      pure (runtime provider (Just probe) (Just stats))+    TracingSdkOtlp -> do+      stats <- newPipelineStats+      exporter <- instrumentExporter stats <$> OtlpSpan.otlpExporter (otlpExporterConfig spec)+      processor <- makeProcessor spec.processor exporter+      provider <- createTracerProvider [countingProcessor stats, processor] (providerOptions spec)+      pure (runtime provider Nothing (Just stats))+  where+    runtime provider probe pipeline = TracingRuntime (Just (makeTracer provider (instrumentationLibrary "kenshou" "0.1.0.0") tracerOptions)) (Just provider) probe pipeline++flushTracing :: Int -> TracingRuntime -> IO (Maybe FlushResult)+flushTracing timeoutMs runtime = traverse (\provider -> forceFlushTracerProvider provider (Just (timeoutMs * 1000))) runtime.provider++stopTracing :: Int -> TracingRuntime -> IO (Maybe ShutdownResult)+stopTracing timeoutMs runtime = traverse (\provider -> shutdownTracerProvider provider (Just (timeoutMs * 1000))) runtime.provider++providerOptions :: TelemetrySpec -> TracerProviderOptions+providerOptions spec =+  emptyTracerProviderOptions+    { tracerProviderOptionsIdGenerator = defaultIdGenerator,+      tracerProviderOptionsSampler = sampler spec.sampler,+      tracerProviderOptionsResources = materializeResources (mkResource ["service.name" .= spec.serviceName]),+      tracerProviderOptionsPropagators = w3cTraceContextPropagator+    }++sampler :: SamplerSpec -> Sampler.Sampler+sampler AlwaysOn = Sampler.alwaysOn+sampler AlwaysOff = Sampler.alwaysOff+sampler ParentBasedAlwaysOn = Sampler.parentBased (Sampler.parentBasedOptions Sampler.alwaysOn)+sampler (TraceIdRatio ratio) = Sampler.traceIdRatioBased ratio+sampler (ParentBasedTraceIdRatio ratio) = Sampler.parentBased (Sampler.parentBasedOptions (Sampler.traceIdRatioBased ratio))++makeProcessor :: ProcessorSpec -> SpanExporter -> IO SpanProcessor+makeProcessor (SimpleProcessor timeoutMs) exporter = Simple.simpleProcessor (Simple.SimpleProcessorConfig exporter (timeoutMs * 1000))+makeProcessor (BatchProcessor queue delay batch timeoutMs) exporter =+  Batch.batchProcessor+    Batch.batchTimeoutConfig+      { Batch.maxQueueSize = queue,+        Batch.scheduledDelayMillis = delay,+        Batch.maxExportBatchSize = batch,+        Batch.exportTimeoutMillis = timeoutMs+      }+    exporter++otlpExporterConfig :: TelemetrySpec -> OtlpSpan.OTLPExporterConfig+otlpExporterConfig spec =+  OtlpSpan.OTLPExporterConfig+    { OtlpSpan.otlpEndpoint = Just (Text.unpack (endpointBase spec.endpoint)),+      OtlpSpan.otlpTracesEndpoint = Nothing,+      OtlpSpan.otlpMetricsEndpoint = Nothing,+      OtlpSpan.otlpInsecure = True,+      OtlpSpan.otlpSpanInsecure = True,+      OtlpSpan.otlpMetricInsecure = True,+      OtlpSpan.otlpCertificate = Nothing,+      OtlpSpan.otlpTracesCertificate = Nothing,+      OtlpSpan.otlpMetricCertificate = Nothing,+      OtlpSpan.otlpHeaders = Nothing,+      OtlpSpan.otlpTracesHeaders = Nothing,+      OtlpSpan.otlpMetricsHeaders = Nothing,+      OtlpSpan.otlpCompression = Just (case spec.compression of CompressionNone -> OtlpSpan.None; CompressionGzip -> OtlpSpan.GZip),+      OtlpSpan.otlpTracesCompression = Nothing,+      OtlpSpan.otlpMetricsCompression = Nothing,+      OtlpSpan.otlpTimeout = Just (processorTimeout spec.processor),+      OtlpSpan.otlpTracesTimeout = Nothing,+      OtlpSpan.otlpMetricsTimeout = Nothing,+      OtlpSpan.otlpProtocol = Just OtlpSpan.HttpProtobuf,+      OtlpSpan.otlpTracesProtocol = Nothing,+      OtlpSpan.otlpMetricsProtocol = Nothing+    }++endpointBase :: OtlpEndpoint -> Text.Text+endpointBase (ExternalEndpoint url) = url+endpointBase (BuiltinSink _) = "http://127.0.0.1:4318"++processorTimeout :: ProcessorSpec -> Int+processorTimeout (SimpleProcessor timeoutMs) = timeoutMs+processorTimeout (BatchProcessor _ _ _ timeoutMs) = timeoutMs
+ src/Kenshou/Telemetry/Tracing/Pipeline.hs view
@@ -0,0 +1,187 @@+module Kenshou.Telemetry.Tracing.Pipeline+  ( PipelineStats,+    PipelineSnapshot (..),+    LatencySummary (..),+    newPipelineStats,+    countingProcessor,+    instrumentProcessorSuccess,+    instrumentExporter,+    snapshotPipeline,+    startPipelineSampler,+  )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Exception (displayException, finally, onException)+import Control.Monad (void)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.HashMap.Strict qualified as HashMap+import Data.IORef+import Data.List (sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Vector qualified as Vector+import GHC.Clock (getMonotonicTimeNSec)+import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))+import OpenTelemetry.Internal.Common.Types (FlushResult (FlushSuccess), ShutdownResult (ShutdownSuccess))+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO (BufferMode (LineBuffering), IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)++data PipelineStats = PipelineStats+  { started :: IORef Int,+    ended :: IORef Int,+    exportedOk :: IORef Int,+    exportFailed :: IORef Int,+    exportCalls :: IORef Int,+    exportLatencies :: IORef [Integer],+    maxQueueDepth :: IORef Int,+    lastExportError :: IORef (Maybe Text)+  }++data LatencySummary = LatencySummary {p50 :: Integer, p99 :: Integer, maximum :: Integer}+  deriving stock (Eq, Show)++data PipelineSnapshot = PipelineSnapshot+  { spansStarted :: Int,+    spansEnded :: Int,+    spansExportedOk :: Int,+    spansExportFailed :: Int,+    spansDropped :: Int,+    maxQueueDepth :: Int,+    exportCalls :: Int,+    exportLatencyNs :: LatencySummary,+    lastExportError :: Maybe Text+  }+  deriving stock (Eq, Show)++instance ToJSON LatencySummary where+  toJSON summary = object ["p50" .= summary.p50, "p99" .= summary.p99, "max" .= summary.maximum]++instance ToJSON PipelineSnapshot where+  toJSON snapshot =+    object+      [ "spansStarted" .= snapshot.spansStarted,+        "spansEnded" .= snapshot.spansEnded,+        "spansExportedOk" .= snapshot.spansExportedOk,+        "spansExportFailed" .= snapshot.spansExportFailed,+        "spansDropped" .= snapshot.spansDropped,+        "maxQueueDepth" .= snapshot.maxQueueDepth,+        "exportCalls" .= snapshot.exportCalls,+        "exportLatencyNs" .= snapshot.exportLatencyNs,+        "lastExportError" .= snapshot.lastExportError+      ]++newPipelineStats :: IO PipelineStats+newPipelineStats = PipelineStats <$> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef [] <*> newIORef 0 <*> newIORef Nothing++countingProcessor :: PipelineStats -> SpanProcessor+countingProcessor stats =+  SpanProcessor+    { spanProcessorOnStart = \_ _ -> increment stats.started 1,+      spanProcessorOnEnd = \_ -> increment stats.ended 1 >> updateQueueHighWater stats,+      spanProcessorShutdown = pure ShutdownSuccess,+      spanProcessorForceFlush = pure FlushSuccess+    }++instrumentProcessorSuccess :: PipelineStats -> SpanProcessor -> SpanProcessor+instrumentProcessorSuccess stats processor =+  processor+    { spanProcessorOnEnd = \spanValue -> processor.spanProcessorOnEnd spanValue >> increment stats.exportedOk 1+    }++instrumentExporter :: PipelineStats -> SpanExporter -> SpanExporter+instrumentExporter stats exporter =+  exporter+    { spanExporterExport = \batch -> do+        let spanCount = sum (fmap Vector.length (HashMap.elems batch))+        startedAt <- getMonotonicTimeNSec+        increment stats.exportCalls 1+        result <-+          exporter.spanExporterExport batch+            `onException` do+              increment stats.exportFailed spanCount+              writeIORef stats.lastExportError (Just "exporter raised an exception")+        endedAt <- getMonotonicTimeNSec+        modifyIORef' stats.exportLatencies (fromIntegral (endedAt - startedAt) :)+        case result of+          Success -> increment stats.exportedOk spanCount+          Failure exception -> do+            increment stats.exportFailed spanCount+            writeIORef stats.lastExportError (fmap (Text.pack . displayException) exception)+        pure result+    }++snapshotPipeline :: PipelineStats -> IO PipelineSnapshot+snapshotPipeline stats = do+  spansStarted <- readIORef stats.started+  spansEnded <- readIORef stats.ended+  spansExportedOk <- readIORef stats.exportedOk+  spansExportFailed <- readIORef stats.exportFailed+  exportCalls <- readIORef stats.exportCalls+  latencies <- readIORef stats.exportLatencies+  maxQueueDepth <- readIORef stats.maxQueueDepth+  lastExportError <- readIORef stats.lastExportError+  let spansDropped = max 0 (spansEnded - spansExportedOk - spansExportFailed)+  pure+    PipelineSnapshot+      { spansStarted,+        spansEnded,+        spansExportedOk,+        spansExportFailed,+        spansDropped,+        maxQueueDepth,+        exportCalls,+        exportLatencyNs = summarize latencies,+        lastExportError+      }++startPipelineSampler :: FilePath -> PipelineStats -> IO (IO [Int])+startPipelineSampler outputDir stats = do+  createDirectoryIfMissing True (outputDir </> "series")+  samples <- newIORef []+  worker <- async do+    handle <- openFile (outputDir </> "series" </> "otel-pipeline.csv") WriteMode+    hSetBuffering handle LineBuffering+    hPutStrLn handle "t_mono_ns,spans_started,spans_ended,exported_ok,export_failed,queue_depth,export_calls"+    let loop = do+          threadDelay 1_000_000+          timestamp <- getMonotonicTimeNSec+          started <- readIORef stats.started+          ended <- readIORef stats.ended+          exportedOk <- readIORef stats.exportedOk+          exportFailed <- readIORef stats.exportFailed+          exportCalls <- readIORef stats.exportCalls+          let queueDepth = max 0 (ended - exportedOk - exportFailed)+          modifyIORef' samples (queueDepth :)+          hPutStrLn handle (comma [timestamp, fromIntegral started, fromIntegral ended, fromIntegral exportedOk, fromIntegral exportFailed, fromIntegral queueDepth, fromIntegral exportCalls])+          loop+    loop `finally` hClose handle+  pure do+    cancel worker+    void (waitCatch worker)+    reverse <$> readIORef samples+  where+    comma = foldr1 (\left right -> left <> "," <> right) . fmap show++increment :: IORef Int -> Int -> IO ()+increment ref amount = void (atomicModifyIORef' ref (\value -> let updated = value + amount in (updated, updated)))++updateQueueHighWater :: PipelineStats -> IO ()+updateQueueHighWater stats = do+  ended <- readIORef stats.ended+  exportedOk <- readIORef stats.exportedOk+  exportFailed <- readIORef stats.exportFailed+  let queued = max 0 (ended - exportedOk - exportFailed)+  atomicModifyIORef' stats.maxQueueDepth (\high -> (max high queued, ()))++summarize :: [Integer] -> LatencySummary+summarize [] = LatencySummary 0 0 0+summarize values =+  let ordered = sort values+   in LatencySummary (quantile 0.50 ordered) (quantile 0.99 ordered) (last ordered)++quantile :: Double -> [Integer] -> Integer+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))
+ src/Kenshou/Telemetry/Tracing/Probe.hs view
@@ -0,0 +1,82 @@+module Kenshou.Telemetry.Tracing.Probe+  ( SpanView (..),+    SpanProbe,+    newSpanProbe,+    readSpans,+    spansSeen,+  )+where++import Data.Foldable qualified as Foldable+import Data.IORef+import Data.Sequence (Seq (..), (|>))+import Data.Sequence qualified as Seq+import Data.Text (Text)+import Data.Word (Word64)+import OpenTelemetry.Attributes (Attributes)+import OpenTelemetry.Common (optionalTimestampToMaybe, timestampToNanoseconds)+import OpenTelemetry.Internal.Common.Types (FlushResult (FlushSuccess), ShutdownResult (ShutdownSuccess))+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import OpenTelemetry.Trace.Core (ImmutableSpan (..), SpanContext (..), SpanHot (..), SpanKind, SpanStatus, getSpanContext)+import OpenTelemetry.Trace.Id (SpanId, TraceId)++data SpanView = SpanView+  { name :: Text,+    kind :: SpanKind,+    traceId :: TraceId,+    spanId :: SpanId,+    parentSpanId :: Maybe SpanId,+    attributes :: Attributes,+    startNs :: Word64,+    endNs :: Word64,+    status :: SpanStatus+  }+  deriving stock (Eq, Show)++-- Evict during capture, even when nobody reads the probe during a long run.+-- A strict IORef update alone only forces the outer constructor.+data ProbeState = ProbeState {seen :: !Int, retained :: !(Seq SpanView)}++data SpanProbe = SpanProbe {capacity :: Int, state :: IORef ProbeState}++newSpanProbe :: Int -> IO (SpanProbe, SpanProcessor)+newSpanProbe capacity = do+  state <- newIORef (ProbeState 0 Seq.empty)+  let probe = SpanProbe (max 0 capacity) state+      processor =+        SpanProcessor+          { spanProcessorOnStart = \_ _ -> pure (),+            spanProcessorOnEnd = capture probe,+            spanProcessorShutdown = pure ShutdownSuccess,+            spanProcessorForceFlush = pure FlushSuccess+          }+  pure (probe, processor)++readSpans :: SpanProbe -> IO [SpanView]+readSpans probe = toList . (.retained) <$> readIORef probe.state+  where+    toList = Foldable.toList++spansSeen :: SpanProbe -> IO Int+spansSeen probe = (.seen) <$> readIORef probe.state++capture :: SpanProbe -> ImmutableSpan -> IO ()+capture probe spanValue = do+  hot <- readIORef spanValue.spanHot+  parent <- traverse getSpanContext spanValue.spanParent+  let view =+        SpanView+          { name = hot.hotName,+            kind = spanValue.spanKind,+            traceId = spanValue.spanContext.traceId,+            spanId = spanValue.spanContext.spanId,+            parentSpanId = (.spanId) <$> parent,+            attributes = hot.hotAttributes,+            startNs = timestampToNanoseconds spanValue.spanStart,+            endNs = maybe (timestampToNanoseconds spanValue.spanStart) timestampToNanoseconds (optionalTimestampToMaybe hot.hotEnd),+            status = hot.hotStatus+          }+  atomicModifyIORef' probe.state \current ->+    let appended = current.retained |> view+        trimmed = if Seq.length appended > probe.capacity then Seq.drop 1 appended else appended+     in (ProbeState (current.seen + 1) trimmed, ())
+ test/Main.hs view
@@ -0,0 +1,366 @@+module Main (main) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, readMVar, tryPutMVar)+import Control.Exception (bracket)+import Control.Monad (forM_, replicateM_, void)+import Data.IORef+import Data.List (find)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (fromJust)+import Data.Text qualified as Text+import Kenshou.Core.Knob (Allowed (..), KnobSpec (..), renderKnobName)+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Telemetry.Compose+import Kenshou.Telemetry.Continuity+import Kenshou.Telemetry.Detect (Finding (..), FindingSeverity (Degraded), FindingStatus (Fired), handlerFinding, pipelineFindings, queueGrowthFinding)+import Kenshou.Telemetry.Endpoint+import Kenshou.Telemetry.Metrics+import Kenshou.Telemetry.Overhead+import Kenshou.Telemetry.Scrape+import Kenshou.Telemetry.SelfTest.Arms (armsScenario)+import Kenshou.Telemetry.SelfTest.Service+import Kenshou.Telemetry.Sink qualified as Sink+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Tracing+import Kenshou.Telemetry.Tracing.Pipeline+import Kenshou.Telemetry.Tracing.Probe+import Network.HTTP.Types (status200, status404)+import Network.Socket (close)+import Network.Wai (Application, responseLBS)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import Network.Wai.Handler.WebSockets (websocketsOr)+import Network.WebSockets qualified as WebSockets+import OpenTelemetry.Attributes (lookupAttribute)+import OpenTelemetry.Context.ThreadLocal (getContext)+import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))+import OpenTelemetry.Processor.Batch.Span qualified as Batch+import OpenTelemetry.Processor.Simple.Span qualified as Simple+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import OpenTelemetry.Propagator (emptyTextMap, inject, textMapToList)+import OpenTelemetry.Trace.Core+import System.Directory (createDirectoryIfMissing)+import System.Mem (performMajorGC)+import System.Mem.Weak (Weak, deRefWeak, mkWeakPtr)+import System.Timeout (timeout)+import Test.Hspec++main :: IO ()+main = hspec do+  describe "overhead planning" do+    it "expands the headline one-factor matrix into four arms and twelve slots" do+      let request = overheadRequest (Map.fromList [("telemetry.tracing", ["off", "noop", "sdk-otlp"]), ("telemetry.metrics", ["off", "serve-scraped"])])+      case planOverhead request armsScenario of+        Left err -> expectationFailure (show err)+        Right plan -> do+          length plan.arms `shouldBe` 4+          length plan.slots `shouldBe` 12+          map (length . unique . fmap (.arm.armId)) (blocks plan.slots) `shouldBe` [4, 4, 4]++    it "is deterministic and gives two arms ABBA order across the first two blocks" do+      let request = overheadRequest (Map.singleton "telemetry.tracing" ["off", "noop"])+      case (planOverhead request armsScenario, planOverhead request armsScenario) of+        (Right first, Right second) -> do+          first `shouldBe` second+          let order = fmap (.arm.armId) first.slots+          take 2 order `shouldBe` reverse (take 2 (drop 2 order))+        result -> expectationFailure (show result)++    it "rejects repeated arm values" do+      planOverhead (overheadRequest (Map.singleton "telemetry.tracing" ["off", "off"])) armsScenario `shouldSatisfy` isLeft++  describe "telemetry problem detectors" do+    it "detects synchronous handler stalls while a bounded async wrapper drops instead of blocking" do+      synchronousStats <- newHandlerStats+      timedHandler synchronousStats "sync" (slowHandler 6_000) ()+      synchronous <- snapshotHandlerStats "sync" synchronousStats+      (submit, snapshotAsync) <- asyncHandler 1 (slowHandler 20_000)+      replicateM_ 100 (submit ())+      queued <- snapshotAsync+      handlerFinding synchronous `shouldSatisfy` ((== Fired) . (.status))+      queued.dropped `shouldSatisfy` (> 0)++    it "marks dropped spans as degraded above the policy fraction" do+      let snapshot = PipelineSnapshot 100 100 80 0 20 20 1 (LatencySummary 1 1 1) Nothing+      pipelineFindings 0.01 (Just snapshot) [] `shouldSatisfy` any (\finding -> finding.detector == "span-drop" && finding.status == Fired && finding.severity == Degraded)++    it "fires queue growth only after more than ten consecutive samples above ninety percent" do+      queueGrowthFinding (Just 100) (replicate 10 91) `shouldSatisfy` ((/= Fired) . (.status))+      queueGrowthFinding (Just 100) (replicate 11 91) `shouldSatisfy` ((== Fired) . (.status))++  describe "telemetryKnobs" do+    it "declares each shared knob once" do+      let names = fmap (renderKnobName . (.name)) telemetryKnobs+      length names `shouldBe` length (unique names)+    it "bounds the trace-id ratio to zero through one" do+      fmap (.allowed) (find ((== "otel.sampler-arg") . renderKnobName . (.name)) telemetryKnobs)+        `shouldBe` Just (DoubleRange 0 1)++  describe "tracing arms" do+    it "does not construct a provider under off" do+      runtime <- startTracing (testSpec TracingOff)+      isNothing runtime.tracer `shouldBe` True+      isNothing runtime.provider `shouldBe` True++    it "constructs a disabled tracer under noop" do+      runtime <- startTracing (testSpec TracingNoop)+      let tracer = fromJust runtime.tracer+      tracerIsEnabled tracer `shouldBe` False+      inSpan tracer "ignored" defaultSpanArguments (pure ())+      _ <- stopTracing 1000 runtime+      pure ()++    it "retains a bounded, parented trace and installs W3C propagation" do+      runtime <- startTracing ((testSpec TracingSdkInMemory) {probeRetain = 4})+      let tracer = fromJust runtime.tracer+          provider = fromJust runtime.provider+      replicateM_ 5 (inSpan tracer "discarded" defaultSpanArguments (pure ()))+      headers <- inSpan' tracer "parent" defaultSpanArguments \parentSpan -> do+        addAttribute parentSpan "message.id" ("one" :: Text.Text)+        nested <- inSpan' tracer "child" defaultSpanArguments \childSpan -> addAttribute childSpan "message.id" ("one" :: Text.Text)+        activeContext <- getContext+        carrier <- inject (getTracerProviderPropagators provider) activeContext emptyTextMap+        pure (nested, textMapToList carrier)+      _ <- flushTracing 1000 runtime+      let probe = fromJust runtime.probe+      views <- readSpans probe+      seen <- spansSeen probe+      seen `shouldBe` 7+      length views `shouldBe` 4+      let child = fromJust (find ((== "child") . (.name)) views)+          parent = fromJust (find ((== "parent") . (.name)) views)+      child.traceId `shouldBe` parent.traceId+      child.parentSpanId `shouldBe` Just parent.spanId+      let correlation spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing+          continuity = checkContinuity (SpanSelector (Just "parent") Nothing []) (SpanSelector (Just "child") Nothing []) correlation views+      continuity.violations `shouldBe` 0+      snd headers `shouldSatisfy` any ((== "traceparent") . fst)+      snapshot <- snapshotPipeline (fromJust runtime.pipeline)+      snapshot.spansEnded `shouldBe` 7+      snapshot.spansExportedOk `shouldBe` 7+      snapshot.spansDropped `shouldBe` 0+      _ <- stopTracing 1000 runtime+      pure ()++    it "releases evicted spans before anyone reads the bounded probe" do+      forM_ [0, 1] \capacity -> do+        (probe, provider, weak) <- probeWithEvictedSpan capacity+        performMajorGC+        (isNothing <$> deRefWeak weak) `shouldReturn` True+        spansSeen probe `shouldReturn` 10+        (length <$> readSpans probe) `shouldReturn` capacity+        _ <- shutdownTracerProvider provider (Just 1000000)+        pure ()++    it "exports plain and gzip OTLP requests to the built-in sink" do+      forCompression CompressionNone+      forCompression CompressionGzip++    it "accounts for alternating exporter failures exactly" do+      stats <- newPipelineStats+      calls <- newIORef (0 :: Int)+      let exporter =+            SpanExporter+              { spanExporterExport = \_ -> do+                  call <- atomicModifyIORef' calls (\value -> let next = value + 1 in (next, next))+                  pure (if even call then Failure Nothing else Success),+                spanExporterShutdown = pure ShutdownSuccess,+                spanExporterForceFlush = pure FlushSuccess+              }+      processor <- Simple.simpleProcessor (Simple.SimpleProcessorConfig (instrumentExporter stats exporter) 1_000_000)+      provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions+      let tracer = makeTracer provider (instrumentationLibrary "pipeline-test" "1") tracerOptions+      replicateM_ 10 (inSpan tracer "alternating" defaultSpanArguments (pure ()))+      _ <- shutdownTracerProvider provider (Just 1_000_000)+      snapshot <- snapshotPipeline stats+      snapshot.spansEnded `shouldBe` snapshot.spansExportedOk + snapshot.spansExportFailed + snapshot.spansDropped+      snapshot.spansExportedOk `shouldBe` 5+      snapshot.spansExportFailed `shouldBe` 5++    it "drops instead of blocking when a batch queue is full" do+      stats <- newPipelineStats+      releaseExporter <- newEmptyMVar+      let exporter =+            SpanExporter+              { spanExporterExport = \_ -> readMVar releaseExporter >> pure Success,+                spanExporterShutdown = pure ShutdownSuccess,+                spanExporterForceFlush = pure FlushSuccess+              }+      processor <-+        Batch.batchProcessor+          Batch.batchTimeoutConfig+            { Batch.maxQueueSize = 8,+              Batch.scheduledDelayMillis = 10,+              Batch.maxExportBatchSize = 4,+              Batch.exportTimeoutMillis = 5_000+            }+          (instrumentExporter stats exporter)+      provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions+      let tracer = makeTracer provider (instrumentationLibrary "backpressure-test" "1") tracerOptions+      completed <- timeout 1_000_000 (replicateM_ 10_000 (inSpan tracer "queued" defaultSpanArguments (pure ())))+      completed `shouldBe` Just ()+      beforeRelease <- snapshotPipeline stats+      beforeRelease.spansDropped `shouldSatisfy` (> 0)+      putMVar releaseExporter ()+      void (shutdownTracerProvider provider (Just 1_000_000))++  describe "metrics arms" do+    it "constructs no provider under off" do+      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsOff})+      isNothing runtime.meter `shouldBe` True+      isNothing runtime.provider `shouldBe` True+      isNothing runtime.endpoint `shouldBe` True++    it "collects instruments without opening an endpoint" do+      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsCollect})+      let meter = fromJust runtime.meter+      metrics <- newSyntheticMetrics meter+      void (runSyntheticOperation Nothing (Just metrics) 0 0 1)+      (snapshot, _) <- stopMetrics 1000 runtime+      runtime.endpoint `shouldBe` Nothing+      fmap (.instruments) snapshot `shouldBe` Just 2++    it "serves its Prometheus reader only for a serving arm" do+      runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsServe, otelReader = ReaderPrometheus})+      runtime.endpoint `shouldSatisfy` maybe False ((== PrometheusText) . (.kind))+      traverse_ (\endpoint -> awaitHttpReady endpoint.url 1000 >>= (`shouldBe` True)) runtime.endpoint+      void (stopMetrics 1000 runtime)++  describe "metrics scraper" do+    it "skips fixed-schedule ticks when an endpoint is slower than its interval" do+      withSlowHttpServer 75_000 \endpoint -> do+        let output = ".tmp/telemetry-test/slow-scrape"+        createDirectoryIfMissing True output+        scraper <- startScraperInProcess output 20 0+        scraper.register endpoint+        threadDelay 260_000+        summaries <- scraper.finish+        fmap (.skippedTicks) summaries `shouldSatisfy` any (> 0)++    it "detects exhausted WebSocket slots and accepts a correct server" do+      withWebSocketServer (Just 2) \leaky -> do+        result <- wsSlotLeakProbe leaky 2+        result.freshAccepted `shouldBe` False+      withWebSocketServer Nothing \correct -> do+        result <- wsSlotLeakProbe correct 2+        result.freshAccepted `shouldBe` True++testSpec :: TracingArm -> TelemetrySpec+testSpec tracing =+  TelemetrySpec+    { tracing,+      metrics = MetricsOff,+      serviceName = "kenshou.test",+      sampler = AlwaysOn,+      processor = SimpleProcessor 1000,+      exporter = HttpProtobuf,+      endpoint = BuiltinSink SinkHealthy,+      compression = CompressionNone,+      probeRetain = 16,+      shutdownMs = 1000,+      otelReader = ReaderNone,+      otelExportMs = 1000,+      scrapeMs = 1000,+      wsSubscribers = 0,+      helpers = HelperInProcess,+      workerContext = Nothing,+      outDir = ".tmp/telemetry-test",+      report = const (pure ())+    }++forCompression :: OtlpCompression -> IO ()+forCompression compression = Sink.withSink SinkHealthy \sink -> do+  let spec = (testSpec TracingSdkOtlp) {endpoint = ExternalEndpoint sink.endpoint, compression}+  runtime <- startTracing spec+  let tracer = fromJust runtime.tracer+  inSpan tracer "exported" defaultSpanArguments (pure ())+  _ <- flushTracing 2000 runtime+  _ <- stopTracing 2000 runtime+  stats <- sink.snapshot+  stats.spansReceived `shouldBe` 1++isNothing :: Maybe value -> Bool+isNothing Nothing = True+isNothing (Just _) = False++unique :: (Eq value) => [value] -> [value]+unique [] = []+unique (value : values) = value : unique (filter (/= value) values)++withSlowHttpServer :: Int -> (Endpoint -> IO value) -> IO value+withSlowHttpServer delayMicros action = withWarp application \port ->+  action (Endpoint "slow-json" JsonDocument (Text.pack ("http://127.0.0.1:" <> show port <> "/metrics")) Nothing)+  where+    application _ respond = threadDelay delayMicros >> respond (responseLBS status200 [] "delayed")++withWebSocketServer :: Maybe Int -> (Endpoint -> IO value) -> IO value+withWebSocketServer capacity action = do+  accepted <- newIORef (0 :: Int)+  withWarp (websocketsOr WebSockets.defaultConnectionOptions (server accepted) fallback) \port ->+    action (Endpoint "slot-probe" WebSocketPush (Text.pack ("ws://127.0.0.1:" <> show port <> "/ws")) Nothing)+  where+    server accepted pendingConnection = do+      count <- atomicModifyIORef' accepted (\value -> let next = value + 1 in (next, next))+      case capacity of+        Just limit | count > limit -> WebSockets.rejectRequest pendingConnection "slots exhausted"+        _ -> void (WebSockets.acceptRequest pendingConnection)+    fallback _ respond = respond (responseLBS status404 [] "not found")++withWarp :: Application -> (Int -> IO value) -> IO value+withWarp application action = bracket acquire release (\(port, _, _) -> action port)+  where+    acquire = do+      (port, socket) <- openFreePort+      server <- async (runSettingsSocket defaultSettings socket application)+      pure (port, server, socket)+    release (_, server, socket) = cancel server >> void (waitCatch server) >> close socket++traverse_ :: (a -> IO b) -> Maybe a -> IO ()+traverse_ _ Nothing = pure ()+traverse_ action (Just value) = void (action value)++overheadRequest :: Map Text.Text [Text.Text] -> OverheadRequest+overheadRequest factors =+  OverheadRequest+    { requestId = "test-overhead",+      scenarioId = armsScenario.id,+      factors,+      mode = OneFactor,+      includeControl = False,+      trials = 3,+      fixedKnobs = [],+      fixedDimensions = Map.empty,+      seed = 7,+      settleSeconds = 0,+      retries = 1+    }++blocks :: [Slot] -> [[Slot]]+blocks slots = [[slot | slot <- slots, slot.block == block] | block <- unique (fmap (.block) slots)]++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft (Right _) = False++-- Keeping the provider and probe alive must not keep a span outside the+-- retention window alive. Do not force readSpans/spansSeen before the GC.+{-# NOINLINE probeWithEvictedSpan #-}+probeWithEvictedSpan :: Int -> IO (SpanProbe, TracerProvider, Weak SpanHot)+probeWithEvictedSpan capacity = do+  (probe, processor) <- newSpanProbe capacity+  first <- newEmptyMVar+  let observing =+        processor+          { spanProcessorOnEnd = \spanValue -> do+              hot <- readIORef spanValue.spanHot+              weak <- mkWeakPtr hot Nothing+              void (tryPutMVar first weak)+              processor.spanProcessorOnEnd spanValue+          }+  provider <- createTracerProvider [observing] emptyTracerProviderOptions+  let tracer = makeTracer provider (instrumentationLibrary "probe-retention-test" "0.1.0.0") tracerOptions+  replicateM_ 10 (inSpan tracer "evicted" defaultSpanArguments (pure ()))+  weak <- readMVar first+  pure (probe, provider, weak)
+ test/golden/overhead-report.minimal.json view
@@ -0,0 +1,21 @@+{+  "schema": "kenshou.overhead-report/v1",+  "id": "01a0c631-ff36-77c3-8772-216b4ab24eed",+  "scenario": "selftest/telemetry/benchmark/arms-on-synthetic-service",+  "mode": "one-factor",+  "baseline": {"telemetry.tracing": "off", "telemetry.metrics": "off"},+  "fixed": {"knobs": {}, "dimensions": {}},+  "trials": 3,+  "seed": 7,+  "validBlocks": 3,+  "order": [["a0", "a1"], ["a1", "a0"], ["a0", "a1"]],+  "arms": [+    {"arm": "a0", "dimensions": {"telemetry.tracing": "off"}, "runs": [], "telemetry": null, "leak": null, "findings": []}+  ],+  "comparisons": [],+  "findings": [],+  "policy": {"name": "telemetry-overhead-default", "sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},+  "cohort": {},+  "verdict": "pass",+  "algorithm": {"name": "kenshou-overhead", "version": 1}+}