kenshou-telemetry (empty) → 0.1.0.0
raw patch · 24 files changed
+3387/−0 lines, 24 filesdep +aesondep +asyncdep +base
Dependencies added: aeson, async, base, bytestring, containers, directory, filepath, hs-opentelemetry-api, hs-opentelemetry-exporter-otlp, hs-opentelemetry-exporter-prometheus, hs-opentelemetry-otlp, hs-opentelemetry-propagator-w3c, hs-opentelemetry-sdk, hspec, http-client, http-types, kenshou-core, kenshou-measure, kenshou-telemetry, microlens, network, proto-lens, proto-lens-runtime, splitmix, stm, text, time, unordered-containers, vector, wai, wai-websockets, warp, websockets, zlib
Files
- CHANGELOG.md +11/−0
- LICENSE +28/−0
- kenshou-telemetry.cabal +135/−0
- src/Kenshou/Telemetry.hs +287/−0
- src/Kenshou/Telemetry/Compose.hs +96/−0
- src/Kenshou/Telemetry/Continuity.hs +74/−0
- src/Kenshou/Telemetry/Detect.hs +103/−0
- src/Kenshou/Telemetry/Endpoint.hs +71/−0
- src/Kenshou/Telemetry/Metrics.hs +116/−0
- src/Kenshou/Telemetry/Overhead.hs +648/−0
- src/Kenshou/Telemetry/Overhead/Policy.hs +50/−0
- src/Kenshou/Telemetry/Scrape.hs +301/−0
- src/Kenshou/Telemetry/SelfTest.hs +28/−0
- src/Kenshou/Telemetry/SelfTest/Arms.hs +78/−0
- src/Kenshou/Telemetry/SelfTest/Endpoints.hs +41/−0
- src/Kenshou/Telemetry/SelfTest/Problems.hs +146/−0
- src/Kenshou/Telemetry/SelfTest/Service.hs +55/−0
- src/Kenshou/Telemetry/Sink.hs +111/−0
- src/Kenshou/Telemetry/Spec.hs +230/−0
- src/Kenshou/Telemetry/Tracing.hs +122/−0
- src/Kenshou/Telemetry/Tracing/Pipeline.hs +187/−0
- src/Kenshou/Telemetry/Tracing/Probe.hs +82/−0
- test/Main.hs +366/−0
- test/golden/overhead-report.minimal.json +21/−0
+ CHANGELOG.md view
@@ -0,0 +1,11 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of tracing and metrics arms, endpoint scraping, and observability-overhead reports.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+ may be used to endorse or promote products derived from this software without+ specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ kenshou-telemetry.cabal view
@@ -0,0 +1,135 @@+cabal-version: 3.0+name: kenshou-telemetry+version: 0.1.0.0+synopsis: Telemetry arms and overhead analysis for Kenshou scenarios+description: Explicit tracing and metrics providers, endpoint scrapers, pipeline diagnostics, and paired observability-overhead reports.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+ test/golden/*.json++flag otlp-grpc+ description: Enable the optional OTLP gRPC exporter+ default: False+ manual: True++common warnings+ ghc-options:+ -Wall+ -Wcompat+ -Widentities+ -Wincomplete-record-updates+ -Wincomplete-uni-patterns+ -Wpartial-fields+ -Wredundant-constraints++ default-language: GHC2024+ default-extensions:+ BlockArguments+ CPP+ DuplicateRecordFields+ ImportQualifiedPost+ NoFieldSelectors+ OverloadedLabels+ OverloadedRecordDot+ OverloadedStrings++library+ import: warnings+ hs-source-dirs: src+ exposed-modules:+ Kenshou.Telemetry+ Kenshou.Telemetry.Compose+ Kenshou.Telemetry.Continuity+ Kenshou.Telemetry.Detect+ Kenshou.Telemetry.Endpoint+ Kenshou.Telemetry.Metrics+ Kenshou.Telemetry.Overhead+ Kenshou.Telemetry.Overhead.Policy+ Kenshou.Telemetry.Scrape+ Kenshou.Telemetry.SelfTest+ Kenshou.Telemetry.SelfTest.Arms+ Kenshou.Telemetry.SelfTest.Endpoints+ Kenshou.Telemetry.SelfTest.Problems+ Kenshou.Telemetry.SelfTest.Service+ Kenshou.Telemetry.Sink+ Kenshou.Telemetry.Spec+ Kenshou.Telemetry.Tracing+ Kenshou.Telemetry.Tracing.Pipeline+ Kenshou.Telemetry.Tracing.Probe++ build-depends:+ aeson >=2.2 && <2.3,+ async >=2.2 && <2.3,+ base >=4.21 && <4.22,+ bytestring >=0.12 && <0.13,+ containers >=0.7 && <0.8,+ directory >=1.3 && <1.4,+ filepath >=1.5 && <1.6,+ hs-opentelemetry-api ==1.0.0.0,+ hs-opentelemetry-exporter-otlp ==1.0.0.0,+ hs-opentelemetry-exporter-prometheus ==1.0.0.0,+ hs-opentelemetry-otlp ==1.0.0.0,+ hs-opentelemetry-propagator-w3c ==1.0.0.0,+ hs-opentelemetry-sdk ==1.0.0.0,+ http-client >=0.7 && <0.8,+ http-types >=0.12 && <0.13,+ kenshou-core ^>=0.1.0.0,+ kenshou-measure ^>=0.1.0.0,+ microlens >=0.4 && <0.5,+ network >=3.2 && <3.3,+ proto-lens >=0.7 && <0.8,+ proto-lens-runtime >=0.7 && <0.8,+ splitmix >=0.1 && <0.2,+ stm >=2.5 && <2.6,+ text >=2.1 && <2.2,+ time >=1.14 && <1.15,+ unordered-containers >=0.2 && <0.3,+ vector >=0.13 && <0.14,+ wai >=3.2 && <3.3,+ wai-websockets >=3.0 && <3.1,+ warp >=3.4 && <3.5,+ websockets >=0.13 && <0.14,+ zlib >=0.7 && <0.8,++ if flag(otlp-grpc)+ cpp-options: -DWITH_OTLP_GRPC++test-suite kenshou-telemetry-test+ import: warnings+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Main.hs+ ghc-options:+ -threaded+ -rtsopts++ build-depends:+ async,+ base >=4.21 && <4.22,+ containers,+ directory >=1.3 && <1.4,+ hs-opentelemetry-api,+ hs-opentelemetry-sdk,+ hspec >=2.11 && <2.12,+ http-types >=0.12 && <0.13,+ kenshou-core ^>=0.1.0.0,+ kenshou-telemetry,+ network >=3.2 && <3.3,+ text,+ wai >=3.2 && <3.3,+ wai-websockets >=3.0 && <3.1,+ warp >=3.4 && <3.5,+ websockets >=0.13 && <0.14,++source-repository head+ type: git+ location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Telemetry.hs view
@@ -0,0 +1,287 @@+module Kenshou.Telemetry+ ( TelemetryHandles (..),+ FlushReport (..),+ ProviderCallReport (..),+ withTelemetry,+ module Kenshou.Telemetry.Spec,+ )+where++import Control.Exception (SomeException, displayException, mask, throwIO, try)+import Control.Monad (void)+import Data.Aeson (FromJSON (parseJSON), ToJSON (..), Value (..), object, (.=))+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseMaybe, withObject, (.:))+import Data.IORef+import Data.Maybe (catMaybes)+import Data.Text (Text)+import Data.Text qualified as Text+import GHC.Clock (getMonotonicTimeNSec)+import Kenshou.Core.Dimension (renderMetrics, renderTracing)+import Kenshou.Core.Role (ControlMessage (..), WorkerMessage (..), mkRoleName)+import Kenshou.Core.Role.Spawn (WorkerHandle (..), withWorker)+import Kenshou.Telemetry.Compose (HandlerStatsSnapshot)+import Kenshou.Telemetry.Continuity (ContinuityResult, IsolationResult)+import Kenshou.Telemetry.Detect+import Kenshou.Telemetry.Endpoint+import Kenshou.Telemetry.Metrics+import Kenshou.Telemetry.Metrics qualified as Metrics+import Kenshou.Telemetry.Scrape+import Kenshou.Telemetry.Sink+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Spec qualified as Spec+import Kenshou.Telemetry.Tracing+import Kenshou.Telemetry.Tracing.Pipeline+import Kenshou.Telemetry.Tracing.Probe+import OpenTelemetry.Metric.Core (Meter, MeterProvider)+import OpenTelemetry.Trace.Core (Tracer, TracerProvider)+import System.Environment (getEnvironment)++data ProviderCallReport = ProviderCallReport+ { result :: Text,+ durationMs :: Double+ }+ deriving stock (Eq, Show)++instance ToJSON ProviderCallReport where+ toJSON report = object ["result" .= report.result, "durationMs" .= report.durationMs]++data FlushReport = FlushReport+ { tracing :: Maybe ProviderCallReport,+ metrics :: Maybe ProviderCallReport+ }+ deriving stock (Eq, Show)++instance ToJSON FlushReport where+ toJSON report = object ["tracing" .= report.tracing, "metrics" .= report.metrics]++data TelemetryHandles = TelemetryHandles+ { tracer :: Maybe Tracer,+ tracerProvider :: Maybe TracerProvider,+ meter :: Maybe Meter,+ meterProvider :: Maybe MeterProvider,+ spans :: Maybe SpanProbe,+ pipeline :: Maybe PipelineStats,+ metricsLive :: Bool,+ readMetricSums :: IO [(Text, Double)],+ readMetricGauges :: IO [(Text, Double)],+ servesEndpoints :: Bool,+ metricEndpoint :: Maybe Endpoint,+ registerEndpoint :: Endpoint -> IO (),+ setSinkFault :: SinkFault -> IO (),+ recordContinuity :: ContinuityResult -> IsolationResult -> IO (),+ recordHandlerStats :: HandlerStatsSnapshot -> IO (),+ flushTelemetry :: IO FlushReport+ }++withTelemetry :: TelemetrySpec -> (TelemetryHandles -> IO value) -> IO value+withTelemetry spec action = withConfiguredScraper spec \scraper -> case spec.endpoint of+ BuiltinSink fault | requiresSink spec -> withConfiguredSink spec fault (\sink -> runWith scraper (spec {Spec.endpoint = ExternalEndpoint sink.endpoint}) (Just sink))+ _ -> runWith scraper spec Nothing+ where+ runWith scraper effectiveSpec sink = mask \restore -> do+ runtime <- startTracing effectiveSpec+ metricsRuntime <- startMetrics effectiveSpec+ stopPipelineSampler <- maybe (pure (pure [])) (startPipelineSampler spec.outDir) runtime.pipeline+ endpointsRef <- newIORef (maybe [] pure metricsRuntime.endpoint)+ recordedFindingsRef <- newIORef []+ handlerStatsRef <- newIORef []+ traverse_ (\endpoint -> traverse_ (\active -> active.register endpoint) scraper) metricsRuntime.endpoint+ let register endpoint+ | spec.metrics `notElem` [MetricsServe, MetricsServeScraped] = ioError (userError "metrics endpoint registered while telemetry.metrics does not serve endpoints")+ | otherwise = do+ ready <- if endpoint.kind == WebSocketPush then pure True else awaitHttpReady endpoint.url 10_000+ if ready+ then do+ modifyIORef' endpointsRef (<> [endpoint])+ traverse_ (\active -> active.register endpoint) scraper+ else ioError (userError ("metrics endpoint did not become ready: " <> Text.unpack endpoint.url))+ let handles =+ TelemetryHandles+ { tracer = runtime.tracer,+ tracerProvider = runtime.provider,+ meter = metricsRuntime.meter,+ meterProvider = metricsRuntime.provider,+ spans = runtime.probe,+ pipeline = runtime.pipeline,+ metricsLive = spec.metrics /= MetricsOff,+ readMetricSums = Metrics.readMetricSums metricsRuntime,+ readMetricGauges = Metrics.readMetricGauges metricsRuntime,+ servesEndpoints = spec.metrics `elem` [MetricsServe, MetricsServeScraped],+ metricEndpoint = metricsRuntime.endpoint,+ registerEndpoint = register,+ setSinkFault = maybe (const (pure ())) (.setFault) sink,+ recordContinuity = \continuity isolation -> modifyIORef' recordedFindingsRef (<> continuityFindings continuity isolation),+ recordHandlerStats = \snapshot -> modifyIORef' handlerStatsRef (<> [snapshot]),+ flushTelemetry = FlushReport <$> timedCall (flushTracing spec.shutdownMs runtime) <*> timedCall (flushMetrics spec.shutdownMs metricsRuntime)+ }+ bodyResult <- tryAny (restore (action handles))+ flushResult <- timedCall (flushTracing spec.shutdownMs runtime)+ metricsFlushResult <- timedCall (flushMetrics spec.shutdownMs metricsRuntime)+ endpointSummaries <- maybe (pure Nothing) (fmap Just . (.finish)) scraper+ (metricsSnapshot, metricsShutdownResult) <- timedStopMetrics spec.shutdownMs metricsRuntime+ shutdownResult <- timedCall (stopTracing spec.shutdownMs runtime)+ queueSamples <- stopPipelineSampler+ rawPipelineSnapshot <- traverse snapshotPipeline runtime.pipeline+ let pipelineSnapshot = fmap (normalizeQueueDepth spec) rawPipelineSnapshot+ sinkSnapshot <- traverse (.snapshot) sink+ ambient <- ambientOtelEnvironment+ endpoints <- readIORef endpointsRef+ recordedFindings <- readIORef recordedFindingsRef+ handlerStats <- readIORef handlerStatsRef+ let endpointValues = maybe (fmap toJSON endpoints) (fmap toJSON) endpointSummaries+ calls = catMaybes [providerCall "trace-flush" spec.shutdownMs flushResult, providerCall "metrics-flush" spec.shutdownMs metricsFlushResult, providerCall "trace-shutdown" spec.shutdownMs shutdownResult, providerCall "metrics-shutdown" spec.shutdownMs metricsShutdownResult]+ findings = pipelineFindings 0.01 pipelineSnapshot calls <> [queueGrowthFinding (pipelineQueueLimit spec runtime.pipeline) queueSamples, endpointFinding (maybe [] id endpointSummaries)] <> fmap handlerFinding handlerStats <> recordedFindings+ summary = telemetrySummary spec ambient pipelineSnapshot sinkSnapshot metricsSnapshot endpointValues (fmap toJSON handlerStats) findings flushResult metricsFlushResult shutdownResult metricsShutdownResult+ _ <- tryAny (spec.report summary)+ either throwIO pure bodyResult++requiresSink :: TelemetrySpec -> Bool+requiresSink spec = spec.tracing == TracingSdkOtlp || (spec.metrics /= MetricsOff && spec.otelReader == ReaderOtlpPeriodic)++withConfiguredSink :: TelemetrySpec -> SinkFault -> (SinkHandle -> IO value) -> IO value+withConfiguredSink spec fault action = case (spec.helpers, spec.workerContext) of+ (HelperProcess _, Just context) -> do+ roleName <- either (ioError . userError . Text.unpack) pure (mkRoleName "selftest/telemetry-otlp-sink")+ withWorker context roleName "telemetry-otlp-sink" (object ["fault" .= renderSinkFault fault]) \worker -> do+ ready <- worker.receive 10_000+ endpoint <- case ready of+ Just (WrkCustom "ready" payload) -> maybe (ioError (userError "telemetry sink sent an invalid ready message")) pure (parseMaybe (withObject "sink ready" (.: "endpoint")) payload)+ Just (WrkError message) -> ioError (userError (Text.unpack message))+ _ -> ioError (userError "telemetry sink did not become ready within 10 seconds")+ let setFault updated = worker.send (CtlCustom "fault" (String (renderSinkFault updated)))+ snapshot = do+ worker.send (CtlCustom "snapshot" Null)+ response <- worker.receive 10_000+ case response of+ Just (WrkCustom "sink-stats" payload) -> maybe (ioError (userError "telemetry sink sent invalid statistics")) pure (parseMaybe parseJSON payload)+ _ -> ioError (userError "telemetry sink did not return statistics")+ action (SinkHandle endpoint setFault snapshot)+ _ -> withSink fault action++withConfiguredScraper :: TelemetrySpec -> (Maybe ScraperHandle -> IO value) -> IO value+withConfiguredScraper spec action+ | spec.metrics /= MetricsServeScraped = action Nothing+ | otherwise = case (spec.helpers, spec.workerContext) of+ (HelperProcess _, Just context) -> do+ roleName <- either (ioError . userError . Text.unpack) pure (mkRoleName "selftest/telemetry-scraper")+ withWorker context roleName "telemetry-scraper" (object ["intervalMs" .= spec.scrapeMs, "wsSubscribers" .= spec.wsSubscribers]) \worker -> do+ ready <- worker.receive 10_000+ case ready of+ Just WrkReady -> pure ()+ Just (WrkError message) -> ioError (userError (Text.unpack message))+ _ -> ioError (userError "telemetry scraper did not become ready within 10 seconds")+ let register endpoint = worker.send (CtlCustom "endpoint" (toJSON endpoint))+ finish = do+ worker.send (CtlCustom "finish" Null)+ response <- worker.receive 10_000+ case response of+ Just (WrkCustom "scrape-summary" payload) -> maybe (ioError (userError "telemetry scraper sent an invalid summary")) pure (parseMaybe parseJSON payload)+ _ -> ioError (userError "telemetry scraper did not return its summary")+ action (Just (ScraperHandle register finish))+ _ -> do+ scraper <- startScraperInProcess spec.outDir spec.scrapeMs spec.wsSubscribers+ action (Just scraper)++telemetrySummary :: TelemetrySpec -> [(Text, Text)] -> Maybe PipelineSnapshot -> Maybe SinkStats -> Maybe MetricsSnapshot -> [Value] -> [Value] -> [Finding] -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Value+telemetrySummary spec ambient pipelineSnapshot sinkSnapshot metricsSnapshot endpoints handlers findings flushResult metricsFlushResult shutdownResult metricsShutdownResult =+ object+ [ "schema" .= ("kenshou.telemetry-summary/v1" :: Text),+ "arms" .= object ["tracing" .= renderTracing spec.tracing, "metrics" .= renderMetrics spec.metrics],+ "settings" .= settingsValue spec,+ "ambientEnv" .= object [Key.fromText key .= value | (key, value) <- ambient],+ "pipeline" .= pipelineValue pipelineSnapshot flushResult shutdownResult,+ "sink" .= sinkSnapshot,+ "metrics" .= object ["snapshot" .= metricsSnapshot, "flush" .= metricsFlushResult, "shutdown" .= metricsShutdownResult],+ "endpoints" .= endpoints,+ "handlers" .= handlers,+ "findings" .= findings+ ]++providerCall :: Text -> Int -> Maybe ProviderCallReport -> Maybe (Text, Text, Double, Double)+providerCall _ _ Nothing = Nothing+providerCall name limit (Just report) = Just (name, report.result, report.durationMs, fromIntegral limit)++normalizeQueueDepth :: TelemetrySpec -> PipelineSnapshot -> PipelineSnapshot+normalizeQueueDepth spec snapshot = case spec.processor of+ BatchProcessor queue _ batch _ ->+ -- The public SDK does not expose its private queue count. The accounting+ -- backlog includes SDK-dropped spans, so cap its high-water mark at the+ -- configured queue plus the one batch that may already be exporting.+ PipelineSnapshot snapshot.spansStarted snapshot.spansEnded snapshot.spansExportedOk snapshot.spansExportFailed snapshot.spansDropped (min snapshot.maxQueueDepth (queue + batch)) snapshot.exportCalls snapshot.exportLatencyNs snapshot.lastExportError+ SimpleProcessor _ -> snapshot++pipelineQueueLimit :: TelemetrySpec -> Maybe PipelineStats -> Maybe Int+pipelineQueueLimit spec pipeline = case (pipeline, spec.processor) of+ (Just _, BatchProcessor queue _ _ _) -> Just queue+ _ -> Nothing++pipelineValue :: Maybe PipelineSnapshot -> Maybe ProviderCallReport -> Maybe ProviderCallReport -> Value+pipelineValue Nothing _ _ = Null+pipelineValue (Just snapshot) flushResult shutdownResult = case toJSON snapshot of+ Object fields -> Object (KeyMap.insert "shutdown" (toJSON shutdownResult) (KeyMap.insert "flush" (toJSON flushResult) fields))+ value -> value++settingsValue :: TelemetrySpec -> Value+settingsValue spec =+ object+ [ "sampler" .= renderSampler spec.sampler,+ "processor" .= renderProcessor spec.processor,+ "exporter" .= (case spec.exporter of HttpProtobuf -> "http-protobuf" :: Text; Grpc -> "grpc"),+ "endpoint" .= (case spec.endpoint of BuiltinSink _ -> "builtin-sink" :: Text; ExternalEndpoint value -> value),+ "sinkFault" .= (case spec.endpoint of BuiltinSink fault -> renderSinkFault fault; ExternalEndpoint _ -> "none"),+ "compression" .= (case spec.compression of CompressionNone -> "none" :: Text; CompressionGzip -> "gzip")+ ]++renderSampler :: SamplerSpec -> Text+renderSampler AlwaysOn = "always-on"+renderSampler AlwaysOff = "always-off"+renderSampler ParentBasedAlwaysOn = "parentbased-always-on"+renderSampler (TraceIdRatio ratio) = "traceidratio:" <> Text.pack (show ratio)+renderSampler (ParentBasedTraceIdRatio ratio) = "parentbased-traceidratio:" <> Text.pack (show ratio)++renderProcessor :: ProcessorSpec -> Value+renderProcessor (SimpleProcessor timeoutMs) = object ["kind" .= ("simple" :: Text), "exportTimeoutMs" .= timeoutMs]+renderProcessor (BatchProcessor queue delay batch timeoutMs) =+ object+ [ "kind" .= ("batch" :: Text),+ "maxQueue" .= queue,+ "scheduleDelayMs" .= delay,+ "maxExportBatch" .= batch,+ "exportTimeoutMs" .= timeoutMs+ ]++timedCall :: (Show result) => IO (Maybe result) -> IO (Maybe ProviderCallReport)+timedCall operation = do+ started <- getMonotonicTimeNSec+ outcome <- tryAny operation+ finished <- getMonotonicTimeNSec+ let elapsed = fromIntegral (finished - started) / 1_000_000+ pure . Just $ case outcome of+ Left exception -> ProviderCallReport ("exception: " <> Text.pack (displayException exception)) elapsed+ Right Nothing -> ProviderCallReport "not-applicable" elapsed+ Right (Just result) -> ProviderCallReport (Text.pack (show result)) elapsed++timedStopMetrics :: Int -> MetricsRuntime -> IO (Maybe MetricsSnapshot, Maybe ProviderCallReport)+timedStopMetrics timeoutMs runtime = do+ started <- getMonotonicTimeNSec+ outcome <- tryAny (stopMetrics timeoutMs runtime)+ finished <- getMonotonicTimeNSec+ let elapsed = fromIntegral (finished - started) / 1_000_000+ pure $ case outcome of+ Left exception -> (Nothing, Just (ProviderCallReport ("exception: " <> Text.pack (displayException exception)) elapsed))+ Right (snapshot, Nothing) -> (snapshot, Just (ProviderCallReport "not-applicable" elapsed))+ Right (snapshot, Just result) -> (snapshot, Just (ProviderCallReport (Text.pack (show result)) elapsed))++ambientOtelEnvironment :: IO [(Text, Text)]+ambientOtelEnvironment = do+ environment <- getEnvironment+ pure [(Text.pack key, Text.pack value) | (key, value) <- environment, "OTEL_" `Text.isPrefixOf` Text.pack key]++tryAny :: IO value -> IO (Either SomeException value)+tryAny = try++traverse_ :: (a -> IO b) -> Maybe a -> IO ()+traverse_ _ Nothing = pure ()+traverse_ action (Just value) = void (action value)
+ src/Kenshou/Telemetry/Compose.hs view
@@ -0,0 +1,96 @@+module Kenshou.Telemetry.Compose+ ( HandlerStats,+ HandlerStatsSnapshot (..),+ AsyncHandlerStats (..),+ newHandlerStats,+ snapshotHandlerStats,+ composeHandlers,+ timedHandler,+ slowHandler,+ asyncHandler,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async)+import Control.Concurrent.STM+import Control.Monad (forever)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.IORef+import Data.List (sort)+import Data.Text (Text)+import GHC.Clock (getMonotonicTimeNSec)++newtype HandlerStats = HandlerStats (IORef [Integer])++data HandlerStatsSnapshot = HandlerStatsSnapshot+ { name :: Text,+ calls :: Int,+ durationP50Ns :: Integer,+ durationP99Ns :: Integer,+ durationMaxNs :: Integer+ }+ deriving stock (Eq, Show)++data AsyncHandlerStats = AsyncHandlerStats+ { accepted :: Int,+ dropped :: Int,+ handled :: Int,+ queueDepth :: Int+ }+ deriving stock (Eq, Show)++instance ToJSON HandlerStatsSnapshot where+ toJSON value = object ["name" .= value.name, "calls" .= value.calls, "durationNs" .= object ["p50" .= value.durationP50Ns, "p99" .= value.durationP99Ns, "max" .= value.durationMaxNs]]++instance ToJSON AsyncHandlerStats where+ toJSON value = object ["accepted" .= value.accepted, "dropped" .= value.dropped, "handled" .= value.handled, "queueDepth" .= value.queueDepth]++newHandlerStats :: IO HandlerStats+newHandlerStats = HandlerStats <$> newIORef []++snapshotHandlerStats :: Text -> HandlerStats -> IO HandlerStatsSnapshot+snapshotHandlerStats name (HandlerStats ref) = do+ durations <- sort <$> readIORef ref+ pure (HandlerStatsSnapshot name (length durations) (quantile 0.50 durations) (quantile 0.99 durations) (maybe 0 last (nonEmpty durations)))++composeHandlers :: [a -> IO ()] -> a -> IO ()+composeHandlers [] = const (pure ())+composeHandlers handlers = \value -> mapM_ ($ value) handlers++timedHandler :: HandlerStats -> Text -> (a -> IO ()) -> a -> IO ()+timedHandler (HandlerStats ref) _name handler value = do+ started <- getMonotonicTimeNSec+ handler value+ ended <- getMonotonicTimeNSec+ modifyIORef' ref (fromIntegral (ended - started) :)++slowHandler :: Int -> a -> IO ()+slowHandler micros _ = threadDelay (max 0 micros)++asyncHandler :: Int -> (a -> IO ()) -> IO (a -> IO (), IO AsyncHandlerStats)+asyncHandler requestedCapacity handler = do+ let capacity = max 1 requestedCapacity+ queue <- newTBQueueIO (fromIntegral capacity)+ acceptedRef <- newIORef 0+ droppedRef <- newIORef 0+ handledRef <- newIORef 0+ _ <- async . forever $ do+ value <- atomically (readTBQueue queue)+ handler value+ modifyIORef' handledRef (+ 1)+ let submit value = do+ accepted <- atomically do+ full <- isFullTBQueue queue+ if full then pure False else writeTBQueue queue value >> pure True+ modifyIORef' (if accepted then acceptedRef else droppedRef) (+ 1)+ snapshot = AsyncHandlerStats <$> readIORef acceptedRef <*> readIORef droppedRef <*> readIORef handledRef <*> (fromIntegral <$> atomically (lengthTBQueue queue))+ pure (submit, snapshot)++quantile :: Double -> [Integer] -> Integer+quantile _ [] = 0+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))++nonEmpty :: [a] -> Maybe [a]+nonEmpty [] = Nothing+nonEmpty values = Just values
+ src/Kenshou/Telemetry/Continuity.hs view
@@ -0,0 +1,74 @@+module Kenshou.Telemetry.Continuity+ ( SpanSelector (..),+ ContinuityResult (..),+ IsolationResult (..),+ checkContinuity,+ checkIsolation,+ )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.List (find)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Telemetry.Tracing.Probe (SpanView (..))+import OpenTelemetry.Attributes (Attribute, lookupAttribute)+import OpenTelemetry.Trace.Core (SpanKind)++data SpanSelector = SpanSelector+ { nameIs :: Maybe Text,+ kindIs :: Maybe SpanKind,+ hasAttr :: [(Text, Attribute)]+ }+ deriving stock (Eq, Show)++data ContinuityResult = ContinuityResult+ { producers :: Int,+ consumers :: Int,+ matched :: Int,+ violations :: Int,+ counterExamples :: [Text]+ }+ deriving stock (Eq, Show)++data IsolationResult = IsolationResult+ { checked :: Int,+ violations :: Int,+ counterExamples :: [Text]+ }+ deriving stock (Eq, Show)++instance ToJSON ContinuityResult where+ toJSON value = object ["producers" .= value.producers, "consumers" .= value.consumers, "matched" .= value.matched, "violations" .= value.violations, "counterExamples" .= value.counterExamples]++instance ToJSON IsolationResult where+ toJSON value = object ["checked" .= value.checked, "violations" .= value.violations, "counterExamples" .= value.counterExamples]++checkContinuity :: SpanSelector -> SpanSelector -> (SpanView -> Maybe Text) -> [SpanView] -> ContinuityResult+checkContinuity producerSelector consumerSelector correlation spans =+ let producerSpans = filter (matches producerSelector) spans+ consumerSpans = filter (matches consumerSelector) spans+ outcomes = fmap (checkConsumer producerSpans) consumerSpans+ failures = take 10 [message | Left message <- outcomes]+ in ContinuityResult (length producerSpans) (length consumerSpans) (length [() | Right () <- outcomes]) (length [() | Left _ <- outcomes]) failures+ where+ checkConsumer producerSpans consumer = case correlation consumer of+ Nothing -> Left (consumer.name <> ": missing correlation key")+ Just key -> case find ((== Just key) . correlation) producerSpans of+ Nothing -> Left (consumer.name <> ": no producer for " <> key)+ Just producer+ | consumer.traceId /= producer.traceId -> Left (consumer.name <> ": trace differs for " <> key)+ | consumer.parentSpanId /= Just producer.spanId -> Left (consumer.name <> ": parent differs for " <> key)+ | otherwise -> Right ()++checkIsolation :: SpanSelector -> (SpanView -> Bool) -> [SpanView] -> IsolationResult+checkIsolation selector expectedRoot spans =+ let checkedSpans = filter (\span -> matches selector span && expectedRoot span) spans+ failures = [span.name <> ": inherited parent " <> Text.pack (show parent) | span <- checkedSpans, Just parent <- [span.parentSpanId]]+ in IsolationResult (length checkedSpans) (length failures) (take 10 failures)++matches :: SpanSelector -> SpanView -> Bool+matches selector span =+ maybe True (== span.name) selector.nameIs+ && maybe True (== span.kind) selector.kindIs+ && all (\(key, expected) -> lookupAttribute span.attributes key == Just expected) selector.hasAttr
+ src/Kenshou/Telemetry/Detect.hs view
@@ -0,0 +1,103 @@+module Kenshou.Telemetry.Detect+ ( FindingStatus (..),+ FindingSeverity (..),+ Finding (..),+ pipelineFindings,+ queueGrowthFinding,+ endpointFinding,+ handlerFinding,+ continuityFindings,+ )+where++import Data.Aeson (ToJSON (..), Value, object, (.=))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Telemetry.Compose (HandlerStatsSnapshot (..))+import Kenshou.Telemetry.Continuity (ContinuityResult (..), IsolationResult (..))+import Kenshou.Telemetry.Scrape (EndpointSummary (..))+import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..))++data FindingStatus = Fired | Quiet | NotApplicable deriving stock (Eq, Show)++data FindingSeverity = Info | Degraded | Failure deriving stock (Eq, Ord, Show)++data Finding = Finding+ { detector :: Text,+ status :: FindingStatus,+ severity :: FindingSeverity,+ evidence :: Value+ }+ deriving stock (Eq, Show)++instance ToJSON FindingStatus where+ toJSON Fired = toJSON ("fired" :: Text)+ toJSON Quiet = toJSON ("quiet" :: Text)+ toJSON NotApplicable = toJSON ("not-applicable" :: Text)++instance ToJSON FindingSeverity where+ toJSON Info = toJSON ("info" :: Text)+ toJSON Degraded = toJSON ("degraded" :: Text)+ toJSON Failure = toJSON ("failure" :: Text)++instance ToJSON Finding where+ toJSON value = object ["detector" .= value.detector, "status" .= value.status, "severity" .= value.severity, "evidence" .= value.evidence]++pipelineFindings :: Double -> Maybe PipelineSnapshot -> [(Text, Text, Double, Double)] -> [Finding]+pipelineFindings _ Nothing _ =+ [ Finding "span-drop" NotApplicable Info (object []),+ Finding "export-failure" NotApplicable Info (object []),+ Finding "shutdown-blocked" NotApplicable Info (object [])+ ]+pipelineFindings degradedFraction (Just snapshot) providerCalls =+ [ Finding "span-drop" (fired snapshot.spansDropped) dropSeverity (object ["spansEnded" .= snapshot.spansEnded, "spansDropped" .= snapshot.spansDropped, "fraction" .= dropFraction]),+ Finding "export-failure" (fired snapshot.spansExportFailed) (if snapshot.spansExportFailed > 0 then Degraded else Info) (object ["spansExportFailed" .= snapshot.spansExportFailed, "lastError" .= snapshot.lastExportError]),+ Finding "shutdown-blocked" (if null blocked then Quiet else Fired) (if null blocked then Info else Degraded) (object ["calls" .= [object ["call" .= name, "result" .= result, "durationMs" .= duration, "limitMs" .= limit] | (name, result, duration, limit) <- providerCalls], "blocked" .= fmap (\(name, _, _, _) -> name) blocked])+ ]+ where+ dropFraction = fromIntegral snapshot.spansDropped / fromIntegral (max 1 snapshot.spansEnded)+ dropSeverity = if dropFraction > degradedFraction then Degraded else Info+ blocked = filter (\(_, result, duration, limit) -> duration > limit || "timeout" `Text.isInfixOf` Text.toCaseFold result) providerCalls+ fired count = if count > 0 then Fired else Quiet++queueGrowthFinding :: Maybe Int -> [Int] -> Finding+queueGrowthFinding Nothing _ = Finding "queue-growth" NotApplicable Info (object [])+queueGrowthFinding (Just queueLimit) samples =+ Finding+ "queue-growth"+ (if longest > 10 then Fired else Quiet)+ (if longest > 10 then Degraded else Info)+ (object ["queueLimit" .= queueLimit, "threshold" .= threshold, "longestConsecutiveSamples" .= longest, "samples" .= length samples])+ where+ threshold = fromIntegral queueLimit * (0.9 :: Double)+ longest = longestRun ((> threshold) . fromIntegral) samples++longestRun :: (value -> Bool) -> [value] -> Int+longestRun predicate = snd . foldl step (0, 0)+ where+ step (current, best) value =+ let next = if predicate value then current + 1 else 0+ in (next, max best next)++endpointFinding :: [EndpointSummary] -> Finding+endpointFinding [] = Finding "endpoint-failure" NotApplicable Info (object [])+endpointFinding summaries = Finding "endpoint-failure" status severity (object ["failures" .= failures, "attempted" .= attempted, "fraction" .= fraction])+ where+ attempted = sum [summary.scrapes + summary.skippedTicks | summary <- summaries]+ failures = sum [summary.failures + summary.skippedTicks | summary <- summaries]+ fraction :: Double+ fraction = fromIntegral failures / fromIntegral (max 1 attempted)+ status = if fraction > 0.01 then Fired else Quiet+ severity = if status == Fired then Degraded else Info++handlerFinding :: HandlerStatsSnapshot -> Finding+handlerFinding snapshot = Finding "handler-stall" status severity (object ["handler" .= snapshot.name, "calls" .= snapshot.calls, "durationP99Ns" .= snapshot.durationP99Ns, "limitNs" .= (5_000_000 :: Integer)])+ where+ status = if snapshot.durationP99Ns > 5_000_000 then Fired else Quiet+ severity = if status == Fired then Degraded else Info++continuityFindings :: ContinuityResult -> IsolationResult -> [Finding]+continuityFindings continuity isolation =+ [ Finding "trace-discontinuity" (if continuity.violations > 0 then Fired else Quiet) (if continuity.violations > 0 then Failure else Info) (toJSON continuity),+ Finding "context-leak" (if isolation.violations > 0 then Fired else Quiet) (if isolation.violations > 0 then Failure else Info) (toJSON isolation)+ ]
+ src/Kenshou/Telemetry/Endpoint.hs view
@@ -0,0 +1,71 @@+module Kenshou.Telemetry.Endpoint+ ( EndpointKind (..),+ Endpoint (..),+ reserveFreePort,+ awaitHttpReady,+ )+where++import Control.Concurrent (threadDelay)+import Control.Exception (SomeException, try)+import Data.Aeson (FromJSON (..), ToJSON (..), Value, object, withObject, withText, (.:), (.:?), (.=))+import Data.Text (Text)+import Data.Text qualified as Text+import Network.HTTP.Client (defaultManagerSettings, httpLbs, newManager, parseRequest, responseStatus)+import Network.HTTP.Types.Status (statusCode)+import Network.Socket (close)+import Network.Wai.Handler.Warp (openFreePort)++data EndpointKind = PrometheusText | JsonDocument | HealthProbe | WebSocketPush+ deriving stock (Eq, Ord, Show)++data Endpoint = Endpoint+ { name :: Text,+ kind :: EndpointKind,+ url :: Text,+ wsHello :: Maybe Value+ }+ deriving stock (Eq, Show)++instance ToJSON EndpointKind where+ toJSON = toJSON . renderKind++instance FromJSON EndpointKind where+ parseJSON = withText "EndpointKind" (maybe (fail "unknown endpoint kind") pure . parseKind)++instance ToJSON Endpoint where+ toJSON endpoint = object ["name" .= endpoint.name, "kind" .= endpoint.kind, "url" .= endpoint.url, "wsHello" .= endpoint.wsHello]++instance FromJSON Endpoint where+ parseJSON = withObject "Endpoint" (\value -> Endpoint <$> value .: "name" <*> value .: "kind" <*> value .: "url" <*> value .:? "wsHello")++reserveFreePort :: IO Int+reserveFreePort = do+ (port, socket) <- openFreePort+ close socket+ pure port++awaitHttpReady :: Text -> Int -> IO Bool+awaitHttpReady url timeoutMs = do+ manager <- newManager defaultManagerSettings+ request <- parseRequest (Text.unpack url)+ let attempts = max 1 (timeoutMs `div` 50)+ loop 0 = pure False+ loop remaining = do+ response <- tryAny (httpLbs request manager)+ case response of+ Right value | statusCode (responseStatus value) < 500 -> pure True+ _ -> threadDelay 50_000 >> loop (remaining - 1)+ loop attempts++renderKind :: EndpointKind -> Text+renderKind PrometheusText = "prometheus-text"+renderKind JsonDocument = "json-document"+renderKind HealthProbe = "health-probe"+renderKind WebSocketPush = "websocket-push"++parseKind :: Text -> Maybe EndpointKind+parseKind value = lookup value [(renderKind kind, kind) | kind <- [PrometheusText, JsonDocument, HealthProbe, WebSocketPush]]++tryAny :: IO value -> IO (Either SomeException value)+tryAny = try
+ src/Kenshou/Telemetry/Metrics.hs view
@@ -0,0 +1,116 @@+module Kenshou.Telemetry.Metrics+ ( MetricsRuntime (..),+ MetricsSnapshot (..),+ startMetrics,+ flushMetrics,+ readMetricSums,+ readMetricGauges,+ stopMetrics,+ )+where++import Control.Concurrent.Async (Async, async, cancel, waitCatch)+import Control.Monad (void)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.Text qualified as Text+import Data.Vector qualified as Vector+import Kenshou.Telemetry.Endpoint+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Tracing (otlpExporterConfig)+import Network.Socket (Socket, close)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import OpenTelemetry.Exporter.Metric (GaugeDataPoint (..), MetricExport (..), NumberValue (..), ResourceMetricsExport (..), ScopeMetricsExport (..), SumDataPoint (..))+import OpenTelemetry.Exporter.OTLP.Metric qualified as OtlpMetric+import OpenTelemetry.Exporter.Prometheus.WAI (prometheusApplication)+import OpenTelemetry.MeterProvider (SdkMeterEnv, collectResourceMetrics, createMeterProvider, defaultSdkMeterProviderOptions)+import OpenTelemetry.Metric.Core (Meter, MeterProvider, forceFlushMeterProvider, getMeter, shutdownMeterProvider)+import OpenTelemetry.MetricReader (PeriodicMetricReaderHandle (..), PeriodicMetricReaderOptions (..), forkPeriodicMetricReader)+import OpenTelemetry.Resource (emptyMaterializedResources)+import OpenTelemetry.Trace.Core (FlushResult, ShutdownResult, instrumentationLibrary)++data MetricsRuntime = MetricsRuntime+ { meter :: Maybe Meter,+ provider :: Maybe MeterProvider,+ env :: Maybe SdkMeterEnv,+ endpoint :: Maybe Endpoint,+ server :: Maybe (Async (), Socket),+ reader :: Maybe PeriodicMetricReaderHandle+ }++data MetricsSnapshot = MetricsSnapshot+ { resourceBatches :: Int,+ instruments :: Int+ }+ deriving stock (Eq, Show)++instance ToJSON MetricsSnapshot where+ toJSON snapshot = object ["resourceBatches" .= snapshot.resourceBatches, "instruments" .= snapshot.instruments]++startMetrics :: TelemetrySpec -> IO MetricsRuntime+startMetrics spec = case spec.metrics of+ MetricsOff -> pure (MetricsRuntime Nothing Nothing Nothing Nothing Nothing Nothing)+ _ -> do+ (provider, env) <- createMeterProvider emptyMaterializedResources defaultSdkMeterProviderOptions+ meter <- getMeter provider (instrumentationLibrary "kenshou" "0.1.0.0")+ (endpoint, server) <- case (spec.metrics, spec.otelReader) of+ (arm, ReaderPrometheus) | arm `elem` [MetricsServe, MetricsServeScraped] -> do+ (port, socket) <- openFreePort+ running <- async (runSettingsSocket defaultSettings socket (prometheusApplication (Vector.fromList <$> collectResourceMetrics env)))+ let url = "http://127.0.0.1:" <> Text.pack (show port) <> "/metrics"+ pure (Just (Endpoint "otel-prometheus" PrometheusText url Nothing), Just (running, socket))+ _ -> pure (Nothing, Nothing)+ reader <- case spec.otelReader of+ ReaderOtlpPeriodic -> do+ exporter <- OtlpMetric.otlpMetricExporter (otlpExporterConfig spec)+ Just <$> forkPeriodicMetricReader env exporter (PeriodicMetricReaderOptions (spec.otelExportMs * 1000))+ _ -> pure Nothing+ pure (MetricsRuntime (Just meter) (Just provider) (Just env) endpoint server reader)++flushMetrics :: Int -> MetricsRuntime -> IO (Maybe FlushResult)+flushMetrics timeoutMs runtime = traverse (\provider -> forceFlushMeterProvider provider (Just (timeoutMs * 1000))) runtime.provider++readMetricSums :: MetricsRuntime -> IO [(Text.Text, Double)]+readMetricSums runtime = do+ batches <- maybe (pure []) collectResourceMetrics runtime.env+ pure+ [ (metric.mesName, sum (fmap pointValue metric.mesSumPoints))+ | resource <- batches,+ scope <- Vector.toList resource.resourceMetricsScopes,+ metric@MetricExportSum {} <- Vector.toList scope.scopeMetricsExports+ ]+ where+ pointValue point = case point.sumDataPointValue of+ IntNumber value -> fromIntegral value+ DoubleNumber value -> value++readMetricGauges :: MetricsRuntime -> IO [(Text.Text, Double)]+readMetricGauges runtime = do+ batches <- maybe (pure []) collectResourceMetrics runtime.env+ pure+ [ (metric.megName, pointValue point.gaugeDataPointValue)+ | resource <- batches,+ scope <- Vector.toList resource.resourceMetricsScopes,+ metric@MetricExportGauge {} <- Vector.toList scope.scopeMetricsExports,+ point <- Vector.toList metric.megGaugePoints+ ]+ where+ pointValue = \case+ IntNumber value -> fromIntegral value+ DoubleNumber value -> value++stopMetrics :: Int -> MetricsRuntime -> IO (Maybe MetricsSnapshot, Maybe ShutdownResult)+stopMetrics timeoutMs runtime = do+ traverse_ (.stopPeriodicMetricReader) runtime.reader+ batches <- maybe (pure []) collectResourceMetrics runtime.env+ let snapshot = case runtime.env of+ Nothing -> Nothing+ Just _ -> Just (MetricsSnapshot (length batches) (sum [Vector.length scope.scopeMetricsExports | resource <- batches, scope <- Vector.toList resource.resourceMetricsScopes]))+ shutdown <- traverse (\provider -> shutdownMeterProvider provider (Just (timeoutMs * 1000))) runtime.provider+ case runtime.server of+ Nothing -> pure ()+ Just (running, socket) -> cancel running >> void (waitCatch running) >> close socket+ pure (snapshot, shutdown)++traverse_ :: (a -> IO b) -> Maybe a -> IO ()+traverse_ _ Nothing = pure ()+traverse_ action (Just value) = void (action value)
+ src/Kenshou/Telemetry/Overhead.hs view
@@ -0,0 +1,648 @@+module Kenshou.Telemetry.Overhead+ ( Arm (..),+ Slot (..),+ OverheadMode (..),+ OverheadRequest (..),+ OverheadPlan (..),+ SlotRun (..),+ OverheadState (..),+ OverheadHooks (..),+ OverheadComparison (..),+ OverheadReport (..),+ UsageError (..),+ planOverhead,+ executeOverhead,+ analyseOverhead,+ loadOverheadState,+ overheadVerdictExitCode,+ )+where++import Control.Concurrent (threadDelay)+import Control.Exception (SomeException, try)+import Control.Monad (foldM, when)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (nub, sort, sortOn)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, fromMaybe, listToMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Core.Canonical (canonicalEncode, sha256Hex)+import Kenshou.Core.Dimension+import Kenshou.Core.Id+import Kenshou.Core.Knob+import Kenshou.Core.Manifest (verifyManifest)+import Kenshou.Core.RunSpec+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility (VaryingAxis (..))+import Kenshou.Measure.Compare.Policy (Policy (..))+import Kenshou.Measure.Stats (Interval (..))+import Kenshou.Telemetry.Overhead.Policy+import System.Directory (createDirectoryIfMissing, doesFileExist, renameFile)+import System.Exit (ExitCode (..))+import System.FilePath ((</>))+import System.Random.SplitMix (mkSMGen, nextWord64)++data Arm = Arm+ { armId :: Text,+ dimensions :: Map Text Text,+ knobs :: Map Text KnobValue,+ control :: Bool+ }+ deriving stock (Eq, Show)++data Slot = Slot+ { block :: Int,+ position :: Int,+ arm :: Arm,+ spec :: RunSpec+ }+ deriving stock (Eq, Show)++data OverheadMode = OneFactor | FullFactorial deriving stock (Eq, Show)++data OverheadRequest = OverheadRequest+ { requestId :: Text,+ scenarioId :: ScenarioId,+ factors :: Map Text [Text],+ mode :: OverheadMode,+ includeControl :: Bool,+ trials :: Int,+ fixedKnobs :: [(KnobName, RawKnob)],+ fixedDimensions :: Map Text Text,+ seed :: Word64,+ settleSeconds :: Int,+ retries :: Int+ }+ deriving stock (Eq, Show)++data OverheadPlan = OverheadPlan+ { planId :: Text,+ scenarioId :: ScenarioId,+ scenarioRevision :: Int,+ mode :: OverheadMode,+ factorValues :: Map Text [Text],+ baselineArm :: Text,+ arms :: [Arm],+ trials :: Int,+ seed :: Word64,+ settleSeconds :: Int,+ retries :: Int,+ fixedDimensions :: Map Text Text,+ fixedKnobs :: Map Text KnobValue,+ slots :: [Slot]+ }+ deriving stock (Eq, Show)++data SlotRun = SlotRun+ { block :: Int,+ position :: Int,+ armId :: Text,+ runIds :: [RunId],+ exitCode :: Maybe Int,+ complete :: Bool+ }+ deriving stock (Eq, Show)++data OverheadState = OverheadState+ { plan :: OverheadPlan,+ slots :: [SlotRun]+ }+ deriving stock (Eq, Show)++data OverheadHooks = OverheadHooks+ { runChild :: RunSpec -> FilePath -> IO ExitCode,+ compare :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison),+ leakCheck :: Maybe (FilePath -> IO (Maybe Value))+ }++data OverheadComparison = OverheadComparison+ { candidate :: Text,+ baseline :: Text,+ factor :: Text,+ fromValue :: Text,+ toValue :: Text,+ comparisonPath :: Maybe FilePath,+ verdict :: Verdict,+ degraded :: Bool,+ deltas :: Value,+ reasons :: [Text]+ }+ deriving stock (Eq, Show)++data OverheadReport = OverheadReport+ { reportId :: Text,+ scenarioId :: ScenarioId,+ mode :: OverheadMode,+ baselineDimensions :: Map Text Text,+ fixedDimensions :: Map Text Text,+ fixedKnobs :: Map Text KnobValue,+ trials :: Int,+ seed :: Word64,+ validBlocks :: Int,+ order :: [[Text]],+ armRuns :: [ArmEvidence],+ comparisons :: [OverheadComparison],+ reportFindings :: [Value],+ policyIdentity :: Value,+ verdict :: Verdict+ }+ deriving stock (Eq, Show)++data ArmEvidence = ArmEvidence+ { arm :: Arm,+ runs :: [RunId],+ telemetry :: Value,+ leak :: Maybe Value,+ findings :: [Value],+ degraded :: Bool+ }+ deriving stock (Eq, Show)++newtype UsageError = UsageError Text deriving stock (Eq, Show)++instance ToJSON Arm where+ toJSON value = object ["arm" .= value.armId, "dimensions" .= value.dimensions, "knobs" .= value.knobs, "control" .= value.control]++instance FromJSON Arm where+ parseJSON = withObject "Arm" \value -> Arm <$> value .: "arm" <*> value .: "dimensions" <*> value .: "knobs" <*> value .:? "control" .!= False++instance ToJSON OverheadMode where+ toJSON OneFactor = String "one-factor"+ toJSON FullFactorial = String "full"++instance FromJSON OverheadMode where+ parseJSON = withText "OverheadMode" \case+ "one-factor" -> pure OneFactor+ "full" -> pure FullFactorial+ _ -> fail "mode must be one-factor or full"++instance ToJSON Slot where+ toJSON value = object ["block" .= value.block, "position" .= value.position, "arm" .= value.arm, "spec" .= value.spec]++instance FromJSON Slot where+ parseJSON = withObject "Slot" \value -> Slot <$> value .: "block" <*> value .: "position" <*> value .: "arm" <*> value .: "spec"++instance ToJSON OverheadPlan where+ toJSON value =+ object+ [ "schema" .= ("kenshou.overhead-plan/v1" :: Text),+ "id" .= value.planId,+ "scenario" .= value.scenarioId,+ "scenarioRevision" .= value.scenarioRevision,+ "mode" .= value.mode,+ "factorValues" .= value.factorValues,+ "baselineArm" .= value.baselineArm,+ "arms" .= value.arms,+ "trials" .= value.trials,+ "seed" .= value.seed,+ "settleSeconds" .= value.settleSeconds,+ "retries" .= value.retries,+ "fixedDimensions" .= value.fixedDimensions,+ "fixedKnobs" .= value.fixedKnobs,+ "slots" .= value.slots+ ]++instance FromJSON OverheadPlan where+ parseJSON = withObject "OverheadPlan" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.overhead-plan/v1" :: Text) then fail "unsupported overhead plan" else pure ()+ OverheadPlan <$> value .: "id" <*> value .: "scenario" <*> value .: "scenarioRevision" <*> value .: "mode" <*> value .: "factorValues" <*> value .: "baselineArm" <*> value .: "arms" <*> value .: "trials" <*> value .: "seed" <*> value .: "settleSeconds" <*> value .: "retries" <*> value .: "fixedDimensions" <*> value .: "fixedKnobs" <*> value .: "slots"++instance ToJSON SlotRun where+ toJSON value = object ["block" .= value.block, "position" .= value.position, "arm" .= value.armId, "runIds" .= value.runIds, "exitCode" .= value.exitCode, "complete" .= value.complete]++instance FromJSON SlotRun where+ parseJSON = withObject "SlotRun" \value -> SlotRun <$> value .: "block" <*> value .: "position" <*> value .: "arm" <*> value .:? "runIds" .!= [] <*> value .:? "exitCode" <*> value .:? "complete" .!= False++instance ToJSON OverheadState where+ toJSON value = object ["schema" .= ("kenshou.overhead-state/v1" :: Text), "plan" .= value.plan, "slots" .= value.slots]++instance FromJSON OverheadState where+ parseJSON = withObject "OverheadState" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.overhead-state/v1" :: Text) then fail "unsupported overhead state" else pure ()+ OverheadState <$> value .: "plan" <*> value .:? "slots" .!= []++instance ToJSON OverheadComparison where+ toJSON value =+ object+ [ "candidate" .= value.candidate,+ "baseline" .= value.baseline,+ "factor" .= value.factor,+ "from" .= value.fromValue,+ "to" .= value.toValue,+ "comparison" .= value.comparisonPath,+ "verdict" .= value.verdict,+ "degraded" .= value.degraded,+ "deltas" .= value.deltas,+ "reasons" .= value.reasons+ ]++instance ToJSON OverheadReport where+ toJSON value =+ object+ [ "schema" .= ("kenshou.overhead-report/v1" :: Text),+ "id" .= value.reportId,+ "scenario" .= value.scenarioId,+ "mode" .= value.mode,+ "baseline" .= value.baselineDimensions,+ "fixed" .= object ["knobs" .= value.fixedKnobs, "dimensions" .= value.fixedDimensions],+ "trials" .= value.trials,+ "seed" .= value.seed,+ "validBlocks" .= value.validBlocks,+ "order" .= value.order,+ "arms" .= [object ["arm" .= row.arm.armId, "dimensions" .= row.arm.dimensions, "runs" .= row.runs, "telemetry" .= row.telemetry, "leak" .= row.leak, "findings" .= row.findings] | row <- value.armRuns],+ "comparisons" .= value.comparisons,+ "findings" .= value.reportFindings,+ "policy" .= value.policyIdentity,+ "cohort" .= object [],+ "verdict" .= value.verdict,+ "algorithm" .= object ["name" .= ("kenshou-overhead" :: Text), "version" .= (1 :: Int)]+ ]++planOverhead :: OverheadRequest -> Scenario -> Either UsageError OverheadPlan+planOverhead request scenario = do+ whenEither (request.scenarioId /= scenario.id) "scenario request does not match the selected scenario"+ whenEither (request.trials < 3) "trials must be at least 3"+ whenEither (request.settleSeconds < 0) "settle-seconds must be non-negative"+ whenEither (request.retries < 0) "retries must be non-negative"+ whenEither (Map.null request.factors) "at least one --arms factor is required"+ traverse_ validateFactor (Map.toList request.factors)+ whenEither (any (`Map.member` request.fixedDimensions) (Map.keys request.factors)) "a varied factor cannot also be fixed with --dim"+ resolvedKnobs <- firstUsage (resolveKnobs scenario.knobs request.fixedKnobs)+ let knobMap = Map.fromList [(renderKnobName name, value) | (name, value) <- Map.toList (resolvedKnobsMap resolvedKnobs)]+ baselineCell = Map.map (fromMaybe "" . listToMaybe) request.factors+ cells = case request.mode of+ OneFactor -> baselineCell : [Map.insert factor armValue baselineCell | (factor, values) <- Map.toAscList request.factors, armValue <- drop 1 values]+ FullFactorial -> cartesian (Map.toAscList request.factors)+ plannedArms <- traverse (makeArm scenario knobMap) (zip [0 :: Int ..] (nub cells))+ baselinePlanned <- maybe (Left (UsageError "overhead planning produced no baseline arm")) Right (listToMaybe plannedArms)+ let withControl = plannedArms <> [Arm "control" baselinePlanned.dimensions knobMap True | request.includeControl]+ shuffled = shuffle request.seed withControl+ template =+ OverheadPlan+ request.requestId+ scenario.id+ scenario.revision+ request.mode+ request.factors+ baselinePlanned.armId+ withControl+ request.trials+ request.seed+ request.settleSeconds+ request.retries+ request.fixedDimensions+ knobMap+ []+ initialSlots = concatMap (blockSlots template shuffled) [0 .. request.trials - 1]+ pure (setPlanSlots initialSlots template)+ where+ validateFactor (factor, values) = do+ whenEither (factor `notElem` ["telemetry.tracing", "telemetry.metrics"]) ("unknown overhead factor " <> factor)+ whenEither (null values) (factor <> " has no values")+ whenEither (length values /= length (nub values)) (factor <> " repeats a value")+ makeArm selected knobMap (index, cell) = do+ let assignments = Map.toList (request.fixedDimensions <> cell)+ dimensions <- firstUsage (resolveDimensions selected.dimensions assignments)+ let armName = "a" <> Text.pack (show index)+ pure (Arm armName (Map.fromList (renderDimensions dimensions)) knobMap False)++blockSlots :: OverheadPlan -> [Arm] -> Int -> [Slot]+blockSlots plan shuffled blockNumber = zipWith make [0 ..] ordered+ where+ count = length shuffled+ rotated = rotate ((blockNumber `div` 2) `mod` max 1 count) shuffled+ ordered = if even blockNumber then rotated else reverse rotated+ make position arm =+ let rawKnobs = [(name, RawJson (toJSON value)) | (nameText, value) <- Map.toList arm.knobs, Right name <- [mkKnobName nameText]]+ blockSeed = (plan.seed + fromIntegral blockNumber) `mod` 9007199254740992+ seedValue = either (const Nothing) Just (mkSeed blockSeed)+ base :: RunSpec+ base = minimalRunSpec plan.scenarioId+ runSpec =+ RunSpec+ base.runId+ base.scenario+ (Just plan.scenarioRevision)+ rawKnobs+ (Map.toList arm.dimensions)+ seedValue+ base.phases+ base.timeoutSeconds+ base.environment+ base.cohortExpectation+ (Just (ComparisonMembership plan.planId arm.armId blockNumber (blockNumber * count + position)))+ (Map.fromList [("overhead", plan.planId), ("overhead-arm", arm.armId)])+ in Slot blockNumber position arm runSpec++executeOverhead :: OverheadHooks -> OverheadPlan -> FilePath -> IO OverheadState+executeOverhead hooks plan output = do+ createDirectoryIfMissing True (output </> "runs")+ createDirectoryIfMissing True (output </> "comparisons")+ loaded <- loadOverheadState output+ initial <- case loaded of+ Right state | state.plan.planId == plan.planId -> pure state+ Right _ -> fail "state.json belongs to a different overhead plan"+ Left _ -> pure (OverheadState plan [])+ writeState output initial+ runBlocks initial 0+ where+ shuffled = shuffle plan.seed plan.arms+ maximumBlocks = plan.trials * 2+ runBlocks state blockNumber+ | validBlockCount state >= plan.trials = pure state+ | blockNumber >= maximumBlocks = pure state+ | otherwise = do+ let slotsForBlock = blockSlots plan shuffled blockNumber+ stateWithSlots <- ensureSlots output state slotsForBlock+ finished <- foldM (runSlot hooks output plan) stateWithSlots slotsForBlock+ runBlocks finished (blockNumber + 1)++analyseOverhead :: OverheadHooks -> OverheadPolicy -> OverheadPlan -> OverheadState -> FilePath -> IO OverheadReport+analyseOverhead hooks policy plan state output = do+ createDirectoryIfMissing True (output </> "comparisons")+ baseline <- maybe (fail "overhead plan has no baseline arm") pure (findArm plan.baselineArm plan.arms)+ let valid = validBlockNumbers state+ candidates = filter ((/= plan.baselineArm) . (.armId)) plan.arms+ armRows <- traverse (armRow valid) plan.arms+ let degradedArms = Map.fromList [(row.arm.armId, row.degraded) | row <- armRows]+ comparisons <- traverse (compareArm degradedArms valid baseline) candidates+ let overall = if length valid < 3 then VerdictInfrastructureFailure else worstVerdict (fmap (.verdict) comparisons)+ report =+ OverheadReport+ plan.planId+ plan.scenarioId+ plan.mode+ baseline.dimensions+ plan.fixedDimensions+ plan.fixedKnobs+ plan.trials+ plan.seed+ (length valid)+ [[slot.armId | slot <- sortOn (.position) state.slots, slot.block == blockNumber] | blockNumber <- sort valid]+ armRows+ comparisons+ (concatMap (.findings) armRows)+ (object ["name" .= policy.defaultPolicy.name, "sha256" .= sha256Hex (canonicalEncode (toJSON policy))])+ overall+ LazyByteString.writeFile (output </> "overhead-report.json") (encode report)+ pure report+ where+ compareArm degradedArms valid baseline candidate+ | length valid < 3 = pure (failedComparison candidate baseline VerdictInfrastructureFailure ["fewer than three valid blocks"])+ | otherwise = do+ let baselineDirs = mapMaybeRunDirs output state valid baseline.armId+ candidateDirs = mapMaybeRunDirs output state valid candidate.armId+ changes = [(name, from, to) | (name, from) <- Map.toList baseline.dimensions, Just to <- [Map.lookup name candidate.dimensions], from /= to]+ transition = case changes of [change] -> Just change; _ -> Nothing+ axes =+ if candidate.control+ then VaryControl :| []+ else case changes of+ [] -> VaryControl :| []+ (name, _, _) : rest -> VaryDimension name :| [VaryDimension other | (other, _, _) <- rest]+ selectedPolicy = if candidate.control then policy.controlPolicy else policyForTransition policy transition+ compared <- hooks.compare selectedPolicy axes baselineDirs candidateDirs+ case compared of+ Left err -> pure (failedComparison candidate baseline VerdictInfrastructureFailure [Text.pack (show err)])+ Right comparison -> do+ let relative = "comparisons" </> Text.unpack candidate.armId <> "-vs-" <> Text.unpack baseline.armId <> ".json"+ candidateDegraded = Map.findWithDefault False candidate.armId degradedArms+ reportedVerdict+ | candidateDegraded && comparison.verdict `elem` [VerdictPass, VerdictRegression] = VerdictInconclusive+ | candidate.control && comparison.verdict /= VerdictPass = VerdictInconclusive+ | otherwise = comparison.verdict+ LazyByteString.writeFile (output </> relative) (encode comparison)+ pure+ OverheadComparison+ { candidate = candidate.armId,+ baseline = baseline.armId,+ factor = maybe (if candidate.control then "control" else "multiple") (\(name, _, _) -> name) transition,+ fromValue = maybe "baseline" (\(_, value, _) -> value) transition,+ toValue = maybe (if candidate.control then "control" else "candidate") (\(_, _, value) -> value) transition,+ comparisonPath = Just relative,+ verdict = reportedVerdict,+ degraded = candidateDegraded,+ deltas = metricDeltas comparison,+ reasons = comparison.reasons+ }+ armRow valid arm = do+ let runIds = mapMaybeRunIds state valid arm.armId+ directories = fmap (\runId -> output </> "runs" </> Text.unpack (renderRunId runId)) runIds+ leak <- case (hooks.leakCheck, reverse directories) of (Just check, directory : _) -> check directory; _ -> pure Nothing+ telemetryValues <- catMaybes <$> traverse readTelemetry directories+ let findings = concatMap findingsFromTelemetry telemetryValues+ pure (ArmEvidence arm runIds (aggregateTelemetry telemetryValues) leak findings (any degradedFinding findings))++loadOverheadState :: FilePath -> IO (Either String OverheadState)+loadOverheadState output = do+ let path = output </> "state.json"+ exists <- doesFileExist path+ if exists then eitherDecodeFileStrict' path else pure (Left "state.json does not exist")++overheadVerdictExitCode :: Verdict -> Int+overheadVerdictExitCode = verdictExitCode++ensureSlots :: FilePath -> OverheadState -> [Slot] -> IO OverheadState+ensureSlots output state wanted = do+ let existingKeys = [(slot.block, slot.position) | slot <- state.slots]+ additions = [SlotRun slot.block slot.position slot.arm.armId [] Nothing False | slot <- wanted, (slot.block, slot.position) `notElem` existingKeys]+ updated = setStateSlots (state.slots <> additions) state+ writeState output updated+ pure updated++runSlot :: OverheadHooks -> FilePath -> OverheadPlan -> OverheadState -> Slot -> IO OverheadState+runSlot hooks output plan state slot = case findSlotRun slot state.slots of+ Just recorded | recorded.complete -> do+ valid <- completedRunIsValid output recorded+ if valid then pure state else attempt state 0+ _ -> attempt state 0+ where+ attempt current retryNumber = do+ runId <- newRunId+ let spec = setRunId runId slot.spec+ prepared = updateSlot slot (\old -> old {runIds = old.runIds <> [runId], exitCode = Nothing, complete = False}) current+ writeState output prepared+ result <- try (hooks.runChild spec (output </> "runs")) :: IO (Either SomeException ExitCode)+ let code = either (const 4) exitNumber result+ runDirectory = output </> "runs" </> Text.unpack (renderRunId runId)+ complete <- if code == 0 then either (const False) (const True) <$> verifyManifest runDirectory else pure False+ let finished = updateSlot slot (\old -> old {exitCode = Just code, complete}) prepared+ writeState output finished+ when (plan.settleSeconds > 0) (threadDelay (plan.settleSeconds * 1_000_000))+ if code == 4 && retryNumber < plan.retries then attempt finished (retryNumber + 1) else pure finished++writeState :: FilePath -> OverheadState -> IO ()+writeState output state = do+ let temporary = output </> "state.json.tmp"+ LazyByteString.writeFile temporary (encode state)+ renameFile temporary (output </> "state.json")++validBlockCount :: OverheadState -> Int+validBlockCount = length . validBlockNumbers++validBlockNumbers :: OverheadState -> [Int]+validBlockNumbers state =+ [ blockNumber+ | blockNumber <- nub (fmap (.block) state.slots),+ let blockRuns = filter ((== blockNumber) . (.block)) state.slots,+ length blockRuns == length state.plan.arms,+ all (.complete) blockRuns+ ]++completedRunIsValid :: FilePath -> SlotRun -> IO Bool+completedRunIsValid output recorded = case reverse recorded.runIds of+ [] -> pure False+ runId : _ -> either (const False) (const True) <$> verifyManifest (output </> "runs" </> Text.unpack (renderRunId runId))++findSlotRun :: Slot -> [SlotRun] -> Maybe SlotRun+findSlotRun slot = listToMaybe . filter (\recorded -> recorded.block == slot.block && recorded.position == slot.position)++updateSlot :: Slot -> (SlotRun -> SlotRun) -> OverheadState -> OverheadState+updateSlot slot change state = setStateSlots (fmap update state.slots) state+ where+ update current | current.block == slot.block && current.position == slot.position = change current+ update current = current++mapMaybeRunIds :: OverheadState -> [Int] -> Text -> [RunId]+mapMaybeRunIds state blocks armName =+ catMaybes+ [ listToMaybe (reverse slot.runIds)+ | blockNumber <- sort blocks,+ slot <- state.slots,+ slot.block == blockNumber,+ slot.armId == armName,+ slot.complete+ ]++mapMaybeRunDirs :: FilePath -> OverheadState -> [Int] -> Text -> [FilePath]+mapMaybeRunDirs output state blocks armName = fmap (\runId -> output </> "runs" </> Text.unpack (renderRunId runId)) (mapMaybeRunIds state blocks armName)++readTelemetry :: FilePath -> IO (Maybe Value)+readTelemetry directory = do+ decoded <- eitherDecodeFileStrict' (directory </> "run-result.json") :: IO (Either String Value)+ pure (either (const Nothing) (valueAt ["summaries", "telemetry", "telemetry"]) decoded)++aggregateTelemetry :: [Value] -> Value+aggregateTelemetry [] = Null+aggregateTelemetry values =+ object+ [ "runs" .= length values,+ "spansEnded" .= sumNumbers ["pipeline", "spansEnded"],+ "spansDropped" .= sumNumbers ["pipeline", "spansDropped"],+ "exportFailures" .= sumNumbers ["pipeline", "spansExportFailed"],+ "shutdownMsMax" .= maximumOrZero (mapMaybeNumber ["pipeline", "shutdown", "durationMs"] values),+ "sinkSpansReceived" .= sumNumbers ["sink", "spansReceived"]+ ]+ where+ sumNumbers path = sum (mapMaybeNumber path values)++findingsFromTelemetry :: Value -> [Value]+findingsFromTelemetry value = case valueAt ["findings"] value of Just (Array findings) -> toList findings; _ -> []++degradedFinding :: Value -> Bool+degradedFinding value = textAt ["status"] value == Just "fired" && textAt ["severity"] value == Just "degraded"++valueAt :: [Text] -> Value -> Maybe Value+valueAt [] value = Just value+valueAt (key : rest) (Object values) = KeyMap.lookup (Key.fromText key) values >>= valueAt rest+valueAt _ _ = Nothing++textAt :: [Text] -> Value -> Maybe Text+textAt path value = case valueAt path value of Just (String textValue) -> Just textValue; _ -> Nothing++mapMaybeNumber :: [Text] -> [Value] -> [Double]+mapMaybeNumber path = foldr (\value numbers -> maybe numbers (: numbers) (numberAt path value)) []++numberAt :: [Text] -> Value -> Maybe Double+numberAt path value = case valueAt path value of Just (Number number) -> Just (realToFrac number); _ -> Nothing++maximumOrZero :: [Double] -> Double+maximumOrZero [] = 0+maximumOrZero values = maximum values++toList :: (Foldable collection) => collection value -> [value]+toList = foldr (:) []++metricDeltas :: Comparison -> Value+metricDeltas comparison =+ toJSON+ ( Map.map+ (\metric -> object ["relative" .= (metric.ratio.estimate - 1), "ci95" .= [metric.ratio.low - 1, metric.ratio.high - 1], "absolute" .= metric.delta.estimate, "unit" .= metric.unit])+ comparison.metrics+ )++setPlanSlots :: [Slot] -> OverheadPlan -> OverheadPlan+setPlanSlots value plan = OverheadPlan plan.planId plan.scenarioId plan.scenarioRevision plan.mode plan.factorValues plan.baselineArm plan.arms plan.trials plan.seed plan.settleSeconds plan.retries plan.fixedDimensions plan.fixedKnobs value++setStateSlots :: [SlotRun] -> OverheadState -> OverheadState+setStateSlots value state = OverheadState state.plan value++setRunId :: RunId -> RunSpec -> RunSpec+setRunId value spec =+ RunSpec+ (Just value)+ spec.scenario+ spec.scenarioRevision+ spec.knobs+ spec.dimensions+ spec.seed+ spec.phases+ spec.timeoutSeconds+ spec.environment+ spec.cohortExpectation+ spec.comparison+ spec.labels++failedComparison :: Arm -> Arm -> Verdict -> [Text] -> OverheadComparison+failedComparison candidate baseline verdict reasons =+ OverheadComparison candidate.armId baseline.armId (if candidate.control then "control" else "unknown") "baseline" "candidate" Nothing verdict False (object []) reasons++worstVerdict :: [Verdict] -> Verdict+worstVerdict [] = VerdictInfrastructureFailure+worstVerdict values = maximumByRank values+ where+ rank VerdictPass = 0 :: Int+ rank VerdictInconclusive = 1+ rank VerdictRegression = 2+ rank VerdictInfrastructureFailure = 3+ maximumByRank (first : rest) = foldl (\current candidate -> if rank candidate > rank current then candidate else current) first rest+ maximumByRank [] = VerdictInfrastructureFailure++findArm :: Text -> [Arm] -> Maybe Arm+findArm name = listToMaybe . filter ((== name) . (.armId))++shuffle :: Word64 -> [value] -> [value]+shuffle seed values = fmap snd (sortOn fst (zip (randomWords (mkSMGen seed)) values))+ where+ randomWords generator = let (word, next) = nextWord64 generator in word : randomWords next++rotate :: Int -> [value] -> [value]+rotate _ [] = []+rotate count values = drop offset values <> take offset values where offset = count `mod` length values++cartesian :: [(Text, [Text])] -> [Map Text Text]+cartesian [] = [Map.empty]+cartesian ((name, values) : rest) = [Map.insert name value suffix | value <- values, suffix <- cartesian rest]++whenEither :: Bool -> Text -> Either UsageError ()+whenEither condition message = if condition then Left (UsageError message) else Right ()++firstUsage :: (Show error) => Either error value -> Either UsageError value+firstUsage = either (Left . UsageError . Text.pack . show) Right++exitNumber :: ExitCode -> Int+exitNumber ExitSuccess = 0+exitNumber (ExitFailure code) = code++traverse_ :: (a -> Either error b) -> [a] -> Either error ()+traverse_ action = foldM (\() value -> action value >> pure ()) ()
+ src/Kenshou/Telemetry/Overhead/Policy.hs view
@@ -0,0 +1,50 @@+module Kenshou.Telemetry.Overhead.Policy+ ( OverheadPolicy (..),+ decodeOverheadPolicy,+ policyForTransition,+ )+where++import Data.Aeson+import Data.ByteString (ByteString)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Measure.Compare.Policy (MetricRule (..), Policy (..))++data OverheadPolicy = OverheadPolicy+ { maxDegradedSpanFraction :: Double,+ defaultPolicy :: Policy,+ controlPolicy :: Policy,+ transitions :: Map Text Policy+ }+ deriving stock (Eq, Show)++instance FromJSON OverheadPolicy where+ parseJSON = withObject "OverheadPolicy" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.overhead-policy/v1" :: Text)+ then fail "unsupported overhead policy"+ else OverheadPolicy <$> value .: "maxDegradedSpanFraction" <*> value .: "default" <*> value .: "control" <*> value .:? "transitions" .!= Map.empty++instance ToJSON OverheadPolicy where+ toJSON value =+ object+ [ "schema" .= ("kenshou.overhead-policy/v1" :: Text),+ "maxDegradedSpanFraction" .= value.maxDegradedSpanFraction,+ "default" .= value.defaultPolicy,+ "control" .= value.controlPolicy,+ "transitions" .= value.transitions+ ]++decodeOverheadPolicy :: ByteString -> Either String OverheadPolicy+decodeOverheadPolicy = eitherDecodeStrict'++policyForTransition :: OverheadPolicy -> Maybe (Text, Text, Text) -> Policy+policyForTransition policy transition = case transition of+ Nothing -> policy.defaultPolicy+ Just (factor, fromValue, toValue) -> case Map.lookup (factor <> ":" <> fromValue <> "->" <> toValue) policy.transitions of+ Nothing -> policy.defaultPolicy+ Just selected ->+ let selectedMatches = fmap (.match) selected.metricRules+ in selected {metricRules = selected.metricRules <> filter ((`notElem` selectedMatches) . (.match)) policy.defaultPolicy.metricRules}
+ src/Kenshou/Telemetry/Scrape.hs view
@@ -0,0 +1,301 @@+module Kenshou.Telemetry.Scrape+ ( EndpointSummary (..),+ ScraperHandle (..),+ startScraperInProcess,+ runScraperInProcess,+ runScraperRole,+ SlotLeakResult (..),+ wsSlotLeakProbe,+ )+where++import Control.Concurrent.Async (Async, async, cancel, mapConcurrently_, waitCatch)+import Control.Concurrent.MVar (MVar, newMVar, withMVar)+import Control.Exception (SomeAsyncException, SomeException, displayException, finally, fromException, throwIO, try)+import Control.Monad (forM_, replicateM_, void)+import Data.Aeson (FromJSON (..), ToJSON (..), encode, object, withObject, (.:), (.=))+import Data.Aeson.Types (parseMaybe, (.!=), (.:?))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Foldable (traverse_)+import Data.IORef+import Data.List (sort)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import GHC.Clock (getMonotonicTimeNSec)+import Kenshou.Core.Role (ControlMessage (..), RoleContext (..), WorkerInit (..), WorkerMessage (..))+import Kenshou.Measure.Clock (Origin (..), captureOrigin, sleepUntilNs)+import Kenshou.Telemetry.Endpoint+import Network.HTTP.Client (Manager, defaultManagerSettings, httpLbs, newManager, parseRequest, responseBody, responseStatus)+import Network.HTTP.Types.Status (statusCode)+import Network.WebSockets qualified as WebSockets+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO (BufferMode (LineBuffering), Handle, IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)+import Text.Read (readMaybe)++data ScrapeSample = ScrapeSample+ { status :: Int,+ latencyNs :: Integer,+ bodyBytes :: Integer,+ skipped :: Int,+ errorMessage :: Maybe Text+ }++data EndpointSummary = EndpointSummary+ { endpoint :: Text,+ kind :: EndpointKind,+ scrapes :: Int,+ failures :: Int,+ skippedTicks :: Int,+ latencyP50Ns :: Integer,+ latencyP99Ns :: Integer,+ latencyMaxNs :: Integer,+ meanBodyBytes :: Double,+ maxBodyBytes :: Integer+ }+ deriving stock (Eq, Show)++instance ToJSON EndpointSummary where+ toJSON summary =+ object+ [ "endpoint" .= summary.endpoint,+ "kind" .= summary.kind,+ "scrapes" .= summary.scrapes,+ "failures" .= summary.failures,+ "skippedTicks" .= summary.skippedTicks,+ "latencyNs" .= object ["p50" .= summary.latencyP50Ns, "p99" .= summary.latencyP99Ns, "max" .= summary.latencyMaxNs],+ "bodyBytes" .= object ["mean" .= summary.meanBodyBytes, "max" .= summary.maxBodyBytes]+ ]++instance FromJSON EndpointSummary where+ parseJSON = withObject "EndpointSummary" \value -> do+ latency <- value .: "latencyNs"+ body <- value .: "bodyBytes"+ EndpointSummary+ <$> value .: "endpoint"+ <*> value .: "kind"+ <*> value .: "scrapes"+ <*> value .: "failures"+ <*> value .: "skippedTicks"+ <*> latency .: "p50"+ <*> latency .: "p99"+ <*> latency .: "max"+ <*> body .: "mean"+ <*> body .: "max"++data ScraperHandle = ScraperHandle+ { register :: Endpoint -> IO (),+ finish :: IO [EndpointSummary]+ }++data ScraperState = ScraperState+ { manager :: Manager,+ outputDir :: FilePath,+ intervalNs :: Integer,+ wsSubscribers :: Int,+ workers :: IORef [(Endpoint, Async ())],+ endpoints :: IORef [Endpoint],+ samples :: IORef (Map Text [ScrapeSample])+ }++startScraperInProcess :: FilePath -> Int -> Int -> IO ScraperHandle+startScraperInProcess outputDir intervalMs wsSubscribers = do+ createDirectoryIfMissing True (outputDir </> "series")+ manager <- newManager defaultManagerSettings+ workers <- newIORef []+ endpoints <- newIORef []+ samples <- newIORef Map.empty+ let state = ScraperState manager outputDir (fromIntegral intervalMs * 1_000_000) wsSubscribers workers endpoints samples+ pure (ScraperHandle (registerEndpoint state) (finishScraper state))++runScraperInProcess :: FilePath -> Int -> Int -> [Endpoint] -> IO (IO [EndpointSummary])+runScraperInProcess outputDir intervalMs wsSubscribers endpoints = do+ scraper <- startScraperInProcess outputDir intervalMs wsSubscribers+ traverse_ scraper.register endpoints+ pure scraper.finish++runScraperRole :: RoleContext -> IO ()+runScraperRole context = do+ let (intervalMs, subscribers) = maybe (15_000, 0) id (parseMaybe parseConfig context.init.args)+ scraper <- startScraperInProcess context.init.outDir intervalMs subscribers+ context.send WrkReady+ loop scraper+ where+ loop scraper =+ context.receive >>= \case+ Just (CtlCustom "endpoint" payload) -> case parseMaybe parseJSON payload of+ Nothing -> context.send (WrkError "invalid scraper endpoint") >> loop scraper+ Just endpoint -> scraper.register endpoint >> loop scraper+ Just (CtlCustom "finish" _) -> scraper.finish >>= context.send . WrkCustom "scrape-summary" . toJSON >> loop scraper+ Just (CtlStop _) -> void scraper.finish+ Just _ -> loop scraper+ Nothing -> void scraper.finish+ parseConfig = withObject "scraper config" (\value -> (,) <$> value .: "intervalMs" <*> value .:? "wsSubscribers" .!= 0)++registerEndpoint :: ScraperState -> Endpoint -> IO ()+registerEndpoint state endpoint = do+ modifyIORef' state.endpoints (<> [endpoint])+ if endpoint.kind == WebSocketPush+ then do+ modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [])+ if state.wsSubscribers <= 0+ then pure ()+ else do+ worker <- async (websocketSubscribers state endpoint)+ modifyIORef' state.workers (<> [(endpoint, worker)])+ else do+ worker <- async (scrapeLoop state endpoint)+ modifyIORef' state.workers (<> [(endpoint, worker)])++scrapeLoop :: ScraperState -> Endpoint -> IO ()+scrapeLoop state endpoint = do+ request <- parseRequest (Text.unpack endpoint.url)+ handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> ".csv")) WriteMode+ hSetBuffering handle LineBuffering+ hPutStrLn handle "t_wall_ns,endpoint,kind,status,latency_ns,body_bytes,skipped,error"+ started <- getMonotonicTimeNSec+ let loop tick = do+ sleepUntilNs (started + fromIntegral tick * fromIntegral state.intervalNs)+ origin <- captureOrigin+ callStarted <- getMonotonicTimeNSec+ result <- trySync (httpLbs request state.manager)+ callEnded <- getMonotonicTimeNSec+ let elapsed = fromIntegral (callEnded - callStarted)+ behind = max 0 (fromIntegral (callEnded - started) `div` state.intervalNs - tick)+ sample = case result of+ Left exception -> ScrapeSample 0 elapsed 0 (fromIntegral behind) (Just (Text.pack (displayException exception)))+ Right response -> ScrapeSample (statusCode (responseStatus response)) elapsed (fromIntegral (LazyByteString.length (responseBody response))) (fromIntegral behind) Nothing+ modifyIORef' state.samples (Map.insertWith (<>) endpoint.name [sample])+ hPutStrLn handle (csvRow origin endpoint sample)+ loop (tick + 1 + behind)+ loop 1 `finally` hClose handle++websocketSubscribers :: ScraperState -> Endpoint -> IO ()+websocketSubscribers state endpoint = do+ (host, port, path) <- either (ioError . userError . Text.unpack) pure (parseWebSocketUrl endpoint.url)+ handle <- openFile (state.outputDir </> "series" </> ("scrape-" <> Text.unpack endpoint.name <> "-ws.csv")) WriteMode+ hSetBuffering handle LineBuffering+ hPutStrLn handle "t_wall_ns,subscriber,event,frames,bytes,gap_ns,error"+ lock <- newMVar ()+ mapConcurrently_ (subscriber host port path handle lock) [1 .. state.wsSubscribers] `finally` hClose handle+ where+ subscriber host port path handle lock subscriberId = do+ outcome <- trySync $ WebSockets.runClient (Text.unpack host) port (Text.unpack path) \connection -> do+ traverse_ (WebSockets.sendTextData connection . encode) endpoint.wsHello+ writeWsRow handle lock subscriberId "connect" 0 0 0 Nothing+ previous <- newIORef Nothing+ let loop frames = do+ payload <- WebSockets.receiveData connection :: IO LazyByteString.ByteString+ now <- getMonotonicTimeNSec+ prior <- atomicModifyIORef' previous (\value -> (Just now, value))+ writeWsRow handle lock subscriberId "frame" frames (fromIntegral (LazyByteString.length payload)) (maybe 0 (fromIntegral . (now -)) prior) Nothing+ loop (frames + 1)+ loop 1+ case outcome of+ Left exception -> writeWsRow handle lock subscriberId "error" 0 0 0 (Just (Text.pack (displayException exception)))+ Right () -> writeWsRow handle lock subscriberId "close" 0 0 0 Nothing++writeWsRow :: Handle -> MVar () -> Int -> Text -> Int -> Integer -> Integer -> Maybe Text -> IO ()+writeWsRow handle lock subscriberId event frames bytes gap errorMessage = withMVar lock \_ -> do+ origin <- captureOrigin+ hPutStrLn handle . Text.unpack . Text.intercalate "," $+ [ Text.pack (show origin.wallUnixNs),+ Text.pack (show subscriberId),+ event,+ Text.pack (show frames),+ Text.pack (show bytes),+ Text.pack (show gap),+ maybe "" (Text.replace "," ";" . Text.replace "\n" " ") errorMessage+ ]++finishScraper :: ScraperState -> IO [EndpointSummary]+finishScraper state = do+ workers <- readIORef state.workers+ forM_ workers (cancel . snd)+ forM_ workers (void . waitCatch . snd)+ captured <- readIORef state.samples+ endpoints <- readIORef state.endpoints+ pure [summarize endpoint (Map.findWithDefault [] endpoint.name captured) | endpoint <- endpoints]++summarize :: Endpoint -> [ScrapeSample] -> EndpointSummary+summarize endpoint samples =+ EndpointSummary+ { endpoint = endpoint.name,+ kind = endpoint.kind,+ scrapes = length samples,+ failures = length [() | sample <- samples, sample.status < 200 || sample.status >= 300 || sample.errorMessage /= Nothing],+ skippedTicks = sum (fmap (.skipped) samples),+ latencyP50Ns = quantile 0.50 latencies,+ latencyP99Ns = quantile 0.99 latencies,+ latencyMaxNs = maybe 0 last nonEmptyLatencies,+ meanBodyBytes = if null samples then 0 else fromIntegral (sum (fmap (.bodyBytes) samples)) / fromIntegral (length samples),+ maxBodyBytes = maybe 0 last nonEmptyBodies+ }+ where+ latencies = sort (fmap (.latencyNs) samples)+ bodies = sort (fmap (.bodyBytes) samples)+ nonEmptyLatencies = if null latencies then Nothing else Just latencies+ nonEmptyBodies = if null bodies then Nothing else Just bodies++quantile :: Double -> [Integer] -> Integer+quantile _ [] = 0+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))++data SlotLeakResult = SlotLeakResult+ { attempted :: Int,+ freshAccepted :: Bool+ }+ deriving stock (Eq, Show)++instance ToJSON SlotLeakResult where+ toJSON result = object ["attempted" .= result.attempted, "freshAccepted" .= result.freshAccepted]++wsSlotLeakProbe :: Endpoint -> Int -> IO SlotLeakResult+wsSlotLeakProbe endpoint attempts = case parseWebSocketUrl endpoint.url of+ Left _ -> pure (SlotLeakResult attempts False)+ Right (host, port, path) -> do+ replicateM_ attempts (void (trySync (connect host port path)))+ fresh <- trySync (connect host port path)+ pure (SlotLeakResult attempts (either (const False) (const True) fresh))+ where+ connect host port path = WebSockets.runClient (Text.unpack host) port (Text.unpack path) (const (pure ()))++parseWebSocketUrl :: Text -> Either Text (Text, Int, Text)+parseWebSocketUrl url = do+ remainder <- maybe (Left "WebSocket URL must start with ws://") Right (Text.stripPrefix "ws://" url)+ let (authority, rawPath) = Text.breakOn "/" remainder+ path = if Text.null rawPath then "/" else rawPath+ pieces = Text.splitOn ":" authority+ case pieces of+ [host] -> Right (host, 80, path)+ [host, rawPort] -> maybe (Left "WebSocket URL has an invalid port") (\port -> Right (host, port, path)) (readMaybe (Text.unpack rawPort))+ _ -> Left "WebSocket URL has an invalid authority"++csvRow :: Origin -> Endpoint -> ScrapeSample -> String+csvRow origin endpoint sample =+ Text.unpack . Text.intercalate "," $+ [ Text.pack (show origin.wallUnixNs),+ endpoint.name,+ Text.pack (show endpoint.kind),+ Text.pack (show sample.status),+ Text.pack (show sample.latencyNs),+ Text.pack (show sample.bodyBytes),+ Text.pack (show sample.skipped),+ maybe "" sanitize sample.errorMessage+ ]+ where+ sanitize = Text.replace "," ";" . Text.replace "\n" " "++tryAny :: IO value -> IO (Either SomeException value)+tryAny = try++trySync :: IO value -> IO (Either SomeException value)+trySync action = do+ outcome <- tryAny action+ case outcome of+ Left exception -> case fromException exception :: Maybe SomeAsyncException of+ Just _ -> throwIO exception+ Nothing -> pure outcome+ Right _ -> pure outcome
+ src/Kenshou/Telemetry/SelfTest.hs view
@@ -0,0 +1,28 @@+module Kenshou.Telemetry.SelfTest (selfTestBundle) where++import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Id (selftestLayer)+import Kenshou.Core.Role (WorkerRole (..), mkRoleName)+import Kenshou.Telemetry.Scrape (runScraperRole)+import Kenshou.Telemetry.SelfTest.Arms (armsScenario)+import Kenshou.Telemetry.SelfTest.Problems (slowExporterScenario, traceContinuityScenario)+import Kenshou.Telemetry.Sink (runSinkRole)++selfTestBundle :: LayerBundle+selfTestBundle = mkBundle selftestLayer [armsScenario, traceContinuityScenario, slowExporterScenario] [sinkRole, scraperRole]++sinkRole :: WorkerRole+sinkRole =+ WorkerRole+ { name = either (error . show) id (mkRoleName "selftest/telemetry-otlp-sink"),+ summary = "Receives and counts OTLP payloads for telemetry scenarios.",+ run = runSinkRole+ }++scraperRole :: WorkerRole+scraperRole =+ WorkerRole+ { name = either (error . show) id (mkRoleName "selftest/telemetry-scraper"),+ summary = "Scrapes telemetry endpoints on a fixed schedule.",+ run = runScraperRole+ }
+ src/Kenshou/Telemetry/SelfTest/Arms.hs view
@@ -0,0 +1,78 @@+module Kenshou.Telemetry.SelfTest.Arms (armsScenario) where++import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Dimension (allTelemetryArms, noDimensions)+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Clock (Nanos (..))+import Kenshou.Measure.Health (HealthConfig (..))+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Phase qualified as Measure+import Kenshou.Measure.Recorder (OpName (..))+import Kenshou.Measure.Session+import Kenshou.Telemetry+import Kenshou.Telemetry.SelfTest.Endpoints (withSyntheticEndpoints)+import Kenshou.Telemetry.SelfTest.Service++armsScenario :: Scenario+armsScenario =+ Scenario+ { id = either (error . Text.unpack) id (parseScenarioId "selftest/telemetry/benchmark/arms-on-synthetic-service"),+ revision = 1,+ summary = "Measures one synthetic service under every tracing and metrics arm.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = loadKnobs defaults <> measureKnobs Benchmark <> workKnobs <> telemetryKnobs,+ dimensions = allTelemetryArms noDimensions,+ phases = Core.PhasePlan 0.1 10 0.1,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = runArms+ }+ where+ defaults = defaultLoadDefaults {model = "closed", workers = 4, thinkTimeUs = 0}++runArms :: RunContext -> IO ScenarioReport+runArms context = case (telemetrySpecFromContext context, loadModelFromKnobs context.knobs, measureConfigFromKnobs context phases) of+ (Left message, _, _) -> pure (failedWith ["invalid-telemetry-config"] message)+ (_, Left message, _) -> pure (failedWith ["invalid-load-config"] message)+ (_, _, Left message) -> pure (failedWith ["invalid-measure-config"] message)+ (Right telemetrySpec, Right loadModel, Right measureConfig) ->+ withTelemetry telemetrySpec \telemetry -> do+ withSyntheticEndpoints telemetry.servesEndpoints telemetry.registerEndpoint do+ syntheticMetrics <- traverse newSyntheticMetrics telemetry.meter+ let spanCount = integer "work.spans-per-op"+ attributeCount = integer "work.attributes-per-span"+ cpuMicros = integer "work.cpu-micros"+ operation = Operation (OpName "synthetic") (\_ _ -> runSyntheticOperation telemetry.tracer syntheticMetrics spanCount attributeCount cpuMicros)+ -- This closed-loop benchmark intentionally saturates the measured+ -- process when work.cpu-micros is zero. That CPU is the subject under+ -- test, rather than an overloaded open-loop driver.+ let closedLoopConfig = measureConfig {healthConfig = measureConfig.healthConfig {cpuSoft = 1.1, cpuHard = 1.2}}+ (_, measurement) <- withMeasurement context closedLoopConfig (\session -> runLoad session loadModel operation)+ pure (passed {outcome = measuredOutcome measurement passed.outcome})+ where+ duration = fromIntegral (knobInt context.knobs (name "load.duration-seconds")) * 1_000_000_000+ phases = Measure.PhasePlan (Nanos 100_000_000) (Measure.SteadyFor (Nanos duration)) (Nanos 100_000_000)+ integer key = fromIntegral (knobInt context.knobs (name key))++workKnobs :: [KnobSpec]+workKnobs =+ [ intKnob "work.cpu-micros" "Synthetic CPU work per operation" 1000 0 100000,+ intKnob "work.spans-per-op" "Spans emitted per operation" 2 1 16,+ intKnob "work.attributes-per-span" "Attributes attached to every span" 6 0 64,+ intKnob "load.duration-seconds" "Steady measurement duration" 10 1 3600+ ]++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []++name :: Text -> KnobName+name = either (error . Text.unpack) id . mkKnobName
+ src/Kenshou/Telemetry/SelfTest/Endpoints.hs view
@@ -0,0 +1,41 @@+module Kenshou.Telemetry.SelfTest.Endpoints (withSyntheticEndpoints) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Exception (bracket)+import Control.Monad (forever, void)+import Data.Aeson (encode, object, (.=))+import Data.Text qualified as Text+import Kenshou.Telemetry.Endpoint+import Network.HTTP.Types (status200, status404)+import Network.Socket (close)+import Network.Wai (Application, pathInfo, responseLBS)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import Network.Wai.Handler.WebSockets (websocketsOr)+import Network.WebSockets qualified as WebSockets++withSyntheticEndpoints :: Bool -> (Endpoint -> IO ()) -> IO value -> IO value+withSyntheticEndpoints False _ action = action+withSyntheticEndpoints True register action = bracket acquire release \(port, _) -> do+ let base = Text.pack ("127.0.0.1:" <> show port)+ register (Endpoint "synthetic-json" JsonDocument ("http://" <> base <> "/metrics") Nothing)+ register (Endpoint "synthetic-push" WebSocketPush ("ws://" <> base <> "/ws/metrics") Nothing)+ action+ where+ acquire = do+ (port, socket) <- openFreePort+ server <- async (runSettingsSocket defaultSettings socket application)+ pure (port, (server, socket))+ release (_, (server, socket)) = cancel server >> void (waitCatch server) >> close socket++application :: Application+application = websocketsOr WebSockets.defaultConnectionOptions websocket http+ where+ http request respond+ | pathInfo request == ["metrics"] = respond (responseLBS status200 [("Content-Type", "application/json")] (encode (object ["operations" .= (1 :: Int)])))+ | otherwise = respond (responseLBS status404 [] "not found")+ websocket pending = do+ connection <- WebSockets.acceptRequest pending+ forever do+ WebSockets.sendTextData connection (encode (object ["operations" .= (1 :: Int)]))+ threadDelay 100_000
+ src/Kenshou/Telemetry/SelfTest/Problems.hs view
@@ -0,0 +1,146 @@+module Kenshou.Telemetry.SelfTest.Problems+ ( traceContinuityScenario,+ slowExporterScenario,+ )+where++import Control.Concurrent (threadDelay)+import Control.Monad (forM_, replicateM_, void)+import Data.Aeson (object, (.=))+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (RunContext (..), SummarySection (Verdicts), putSummary)+import Kenshou.Core.Dimension+import Kenshou.Core.Dimension qualified as Dimension+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (ScenarioId, parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase (PhasePlan (..))+import Kenshou.Core.Scenario+import Kenshou.Telemetry+import Kenshou.Telemetry.Compose+import Kenshou.Telemetry.Continuity+import Kenshou.Telemetry.Detect+import Kenshou.Telemetry.Tracing.Pipeline (PipelineSnapshot (..), snapshotPipeline)+import Kenshou.Telemetry.Tracing.Probe (SpanView (..), readSpans)+import OpenTelemetry.Attributes (Attribute (AttributeValue), PrimitiveAttribute (TextAttribute), lookupAttribute)+import OpenTelemetry.Trace.Core (Span, addAttribute, defaultSpanArguments, inSpan, inSpan')+import System.Timeout (timeout)++traceContinuityScenario :: Scenario+traceContinuityScenario =+ Scenario+ { id = scenarioId "selftest/telemetry/correctness/trace-continuity",+ revision = 1,+ summary = "Proves trace discontinuity and context-leak checks are non-vacuous.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = [intKnob "continuity.messages" "Synthetic messages in each continuity case" 1000 10 100000] <> telemetryKnobs,+ dimensions = onlyArms TracingSdkInMemory MetricsOff,+ phases = PhasePlan 0 0 0,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = runTraceContinuity+ }++slowExporterScenario :: Scenario+slowExporterScenario =+ Scenario+ { id = scenarioId "selftest/telemetry/concurrency/slow-exporter-backpressure",+ revision = 1,+ summary = "Proves slow exporters and handlers are detected without blocking the emitter.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs =+ [ intKnob "backpressure.phase-seconds" "Maximum synthetic phase duration" 8 2 600,+ intKnob "handler.delay-micros" "Injected handler delay" 5000 0 1000000+ ]+ <> telemetryKnobs,+ dimensions = onlyArms TracingSdkOtlp MetricsOff,+ phases = PhasePlan 0 0 0,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = runSlowExporter+ }++runTraceContinuity :: RunContext -> IO ScenarioReport+runTraceContinuity context = case telemetrySpecFromContext context of+ Left message -> pure (failedWith ["invalid-telemetry-config"] message)+ Right spec -> withTelemetry spec \telemetry -> case telemetry.tracer of+ Nothing -> pure (failedWith ["missing-tracer"] "sdk-inmemory did not provide a tracer")+ Just tracer -> do+ let messages = fromIntegral (knobInt context.knobs (name "continuity.messages"))+ forM_ [1 .. messages] \index ->+ inSpan' tracer "intact.producer" defaultSpanArguments \producer -> do+ addMessageId producer ("intact-" <> tshow index)+ inSpan' tracer "intact.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("intact-" <> tshow index)+ forM_ [10, 20 .. messages] \index -> do+ inSpan' tracer "broken.producer" defaultSpanArguments \producer -> addMessageId producer ("broken-" <> tshow index)+ inSpan' tracer "broken.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("broken-" <> tshow index)+ inSpan tracer "leak.carrier" defaultSpanArguments do+ forM_ [10, 20 .. messages] \index ->+ inSpan' tracer "leaky.consumer" defaultSpanArguments \consumer -> addMessageId consumer ("leaky-" <> tshow index)+ void telemetry.flushTelemetry+ spans <- maybe (pure []) readSpans telemetry.spans+ let correlate spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing+ intact = checkContinuity (named "intact.producer") (named "intact.consumer") correlate spans+ broken = checkContinuity (named "broken.producer") (named "broken.consumer") correlate spans+ isolated = checkIsolation (named "leaky.consumer") (const True) spans+ expected = messages `div` 10+ ok = intact.violations == 0 && intact.consumers == messages && broken.violations == expected && isolated.violations == expected+ telemetry.recordContinuity intact (IsolationResult 0 0 [])+ telemetry.recordContinuity broken isolated+ putSummary context Verdicts "trace-continuity" (object ["intact" .= intact, "broken" .= broken, "leaky" .= isolated, "expectedInjected" .= expected])+ pure (if ok then passed else failedWith ["trace-detector-non-vacuity"] "continuity detectors did not identify exactly the injected defects")++runSlowExporter :: RunContext -> IO ScenarioReport+runSlowExporter context = case telemetrySpecFromContext context of+ Left message -> pure (failedWith ["invalid-telemetry-config"] message)+ Right spec -> withTelemetry (spec {processor = BatchProcessor 256 200 128 2000, shutdownMs = 1000}) \telemetry -> case (telemetry.tracer, telemetry.pipeline) of+ (Just tracer, Just pipeline) -> do+ let delayMicros = fromIntegral (knobInt context.knobs (name "handler.delay-micros"))+ phaseMicros = fromIntegral (knobInt context.knobs (name "backpressure.phase-seconds")) * 1_000_000+ synchronousStats <- newHandlerStats+ replicateM_ 20 (timedHandler synchronousStats "synchronous" (slowHandler delayMicros) ())+ synchronous <- snapshotHandlerStats "synchronous" synchronousStats+ asyncStats <- newHandlerStats+ (submit, asyncSnapshot) <- asyncHandler 64 (slowHandler delayMicros)+ completed <- timeout phaseMicros (replicateM_ 10000 (timedHandler asyncStats "async-submit" submit ()))+ threadDelay (min 250_000 phaseMicros)+ asynchronous <- snapshotHandlerStats "async-submit" asyncStats+ asyncQueue <- asyncSnapshot+ telemetry.recordHandlerStats synchronous+ telemetry.recordHandlerStats asynchronous+ telemetry.setSinkFault SinkHang+ emitted <- timeout phaseMicros (replicateM_ 10000 (inSpan tracer "backpressure" defaultSpanArguments (pure ())))+ snapshot <- snapshotPipeline pipeline+ let syncFired = (handlerFinding synchronous).status == Fired+ asyncQuiet = (handlerFinding asynchronous).status == Quiet+ ok = completed /= Nothing && emitted /= Nothing && syncFired && asyncQuiet && asyncQueue.dropped > 0 && snapshot.spansDropped > 0+ putSummary context Verdicts "slow-exporter-backpressure" (object ["synchronous" .= synchronous, "asynchronous" .= asynchronous, "asyncQueue" .= asyncQueue, "pipeline" .= snapshot, "emitterCompleted" .= (emitted /= Nothing)])+ pure (if ok then passed else failedWith ["backpressure-detector-non-vacuity"] "slow exporter or handler detector did not fire as expected")+ _ -> pure (failedWith ["missing-sdk"] "sdk-otlp did not provide a tracer and pipeline")++onlyArms :: TracingArm -> MetricsArm -> DimensionSupport+onlyArms tracing metrics =+ DimensionSupport (Supported (Support (tracing :| []) tracing)) (Supported (Support (metrics :| []) metrics)) Dimension.NotApplicable Dimension.NotApplicable++named :: Text -> SpanSelector+named spanName = SpanSelector (Just spanName) Nothing []++addMessageId :: Span -> Text -> IO ()+addMessageId spanValue value = addAttribute spanValue "message.id" value++scenarioId :: Text -> ScenarioId+scenarioId text = either (error . Text.unpack) id (parseScenarioId text)++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []++name :: Text -> KnobName+name = either (error . Text.unpack) id . mkKnobName++tshow :: (Show value) => value -> Text+tshow = Text.pack . show
+ src/Kenshou/Telemetry/SelfTest/Service.hs view
@@ -0,0 +1,55 @@+module Kenshou.Telemetry.SelfTest.Service+ ( SyntheticMetrics,+ newSyntheticMetrics,+ runSyntheticOperation,+ )+where++import Control.Concurrent (threadDelay)+import Control.Exception (evaluate)+import Control.Monad (forM_)+import Data.Bits (xor)+import Data.Int (Int64)+import Data.Text qualified as Text+import Kenshou.Measure.Recorder (OpResult (OpOk))+import OpenTelemetry.Attributes (emptyAttributes)+import OpenTelemetry.Metric.Core (Counter (..), Histogram (..), Meter (..), defaultAdvisoryParameters)+import OpenTelemetry.Trace.Core (Tracer, addAttribute, defaultSpanArguments, inSpan')++data SyntheticMetrics = SyntheticMetrics (Counter Int64) Histogram++newSyntheticMetrics :: Meter -> IO SyntheticMetrics+newSyntheticMetrics meter =+ SyntheticMetrics+ <$> meter.meterCreateCounterInt64 "synthetic.operations" (Just "operations") Nothing defaultAdvisoryParameters+ <*> meter.meterCreateHistogram "synthetic.work.duration" (Just "us") Nothing defaultAdvisoryParameters++runSyntheticOperation :: Maybe Tracer -> Maybe SyntheticMetrics -> Int -> Int -> Int -> IO OpResult+runSyntheticOperation tracer metrics spanCount attributeCount cpuMicros = do+ withSpans tracer spanCount attributeCount do+ _ <- evaluate (burnCpu (max 1 (cpuMicros * 40)) 0x9e3779b9)+ threadDelay cpuMicros+ pure ()+ case metrics of+ Nothing -> pure ()+ Just (SyntheticMetrics counter histogram) -> do+ counter.counterAdd 1 emptyAttributes+ histogram.histogramRecord (fromIntegral cpuMicros) emptyAttributes+ pure (OpOk 1)++withSpans :: Maybe Tracer -> Int -> Int -> IO value -> IO value+withSpans Nothing _ _ action = action+withSpans (Just tracer) count attributeCount action = go count+ where+ go remaining+ | remaining <= 0 = action+ | otherwise =+ inSpan' tracer ("synthetic-" <> Text.pack (show remaining)) defaultSpanArguments \spanValue -> do+ forM_ [1 .. attributeCount] \index -> addAttribute spanValue ("work.attribute." <> Text.pack (show index)) index+ go (remaining - 1)++burnCpu :: Int -> Int -> Int+burnCpu 0 accumulator = accumulator+burnCpu remaining accumulator =+ let mixed = (accumulator * 1664525 + 1013904223) `xor` remaining+ in burnCpu (remaining - 1) mixed
+ src/Kenshou/Telemetry/Sink.hs view
@@ -0,0 +1,111 @@+module Kenshou.Telemetry.Sink+ ( SinkStats (..),+ SinkHandle (..),+ sinkApplication,+ withSink,+ runSinkRole,+ )+where++import Codec.Compression.GZip qualified as GZip+import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Exception (bracket)+import Control.Monad (void)+import Data.Aeson (FromJSON (..), ToJSON (..), Value (..), object, withObject, (.:), (.=))+import Data.Aeson.Types (parseMaybe)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef+import Data.ProtoLens (decodeMessage)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Vector qualified as Vector+import Kenshou.Core.Role (ControlMessage (..), RoleContext (..), WorkerInit (..), WorkerMessage (..))+import Kenshou.Telemetry.Spec (SinkFault (..), renderSinkFault)+import Lens.Micro ((^.))+import Network.HTTP.Types (status200, status404, status503)+import Network.Socket (close)+import Network.Wai (Application, Request (..), responseLBS, strictRequestBody)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import Proto.Opentelemetry.Proto.Collector.Trace.V1.TraceService (ExportTraceServiceRequest)+import Proto.Opentelemetry.Proto.Collector.Trace.V1.TraceService_Fields qualified as TraceService+import Proto.Opentelemetry.Proto.Trace.V1.Trace_Fields qualified as Trace++data SinkStats = SinkStats {requests :: Int, bytes :: Integer, spansReceived :: Int}+ deriving stock (Eq, Show)++instance ToJSON SinkStats where+ toJSON stats = object ["requests" .= stats.requests, "bytes" .= stats.bytes, "spansReceived" .= stats.spansReceived]++instance FromJSON SinkStats where+ parseJSON = withObject "SinkStats" (\value -> SinkStats <$> value .: "requests" <*> value .: "bytes" <*> value .: "spansReceived")++data SinkHandle = SinkHandle+ { endpoint :: Text,+ setFault :: SinkFault -> IO (),+ snapshot :: IO SinkStats+ }++sinkApplication :: IORef SinkFault -> IORef SinkStats -> Application+sinkApplication faultRef statsRef request respond+ | requestMethod request /= "POST" || pathInfo request `notElem` [["v1", "traces"], ["v1", "metrics"]] = respond (responseLBS status404 [] "not found")+ | otherwise = do+ fault <- readIORef faultRef+ case fault of+ SinkDelay200ms -> threadDelay 200000+ SinkDelay2000ms -> threadDelay 2000000+ SinkHang -> threadDelay 120000000+ _ -> pure ()+ body <- strictRequestBody request+ let decodedBody = if lookup "Content-Encoding" (requestHeaders request) == Just "gzip" then GZip.decompress body else body+ spans = if pathInfo request == ["v1", "traces"] then countSpans decodedBody else 0+ atomicModifyIORef' statsRef \stats -> (SinkStats (stats.requests + 1) (stats.bytes + fromIntegral (LazyByteString.length body)) (stats.spansReceived + spans), ())+ respond (responseLBS (if fault == SinkStatus503 then status503 else status200) [("Content-Type", "application/x-protobuf")] "")++withSink :: SinkFault -> (SinkHandle -> IO value) -> IO value+withSink initialFault action = bracket acquire release use+ where+ acquire = do+ faultRef <- newIORef initialFault+ statsRef <- newIORef (SinkStats 0 0 0)+ (port, socket) <- openFreePort+ if initialFault == SinkRefuse+ then close socket >> pure (SinkHandle (url port) (writeIORef faultRef) (readIORef statsRef), Nothing)+ else do+ server <- async (runSettingsSocket defaultSettings socket (sinkApplication faultRef statsRef))+ pure (SinkHandle (url port) (writeIORef faultRef) (readIORef statsRef), Just (server, socket))+ release (_, Nothing) = pure ()+ release (_, Just (server, socket)) = cancel server >> void (waitCatch server) >> close socket+ use (handle, _) = action handle+ url port = "http://127.0.0.1:" <> Text.pack (show port)++runSinkRole :: RoleContext -> IO ()+runSinkRole context = withSink initialFault \sink -> do+ context.send (WrkCustom "ready" (object ["endpoint" .= sink.endpoint]))+ loop sink+ stats <- sink.snapshot+ context.send (WrkCustom "sink-stats" (toJSON stats))+ where+ loop sink =+ context.receive >>= \case+ Just (CtlCustom "fault" (String raw)) -> case parseFault raw of+ Nothing -> context.send (WrkError ("unknown sink fault: " <> raw))+ Just fault -> sink.setFault fault >> loop sink+ Just (CtlCustom "snapshot" _) -> sink.snapshot >>= context.send . WrkCustom "sink-stats" . toJSON >> loop sink+ Just (CtlStop _) -> pure ()+ Just _ -> loop sink+ Nothing -> pure ()+ initialFault = maybe SinkHealthy id (parseMaybe (withObject "sink arguments" (\value -> value .: "fault" >>= maybe (fail "unknown sink fault") pure . parseFault)) context.init.args)++countSpans :: LazyByteString.ByteString -> Int+countSpans body = case decodeMessage (LazyByteString.toStrict body) :: Either String ExportTraceServiceRequest of+ Left _ -> 0+ Right request ->+ sum+ [ Vector.length (scope ^. Trace.vec'spans)+ | resource <- Vector.toList (request ^. TraceService.vec'resourceSpans),+ scope <- Vector.toList (resource ^. Trace.vec'scopeSpans)+ ]++parseFault :: Text -> Maybe SinkFault+parseFault raw = lookup raw [(renderSinkFault fault, fault) | fault <- [SinkHealthy, SinkDelay200ms, SinkDelay2000ms, SinkStatus503, SinkHang, SinkRefuse]]
+ src/Kenshou/Telemetry/Spec.hs view
@@ -0,0 +1,230 @@+module Kenshou.Telemetry.Spec+ ( TracingArm (..),+ MetricsArm (..),+ SamplerSpec (..),+ ProcessorSpec (..),+ OtlpProtocol (..),+ OtlpEndpoint (..),+ OtlpCompression (..),+ SinkFault (..),+ OtelReader (..),+ HelperPlacement (..),+ TelemetrySpec (..),+ telemetryKnobs,+ telemetrySpecFromContext,+ telemetrySpecFromWorker,+ renderSinkFault,+ )+where++import Data.Aeson (Value, encode)+import Data.ByteString.Lazy qualified as LBS+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Kenshou.Core.Context (RunContext (..), SummarySection (Telemetry), putSummary)+import Kenshou.Core.Dimension (Dimensions (..), MetricsArm (..), TracingArm (..))+import Kenshou.Core.Id (renderScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Role (RoleContext (..), WorkerInit (..))+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))++data SamplerSpec = AlwaysOn | AlwaysOff | ParentBasedAlwaysOn | TraceIdRatio Double | ParentBasedTraceIdRatio Double+ deriving stock (Eq, Show)++data ProcessorSpec+ = BatchProcessor+ { maxQueue :: Int,+ scheduleDelayMs :: Int,+ maxExportBatch :: Int,+ exportTimeoutMs :: Int+ }+ | SimpleProcessor {exportTimeoutMs :: Int}+ deriving stock (Eq, Show)++data OtlpProtocol = HttpProtobuf | Grpc deriving stock (Eq, Show)++data SinkFault = SinkHealthy | SinkDelay200ms | SinkDelay2000ms | SinkStatus503 | SinkHang | SinkRefuse+ deriving stock (Eq, Ord, Show)++data OtlpEndpoint = BuiltinSink SinkFault | ExternalEndpoint Text deriving stock (Eq, Show)++data OtlpCompression = CompressionNone | CompressionGzip deriving stock (Eq, Show)++data OtelReader = ReaderPrometheus | ReaderOtlpPeriodic | ReaderNone deriving stock (Eq, Show)++data HelperPlacement = HelperProcess FilePath | HelperInProcess deriving stock (Eq, Show)++data TelemetrySpec = TelemetrySpec+ { tracing :: TracingArm,+ metrics :: MetricsArm,+ serviceName :: Text,+ sampler :: SamplerSpec,+ processor :: ProcessorSpec,+ exporter :: OtlpProtocol,+ endpoint :: OtlpEndpoint,+ compression :: OtlpCompression,+ probeRetain :: Int,+ shutdownMs :: Int,+ otelReader :: OtelReader,+ otelExportMs :: Int,+ scrapeMs :: Int,+ wsSubscribers :: Int,+ helpers :: HelperPlacement,+ workerContext :: Maybe RunContext,+ outDir :: FilePath,+ report :: Value -> IO ()+ }++telemetryKnobs :: [KnobSpec]+telemetryKnobs =+ [ textKnob "otel.sampler" "OpenTelemetry sampler" "parentbased-always-on" ["always-on", "always-off", "parentbased-always-on", "traceidratio", "parentbased-traceidratio"],+ doubleKnob "otel.sampler-arg" "Trace-id-ratio sampler fraction" 1 0 1,+ textKnob "otel.processor" "Span processor" "batch" ["batch", "simple"],+ intKnob "otel.bsp.max-queue" "Batch processor queue bound" 2048 1 1048576,+ intKnob "otel.bsp.schedule-delay-ms" "Batch export schedule delay" 5000 10 60000,+ intKnob "otel.bsp.max-export-batch" "Maximum export batch" 512 1 1048576,+ intKnob "otel.bsp.export-timeout-ms" "Exporter timeout" 30000 100 120000,+ textKnob "otel.exporter" "OTLP protocol" "http-protobuf" ["http-protobuf", "grpc"],+ KnobSpec (name "otel.endpoint") "External OTLP base URL, empty for built-in sink" KnobText (VText "") AnyValue [],+ textKnob "otel.sink-fault" "Built-in OTLP sink fault" "none" ["none", "delay-200ms", "delay-2000ms", "status-503", "hang", "refuse"],+ textKnob "otel.compression" "OTLP compression" "none" ["none", "gzip"],+ intKnob "otel.inmemory.retain" "Recent spans retained by the bounded probe" 4096 0 1000000,+ intKnob "otel.shutdown-timeout-ms" "Provider flush and shutdown timeout" 5000 100 120000,+ intKnob "metrics.scrape-interval-ms" "Metrics scrape interval" 15000 100 600000,+ intKnob "metrics.ws-subscribers" "WebSocket subscribers per endpoint" 0 0 64,+ textKnob "metrics.otel-reader" "OpenTelemetry metric reader" "prometheus" ["prometheus", "otlp-periodic", "none"],+ intKnob "metrics.otel-export-interval-ms" "Periodic OTLP metric export interval" 60000 1000 600000+ ]++telemetrySpecFromContext :: RunContext -> Either Text TelemetrySpec+telemetrySpecFromContext context =+ makeTelemetrySpec+ context.knobs+ context.dimensions+ (Text.replace "/" "." (renderScenarioId context.scenario))+ (HelperProcess "kenshou")+ (Just context)+ context.outDir+ (putSummary context Telemetry "telemetry")++-- A child owns its providers, endpoints and files. It must never write into+-- the parent's series or retain the parent's in-memory summary state.+telemetrySpecFromWorker :: RoleContext -> FilePath -> Either Text TelemetrySpec+telemetrySpecFromWorker context directory =+ makeTelemetrySpec+ context.init.knobs+ context.init.dimensions+ (Text.replace "/" "." context.init.instanceName)+ HelperInProcess+ Nothing+ directory+ (\value -> createDirectoryIfMissing True (directory </> "logs") >> LBS.writeFile (directory </> "logs" </> "telemetry-summary.json") (encode value))++makeTelemetrySpec :: ResolvedKnobs -> Dimensions -> Text -> HelperPlacement -> Maybe RunContext -> FilePath -> (Value -> IO ()) -> Either Text TelemetrySpec+makeTelemetrySpec knobs dimensions serviceName helpers workerContext outDir report = do+ let Dimensions resolvedTracing resolvedMetrics _ _ = dimensions+ tracing <- maybe (Left "telemetry.tracing is not applicable") Right resolvedTracing+ metrics <- maybe (Left "telemetry.metrics is not applicable") Right resolvedMetrics+ sampler <- parseSampler (knobText knobs (name "otel.sampler")) (knobDouble knobs (name "otel.sampler-arg"))+ processor <- parseProcessor knobs+ exporter <- parseProtocol (knobText knobs (name "otel.exporter"))+ endpoint <- parseEndpoint knobs+ compression <- parseCompression (knobText knobs (name "otel.compression"))+ reader <- parseReader (knobText knobs (name "metrics.otel-reader"))+ pure+ TelemetrySpec+ { tracing,+ metrics,+ serviceName = serviceName,+ sampler,+ processor,+ exporter,+ endpoint,+ compression,+ probeRetain = integer "otel.inmemory.retain",+ shutdownMs = integer "otel.shutdown-timeout-ms",+ otelReader = reader,+ otelExportMs = integer "metrics.otel-export-interval-ms",+ scrapeMs = integer "metrics.scrape-interval-ms",+ wsSubscribers = integer "metrics.ws-subscribers",+ helpers,+ workerContext,+ outDir,+ report+ }+ where+ integer key = fromIntegral (knobInt knobs (name key))++parseSampler :: Text -> Double -> Either Text SamplerSpec+parseSampler "always-on" _ = Right AlwaysOn+parseSampler "always-off" _ = Right AlwaysOff+parseSampler "parentbased-always-on" _ = Right ParentBasedAlwaysOn+parseSampler "traceidratio" ratio = Right (TraceIdRatio ratio)+parseSampler "parentbased-traceidratio" ratio = Right (ParentBasedTraceIdRatio ratio)+parseSampler value _ = Left ("unsupported otel.sampler: " <> value)++parseProcessor :: ResolvedKnobs -> Either Text ProcessorSpec+parseProcessor knobs = case knobText knobs (name "otel.processor") of+ "simple" -> Right (SimpleProcessor timeoutMs)+ "batch"+ | batch > queue -> Left "otel.bsp.max-export-batch must not exceed otel.bsp.max-queue"+ | otherwise -> Right (BatchProcessor queue delay batch timeoutMs)+ value -> Left ("unsupported otel.processor: " <> value)+ where+ queue = integer "otel.bsp.max-queue"+ delay = integer "otel.bsp.schedule-delay-ms"+ batch = integer "otel.bsp.max-export-batch"+ timeoutMs = integer "otel.bsp.export-timeout-ms"+ integer key = fromIntegral (knobInt knobs (name key))++parseProtocol :: Text -> Either Text OtlpProtocol+parseProtocol "http-protobuf" = Right HttpProtobuf+parseProtocol "grpc" = Left "otel.exporter=grpc requires building kenshou-telemetry with -fotlp-grpc"+parseProtocol value = Left ("unsupported otel.exporter: " <> value)++parseEndpoint :: ResolvedKnobs -> Either Text OtlpEndpoint+parseEndpoint knobs = case (knobText knobs (name "otel.endpoint"), parseSinkFault (knobText knobs (name "otel.sink-fault"))) of+ (_, Left message) -> Left message+ ("", Right fault) -> Right (BuiltinSink fault)+ (url, Right SinkHealthy) -> Right (ExternalEndpoint url)+ (_, Right _) -> Left "otel.sink-fault is only valid with the built-in sink"++parseCompression :: Text -> Either Text OtlpCompression+parseCompression "none" = Right CompressionNone+parseCompression "gzip" = Right CompressionGzip+parseCompression value = Left ("unsupported otel.compression: " <> value)++parseReader :: Text -> Either Text OtelReader+parseReader "prometheus" = Right ReaderPrometheus+parseReader "otlp-periodic" = Right ReaderOtlpPeriodic+parseReader "none" = Right ReaderNone+parseReader value = Left ("unsupported metrics.otel-reader: " <> value)++parseSinkFault :: Text -> Either Text SinkFault+parseSinkFault value = maybe (Left ("unsupported otel.sink-fault: " <> value)) Right (lookup value values)+ where+ values = [(renderSinkFault fault, fault) | fault <- [SinkHealthy, SinkDelay200ms, SinkDelay2000ms, SinkStatus503, SinkHang, SinkRefuse]]++renderSinkFault :: SinkFault -> Text+renderSinkFault SinkHealthy = "none"+renderSinkFault SinkDelay200ms = "delay-200ms"+renderSinkFault SinkDelay2000ms = "delay-2000ms"+renderSinkFault SinkStatus503 = "status-503"+renderSinkFault SinkHang = "hang"+renderSinkFault SinkRefuse = "refuse"++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob key summary def low high = KnobSpec (name key) summary KnobInt (VInt def) (IntRange low high) []++doubleKnob :: Text -> Text -> Double -> Double -> Double -> KnobSpec+doubleKnob key summary def low high = KnobSpec (name key) summary KnobDouble (VDouble def) (DoubleRange low high) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob key summary def (first : rest) = KnobSpec (name key) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []+textKnob key _ _ [] = error ("text knob has no values: " <> Text.unpack key)++name :: Text -> KnobName+name = either (error . Text.unpack) id . mkKnobName
+ src/Kenshou/Telemetry/Tracing.hs view
@@ -0,0 +1,122 @@+module Kenshou.Telemetry.Tracing+ ( TracingRuntime (..),+ startTracing,+ flushTracing,+ stopTracing,+ otlpExporterConfig,+ )+where++import Control.Monad (when)+import Data.Text qualified as Text+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Tracing.Pipeline+import Kenshou.Telemetry.Tracing.Probe+import OpenTelemetry.Exporter.OTLP.Span qualified as OtlpSpan+import OpenTelemetry.Exporter.Span (SpanExporter)+import OpenTelemetry.Processor.Batch.Span qualified as Batch+import OpenTelemetry.Processor.Simple.Span qualified as Simple+import OpenTelemetry.Processor.Span (SpanProcessor)+import OpenTelemetry.Propagator (setGlobalTextMapPropagator)+import OpenTelemetry.Propagator.W3CTraceContext (w3cTraceContextPropagator)+import OpenTelemetry.Resource (materializeResources, mkResource, (.=))+import OpenTelemetry.Trace.Core+import OpenTelemetry.Trace.Id.Generator.Default (defaultIdGenerator)+import OpenTelemetry.Trace.Sampler qualified as Sampler++data TracingRuntime = TracingRuntime+ { tracer :: Maybe Tracer,+ provider :: Maybe TracerProvider,+ probe :: Maybe SpanProbe,+ pipeline :: Maybe PipelineStats+ }++startTracing :: TelemetrySpec -> IO TracingRuntime+startTracing spec = do+ when (spec.tracing /= TracingOff) (setGlobalTextMapPropagator w3cTraceContextPropagator)+ case spec.tracing of+ TracingOff -> pure (TracingRuntime Nothing Nothing Nothing Nothing)+ TracingNoop -> do+ provider <- createTracerProvider [] (providerOptions spec)+ pure (runtime provider Nothing Nothing)+ TracingSdkInMemory -> do+ stats <- newPipelineStats+ (probe, probeProcessor) <- newSpanProbe spec.probeRetain+ provider <- createTracerProvider [countingProcessor stats, instrumentProcessorSuccess stats probeProcessor] (providerOptions spec)+ pure (runtime provider (Just probe) (Just stats))+ TracingSdkOtlp -> do+ stats <- newPipelineStats+ exporter <- instrumentExporter stats <$> OtlpSpan.otlpExporter (otlpExporterConfig spec)+ processor <- makeProcessor spec.processor exporter+ provider <- createTracerProvider [countingProcessor stats, processor] (providerOptions spec)+ pure (runtime provider Nothing (Just stats))+ where+ runtime provider probe pipeline = TracingRuntime (Just (makeTracer provider (instrumentationLibrary "kenshou" "0.1.0.0") tracerOptions)) (Just provider) probe pipeline++flushTracing :: Int -> TracingRuntime -> IO (Maybe FlushResult)+flushTracing timeoutMs runtime = traverse (\provider -> forceFlushTracerProvider provider (Just (timeoutMs * 1000))) runtime.provider++stopTracing :: Int -> TracingRuntime -> IO (Maybe ShutdownResult)+stopTracing timeoutMs runtime = traverse (\provider -> shutdownTracerProvider provider (Just (timeoutMs * 1000))) runtime.provider++providerOptions :: TelemetrySpec -> TracerProviderOptions+providerOptions spec =+ emptyTracerProviderOptions+ { tracerProviderOptionsIdGenerator = defaultIdGenerator,+ tracerProviderOptionsSampler = sampler spec.sampler,+ tracerProviderOptionsResources = materializeResources (mkResource ["service.name" .= spec.serviceName]),+ tracerProviderOptionsPropagators = w3cTraceContextPropagator+ }++sampler :: SamplerSpec -> Sampler.Sampler+sampler AlwaysOn = Sampler.alwaysOn+sampler AlwaysOff = Sampler.alwaysOff+sampler ParentBasedAlwaysOn = Sampler.parentBased (Sampler.parentBasedOptions Sampler.alwaysOn)+sampler (TraceIdRatio ratio) = Sampler.traceIdRatioBased ratio+sampler (ParentBasedTraceIdRatio ratio) = Sampler.parentBased (Sampler.parentBasedOptions (Sampler.traceIdRatioBased ratio))++makeProcessor :: ProcessorSpec -> SpanExporter -> IO SpanProcessor+makeProcessor (SimpleProcessor timeoutMs) exporter = Simple.simpleProcessor (Simple.SimpleProcessorConfig exporter (timeoutMs * 1000))+makeProcessor (BatchProcessor queue delay batch timeoutMs) exporter =+ Batch.batchProcessor+ Batch.batchTimeoutConfig+ { Batch.maxQueueSize = queue,+ Batch.scheduledDelayMillis = delay,+ Batch.maxExportBatchSize = batch,+ Batch.exportTimeoutMillis = timeoutMs+ }+ exporter++otlpExporterConfig :: TelemetrySpec -> OtlpSpan.OTLPExporterConfig+otlpExporterConfig spec =+ OtlpSpan.OTLPExporterConfig+ { OtlpSpan.otlpEndpoint = Just (Text.unpack (endpointBase spec.endpoint)),+ OtlpSpan.otlpTracesEndpoint = Nothing,+ OtlpSpan.otlpMetricsEndpoint = Nothing,+ OtlpSpan.otlpInsecure = True,+ OtlpSpan.otlpSpanInsecure = True,+ OtlpSpan.otlpMetricInsecure = True,+ OtlpSpan.otlpCertificate = Nothing,+ OtlpSpan.otlpTracesCertificate = Nothing,+ OtlpSpan.otlpMetricCertificate = Nothing,+ OtlpSpan.otlpHeaders = Nothing,+ OtlpSpan.otlpTracesHeaders = Nothing,+ OtlpSpan.otlpMetricsHeaders = Nothing,+ OtlpSpan.otlpCompression = Just (case spec.compression of CompressionNone -> OtlpSpan.None; CompressionGzip -> OtlpSpan.GZip),+ OtlpSpan.otlpTracesCompression = Nothing,+ OtlpSpan.otlpMetricsCompression = Nothing,+ OtlpSpan.otlpTimeout = Just (processorTimeout spec.processor),+ OtlpSpan.otlpTracesTimeout = Nothing,+ OtlpSpan.otlpMetricsTimeout = Nothing,+ OtlpSpan.otlpProtocol = Just OtlpSpan.HttpProtobuf,+ OtlpSpan.otlpTracesProtocol = Nothing,+ OtlpSpan.otlpMetricsProtocol = Nothing+ }++endpointBase :: OtlpEndpoint -> Text.Text+endpointBase (ExternalEndpoint url) = url+endpointBase (BuiltinSink _) = "http://127.0.0.1:4318"++processorTimeout :: ProcessorSpec -> Int+processorTimeout (SimpleProcessor timeoutMs) = timeoutMs+processorTimeout (BatchProcessor _ _ _ timeoutMs) = timeoutMs
+ src/Kenshou/Telemetry/Tracing/Pipeline.hs view
@@ -0,0 +1,187 @@+module Kenshou.Telemetry.Tracing.Pipeline+ ( PipelineStats,+ PipelineSnapshot (..),+ LatencySummary (..),+ newPipelineStats,+ countingProcessor,+ instrumentProcessorSuccess,+ instrumentExporter,+ snapshotPipeline,+ startPipelineSampler,+ )+where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Exception (displayException, finally, onException)+import Control.Monad (void)+import Data.Aeson (ToJSON (..), object, (.=))+import Data.HashMap.Strict qualified as HashMap+import Data.IORef+import Data.List (sort)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Vector qualified as Vector+import GHC.Clock (getMonotonicTimeNSec)+import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))+import OpenTelemetry.Internal.Common.Types (FlushResult (FlushSuccess), ShutdownResult (ShutdownSuccess))+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO (BufferMode (LineBuffering), IOMode (WriteMode), hClose, hPutStrLn, hSetBuffering, openFile)++data PipelineStats = PipelineStats+ { started :: IORef Int,+ ended :: IORef Int,+ exportedOk :: IORef Int,+ exportFailed :: IORef Int,+ exportCalls :: IORef Int,+ exportLatencies :: IORef [Integer],+ maxQueueDepth :: IORef Int,+ lastExportError :: IORef (Maybe Text)+ }++data LatencySummary = LatencySummary {p50 :: Integer, p99 :: Integer, maximum :: Integer}+ deriving stock (Eq, Show)++data PipelineSnapshot = PipelineSnapshot+ { spansStarted :: Int,+ spansEnded :: Int,+ spansExportedOk :: Int,+ spansExportFailed :: Int,+ spansDropped :: Int,+ maxQueueDepth :: Int,+ exportCalls :: Int,+ exportLatencyNs :: LatencySummary,+ lastExportError :: Maybe Text+ }+ deriving stock (Eq, Show)++instance ToJSON LatencySummary where+ toJSON summary = object ["p50" .= summary.p50, "p99" .= summary.p99, "max" .= summary.maximum]++instance ToJSON PipelineSnapshot where+ toJSON snapshot =+ object+ [ "spansStarted" .= snapshot.spansStarted,+ "spansEnded" .= snapshot.spansEnded,+ "spansExportedOk" .= snapshot.spansExportedOk,+ "spansExportFailed" .= snapshot.spansExportFailed,+ "spansDropped" .= snapshot.spansDropped,+ "maxQueueDepth" .= snapshot.maxQueueDepth,+ "exportCalls" .= snapshot.exportCalls,+ "exportLatencyNs" .= snapshot.exportLatencyNs,+ "lastExportError" .= snapshot.lastExportError+ ]++newPipelineStats :: IO PipelineStats+newPipelineStats = PipelineStats <$> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef 0 <*> newIORef [] <*> newIORef 0 <*> newIORef Nothing++countingProcessor :: PipelineStats -> SpanProcessor+countingProcessor stats =+ SpanProcessor+ { spanProcessorOnStart = \_ _ -> increment stats.started 1,+ spanProcessorOnEnd = \_ -> increment stats.ended 1 >> updateQueueHighWater stats,+ spanProcessorShutdown = pure ShutdownSuccess,+ spanProcessorForceFlush = pure FlushSuccess+ }++instrumentProcessorSuccess :: PipelineStats -> SpanProcessor -> SpanProcessor+instrumentProcessorSuccess stats processor =+ processor+ { spanProcessorOnEnd = \spanValue -> processor.spanProcessorOnEnd spanValue >> increment stats.exportedOk 1+ }++instrumentExporter :: PipelineStats -> SpanExporter -> SpanExporter+instrumentExporter stats exporter =+ exporter+ { spanExporterExport = \batch -> do+ let spanCount = sum (fmap Vector.length (HashMap.elems batch))+ startedAt <- getMonotonicTimeNSec+ increment stats.exportCalls 1+ result <-+ exporter.spanExporterExport batch+ `onException` do+ increment stats.exportFailed spanCount+ writeIORef stats.lastExportError (Just "exporter raised an exception")+ endedAt <- getMonotonicTimeNSec+ modifyIORef' stats.exportLatencies (fromIntegral (endedAt - startedAt) :)+ case result of+ Success -> increment stats.exportedOk spanCount+ Failure exception -> do+ increment stats.exportFailed spanCount+ writeIORef stats.lastExportError (fmap (Text.pack . displayException) exception)+ pure result+ }++snapshotPipeline :: PipelineStats -> IO PipelineSnapshot+snapshotPipeline stats = do+ spansStarted <- readIORef stats.started+ spansEnded <- readIORef stats.ended+ spansExportedOk <- readIORef stats.exportedOk+ spansExportFailed <- readIORef stats.exportFailed+ exportCalls <- readIORef stats.exportCalls+ latencies <- readIORef stats.exportLatencies+ maxQueueDepth <- readIORef stats.maxQueueDepth+ lastExportError <- readIORef stats.lastExportError+ let spansDropped = max 0 (spansEnded - spansExportedOk - spansExportFailed)+ pure+ PipelineSnapshot+ { spansStarted,+ spansEnded,+ spansExportedOk,+ spansExportFailed,+ spansDropped,+ maxQueueDepth,+ exportCalls,+ exportLatencyNs = summarize latencies,+ lastExportError+ }++startPipelineSampler :: FilePath -> PipelineStats -> IO (IO [Int])+startPipelineSampler outputDir stats = do+ createDirectoryIfMissing True (outputDir </> "series")+ samples <- newIORef []+ worker <- async do+ handle <- openFile (outputDir </> "series" </> "otel-pipeline.csv") WriteMode+ hSetBuffering handle LineBuffering+ hPutStrLn handle "t_mono_ns,spans_started,spans_ended,exported_ok,export_failed,queue_depth,export_calls"+ let loop = do+ threadDelay 1_000_000+ timestamp <- getMonotonicTimeNSec+ started <- readIORef stats.started+ ended <- readIORef stats.ended+ exportedOk <- readIORef stats.exportedOk+ exportFailed <- readIORef stats.exportFailed+ exportCalls <- readIORef stats.exportCalls+ let queueDepth = max 0 (ended - exportedOk - exportFailed)+ modifyIORef' samples (queueDepth :)+ hPutStrLn handle (comma [timestamp, fromIntegral started, fromIntegral ended, fromIntegral exportedOk, fromIntegral exportFailed, fromIntegral queueDepth, fromIntegral exportCalls])+ loop+ loop `finally` hClose handle+ pure do+ cancel worker+ void (waitCatch worker)+ reverse <$> readIORef samples+ where+ comma = foldr1 (\left right -> left <> "," <> right) . fmap show++increment :: IORef Int -> Int -> IO ()+increment ref amount = void (atomicModifyIORef' ref (\value -> let updated = value + amount in (updated, updated)))++updateQueueHighWater :: PipelineStats -> IO ()+updateQueueHighWater stats = do+ ended <- readIORef stats.ended+ exportedOk <- readIORef stats.exportedOk+ exportFailed <- readIORef stats.exportFailed+ let queued = max 0 (ended - exportedOk - exportFailed)+ atomicModifyIORef' stats.maxQueueDepth (\high -> (max high queued, ()))++summarize :: [Integer] -> LatencySummary+summarize [] = LatencySummary 0 0 0+summarize values =+ let ordered = sort values+ in LatencySummary (quantile 0.50 ordered) (quantile 0.99 ordered) (last ordered)++quantile :: Double -> [Integer] -> Integer+quantile fraction values = values !! min (length values - 1) (floor (fraction * fromIntegral (length values - 1)))
+ src/Kenshou/Telemetry/Tracing/Probe.hs view
@@ -0,0 +1,82 @@+module Kenshou.Telemetry.Tracing.Probe+ ( SpanView (..),+ SpanProbe,+ newSpanProbe,+ readSpans,+ spansSeen,+ )+where++import Data.Foldable qualified as Foldable+import Data.IORef+import Data.Sequence (Seq (..), (|>))+import Data.Sequence qualified as Seq+import Data.Text (Text)+import Data.Word (Word64)+import OpenTelemetry.Attributes (Attributes)+import OpenTelemetry.Common (optionalTimestampToMaybe, timestampToNanoseconds)+import OpenTelemetry.Internal.Common.Types (FlushResult (FlushSuccess), ShutdownResult (ShutdownSuccess))+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import OpenTelemetry.Trace.Core (ImmutableSpan (..), SpanContext (..), SpanHot (..), SpanKind, SpanStatus, getSpanContext)+import OpenTelemetry.Trace.Id (SpanId, TraceId)++data SpanView = SpanView+ { name :: Text,+ kind :: SpanKind,+ traceId :: TraceId,+ spanId :: SpanId,+ parentSpanId :: Maybe SpanId,+ attributes :: Attributes,+ startNs :: Word64,+ endNs :: Word64,+ status :: SpanStatus+ }+ deriving stock (Eq, Show)++-- Evict during capture, even when nobody reads the probe during a long run.+-- A strict IORef update alone only forces the outer constructor.+data ProbeState = ProbeState {seen :: !Int, retained :: !(Seq SpanView)}++data SpanProbe = SpanProbe {capacity :: Int, state :: IORef ProbeState}++newSpanProbe :: Int -> IO (SpanProbe, SpanProcessor)+newSpanProbe capacity = do+ state <- newIORef (ProbeState 0 Seq.empty)+ let probe = SpanProbe (max 0 capacity) state+ processor =+ SpanProcessor+ { spanProcessorOnStart = \_ _ -> pure (),+ spanProcessorOnEnd = capture probe,+ spanProcessorShutdown = pure ShutdownSuccess,+ spanProcessorForceFlush = pure FlushSuccess+ }+ pure (probe, processor)++readSpans :: SpanProbe -> IO [SpanView]+readSpans probe = toList . (.retained) <$> readIORef probe.state+ where+ toList = Foldable.toList++spansSeen :: SpanProbe -> IO Int+spansSeen probe = (.seen) <$> readIORef probe.state++capture :: SpanProbe -> ImmutableSpan -> IO ()+capture probe spanValue = do+ hot <- readIORef spanValue.spanHot+ parent <- traverse getSpanContext spanValue.spanParent+ let view =+ SpanView+ { name = hot.hotName,+ kind = spanValue.spanKind,+ traceId = spanValue.spanContext.traceId,+ spanId = spanValue.spanContext.spanId,+ parentSpanId = (.spanId) <$> parent,+ attributes = hot.hotAttributes,+ startNs = timestampToNanoseconds spanValue.spanStart,+ endNs = maybe (timestampToNanoseconds spanValue.spanStart) timestampToNanoseconds (optionalTimestampToMaybe hot.hotEnd),+ status = hot.hotStatus+ }+ atomicModifyIORef' probe.state \current ->+ let appended = current.retained |> view+ trimmed = if Seq.length appended > probe.capacity then Seq.drop 1 appended else appended+ in (ProbeState (current.seen + 1) trimmed, ())
+ test/Main.hs view
@@ -0,0 +1,366 @@+module Main (main) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async, cancel, waitCatch)+import Control.Concurrent.MVar (newEmptyMVar, putMVar, readMVar, tryPutMVar)+import Control.Exception (bracket)+import Control.Monad (forM_, replicateM_, void)+import Data.IORef+import Data.List (find)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (fromJust)+import Data.Text qualified as Text+import Kenshou.Core.Knob (Allowed (..), KnobSpec (..), renderKnobName)+import Kenshou.Core.Scenario (Scenario (..))+import Kenshou.Telemetry.Compose+import Kenshou.Telemetry.Continuity+import Kenshou.Telemetry.Detect (Finding (..), FindingSeverity (Degraded), FindingStatus (Fired), handlerFinding, pipelineFindings, queueGrowthFinding)+import Kenshou.Telemetry.Endpoint+import Kenshou.Telemetry.Metrics+import Kenshou.Telemetry.Overhead+import Kenshou.Telemetry.Scrape+import Kenshou.Telemetry.SelfTest.Arms (armsScenario)+import Kenshou.Telemetry.SelfTest.Service+import Kenshou.Telemetry.Sink qualified as Sink+import Kenshou.Telemetry.Spec+import Kenshou.Telemetry.Tracing+import Kenshou.Telemetry.Tracing.Pipeline+import Kenshou.Telemetry.Tracing.Probe+import Network.HTTP.Types (status200, status404)+import Network.Socket (close)+import Network.Wai (Application, responseLBS)+import Network.Wai.Handler.Warp (defaultSettings, openFreePort, runSettingsSocket)+import Network.Wai.Handler.WebSockets (websocketsOr)+import Network.WebSockets qualified as WebSockets+import OpenTelemetry.Attributes (lookupAttribute)+import OpenTelemetry.Context.ThreadLocal (getContext)+import OpenTelemetry.Exporter.Span (ExportResult (..), SpanExporter (..))+import OpenTelemetry.Processor.Batch.Span qualified as Batch+import OpenTelemetry.Processor.Simple.Span qualified as Simple+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import OpenTelemetry.Propagator (emptyTextMap, inject, textMapToList)+import OpenTelemetry.Trace.Core+import System.Directory (createDirectoryIfMissing)+import System.Mem (performMajorGC)+import System.Mem.Weak (Weak, deRefWeak, mkWeakPtr)+import System.Timeout (timeout)+import Test.Hspec++main :: IO ()+main = hspec do+ describe "overhead planning" do+ it "expands the headline one-factor matrix into four arms and twelve slots" do+ let request = overheadRequest (Map.fromList [("telemetry.tracing", ["off", "noop", "sdk-otlp"]), ("telemetry.metrics", ["off", "serve-scraped"])])+ case planOverhead request armsScenario of+ Left err -> expectationFailure (show err)+ Right plan -> do+ length plan.arms `shouldBe` 4+ length plan.slots `shouldBe` 12+ map (length . unique . fmap (.arm.armId)) (blocks plan.slots) `shouldBe` [4, 4, 4]++ it "is deterministic and gives two arms ABBA order across the first two blocks" do+ let request = overheadRequest (Map.singleton "telemetry.tracing" ["off", "noop"])+ case (planOverhead request armsScenario, planOverhead request armsScenario) of+ (Right first, Right second) -> do+ first `shouldBe` second+ let order = fmap (.arm.armId) first.slots+ take 2 order `shouldBe` reverse (take 2 (drop 2 order))+ result -> expectationFailure (show result)++ it "rejects repeated arm values" do+ planOverhead (overheadRequest (Map.singleton "telemetry.tracing" ["off", "off"])) armsScenario `shouldSatisfy` isLeft++ describe "telemetry problem detectors" do+ it "detects synchronous handler stalls while a bounded async wrapper drops instead of blocking" do+ synchronousStats <- newHandlerStats+ timedHandler synchronousStats "sync" (slowHandler 6_000) ()+ synchronous <- snapshotHandlerStats "sync" synchronousStats+ (submit, snapshotAsync) <- asyncHandler 1 (slowHandler 20_000)+ replicateM_ 100 (submit ())+ queued <- snapshotAsync+ handlerFinding synchronous `shouldSatisfy` ((== Fired) . (.status))+ queued.dropped `shouldSatisfy` (> 0)++ it "marks dropped spans as degraded above the policy fraction" do+ let snapshot = PipelineSnapshot 100 100 80 0 20 20 1 (LatencySummary 1 1 1) Nothing+ pipelineFindings 0.01 (Just snapshot) [] `shouldSatisfy` any (\finding -> finding.detector == "span-drop" && finding.status == Fired && finding.severity == Degraded)++ it "fires queue growth only after more than ten consecutive samples above ninety percent" do+ queueGrowthFinding (Just 100) (replicate 10 91) `shouldSatisfy` ((/= Fired) . (.status))+ queueGrowthFinding (Just 100) (replicate 11 91) `shouldSatisfy` ((== Fired) . (.status))++ describe "telemetryKnobs" do+ it "declares each shared knob once" do+ let names = fmap (renderKnobName . (.name)) telemetryKnobs+ length names `shouldBe` length (unique names)+ it "bounds the trace-id ratio to zero through one" do+ fmap (.allowed) (find ((== "otel.sampler-arg") . renderKnobName . (.name)) telemetryKnobs)+ `shouldBe` Just (DoubleRange 0 1)++ describe "tracing arms" do+ it "does not construct a provider under off" do+ runtime <- startTracing (testSpec TracingOff)+ isNothing runtime.tracer `shouldBe` True+ isNothing runtime.provider `shouldBe` True++ it "constructs a disabled tracer under noop" do+ runtime <- startTracing (testSpec TracingNoop)+ let tracer = fromJust runtime.tracer+ tracerIsEnabled tracer `shouldBe` False+ inSpan tracer "ignored" defaultSpanArguments (pure ())+ _ <- stopTracing 1000 runtime+ pure ()++ it "retains a bounded, parented trace and installs W3C propagation" do+ runtime <- startTracing ((testSpec TracingSdkInMemory) {probeRetain = 4})+ let tracer = fromJust runtime.tracer+ provider = fromJust runtime.provider+ replicateM_ 5 (inSpan tracer "discarded" defaultSpanArguments (pure ()))+ headers <- inSpan' tracer "parent" defaultSpanArguments \parentSpan -> do+ addAttribute parentSpan "message.id" ("one" :: Text.Text)+ nested <- inSpan' tracer "child" defaultSpanArguments \childSpan -> addAttribute childSpan "message.id" ("one" :: Text.Text)+ activeContext <- getContext+ carrier <- inject (getTracerProviderPropagators provider) activeContext emptyTextMap+ pure (nested, textMapToList carrier)+ _ <- flushTracing 1000 runtime+ let probe = fromJust runtime.probe+ views <- readSpans probe+ seen <- spansSeen probe+ seen `shouldBe` 7+ length views `shouldBe` 4+ let child = fromJust (find ((== "child") . (.name)) views)+ parent = fromJust (find ((== "parent") . (.name)) views)+ child.traceId `shouldBe` parent.traceId+ child.parentSpanId `shouldBe` Just parent.spanId+ let correlation spanValue = case lookupAttribute spanValue.attributes "message.id" of Just (AttributeValue (TextAttribute value)) -> Just value; _ -> Nothing+ continuity = checkContinuity (SpanSelector (Just "parent") Nothing []) (SpanSelector (Just "child") Nothing []) correlation views+ continuity.violations `shouldBe` 0+ snd headers `shouldSatisfy` any ((== "traceparent") . fst)+ snapshot <- snapshotPipeline (fromJust runtime.pipeline)+ snapshot.spansEnded `shouldBe` 7+ snapshot.spansExportedOk `shouldBe` 7+ snapshot.spansDropped `shouldBe` 0+ _ <- stopTracing 1000 runtime+ pure ()++ it "releases evicted spans before anyone reads the bounded probe" do+ forM_ [0, 1] \capacity -> do+ (probe, provider, weak) <- probeWithEvictedSpan capacity+ performMajorGC+ (isNothing <$> deRefWeak weak) `shouldReturn` True+ spansSeen probe `shouldReturn` 10+ (length <$> readSpans probe) `shouldReturn` capacity+ _ <- shutdownTracerProvider provider (Just 1000000)+ pure ()++ it "exports plain and gzip OTLP requests to the built-in sink" do+ forCompression CompressionNone+ forCompression CompressionGzip++ it "accounts for alternating exporter failures exactly" do+ stats <- newPipelineStats+ calls <- newIORef (0 :: Int)+ let exporter =+ SpanExporter+ { spanExporterExport = \_ -> do+ call <- atomicModifyIORef' calls (\value -> let next = value + 1 in (next, next))+ pure (if even call then Failure Nothing else Success),+ spanExporterShutdown = pure ShutdownSuccess,+ spanExporterForceFlush = pure FlushSuccess+ }+ processor <- Simple.simpleProcessor (Simple.SimpleProcessorConfig (instrumentExporter stats exporter) 1_000_000)+ provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions+ let tracer = makeTracer provider (instrumentationLibrary "pipeline-test" "1") tracerOptions+ replicateM_ 10 (inSpan tracer "alternating" defaultSpanArguments (pure ()))+ _ <- shutdownTracerProvider provider (Just 1_000_000)+ snapshot <- snapshotPipeline stats+ snapshot.spansEnded `shouldBe` snapshot.spansExportedOk + snapshot.spansExportFailed + snapshot.spansDropped+ snapshot.spansExportedOk `shouldBe` 5+ snapshot.spansExportFailed `shouldBe` 5++ it "drops instead of blocking when a batch queue is full" do+ stats <- newPipelineStats+ releaseExporter <- newEmptyMVar+ let exporter =+ SpanExporter+ { spanExporterExport = \_ -> readMVar releaseExporter >> pure Success,+ spanExporterShutdown = pure ShutdownSuccess,+ spanExporterForceFlush = pure FlushSuccess+ }+ processor <-+ Batch.batchProcessor+ Batch.batchTimeoutConfig+ { Batch.maxQueueSize = 8,+ Batch.scheduledDelayMillis = 10,+ Batch.maxExportBatchSize = 4,+ Batch.exportTimeoutMillis = 5_000+ }+ (instrumentExporter stats exporter)+ provider <- createTracerProvider [countingProcessor stats, processor] emptyTracerProviderOptions+ let tracer = makeTracer provider (instrumentationLibrary "backpressure-test" "1") tracerOptions+ completed <- timeout 1_000_000 (replicateM_ 10_000 (inSpan tracer "queued" defaultSpanArguments (pure ())))+ completed `shouldBe` Just ()+ beforeRelease <- snapshotPipeline stats+ beforeRelease.spansDropped `shouldSatisfy` (> 0)+ putMVar releaseExporter ()+ void (shutdownTracerProvider provider (Just 1_000_000))++ describe "metrics arms" do+ it "constructs no provider under off" do+ runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsOff})+ isNothing runtime.meter `shouldBe` True+ isNothing runtime.provider `shouldBe` True+ isNothing runtime.endpoint `shouldBe` True++ it "collects instruments without opening an endpoint" do+ runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsCollect})+ let meter = fromJust runtime.meter+ metrics <- newSyntheticMetrics meter+ void (runSyntheticOperation Nothing (Just metrics) 0 0 1)+ (snapshot, _) <- stopMetrics 1000 runtime+ runtime.endpoint `shouldBe` Nothing+ fmap (.instruments) snapshot `shouldBe` Just 2++ it "serves its Prometheus reader only for a serving arm" do+ runtime <- startMetrics ((testSpec TracingOff) {metrics = MetricsServe, otelReader = ReaderPrometheus})+ runtime.endpoint `shouldSatisfy` maybe False ((== PrometheusText) . (.kind))+ traverse_ (\endpoint -> awaitHttpReady endpoint.url 1000 >>= (`shouldBe` True)) runtime.endpoint+ void (stopMetrics 1000 runtime)++ describe "metrics scraper" do+ it "skips fixed-schedule ticks when an endpoint is slower than its interval" do+ withSlowHttpServer 75_000 \endpoint -> do+ let output = ".tmp/telemetry-test/slow-scrape"+ createDirectoryIfMissing True output+ scraper <- startScraperInProcess output 20 0+ scraper.register endpoint+ threadDelay 260_000+ summaries <- scraper.finish+ fmap (.skippedTicks) summaries `shouldSatisfy` any (> 0)++ it "detects exhausted WebSocket slots and accepts a correct server" do+ withWebSocketServer (Just 2) \leaky -> do+ result <- wsSlotLeakProbe leaky 2+ result.freshAccepted `shouldBe` False+ withWebSocketServer Nothing \correct -> do+ result <- wsSlotLeakProbe correct 2+ result.freshAccepted `shouldBe` True++testSpec :: TracingArm -> TelemetrySpec+testSpec tracing =+ TelemetrySpec+ { tracing,+ metrics = MetricsOff,+ serviceName = "kenshou.test",+ sampler = AlwaysOn,+ processor = SimpleProcessor 1000,+ exporter = HttpProtobuf,+ endpoint = BuiltinSink SinkHealthy,+ compression = CompressionNone,+ probeRetain = 16,+ shutdownMs = 1000,+ otelReader = ReaderNone,+ otelExportMs = 1000,+ scrapeMs = 1000,+ wsSubscribers = 0,+ helpers = HelperInProcess,+ workerContext = Nothing,+ outDir = ".tmp/telemetry-test",+ report = const (pure ())+ }++forCompression :: OtlpCompression -> IO ()+forCompression compression = Sink.withSink SinkHealthy \sink -> do+ let spec = (testSpec TracingSdkOtlp) {endpoint = ExternalEndpoint sink.endpoint, compression}+ runtime <- startTracing spec+ let tracer = fromJust runtime.tracer+ inSpan tracer "exported" defaultSpanArguments (pure ())+ _ <- flushTracing 2000 runtime+ _ <- stopTracing 2000 runtime+ stats <- sink.snapshot+ stats.spansReceived `shouldBe` 1++isNothing :: Maybe value -> Bool+isNothing Nothing = True+isNothing (Just _) = False++unique :: (Eq value) => [value] -> [value]+unique [] = []+unique (value : values) = value : unique (filter (/= value) values)++withSlowHttpServer :: Int -> (Endpoint -> IO value) -> IO value+withSlowHttpServer delayMicros action = withWarp application \port ->+ action (Endpoint "slow-json" JsonDocument (Text.pack ("http://127.0.0.1:" <> show port <> "/metrics")) Nothing)+ where+ application _ respond = threadDelay delayMicros >> respond (responseLBS status200 [] "delayed")++withWebSocketServer :: Maybe Int -> (Endpoint -> IO value) -> IO value+withWebSocketServer capacity action = do+ accepted <- newIORef (0 :: Int)+ withWarp (websocketsOr WebSockets.defaultConnectionOptions (server accepted) fallback) \port ->+ action (Endpoint "slot-probe" WebSocketPush (Text.pack ("ws://127.0.0.1:" <> show port <> "/ws")) Nothing)+ where+ server accepted pendingConnection = do+ count <- atomicModifyIORef' accepted (\value -> let next = value + 1 in (next, next))+ case capacity of+ Just limit | count > limit -> WebSockets.rejectRequest pendingConnection "slots exhausted"+ _ -> void (WebSockets.acceptRequest pendingConnection)+ fallback _ respond = respond (responseLBS status404 [] "not found")++withWarp :: Application -> (Int -> IO value) -> IO value+withWarp application action = bracket acquire release (\(port, _, _) -> action port)+ where+ acquire = do+ (port, socket) <- openFreePort+ server <- async (runSettingsSocket defaultSettings socket application)+ pure (port, server, socket)+ release (_, server, socket) = cancel server >> void (waitCatch server) >> close socket++traverse_ :: (a -> IO b) -> Maybe a -> IO ()+traverse_ _ Nothing = pure ()+traverse_ action (Just value) = void (action value)++overheadRequest :: Map Text.Text [Text.Text] -> OverheadRequest+overheadRequest factors =+ OverheadRequest+ { requestId = "test-overhead",+ scenarioId = armsScenario.id,+ factors,+ mode = OneFactor,+ includeControl = False,+ trials = 3,+ fixedKnobs = [],+ fixedDimensions = Map.empty,+ seed = 7,+ settleSeconds = 0,+ retries = 1+ }++blocks :: [Slot] -> [[Slot]]+blocks slots = [[slot | slot <- slots, slot.block == block] | block <- unique (fmap (.block) slots)]++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft (Right _) = False++-- Keeping the provider and probe alive must not keep a span outside the+-- retention window alive. Do not force readSpans/spansSeen before the GC.+{-# NOINLINE probeWithEvictedSpan #-}+probeWithEvictedSpan :: Int -> IO (SpanProbe, TracerProvider, Weak SpanHot)+probeWithEvictedSpan capacity = do+ (probe, processor) <- newSpanProbe capacity+ first <- newEmptyMVar+ let observing =+ processor+ { spanProcessorOnEnd = \spanValue -> do+ hot <- readIORef spanValue.spanHot+ weak <- mkWeakPtr hot Nothing+ void (tryPutMVar first weak)+ processor.spanProcessorOnEnd spanValue+ }+ provider <- createTracerProvider [observing] emptyTracerProviderOptions+ let tracer = makeTracer provider (instrumentationLibrary "probe-retention-test" "0.1.0.0") tracerOptions+ replicateM_ 10 (inSpan tracer "evicted" defaultSpanArguments (pure ()))+ weak <- readMVar first+ pure (probe, provider, weak)
+ test/golden/overhead-report.minimal.json view
@@ -0,0 +1,21 @@+{+ "schema": "kenshou.overhead-report/v1",+ "id": "01a0c631-ff36-77c3-8772-216b4ab24eed",+ "scenario": "selftest/telemetry/benchmark/arms-on-synthetic-service",+ "mode": "one-factor",+ "baseline": {"telemetry.tracing": "off", "telemetry.metrics": "off"},+ "fixed": {"knobs": {}, "dimensions": {}},+ "trials": 3,+ "seed": 7,+ "validBlocks": 3,+ "order": [["a0", "a1"], ["a1", "a0"], ["a0", "a1"]],+ "arms": [+ {"arm": "a0", "dimensions": {"telemetry.tracing": "off"}, "runs": [], "telemetry": null, "leak": null, "findings": []}+ ],+ "comparisons": [],+ "findings": [],+ "policy": {"name": "telemetry-overhead-default", "sha256": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},+ "cohort": {},+ "verdict": "pass",+ "algorithm": {"name": "kenshou-overhead", "version": 1}+}