diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## Unreleased + +## 0.1.0.0 - 2026-06-13 + +### Added + +- Initial Hackage release of OpenTelemetry export for shikumi trace trees. +- Batch and live export helpers that map shikumi spans to OpenTelemetry spans with GenAI-oriented attributes. diff --git a/shikumi-trace-otel.cabal b/shikumi-trace-otel.cabal new file mode 100644 --- /dev/null +++ b/shikumi-trace-otel.cabal @@ -0,0 +1,78 @@ +cabal-version: 3.4 +name: shikumi-trace-otel +version: 0.1.0.0 +synopsis: + OpenTelemetry export of shikumi hierarchical trace trees (EP-7) + +category: AI +description: + An opt-in OpenTelemetry adapter for shikumi's hierarchical trace trees. + 'Shikumi.Trace.OpenTelemetry.exportTree' walks a finished 'TraceTree' and emits + one OTel span per node, preserving parent/child nesting (each child span is + created in a context carrying its parent span). LM-call spans carry GenAI + semantic-convention attributes; every span carries @shikumi.@-prefixed + span-kind/cost/latency/retry attributes. The heavy @hs-opentelemetry-*@ + dependency tree is isolated in this package, mirroring baikai's + @baikai-trace-otel@ split. + +license: BSD-3-Clause +author: Nadeem Bitar +maintainer: nadeem@gmail.com +build-type: Simple +extra-doc-files: CHANGELOG.md + +common common-options + ghc-options: + -Wall -Wcompat -Widentities -Wincomplete-uni-patterns + -Wincomplete-record-updates -Wredundant-constraints + -fhide-source-paths -Wmissing-export-lists -Wpartial-fields + -Wmissing-deriving-strategies + + default-language: GHC2024 + default-extensions: + DeriveAnyClass + DuplicateRecordFields + OverloadedLabels + OverloadedStrings + +library + import: common-options + hs-source-dirs: src + exposed-modules: + Shikumi.Trace.LiveExport + Shikumi.Trace.OpenTelemetry + + build-depends: + , base >=4.20 && <5 + , containers + , generic-lens + , hs-opentelemetry-api >=1.0 && <1.1 + , hs-opentelemetry-exporter-otlp >=1.0 && <1.1 + , hs-opentelemetry-sdk >=1.0 && <1.1 + , hs-opentelemetry-semantic-conventions >=1.40 && <2 + , lens ^>=5.3 + , scientific + , shikumi-trace ^>=0.1.0.0 + , text ^>=2.1 + , time + , unordered-containers + +test-suite shikumi-trace-otel-test + import: common-options + type: exitcode-stdio-1.0 + hs-source-dirs: test + main-is: Main.hs + ghc-options: -threaded -with-rtsopts=-N + build-depends: + , base + , containers + , hs-opentelemetry-api >=1.0 && <1.1 + , hs-opentelemetry-exporter-in-memory >=1.0 && <1.1 + , hs-opentelemetry-sdk >=1.0 && <1.1 + , shikumi-trace ^>=0.1.0.0 + , shikumi-trace-otel ^>=0.1.0.0 + , tasty + , tasty-hunit + , text + , time + , unordered-containers diff --git a/src/Shikumi/Trace/LiveExport.hs b/src/Shikumi/Trace/LiveExport.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Trace/LiveExport.hs @@ -0,0 +1,57 @@ +-- | Live OpenTelemetry export of a shikumi 'TraceTree' (EP-17). +-- +-- 'Shikumi.Trace.OpenTelemetry.exportTree' turns a finished tree into nested OTel +-- spans against a 'Otel.Tracer', but a bare tracer exports nothing on its own — it +-- needs a 'TracerProvider' holding a span processor that wraps an exporter. This +-- module supplies that orchestration: +-- +-- * 'exportTreeWith' builds a provider from a given 'SpanProcessor', makes a +-- tracer, runs 'exportTree', then flushes and shuts the provider down. Factored +-- on the processor so the hermetic test passes the in-memory recording +-- processor and the live path passes the OTLP batch processor — one code path. +-- * 'exportTreeLive' builds the real OTLP/HTTP exporter from the standard +-- @OTEL_*@ environment variables (default endpoint @http:\/\/localhost:4318@), +-- wraps it in a batch processor, and exports through 'exportTreeWith'. +-- +-- The batch processor requires the @-threaded@ runtime; both this package's test +-- suite and the @shikumi@ executable already pass it. Because the batch processor +-- exports on a timer, 'exportTreeWith' shuts the provider down before returning so +-- the spans are flushed. +module Shikumi.Trace.LiveExport + ( exportTreeWith, + exportTreeLive, + ) +where + +import Control.Monad.IO.Class (MonadIO) +import Data.String (fromString) +import Data.Text (Text) +import Data.Text qualified as T +import OpenTelemetry.Exporter.OTLP.Span (loadExporterEnvironmentVariables, otlpExporter) +import OpenTelemetry.Processor.Batch.Span (batchProcessor, batchTimeoutConfig) +import OpenTelemetry.Processor.Span (SpanProcessor) +import OpenTelemetry.Trace.Core qualified as Otel +import Shikumi.Trace (TraceTree) +import Shikumi.Trace.OpenTelemetry (exportTree) + +-- | Build a 'TracerProvider' around the given processor, export the tree through +-- the shared 'exportTree' walker, then flush and shut the provider down. The +-- shutdown flushes the batch processor, so spans have left by the time this +-- returns. +exportTreeWith :: (MonadIO m) => SpanProcessor -> Text -> TraceTree -> m () +exportTreeWith processor name tree = do + tp <- Otel.createTracerProvider [processor] Otel.emptyTracerProviderOptions + let tracer = Otel.makeTracer tp (fromString (T.unpack name)) Otel.tracerOptions + exportTree tracer tree + _ <- Otel.shutdownTracerProvider tp Nothing + pure () + +-- | The live path: load OTLP config from the standard @OTEL_EXPORTER_OTLP_*@ +-- environment variables (default endpoint @http:\/\/localhost:4318@), build the +-- OTLP/HTTP exporter, wrap it in a batch processor, and export the tree. +exportTreeLive :: (MonadIO m) => Text -> TraceTree -> m () +exportTreeLive name tree = do + cfg <- loadExporterEnvironmentVariables + exporter <- otlpExporter cfg + processor <- batchProcessor batchTimeoutConfig exporter + exportTreeWith processor name tree diff --git a/src/Shikumi/Trace/OpenTelemetry.hs b/src/Shikumi/Trace/OpenTelemetry.hs new file mode 100644 --- /dev/null +++ b/src/Shikumi/Trace/OpenTelemetry.hs @@ -0,0 +1,117 @@ +-- | Export a shikumi 'TraceTree' as nested OpenTelemetry spans (EP-7, M4). +-- +-- Unlike baikai's @baikai-trace-otel@ (one flat span per provider call), +-- 'exportTree' preserves the tree's parent/child nesting: it walks from the root +-- depth-first and creates each child span in a 'Context' carrying its parent span +-- (via 'OpenTelemetry.Context.insertSpan'), the explicit-context approach for +-- reconstructing a recorded tree rather than discovering it from the live call +-- stack. Each span's start/end times come from the recorded 'startedAt'/'endedAt'. +-- +-- LM-call spans carry GenAI semantic-convention attributes +-- (@gen_ai.provider.name@, @gen_ai.request.model@, @gen_ai.response.model@, +-- @gen_ai.usage.input_tokens@, @gen_ai.usage.output_tokens@, +-- @gen_ai.operation.name@); every span carries @shikumi.@-prefixed attributes +-- (@shikumi.span_kind@, @shikumi.retries@, and, on LM-call spans, +-- @shikumi.cost.usd@ / @shikumi.latency_ms@). +module Shikumi.Trace.OpenTelemetry + ( exportTree, + ) +where + +import Control.Lens ((^.)) +import Control.Monad.IO.Class (MonadIO, liftIO) +import Data.Generics.Labels () +import Data.HashMap.Strict (HashMap) +import Data.HashMap.Strict qualified as HashMap +import Data.Int (Int64) +import Data.Map.Strict qualified as Map +import Data.Scientific qualified as Sci +import Data.Text (Text) +import Data.Time (UTCTime) +import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) +import Data.Word (Word64) +import OpenTelemetry.Attributes qualified as Attr +import OpenTelemetry.Attributes.Map qualified as AttrMap +import OpenTelemetry.Common (Timestamp, mkTimestamp) +import OpenTelemetry.Context qualified as Context +import OpenTelemetry.SemanticConventions qualified as SC +import OpenTelemetry.Trace.Core qualified as Otel +import Shikumi.Trace (Span, SpanAttrs, SpanKind (..), TraceTree, childrenOf) +import Shikumi.Trace.Node (renderNodePath) + +-- | Export a finished trace tree to a tracer as a forest of nested spans. Pure +-- structural ('ProgramSpan' / 'ModuleSpan' / 'CombinatorSpan') and LM-call +-- ('LlmCallSpan') nodes alike become spans; nesting is preserved. +exportTree :: (MonadIO m) => Otel.Tracer -> TraceTree -> m () +exportTree tracer tree = liftIO (go Context.empty (tree ^. #root)) + where + smap = tree ^. #spans + go ctx sid = case Map.lookup sid smap of + Nothing -> pure () + Just s -> do + sp <- Otel.createSpan tracer ctx (s ^. #label) (argsFor s) + Otel.addAttributes sp (attrsFor s) + Otel.setStatus sp Otel.Ok + let childCtx = Context.insertSpan sp Context.empty + mapM_ (go childCtx) (childrenOf tree sid) + Otel.endSpan sp (utcToTimestamp <$> (s ^. #endedAt)) + + argsFor :: Span -> Otel.SpanArguments + argsFor s = + Otel.defaultSpanArguments + { Otel.kind = if (s ^. #kind) == LlmCallSpan then Otel.Client else Otel.Internal, + Otel.startTime = Just (utcToTimestamp (s ^. #startedAt)) + } + +-- | The OTel attribute map for a span. Every span carries @shikumi.span_kind@ and +-- @shikumi.retries@, plus — when the source span was produced by +-- @runProgramTraced@ (EP-16) — @shikumi.node_path@, the structural path of the +-- @Program@ node that issued it. LM-call spans additionally carry the GenAI +-- attributes. +attrsFor :: Span -> HashMap Text Attr.Attribute +attrsFor s = + let a = s ^. #attrs + base = + HashMap.fromList + [ ("shikumi.span_kind", Attr.toAttribute (kindText (s ^. #kind))), + ("shikumi.retries", Attr.toAttribute (a ^. #retries)) + ] + withNode = + maybe + id + (\p -> HashMap.insert "shikumi.node_path" (Attr.toAttribute (renderNodePath p))) + (a ^. #nodePath) + in withNode (if (s ^. #kind) == LlmCallSpan then genAiAttrs a base else base) + +-- | Overlay the GenAI semantic-convention attributes (plus shikumi cost/latency) +-- for an LM-call span. +genAiAttrs :: SpanAttrs -> HashMap Text Attr.Attribute -> HashMap Text Attr.Attribute +genAiAttrs a m0 = + maybe id (AttrMap.insertByKey SC.genAi_provider_name) (a ^. #provider) $ + maybe id (AttrMap.insertByKey SC.genAi_request_model) (a ^. #model) $ + maybe id (AttrMap.insertByKey SC.genAi_response_model) (a ^. #model) $ + maybe id (\n -> AttrMap.insertByKey SC.genAi_usage_inputTokens (fromIntegral n :: Int64)) (a ^. #inputTokens) $ + maybe id (\n -> AttrMap.insertByKey SC.genAi_usage_outputTokens (fromIntegral n :: Int64)) (a ^. #outputTokens) $ + maybe id (\u -> HashMap.insert "shikumi.cost.usd" (Attr.toAttribute (Sci.toRealFloat u :: Double))) (a ^. #costUsd) $ + maybe id (\l -> HashMap.insert "shikumi.latency_ms" (Attr.toAttribute (fromIntegral l :: Int))) (a ^. #latencyMs) $ + AttrMap.insertByKey SC.genAi_operation_name ("chat" :: Text) m0 + +kindText :: SpanKind -> Text +kindText = \case + ProgramSpan -> "program" + ModuleSpan -> "module" + CombinatorSpan -> "combinator" + LlmCallSpan -> "llm-call" + +-- | Convert a 'UTCTime' to an OpenTelemetry 'Timestamp' (nanoseconds since the +-- Unix epoch), going via 'Rational' so no precision is dropped (mirrors baikai's +-- adapter). +utcToTimestamp :: UTCTime -> Timestamp +utcToTimestamp t = + let posix :: Rational + posix = toRational (utcTimeToPOSIXSeconds t) + secs :: Word64 + secs = floor posix + nanos :: Word64 + nanos = floor ((posix - toRational secs) * 1_000_000_000) + in mkTimestamp secs nanos diff --git a/test/Main.hs b/test/Main.hs new file mode 100644 --- /dev/null +++ b/test/Main.hs @@ -0,0 +1,153 @@ +{-# LANGUAGE ScopedTypeVariables #-} + +-- | M4 acceptance for shikumi-trace-otel: exporting a fixed two-stage trace tree +-- emits the right number of OTel spans, the LM-call spans carry GenAI attributes, +-- and the parent/child nesting survives (all spans share one trace id; each +-- non-root span's parent is another emitted span). +module Main (main) where + +import Data.HashMap.Strict qualified as HashMap +import Data.IORef (IORef, readIORef) +import Data.Map.Strict qualified as Map +import Data.Maybe (catMaybes) +import Data.Text (Text) +import Data.Time (UTCTime) +import OpenTelemetry.Attributes qualified as Attr +import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter) +import OpenTelemetry.Trace.Core qualified as Otel +import OpenTelemetry.Trace.Id qualified as OtelId +import Shikumi.Trace + ( Span (..), + SpanAttrs (..), + SpanId (..), + SpanKind (..), + TraceTree (..), + emptyAttrs, + ) +import Shikumi.Trace.LiveExport (exportTreeWith) +import Shikumi.Trace.Node (NodePath (..), NodeStep (..)) +import Shikumi.Trace.OpenTelemetry (exportTree) +import Test.Tasty (TestTree, defaultMain, testGroup) +import Test.Tasty.HUnit (assertBool, assertEqual, testCase) + +main :: IO () +main = defaultMain $ testGroup "shikumi-trace-otel" [nestingTest, liveExportInMemoryTest] + +nestingTest :: TestTree +nestingTest = + testCase "exportTree emits nested spans with GenAI attributes" $ do + (tracer, getSpans) <- newTracerWithInMemory + exportTree tracer fixedTree + emitted <- getSpans + -- (a) one span per tree node. + assertEqual "one OTel span per tree node" (Map.size (spans fixedTree)) (length emitted) + -- (b) the two LM-call spans carry gen_ai.request.model. + let llmSpans = [s | s <- emitted, Otel.spanKind s == Otel.Client] + assertEqual "two LM-call (Client) spans" 2 (length llmSpans) + modelled <- + mapM + ( \s -> do + hot <- readIORef (Otel.spanHot s) + pure (HashMap.member "gen_ai.request.model" (Attr.getAttributeMap (Otel.hotAttributes hot))) + ) + llmSpans + assertBool "every LM-call span has gen_ai.request.model" (and modelled) + -- (c) nesting survived: all spans share one trace id; every non-root span's + -- parent is one of the emitted spans. + let traceIds = map (Otel.traceId . Otel.spanContext) emitted + case traceIds of + (t0 : rest) -> assertBool "all spans share one trace id" (all (== t0) rest) + [] -> assertBool "no spans" False + let emittedSpanIds = map (Otel.spanId . Otel.spanContext) emitted + parentIds <- mapM parentSpanId emitted + let presentParents = catMaybes parentIds + assertEqual "exactly one root (no parent)" 1 (length emitted - length presentParents) + assertBool + "every parent is an emitted span" + (all (`elem` emittedSpanIds) presentParents) + +-- | EP-17 M1+M3: the full live orchestration (provider + batch processor + export +-- + shutdown-flush) driven through 'exportTreeWith' with the in-memory recording +-- processor swapped in for the OTLP one — the exact code path the CLI calls, minus +-- the network. Asserts one span per node, the GenAI attributes, and (EP-16 landed) +-- the @shikumi.node_path@ attribute on the model-call spans. +liveExportInMemoryTest :: TestTree +liveExportInMemoryTest = + testCase "exportTreeWith (recording processor) preserves nesting and node attrs" $ do + (proc, spansRef :: IORef [Otel.ImmutableSpan]) <- inMemoryListExporter + exportTreeWith proc "shikumi-live-test" fixedTree + emitted <- reverse <$> readIORef spansRef + assertEqual "one OTel span per tree node" (Map.size (spans fixedTree)) (length emitted) + let llmSpans = [s | s <- emitted, Otel.spanKind s == Otel.Client] + assertEqual "two LM-call (Client) spans" 2 (length llmSpans) + modelled <- mapM (spanHasAttr "gen_ai.request.model") llmSpans + assertBool "every LM-call span has gen_ai.request.model" (and modelled) + tagged <- mapM (spanHasAttr "shikumi.node_path") llmSpans + assertBool "every LM-call span carries shikumi.node_path (EP-16 landed)" (and tagged) + +-- | Whether the recorded span's attribute map contains a key. +spanHasAttr :: Text -> Otel.ImmutableSpan -> IO Bool +spanHasAttr key s = do + hot <- readIORef (Otel.spanHot s) + pure (HashMap.member key (Attr.getAttributeMap (Otel.hotAttributes hot))) + +-- | The recorded parent span id of an immutable span, if it has a parent. +parentSpanId :: Otel.ImmutableSpan -> IO (Maybe OtelId.SpanId) +parentSpanId s = case Otel.spanParent s of + Nothing -> pure Nothing + Just p -> Just . Otel.spanId <$> Otel.getSpanContext p + +-- | A tracer wired to an in-memory span exporter, plus a reader for the captured +-- spans (in emission order). +newTracerWithInMemory :: IO (Otel.Tracer, IO [Otel.ImmutableSpan]) +newTracerWithInMemory = do + (proc, spansRef :: IORef [Otel.ImmutableSpan]) <- inMemoryListExporter + tp <- Otel.createTracerProvider [proc] Otel.emptyTracerProviderOptions + let tracer = Otel.makeTracer tp "shikumi-trace-otel-test" Otel.tracerOptions + pure (tracer, reverse <$> readIORef spansRef) + +-- --------------------------------------------------------------------------- +-- A fixed two-stage tree: program -> {draft module -> draft llm, critique module +-- -> critique llm}. Five spans, two of them LM-call leaves. +-- --------------------------------------------------------------------------- + +fixedTree :: TraceTree +fixedTree = + TraceTree + { root = SpanId "s0", + spans = + Map.fromList + [ (SpanId "s0", node "s0" Nothing ProgramSpan "summarize-and-critique" emptyAttrs 0), + (SpanId "s1", node "s1" (Just "s0") ModuleSpan "predict:Draft" emptyAttrs 1), + (SpanId "s2", node "s2" (Just "s1") LlmCallSpan "anthropic/claude" llmAttrs 2), + (SpanId "s3", node "s3" (Just "s0") ModuleSpan "predict:Critique" emptyAttrs 3), + (SpanId "s4", node "s4" (Just "s3") LlmCallSpan "anthropic/claude" llmAttrs 4) + ] + } + +node :: Text -> Maybe Text -> SpanKind -> Text -> SpanAttrs -> Int -> Span +node sid par k lbl a n = + Span + { spanId = SpanId sid, + parent = SpanId <$> par, + kind = k, + label = lbl, + startedAt = at (2 * n), + endedAt = Just (at (2 * n + 1)), + attrs = a + } + +llmAttrs :: SpanAttrs +llmAttrs = + emptyAttrs + { model = Just "claude-sonnet-4-6", + provider = Just "anthropic", + inputTokens = Just 812, + outputTokens = Just 143, + latencyMs = Just 205, + -- EP-16 landed: model-call spans carry their issuing node's path. + nodePath = Just (NodePath [StepComposeL]) + } + +at :: Int -> UTCTime +at n = read ("2026-06-08 00:00:0" <> show n <> " UTC")