diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +## 0.1.2.0 — 2026-09-08 + +- Add the missing `bytestring` upper bound (`>=0.11 && <0.13`), matching the + rest of the package set. `cabal check` reported it under + `missing-upper-bounds`; the dependency was added this cycle without one. + +- Raise the internal `shikumi` bound to `^>=0.4.0.0` and `shikumi-trace` to `^>=0.3.0.0`, and give the test suite's bare `shikumi`, `shikumi-cache`, `shikumi-eval`, and `shikumi-testing` dependencies explicit PVP bounds. + +- Export separately scoped transport attempts and aggregate counters, including error status and canonical billing quality. Use observed-model evidence for gen_ai.response.model. Preserve existing exporter lifecycle and structural traversal. + ## 0.1.1.1 — 2026-08-29 ### Changed diff --git a/shikumi-trace-otel.cabal b/shikumi-trace-otel.cabal --- a/shikumi-trace-otel.cabal +++ b/shikumi-trace-otel.cabal @@ -1,6 +1,6 @@ cabal-version: 3.4 name: shikumi-trace-otel -version: 0.1.1.1 +version: 0.1.2.0 synopsis: OpenTelemetry export of shikumi hierarchical trace trees (EP-7) @@ -44,7 +44,9 @@ build-depends: , aeson >=2.2 && <2.3 + , baikai >=0.7.0.0 && <0.8 , base >=4.20 && <5 + , bytestring >=0.11 && <0.13 , containers >=0.6 && <0.9 , generic-lens >=2.2 && <2.4 , hs-opentelemetry-api >=1.0 && <1.1 @@ -53,7 +55,8 @@ , hs-opentelemetry-semantic-conventions >=1.40 && <2 , lens ^>=5.3 , scientific >=0.3 && <0.4 - , shikumi-trace ^>=0.2.0.0 + , shikumi ^>=0.4.0.0 + , shikumi-trace ^>=0.3.0.0 , text ^>=2.1 , time >=1.12 && <1.17 , unordered-containers >=0.2 && <0.3 @@ -63,16 +66,26 @@ type: exitcode-stdio-1.0 hs-source-dirs: test main-is: Main.hs + other-modules: BillingSpec ghc-options: -threaded -with-rtsopts=-N build-depends: , aeson + , baikai , base + , bytestring , containers + , effectful + , generic-lens , hs-opentelemetry-api >=1.0 && <1.1 , hs-opentelemetry-exporter-in-memory >=1.0 && <1.1 , hs-opentelemetry-sdk >=1.0 && <1.1 - , shikumi-trace ^>=0.2.0.0 - , shikumi-trace-otel ^>=0.1.1.0 + , lens + , shikumi ^>=0.4.0.0 + , shikumi-cache ^>=0.2.0.0 + , shikumi-eval ^>=0.3.0.0 + , shikumi-testing ^>=0.1.0.0 + , shikumi-trace ^>=0.3.0.0 + , shikumi-trace-otel ^>=0.1.2.0 , tasty , tasty-hunit , text diff --git a/src/Shikumi/Trace/OpenTelemetry.hs b/src/Shikumi/Trace/OpenTelemetry.hs --- a/src/Shikumi/Trace/OpenTelemetry.hs +++ b/src/Shikumi/Trace/OpenTelemetry.hs @@ -13,8 +13,7 @@ -- (@gen_ai.provider.name@, @gen_ai.request.model@, @gen_ai.response.model@, -- @gen_ai.usage.input_tokens@, @gen_ai.usage.output_tokens@, -- @gen_ai.operation.name@). The response model is read from the recorded --- response's echoed @model.modelId@ and omitted when no response model is --- present. Every span carries @shikumi.@-prefixed attributes +-- observed-model evidence and omitted when no such evidence is present. Every span carries @shikumi.@-prefixed attributes -- (@shikumi.span_kind@, @shikumi.retries@, optional @shikumi.incomplete@, and, -- on LM-call spans, @shikumi.cost.usd@ / @shikumi.latency_ms@). module Shikumi.Trace.OpenTelemetry @@ -22,11 +21,14 @@ ) where +import Baikai.Cost qualified as C +import Baikai.Usage qualified as U import Control.Lens ((^.)) -import Control.Monad (foldM) +import Control.Monad (foldM, forM_) import Control.Monad.IO.Class (MonadIO, liftIO) -import Data.Aeson (Value (..)) +import Data.Aeson (ToJSON, Value (..), encode) import Data.Aeson.KeyMap qualified as KM +import Data.ByteString.Lazy qualified as BL import Data.Functor (($>)) import Data.Generics.Labels () import Data.HashMap.Strict (HashMap) @@ -37,7 +39,8 @@ import Data.Scientific qualified as Sci import Data.Set qualified as Set import Data.Text (Text) -import Data.Time (UTCTime) +import Data.Text.Encoding qualified as TE +import Data.Time (UTCTime, getCurrentTime) import Data.Time.Clock.POSIX (utcTimeToPOSIXSeconds) import Data.Word (Word64) import OpenTelemetry.Attributes qualified as Attr @@ -46,6 +49,7 @@ import OpenTelemetry.Context qualified as Context import OpenTelemetry.SemanticConventions qualified as SC import OpenTelemetry.Trace.Core qualified as Otel +import Shikumi.LLM.Observation qualified as B import Shikumi.Trace (Span, SpanAttrs, SpanKind (..), TraceTree, childrenOf) import Shikumi.Trace.Node (renderNodePath) @@ -53,7 +57,9 @@ -- structural ('ProgramSpan' / 'ModuleSpan' / 'CombinatorSpan') and LM-call -- ('LlmCallSpan') nodes alike become spans; nesting is preserved. exportTree :: (MonadIO m) => Otel.Tracer -> TraceTree -> m () -exportTree tracer tree = liftIO (go Set.empty Context.empty (tree ^. #root) $> ()) +exportTree tracer tree = liftIO $ do + go Set.empty Context.empty (tree ^. #root) $> () + forM_ (tree ^. #transportBilling) (exportBilling tracer) where smap = tree ^. #spans go visited ctx sid @@ -104,12 +110,12 @@ genAiAttrs a m0 = maybe id (AttrMap.insertByKey SC.genAi_provider_name) (a ^. #provider) $ maybe id (AttrMap.insertByKey SC.genAi_request_model) (a ^. #model) $ - maybe id (AttrMap.insertByKey SC.genAi_response_model) (responseModelOf a) $ + maybe id (AttrMap.insertByKey SC.genAi_response_model) (a ^. #observedModel) $ maybe id (\n -> AttrMap.insertByKey SC.genAi_usage_inputTokens (fromIntegral n :: Int64)) (a ^. #inputTokens) $ maybe id (\n -> AttrMap.insertByKey SC.genAi_usage_outputTokens (fromIntegral n :: Int64)) (a ^. #outputTokens) $ maybe id (\u -> HashMap.insert "shikumi.cost.usd" (Attr.toAttribute (Sci.toRealFloat u :: Double))) (a ^. #costUsd) $ maybe id (\l -> HashMap.insert "shikumi.latency_ms" (Attr.toAttribute (fromIntegral l :: Int))) (a ^. #latencyMs) $ - AttrMap.insertByKey SC.genAi_operation_name ("chat" :: Text) m0 + AttrMap.insertByKey SC.genAi_operation_name ("chat" :: Text) (qualityAttrs (a ^. #billingQuality) (HashMap.insert "shikumi.accounting.scope" (Attr.toAttribute ("logical-call" :: Text)) m0)) -- | The end instant to export. A span that never closed is exported with its -- own start time rather than the wall clock at export time; 'attrsFor' marks @@ -140,15 +146,70 @@ messageErr _ = False isErrorResponse _ = False --- | The model the provider says actually answered, read from the recorded --- response's echoed @model.modelId@. 'Nothing' when no response was recorded --- or the field is absent. -responseModelOf :: SpanAttrs -> Maybe Text -responseModelOf a = do - Object o <- a ^. #response - Object m <- KM.lookup "model" o - String mid <- KM.lookup "modelId" m - pure mid +-- | Canonical provider basis/availability JSON; absent metadata stays absent. +qualityAttrs :: Maybe B.UsageRecord -> HashMap Text Attr.Attribute -> HashMap Text Attr.Attribute +qualityAttrs Nothing m = m +qualityAttrs (Just (B.UsageRecord u)) m = + maybe id (\b -> HashMap.insert "shikumi.cost.basis" (jsonAttribute b)) (C.nonEmptyBasis (U.cost u)) $ + maybe id (\a -> HashMap.insert "shikumi.usage.availability" (jsonAttribute a)) (U.availability u) m + +jsonAttribute :: (ToJSON a) => a -> Attr.Attribute +jsonAttribute = Attr.toAttribute . TE.decodeUtf8 . BL.toStrict . encode + +-- | Explicitly separate transport spans. Their correlation is callId/attempt; +-- no structural program parent or replay response is manufactured. +exportBilling :: Otel.Tracer -> B.BillingSummary -> IO () +exportBilling tracer summary = do + now <- getCurrentTime + sp <- + Otel.createSpan + tracer + Context.empty + "transport billing summary" + Otel.defaultSpanArguments + { Otel.startTime = Just (utcToTimestamp now) + } + Otel.addAttributes + sp + ( HashMap.fromList + [ ("shikumi.accounting.scope", Attr.toAttribute ("transport-summary" :: Text)), + ("shikumi.billing.completed_attempts", Attr.toAttribute (B.completedAttempts summary)), + ("shikumi.billing.failed_attempts", Attr.toAttribute (B.failedAttempts summary)), + ("shikumi.billing.unknown_usage_attempts", Attr.toAttribute (B.unknownUsageAttempts summary)), + ("shikumi.billing.detail_truncated", Attr.toAttribute (B.detailTruncated summary)) + ] + ) + Otel.endSpan sp (Just (utcToTimestamp now)) + forM_ (B.retainedAttempts summary) $ \o -> do + attemptSpan <- + Otel.createSpan + tracer + Context.empty + "LLM transport attempt" + Otel.defaultSpanArguments + { Otel.kind = Otel.Client, + Otel.startTime = Just (utcToTimestamp (B.startedAt o)) + } + let base = + HashMap.fromList + [ ("shikumi.accounting.scope", Attr.toAttribute ("transport-attempt" :: Text)), + ("shikumi.call_id", Attr.toAttribute (B.callId o)), + ("shikumi.attempt", Attr.toAttribute (B.attempt o)), + ("shikumi.call_kind", jsonAttribute (B.callKind o)), + ("gen_ai.request.model", Attr.toAttribute (B.requestedModel o)), + ("gen_ai.provider.name", Attr.toAttribute (B.requestedProvider o)), + ("shikumi.billing.unknown_usage", Attr.toAttribute (B.usageUnknown (B.usage o))) + ] + observed = maybe id (HashMap.insert "gen_ai.response.model" . Attr.toAttribute) (B.observedModel o) + numbers = case B.usage o of + Nothing -> id + Just (B.UsageRecord u) -> + HashMap.insert "gen_ai.usage.input_tokens" (Attr.toAttribute (fromIntegral (U.inputTokens u) :: Int64)) + . HashMap.insert "gen_ai.usage.output_tokens" (Attr.toAttribute (fromIntegral (U.outputTokens u) :: Int64)) + . HashMap.insert "shikumi.cost.usd" (Attr.toAttribute (fromRational (C.usd (U.cost u)) :: Double)) + Otel.addAttributes attemptSpan (observed (numbers (qualityAttrs (B.usage o) base))) + Otel.setStatus attemptSpan (maybe Otel.Ok Otel.Error (B.terminalError o)) + Otel.endSpan attemptSpan (Just (utcToTimestamp (B.endedAt o))) kindText :: SpanKind -> Text kindText = \case diff --git a/test/BillingSpec.hs b/test/BillingSpec.hs new file mode 100644 --- /dev/null +++ b/test/BillingSpec.hs @@ -0,0 +1,219 @@ +-- | Full offline evaluation through actual transport, runtime, cache and export. +module BillingSpec (tests) where + +import Baikai (Response, StopReason (ErrorReason), emptyContext, emptyOptions) +import Baikai.Cost qualified as C +import Baikai.Error qualified as BE +import Baikai.Evidence qualified as E +import Baikai.Usage qualified as U +import Control.Lens ((&), (.~), (^.)) +import Data.Aeson (eitherDecode, encode) +import Data.Generics.Labels () +import Data.HashMap.Strict qualified as HM +import Data.IORef (readIORef) +import Data.Map.Strict qualified as Map +import Data.Ratio ((%)) +import Data.Set qualified as Set +import Data.Text qualified as T +import Effectful (liftIO, runEff) +import Effectful.Concurrent (runConcurrent) +import Effectful.Concurrent.Async (mapConcurrently) +import Effectful.Error.Static (runErrorNoCallStack) +import Effectful.Prim (runPrim) +import OpenTelemetry.Attributes qualified as Attr +import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter) +import OpenTelemetry.Trace.Core qualified as Otel +import Shikumi.Cache (cachedLLM) +import Shikumi.Cache.Backend.Memory (newMemoryCache, runCacheMemory) +import Shikumi.Effect.Time (runTime) +import Shikumi.Error (ShikumiError) +import Shikumi.Eval.Evaluate (evaluateWith) +import Shikumi.Eval.Report qualified as R +import Shikumi.Eval.Types (Example (..), dataset, scoreOne) +import Shikumi.LLM qualified as L +import Shikumi.LLM.Defaults +import Shikumi.LLM.Observation qualified as B +import Shikumi.Routing (routeLLM, runRouting) +import Shikumi.Testing.Fixtures +import Shikumi.Testing.Transport +import Shikumi.Trace qualified as Tr +import Shikumi.Trace.LiveExport (exportTreeWith) +import Shikumi.Trace.Store (replayIndex) +import Test.Tasty (TestTree, testGroup) +import Test.Tasty.HUnit + +-- An estimate followed by provider-reported zero must retain the estimate. +knownUsage :: Rational -> C.CostBasis -> U.Usage +knownUsage amount basis = + U.zeroUsage + & #inputTokens .~ 10 + & #totalTokens .~ 10 + & #availability .~ Just (U.UsageAvailability Set.empty False Set.empty) + & #cost . #usd .~ amount + & #cost . #basis .~ basis + +responseWith :: U.Usage -> Response +responseWith u = validAnswerResponse & #message . #usage .~ u + +fixtureEvidence :: E.ModelCallEvidence +fixtureEvidence = + E.baseEvidence + (E.evidenceRequest "fixture") + "fixture-call" + (E.EndpointIdentity "fixture" "custom" E.TransportHttpApi Nothing "0.7.0.0" Nothing) + "requested-model" + E.noThinkingRequested + instant + instant + E.CallSucceeded + "digest" + "config" + & #observedModel .~ E.Observed "observed-model" + where + instant = read "2026-09-08 00:00:00 UTC" + +tests :: TestTree +tests = + testGroup + "EP-61 transport billing" + [ testCase "evaluation retry, reported zero, missing usage and cache hit stay separate" $ do + let estimated = C.standardCostBasis <> C.CostBasis Set.empty (Set.singleton C.PricingUnavailable) + failure = + responseWith (knownUsage (1 % 100) estimated) + & #message . #stopReason .~ ErrorReason + & #errorInfo .~ Just ((BE.providerError "private provider output") {BE.category = BE.TransientError}) + success = + responseWith (knownUsage (2 % 100) C.providerReportedBasis) + & #evidence .~ Just fixtureEvidence + zero = responseWith (knownUsage 0 C.providerReportedBasis & #inputTokens .~ 0 & #totalTokens .~ 0) + missing = + responseWith + ( U.zeroUsage + & #availability + .~ Just + ( U.UsageAvailability + (Set.fromList [U.InputUsage, U.OutputUsage, U.CacheReadUsage, U.CacheWriteUsage]) + False + Set.empty + ) + & #cost .~ C.estimateCost [C.UsageNotReported] C.zeroCost + ) + (model, registry, requests) <- scriptedTransport [failure, success, zero, missing] + (observer, snapshot) <- B.newBillingCollectorWithLimit 10 + cache <- newMemoryCache + let cfg = (L.defaultLLMConfig registry) {L.observer = Just observer, L.retryPolicy = L.RetryPolicy 2 0 0} + ds = dataset [Example (Question q) (Answer "It is forty-two." 0.9) | q <- ["retry", "zero", "missing", "retry"]] + (result, tree) <- + runEff + . runPrim + . runTime + . runConcurrent + . Tr.runTrace + . runErrorNoCallStack @ShikumiError + . runRouting model + . runCacheMemory cache + . L.runLLMResilient cfg + . cachedLLM + . Tr.tracedLLM + . withRequestDefaults (emptyRequestDefaults {defaultMaxTokens = Just 123}) + . routeLLM + $ Tr.withSpan Tr.ProgramSpan "billing evaluation" + $ evaluateWith (R.defaultEvalConfig {R.concurrency = 1}) ds (\_ _ -> pure scoreOne) instructedProg + report <- either (\e -> assertFailure (show e) >> fail "unreachable") pure result + summary <- snapshot + let attached = R.attachBillingSummary summary report + attachedTree = Tr.attachBillingSummary summary tree + R.totalCostUsd (R.usage attached) @?= 4 % 100 + C.usd (U.cost (B.getUsage (B.observedUsage summary))) @?= 3 % 100 + B.completedAttempts summary @?= 3 + B.failedAttempts summary @?= 1 + B.unknownUsageAttempts summary @?= 1 + map B.attempt (B.retainedAttempts summary) @?= [1, 2, 1, 1] + assertBool "estimate survives aggregation" (Set.member C.PricingUnavailable (C.estimateReasons (C.basis (U.cost (B.getUsage (B.observedUsage summary)))))) + captured <- requests + length captured @?= 4 + assertBool "observer sees effective defaults" (all (\(_, o) -> o ^. #maxTokens == Just 123) captured) + assertBool "both report views rendered" (all (`T.isInfixOf` R.renderReportText attached) ["logical usage quality", "transport billing", "usage_not_reported", "provider_reported_total"]) + eitherDecode (encode attachedTree) @?= Right attachedTree + replayIndex attachedTree @?= replayIndex tree + case replayIndex attachedTree of + Right index -> Map.size index @?= 3 + Left err -> assertFailure (T.unpack err) + assertBool "failure text never retained" (not ("private provider output" `T.isInfixOf` B.renderBillingSummary summary)) + (processor, ref) <- inMemoryListExporter + exportTreeWith processor "billing-test" attachedTree + emitted <- readIORef ref + attrs <- mapM (fmap (Attr.getAttributeMap . Otel.hotAttributes) . readIORef . Otel.spanHot) emitted + let scope name = filter ((== Just (Attr.toAttribute (name :: T.Text))) . HM.lookup "shikumi.accounting.scope") attrs + length (scope "logical-call") @?= 4 + length (filter (HM.member "gen_ai.response.model") (scope "logical-call")) @?= 1 + length (scope "transport-attempt") @?= 4 + length (scope "transport-summary") @?= 1 + assertBool "requested model is never claimed as observed" (all ((/= Just (Attr.toAttribute ("requested-model" :: T.Text))) . HM.lookup "gen_ai.response.model") attrs) + assertBool "actual observed identity exported" (any ((== Just (Attr.toAttribute ("observed-model" :: T.Text))) . HM.lookup "gen_ai.response.model") attrs) + assertBool "billing basis exported" (any (HM.member "shikumi.cost.basis") (scope "transport-attempt")) + statuses <- mapM (fmap Otel.hotStatus . readIORef . Otel.spanHot) emitted + assertBool "failed attempt is an error span" (Otel.Error "TransientError" `elem` statuses), + testCase "FailScore and FailAbort retain terminal failure billing" $ do + let failure = + responseWith (knownUsage (1 % 100) C.providerReportedBasis) + & #message . #stopReason .~ ErrorReason + & #errorInfo .~ Just ((BE.providerError "refused") {BE.category = BE.ContentFiltered}) + ds = dataset [Example (Question "refuse") (Answer "unused" 1)] + run policy = do + (model, registry, _) <- scriptedTransport [failure] + (observer, snapshot) <- B.newBillingCollector + result <- + runEff + . runPrim + . runTime + . runConcurrent + . runErrorNoCallStack @ShikumiError + . runRouting model + . L.runLLMResilient ((L.defaultLLMConfig registry) {L.observer = Just observer}) + . routeLLM + $ evaluateWith (R.defaultEvalConfig {R.failurePolicy = policy}) ds (\_ _ -> pure scoreOne) instructedProg + summary <- snapshot + B.failedAttempts summary @?= 1 + C.usd (U.cost (B.getUsage (B.observedUsage summary))) @?= 1 % 100 + pure result + scored <- run (R.failurePolicy R.defaultEvalConfig) + case scored of + Right r -> do R.failCount r @?= 1; R.totalCostUsd (R.usage r) @?= 0 + Left e -> assertFailure (show e) + aborted <- run R.FailAbort + assertBool "FailAbort propagates" (either (const True) (const False) aborted), + testCase "bare completion and stream agree; separate concurrent collectors do not mix" $ do + let action amount = do + (model, registry, _) <- scriptedTransport [responseWith (knownUsage amount C.standardCostBasis)] + (observer, snapshot) <- B.newBillingCollectorWithLimit 2 + _ <- + runEff . runErrorNoCallStack @ShikumiError . L.runLLMWithObserver registry observer $ + L.complete model emptyContext emptyOptions + snapshot + summaries <- runEff . runConcurrent $ mapConcurrently (liftIO . action) [1 % 3, 2 % 3] + map (C.usd . U.cost . B.getUsage . B.observedUsage) summaries @?= [1 % 3, 2 % 3] + a <- action (1 % 3) + (model, registry, _) <- scriptedTransport [responseWith (knownUsage (1 % 3) C.standardCostBasis) & #evidence .~ Just fixtureEvidence] + (observer, snapshot) <- B.newBillingCollectorWithLimit 2 + (_, streamTree) <- + runEff + . runPrim + . runTime + . Tr.runTrace + . runErrorNoCallStack @ShikumiError + . L.runLLMWithObserver registry observer + . Tr.tracedLLM + $ L.stream model emptyContext emptyOptions + case Map.elems (Tr.spans streamTree) of + [span] -> do + Tr.billingQuality (Tr.attrs span) @?= Just (B.UsageRecord (knownUsage (1 % 3) C.standardCostBasis)) + Tr.observedModel (Tr.attrs span) @?= Just "observed-model" + Tr.response (Tr.attrs span) @?= Nothing + _ -> assertFailure "expected one streaming span" + replayIndex streamTree @?= Right Map.empty + b <- snapshot + B.observedUsage a @?= B.observedUsage b + B.completedAttempts a @?= 1 + B.completedAttempts b @?= 1 + ] diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -6,6 +6,7 @@ -- non-root span's parent is another emitted span). module Main (main) where +import BillingSpec qualified import Control.Exception (SomeException, try) import Data.Aeson (Value (..), object, (.=)) import Data.HashMap.Strict qualified as HashMap @@ -42,7 +43,8 @@ defaultMain $ testGroup "shikumi-trace-otel" - [ nestingTest, + [ BillingSpec.tests, + nestingTest, liveExportInMemoryTest, exceptionReleasesProviderTest, responseStatusAndModelTest, @@ -132,10 +134,10 @@ other -> assertFailure ("expected Error status, got " <> show other) spanStatus okSpan >>= assertEqual "successful response status" Otel.Ok errHasResponseModel <- spanHasAttr "gen_ai.response.model" errSpan - assertBool "error response without echoed model omits gen_ai.response.model" (not errHasResponseModel) + assertBool "error response without observed model omits gen_ai.response.model" (not errHasResponseModel) spanAttr "gen_ai.response.model" okSpan >>= assertEqual - "response model comes from echoed response model" + "response model comes from observed evidence" (Just (Attr.toAttribute ("claude-sonnet-4-6-20250929" :: Text))) openSpanTest :: TestTree @@ -217,7 +219,8 @@ fixedTree :: TraceTree fixedTree = TraceTree - { root = SpanId "s0", + { transportBilling = Nothing, + root = SpanId "s0", spans = Map.fromList [ (SpanId "s0", node "s0" Nothing ProgramSpan "summarize-and-critique" emptyAttrs 0), @@ -231,7 +234,8 @@ throwingTree :: TraceTree throwingTree = TraceTree - { root = SpanId "s0", + { transportBilling = Nothing, + root = SpanId "s0", spans = Map.fromList [ (SpanId "s0", node "s0" Nothing ProgramSpan (error "trace label exploded") emptyAttrs 0) @@ -241,7 +245,8 @@ responseTree :: TraceTree responseTree = TraceTree - { root = SpanId "s0", + { transportBilling = Nothing, + root = SpanId "s0", spans = Map.fromList [ (SpanId "s0", node "s0" Nothing ProgramSpan "responses" emptyAttrs 0), @@ -263,7 +268,8 @@ rootCyclicTree :: TraceTree rootCyclicTree = TraceTree - { root = SpanId "s0", + { transportBilling = Nothing, + root = SpanId "s0", spans = Map.fromList [ (SpanId "s0", node "s0" (Just "s0") ProgramSpan "self-parented" emptyAttrs 0) @@ -309,7 +315,8 @@ okResponseAttrs :: SpanAttrs okResponseAttrs = llmAttrs - { response = + { observedModel = Just "claude-sonnet-4-6-20250929", + response = Just ( object [ "errorInfo" .= Null,