keiro 0.24.0.0 → 0.25.0.0
raw patch · 33 files changed
+2222/−266 lines, 33 filesdep +asyncdep ~keiro-coredep ~keiro-migrationsdep ~keiro-test-supportPVP ok
version bump matches the API change (PVP)
Dependencies added: async
Dependency ranges changed: keiro-core, keiro-migrations, keiro-test-support, shibuya-core, shibuya-kiroku-adapter
API changes (from Hackage documentation)
- Keiro.Timer: instance GHC.Classes.Eq Keiro.Timer.TimerWorkerOptions
+ Keiro.Outbox: DeriveTraceAfter :: !GlobalPosition -> ProducerTracePolicy
+ Keiro.Outbox: PreserveDraftTrace :: ProducerTracePolicy
+ Keiro.Outbox: [tracePolicy] :: IntegrationProducer e -> !ProducerTracePolicy
+ Keiro.Outbox: data ProducerTracePolicy
+ Keiro.Outbox: instance GHC.Classes.Eq Keiro.Outbox.ProducerTracePolicy
+ Keiro.Outbox: instance GHC.Internal.Generics.Generic Keiro.Outbox.ProducerTracePolicy
+ Keiro.Outbox: instance GHC.Internal.Show.Show Keiro.Outbox.ProducerTracePolicy
+ Keiro.Telemetry: creationLinkFromTraceContext :: TraceContext -> Maybe NewLink
+ Keiro.Telemetry: recordSubscriptionOutcome :: MonadIO m => Maybe Span -> AckDecision -> m ()
+ Keiro.Telemetry: recordWorkflowResumeLinks :: MonadIO m => Maybe Span -> Maybe TraceContext -> Maybe TraceContext -> m ()
+ Keiro.Telemetry: stampTraceContext :: MonadIO m => Maybe Value -> m (Maybe Value)
+ Keiro.Telemetry: traceContextFromMetadata :: Maybe Value -> Maybe TraceContext
+ Keiro.Telemetry: withProducerBatchSpan :: (MonadUnliftIO m, HasCallStack) => Maybe Tracer -> [OutboxRow] -> (Maybe Span -> m a) -> m a
+ Keiro.Telemetry: withSubscriptionProcessSpan :: MonadUnliftIO m => Maybe Tracer -> TraceCorrelation -> Text -> Envelope msg -> (Maybe Span -> m a) -> m a
+ Keiro.Telemetry: withTimerFireSpan :: (MonadUnliftIO m, HasCallStack) => Maybe Tracer -> Text -> Text -> Maybe TraceContext -> (Maybe Span -> m a) -> m a
+ Keiro.Timer: [traceContext] :: TimerRow -> !Maybe TraceContext
+ Keiro.Timer: [tracer] :: TimerWorkerOptions -> !Maybe Tracer
+ Keiro.Timer: captureTimerTraceContext :: MonadIO m => Maybe Tracer -> TimerRequest -> m TimerRequest
+ Keiro.Timer: withTimerTraceContext :: Maybe TraceContext -> TimerRequest -> TimerRequest
+ Keiro.Timer.Schema: [traceContext] :: TimerRow -> !Maybe TraceContext
+ Keiro.Timer.Types: [traceContext] :: TimerRequest -> !Maybe TraceContext
+ Keiro.Timer.Types: withTimerTraceContext :: Maybe TraceContext -> TimerRequest -> TimerRequest
+ Keiro.Workflow: appendJournalEntryReturningIdTraced :: forall (es :: [Effect]). (IOE :> es, Store :> es) => Maybe Tracer -> WorkflowName -> WorkflowId -> WorkflowJournalEvent -> Eff es EventId
+ Keiro.Workflow: appendJournalEntryTraced :: forall (es :: [Effect]). (IOE :> es, Store :> es) => Maybe Tracer -> WorkflowName -> WorkflowId -> WorkflowJournalEvent -> Eff es ()
+ Keiro.Workflow: currentWorkflowTracer :: forall (es :: [Effect]). Workflow :> es => Eff es (Maybe Tracer)
+ Keiro.Workflow: prepareJournalAppendWithMetadata :: forall (es :: [Effect]). IOE :> es => Maybe Value -> WorkflowName -> WorkflowId -> Int -> WorkflowJournalEvent -> Eff es (Transaction JournalAppendOutcome)
+ Keiro.Workflow.Awakeable: signalAwakeableFromTraced :: forall (es :: [Effect]) r. (IOE :> es, Store :> es, ToJSON r) => Maybe Tracer -> AwakeableRow -> r -> Eff es Bool
+ Keiro.Workflow.Awakeable: signalAwakeableTraced :: forall (es :: [Effect]) r. (IOE :> es, Store :> es, ToJSON r) => Maybe Tracer -> AwakeableId -> r -> Eff es Bool
+ Keiro.Workflow.Child: childCompletionHookTraced :: forall (es :: [Effect]). (IOE :> es, Store :> es) => Maybe Tracer -> WorkflowName -> WorkflowId -> Value -> Eff es ()
+ Keiro.Workflow.Instance: lookupTraceAwaitedStep :: forall (es :: [Effect]). Store :> es => WorkflowName -> WorkflowId -> Int -> Eff es (Maybe Text)
+ Keiro.Workflow.Instance: markInstanceSuspendedAwaitingTraced :: forall (es :: [Effect]). Store :> es => Bool -> WorkflowName -> WorkflowId -> Int -> Text -> Eff es ()
+ Keiro.Workflow.Instance: upsertSuspendedInstanceTx :: Text -> Text -> Int32 -> WorkflowStatus -> Maybe Text -> Transaction ()
+ Keiro.Workflow.Schema: currentGenerationTx :: WorkflowName -> WorkflowId -> Transaction Int
+ Keiro.Workflow.Sleep: drainWorkflowSleepTimersWith :: forall (es :: [Effect]). (IOE :> es, Store :> es) => TimerWorkerOptions -> Maybe KeiroMetrics -> UTCTime -> Int -> (TimerRow -> Eff es (Maybe EventId)) -> Eff es Int
+ Keiro.Workflow.Sleep: runWorkflowTimerWorkerWith :: forall (es :: [Effect]). (IOE :> es, Store :> es) => TimerWorkerOptions -> Maybe KeiroMetrics -> UTCTime -> (TimerRow -> Eff es (Maybe EventId)) -> Eff es (Maybe TimerRow)
+ Keiro.Workflow.Sleep: workflowSleepFireActionTraced :: forall (es :: [Effect]). (Store :> es, IOE :> es) => Maybe Tracer -> TimerRow -> Eff es (Maybe EventId)
- Keiro.Outbox: IntegrationProducer :: !Text -> !Text -> !Text -> !RecordedEvent -> e -> Maybe IntegrationEventDraft -> IntegrationProducer e
+ Keiro.Outbox: IntegrationProducer :: !Text -> !Text -> !Text -> !ProducerTracePolicy -> !RecordedEvent -> e -> Maybe IntegrationEventDraft -> IntegrationProducer e
- Keiro.Timer: TimerRequest :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> TimerRequest
+ Keiro.Timer: TimerRequest :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> !Maybe TraceContext -> TimerRequest
- Keiro.Timer: TimerRow :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> !TimerStatus -> !Int -> !Maybe EventId -> TimerRow
+ Keiro.Timer: TimerRow :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> !TimerStatus -> !Int -> !Maybe EventId -> !Maybe TraceContext -> TimerRow
- Keiro.Timer: TimerWorkerOptions :: Maybe Int -> !Maybe NominalDiffTime -> TimerWorkerOptions
+ Keiro.Timer: TimerWorkerOptions :: Maybe Int -> !Maybe NominalDiffTime -> !Maybe Tracer -> TimerWorkerOptions
- Keiro.Timer.Schema: TimerRow :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> !TimerStatus -> !Int -> !Maybe EventId -> TimerRow
+ Keiro.Timer.Schema: TimerRow :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> !TimerStatus -> !Int -> !Maybe EventId -> !Maybe TraceContext -> TimerRow
- Keiro.Timer.Types: TimerRequest :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> TimerRequest
+ Keiro.Timer.Types: TimerRequest :: !TimerId -> !Text -> !Text -> !UTCTime -> !Value -> !Maybe TraceContext -> TimerRequest
Files
- CHANGELOG.md +49/−0
- bench/BenchSetup.hs +37/−2
- bench/DurableTraceBench.hs +126/−0
- bench/Main.hs +104/−5
- bench/ProducerIdentityBench.hs +16/−1
- keiro.cabal +26/−13
- retention/Main.hs +3/−1
- retention/Retention/Fixture.hs +18/−4
- retention/Retention/Legs.hs +122/−11
- retention/Retention/SpanOwnership.hs +238/−0
- retention/Retention/Tracing.hs +25/−0
- src/Keiro/Command.hs +30/−14
- src/Keiro/Outbox.hs +30/−18
- src/Keiro/Outbox/Types.hs +3/−3
- src/Keiro/ProcessManager.hs +34/−10
- src/Keiro/ProcessManager/Reaction.hs +32/−13
- src/Keiro/Router.hs +34/−7
- src/Keiro/Telemetry.hs +240/−25
- src/Keiro/Timer.hs +33/−7
- src/Keiro/Timer/Schema.hs +32/−19
- src/Keiro/Timer/Types.hs +12/−1
- src/Keiro/Workflow.hs +63/−27
- src/Keiro/Workflow/Awakeable.hs +22/−7
- src/Keiro/Workflow/Child.hs +39/−9
- src/Keiro/Workflow/Instance.hs +22/−2
- src/Keiro/Workflow/Instance/Schema.hs +54/−25
- src/Keiro/Workflow/Journal.hs +80/−7
- src/Keiro/Workflow/Resume.hs +10/−8
- src/Keiro/Workflow/Schema.hs +7/−2
- src/Keiro/Workflow/Sleep.hs +37/−14
- test/DurableTracingSpec.hs +335/−0
- test/Main.hs +305/−9
- test/ReactionExample.hs +4/−2
CHANGELOG.md view
@@ -6,6 +6,55 @@ ## [Unreleased] +## 0.25.0.0 — 2026-11-14++### Breaking Changes++- `TimerRequest` and `TimerRow` add strict `traceContext`; use `Nothing` for legacy behavior.+- `TimerWorkerOptions` adds `tracer` (default `Nothing`) and retains `Show` but no longer has `Eq`.+++- `IntegrationProducer` requires `tracePolicy`; use `PreserveDraftTrace` until explicit+ adoption of an immutable `DeriveTraceAfter` cutoff. Strict trace-content equality stays intact.+- Mixed publisher batches use the span name `send`; only singleton batch spans carry+ per-message ID and Kafka key attributes. Uniform batches retain `send <destination>`.+- Require shibuya-core 0.13.0.1 and matching shibuya-kiroku-adapter 0.8.0.0.+ In-memory `Adapter` constructors now supply explicit telemetry defaults.++- Command spans are named `command <category>` (for example `command counter`),+ replacing the full stream name (`counter-42`). The full name remains in `keiro.stream.name`.++### New Features++- Linked root timer-fire spans and workflow origin/exact-wake links across snapshots.+ Traced journal/wake APIs are explicit; legacy APIs remain untraced wrappers.+ Tracing preserves replay identity and event metadata on idempotent retries.++++- Traced subscription workers open `process <worker name>` spans linked according to+ adapter policy; dispatched command spans are children. The Kiroku default is a linked root.+- `withConsumerSpan` adds a creation link alongside its existing W3C parent. Inbox handlers+ inherit the transport span and add no independent span.++- Producer context can derive deterministically from stored source-event metadata above+ an immutable cutoff. Publisher root spans link up to 128 stored creation contexts,+ report batch size and omitted valid links, and preserve creation context on the wire.++- Commands with a configured tracer stamp event metadata with the command span's W3C+ creation context once per run, preserving explicit caller context across retries.++### Bug Fixes++- Retained command spans release typed outcome payloads before attribute inspection;+ bounded outcome projections are forced before SDK ownership.++### Other Changes++- Acquire tracing benchmark fixtures only when selected. Compare fixed performance+ workloads with their immutable recording source on the current machine, retaining+ historical diagnostics, unchanged limits and checksum-bound measurement evidence.+ ## 0.24.0.0 — 2026-11-05 ### Breaking Changes
bench/BenchSetup.hs view
@@ -1,9 +1,15 @@ -- | Selection-aware fixture diagnostics shared by the benchmark groups.-module BenchSetup (selected, selectedWithCleanup, memoSetup, setupTrace, freshCounter, nextIteration, nextBoundedIteration, selectedCounter, selectedCounterWithCleanup) where+module BenchSetup (selectedBracket, withRecordingTracer, selected, selectedWithCleanup, memoSetup, setupTrace, freshCounter, nextIteration, nextBoundedIteration, selectedCounter, selectedCounterWithCleanup) where -import Control.Concurrent.MVar (modifyMVar, newMVar)+import Control.Concurrent.Async (async, cancel, link, wait)+import Control.Concurrent.MVar (modifyMVar, newEmptyMVar, newMVar, putMVar, takeMVar) import Control.DeepSeq (NFData (..))+import Control.Exception (bracket, mask, onException)+import Control.Monad (unless) import Data.IORef (IORef, atomicModifyIORef', newIORef, readIORef)+import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter)+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import OpenTelemetry.Trace.Core import System.Environment (lookupEnv) import System.IO (hPutStrLn, stderr) import Test.Tasty.Bench (Benchmark, envWithCleanup)@@ -23,6 +29,19 @@ envWithCleanup (Handle <$> acquire) (\(Handle value) -> cleanup value) $ \handle -> build (case handle of Handle value -> value) +-- Adapt bracket-only resources to Tasty's selected environment lifecycle. The+-- worker holds the bracket open; cleanup joins it after every resource closes.+selectedBracket :: ((a -> IO ()) -> IO ()) -> (a -> Benchmark) -> Benchmark+selectedBracket scope build = selectedWithCleanup acquire cleanup $ \ ~(value, _, _) -> build value+ where+ acquire = mask $ \restore -> do+ ready <- newEmptyMVar+ release <- newEmptyMVar+ worker <- async $ restore $ scope $ \value -> putMVar ready value >> takeMVar release+ value <- (link worker >> restore (takeMVar ready)) `onException` cancel worker+ pure (value, release, worker)+ cleanup (_, release, worker) = putMVar release () >> wait worker+ memoSetup :: IO a -> IO (IO a) memoSetup acquire = do cell <- newMVar Nothing@@ -70,3 +89,19 @@ cleanup n ) build++-- Benchmark SDK work without exporting I/O or accumulating completed spans.+-- Check recording explicitly: processor-free providers can be non-recording.+withRecordingTracer :: (Tracer -> IO a) -> IO a+withRecordingTracer action = bracket acquire (\(provider, _) -> shutdownTracerProvider provider Nothing) (action . snd)+ where+ acquire = do+ (processor, _) <- inMemoryListExporter+ -- Keep the SDK's recording path but discard completed spans immediately.+ let discard = processor {spanProcessorOnEnd = const (pure ())}+ provider <- createTracerProvider [discard] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-bench" tracerOptions+ inSpan' tracer "recording-check" defaultSpanArguments $ \span -> do+ recording <- isRecording span+ unless recording (fail "benchmark tracer is not recording")+ pure (provider, tracer)
+ bench/DurableTraceBench.hs view
@@ -0,0 +1,126 @@+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE TypeOperators #-}++module DurableTraceBench (durableTraceBenchmarks) where++import BenchSetup (selected, setupTrace)+import Contravariant.Extras (contrazip3)+import Control.Monad (forM_, replicateM_, unless, void)+import Data.Aeson qualified as Aeson+import Data.Int (Int32, Int64)+import Data.Map.Strict qualified as Map+import Data.Text qualified as Text+import Data.Text.IO qualified as TextIO+import Data.Time (UTCTime (..))+import Data.Time.Calendar (Day (ModifiedJulianDay))+import Data.UUID qualified as UUID+import Data.Vector qualified as Vector+import Effectful (Eff, IOE, (:>))+import Effectful.Error.Static (Error)+import Hasql.Decoders qualified as D+import Hasql.Encoders qualified as E+import Hasql.Statement (Statement, preparable)+import Keiro.Prelude (liftIO, (&), (.~), (^.))+import Keiro.Telemetry qualified as Telemetry+import Keiro.Test.Postgres (checkpointBenchmarkStore)+import Keiro.Timer qualified as Timer+import Keiro.Workflow qualified as Workflow+import Keiro.Workflow.Instance qualified as Instance+import Keiro.Workflow.Snapshot (writeWorkflowSnapshot)+import Kiroku.Store qualified as Store+import Kiroku.Store.Types (EventId (..), StreamVersion (..))+import OpenTelemetry.Trace.Core (Tracer, defaultSpanArguments, inSpan')+import System.Environment (lookupEnv)+import Test.Tasty.Bench (Benchmark, bench, bgroup, nfIO)+import "hasql-transaction" Hasql.Transaction qualified as Tx++durableTraceBenchmarks :: Store.KirokuStore -> Tracer -> [Benchmark]+durableTraceBenchmarks store tracer =+ [ bgroup "durable-tracing" $+ [timerCell traced | traced <- [False, True]]+ <> [workflowCell size traced | size <- [1 :: Int, 1000], traced <- [False, True]]+ ]+ where+ handle traced = if traced then Just tracer else Nothing+ suffix traced = if traced then "/tracer" else ""+ now = UTCTime (ModifiedJulianDay 60000) 0+ timerCell traced =+ let label = "timer-100" <> suffix traced+ worker = Timer.defaultTimerWorkerOptions & #tracer .~ handle traced & #requeueStuckAfter .~ Nothing+ action = do+ checked store (Store.runTransaction (Tx.sql "TRUNCATE keiro.keiro_timers"))+ Telemetry.withWorkflowSpan (handle traced) (Workflow.WorkflowName "timerBench") (Workflow.WorkflowId "one") Nothing $ \_ -> do+ requests <- traverse (Timer.captureTimerTraceContext (handle traced) . request) [1 .. 100]+ checked store (Store.runTransaction (mapM_ Timer.scheduleTimerTx requests))+ count <- checked store $ Timer.drainDueTimersWith Nothing worker now 101 (const (pure (Just (EventId UUID.nil))))+ unless (count == 100) (fail "timer benchmark did not fire exactly 100 rows")+ request index = Timer.TimerRequest (Timer.TimerId (UUID.fromWords64 0 (fromIntegral index))) "timerBench" "one" now Aeson.Null Nothing+ in selected (warm label action) $ \_ -> bench label (nfIO action)+ workflowCell size traced =+ let label = "resume-snapshot-" <> show size <> suffix traced+ name = Workflow.WorkflowName (Text.pack label)+ wid = Workflow.WorkflowId "one"+ options = Workflow.defaultWorkflowRunOptions & #tracer .~ handle traced+ seed = do+ forM_ [1 .. size] $ \index -> void $ Workflow.step (key index) (pure index)+ void (awaitValue "wake")+ action = replicateM_ 10 do+ checked store (Instance.markInstanceSuspendedAwaitingTraced traced name wid 0 "wake")+ result <- checked store $ Workflow.runWorkflowWith options name wid $ do+ value <- Workflow.step (key 1) (pure (1 :: Int))+ _ <- awaitValue "wake"+ _ <- awaitValue "hold"+ pure value+ unless (result == Workflow.Suspended) (fail "workflow benchmark failed to suspend")+ setup = do+ result <- checked store (Workflow.runWorkflowWith options name wid seed)+ unless (result == Workflow.Suspended) (fail "workflow seed failed to suspend")+ inSpan' tracer "wake" defaultSpanArguments $ \_ ->+ checked store $ Workflow.appendJournalEntryTraced (handle traced) name wid (Workflow.StepRecorded "wake" Aeson.Null now)+ rows <- checked store $ Store.readStreamForward (Workflow.workflowGenerationStreamName name wid 0) (StreamVersion (fromIntegral size)) 1+ unless (Vector.length rows == 1) (fail "workflow seed wake missing")+ let row = Vector.head rows+ state = Map.insert "wake" Aeson.Null (Map.fromList [("step-" <> Text.pack (show index), Aeson.toJSON index) | index <- [1 .. size]])+ checked store (writeWorkflowSnapshot (row ^. #originalStreamId) (row ^. #originalVersion) state)+ diagnostic <- lookupEnv "KEIRO_BENCH_DURABLE_EXPLAIN"+ let explain tag = do+ plan <- checked store (Store.runTransaction (Tx.statement ((case Workflow.workflowGenerationStreamName name wid 0 of Store.StreamName value -> value), 0, 1) originPlan))+ putStrLn (label <> " " <> tag)+ mapM_ TextIO.putStrLn plan+ whenDiagnostic diagnostic (explain "before ANALYZE")+ checked store (Store.runTransaction (Tx.sql "ANALYZE streams; ANALYZE events; ANALYZE stream_events; ANALYZE keiro.keiro_workflows; ANALYZE keiro.keiro_workflow_steps"))+ whenDiagnostic diagnostic (explain "after ANALYZE")+ warm label action+ in selected setup $ \_ -> bench label (nfIO action)+ key index = Workflow.StepName ("step-" <> Text.pack (show index))+ warm label action = do+ checkpointBenchmarkStore store+ setupTrace "durable-tracing" label "warm-up=3; timer rows reset per invocation; resume=10 fixed snapshots"+ replicateM_ 3 action++checked :: Store.KirokuStore -> Eff '[Store.Store, Error Store.StoreError, IOE] a -> IO a+checked store action = Store.runStoreIO store action >>= either (fail . show) pure++awaitValue :: (Workflow.Workflow :> es) => Text.Text -> Eff es Aeson.Value+awaitValue key = Workflow.awaitStep (Workflow.StepName key) (pure ())++whenDiagnostic :: Maybe String -> IO () -> IO ()+whenDiagnostic (Just "1") action = action+whenDiagnostic _ _ = pure ()++-- Same origin query as the registered Kiroku read API, for plan diagnostics.+originPlan :: Statement (Text.Text, Int64, Int32) [Text.Text]+originPlan =+ preparable+ """+ EXPLAIN (ANALYZE, BUFFERS) SELECT e.event_id, e.event_type,+ se.stream_version, 0::bigint AS global_position, se.original_stream_id,+ se.original_stream_version, e.data, e.metadata, e.causation_id, e.correlation_id, e.created_at+ FROM stream_events se JOIN events e ON e.event_id = se.event_id+ JOIN streams s ON s.stream_id = se.stream_id+ WHERE s.stream_name = $1 AND s.deleted_at IS NULL AND se.stream_version > $2+ AND se.stream_version >= s.truncate_before ORDER BY se.stream_version ASC LIMIT $3+ """+ (contrazip3 (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.int8)) (E.param (E.nonNullable E.int4)))+ (D.rowList (D.column (D.nonNullable D.text)))
bench/Main.hs view
@@ -6,7 +6,7 @@ ) where -import BenchSetup (memoSetup, nextBoundedIteration, nextIteration, selected, selectedCounterWithCleanup, selectedWithCleanup, setupTrace)+import BenchSetup (memoSetup, nextBoundedIteration, nextIteration, selected, selectedBracket, selectedCounterWithCleanup, selectedWithCleanup, setupTrace, withRecordingTracer) import CommandProbe (commandPlanProbe, maintenanceProbe, rebuildCountProbe, tableResetProbe, workCountProbe) import Control.Concurrent (threadDelay) import Control.Monad (replicateM_)@@ -19,6 +19,7 @@ import Data.Time.Calendar (Day (ModifiedJulianDay)) import Data.UUID qualified as UUID import Data.Vector qualified as Vector+import DurableTraceBench (durableTraceBenchmarks) import Effectful (Eff, IOE, (:>)) import Effectful.Error.Static (Error) import GHC.Clock (getMonotonicTimeNSec)@@ -51,7 +52,7 @@ runInboxTransactionBatch, runInboxTransactionWith, )-import Keiro.Integration.Event (IntegrationContentType (..), IntegrationEvent (..))+import Keiro.Integration.Event (IntegrationContentType (..), IntegrationEvent (..), TraceContext (..)) import Keiro.Outbox ( OutboxId (..), OutboxRow,@@ -80,7 +81,7 @@ startCatalogRebuild, ) import Keiro.Telemetry qualified as Telemetry-import Keiro.Test.Postgres (StoreRunner (..), checkpointBenchmarkStore, withFreshResourceStore, withMigratedBenchmarkSuite)+import Keiro.Test.Postgres (Fixture, StoreRunner (..), checkpointBenchmarkStore, withFreshResourceStore, withMigratedBenchmarkSuite) import Kiroku.Store qualified as Store import Kiroku.Store.Effect (Store) import Kiroku.Store.Effect.Resource (KirokuStoreResource)@@ -95,9 +96,11 @@ StreamName (..), StreamVersion (..), )+import Numeric (showHex) import OpenTelemetry.MeterProvider (createMeterProvider, defaultSdkMeterProviderOptions) import OpenTelemetry.Metric.Core (getMeter) import OpenTelemetry.Resource (emptyMaterializedResources)+import OpenTelemetry.Trace.Core (Tracer) import ProducerIdentityBench (producerIdentityBenchmarks, runProducerIdentityProbe) import ReadModelBench ( probeReadModelCounts,@@ -106,7 +109,7 @@ runReadModelLatencyEvidenceIfRequested, setupReadModelBench, )-import Shibuya.Adapter (Adapter (..))+import Shibuya.Adapter (Adapter (..), AdapterTelemetry (traceCorrelation), TraceCorrelation (LinkOnly), defaultAdapterTelemetry) import Shibuya.Core.AckHandle (AckHandle (..)) import Shibuya.Core.Ingested (DeliveryStatus (..), Ingested (..)) import Shibuya.Core.Types (Envelope (..))@@ -191,11 +194,40 @@ then runResetProbes outboxStore inboxStore producerStore else defaultMain- ( benchmarks store runner outboxStore inboxStore rebuildStore metrics (inboxDelegatedBenchmarks delegatedStore metrics)+ ( selectedDurableBenchmarks fixture+ <> selectedTracingBenchmarks fixture+ <> benchmarks store runner outboxStore inboxStore rebuildStore metrics (inboxDelegatedBenchmarks delegatedStore metrics) <> [selectedWithCleanup readModelSetup probeReadModelCounts (\model -> readModelBenchmarks model !! groupIndex) | groupIndex <- [0 .. 2]] <> producerIdentityBenchmarks producerStore producerCounter ) +-- These groups remain visible while listing/filtering, without acquiring their+-- tracer, pools, notifier threads or cloned databases until a case is selected.+selectedTracingBenchmarks :: Fixture -> [Benchmark]+selectedTracingBenchmarks fixture =+ [ selectedBracket acquire $ \ ~(store, dispatch, runner, outbox, tracer) ->+ head (tracingBenchmarks store dispatch runner outbox tracer)+ ]+ where+ acquire use = withRecordingTracer $ \tracer ->+ withFreshResourceStore fixture $ \(store, _) ->+ withFreshResourceStore fixture $ \(dispatch, runner) ->+ withFreshResourceStore fixture $ \(outbox, _) -> do+ setupTrace "tracing-fixture" "tracing" "acquired"+ use (store, dispatch, runner, outbox, tracer)+ setupTrace "tracing-fixture" "tracing" "closing"++selectedDurableBenchmarks :: Fixture -> [Benchmark]+selectedDurableBenchmarks fixture =+ [ selectedBracket acquire $ \ ~(store, tracer) -> head (durableTraceBenchmarks store tracer)+ ]+ where+ acquire use = withRecordingTracer $ \tracer ->+ withFreshResourceStore fixture $ \(store, _) -> do+ setupTrace "tracing-fixture" "durable-tracing" "acquired"+ use (store, tracer)+ setupTrace "tracing-fixture" "durable-tracing" "closing"+ benchmarks :: Store.KirokuStore -> StoreRunner -> Store.KirokuStore -> Store.KirokuStore -> Store.KirokuStore -> Telemetry.KeiroMetrics -> [Benchmark] -> [Benchmark] benchmarks store runner outboxStore inboxStore rebuildStore metrics delegatedInboxBenchmarks = [ bgroup@@ -1163,6 +1195,7 @@ fanoutAdapter input = Adapter { adapterName = "bench-domain-coordinator",+ telemetry = defaultAdapterTelemetry, source = Streamly.fromList [ Ingested@@ -1250,3 +1283,69 @@ tableResetProbe outboxStore "keiro.keiro_outbox" 2000 (outboxSeed outboxStore 2000) tableResetProbe inboxStore "keiro.keiro_inbox" 2000 inboxSeed tableResetProbe producerStore "keiro.keiro_outbox" 1000 (outboxSeed producerStore 1000)++-- Fixed 100-command workloads; each scenario owns its own bounded namespace.+tracingBenchmarks :: Store.KirokuStore -> Store.KirokuStore -> StoreRunner -> Store.KirokuStore -> Tracer -> [Benchmark]+tracingBenchmarks store dispatchStore runner outboxStore tracer =+ [ bgroup "tracing" $+ [selected (prepareCommandStore store) $ \_ -> bgroup "command" ([commandCell distinct traced | distinct <- [True, False], traced <- [False, True]] <> [batchCommandCell traced | traced <- [False, True]])]+ <> [publishCell withContext traced | withContext <- [False, True], traced <- [False, True]]+ <> [selected (prepareCommandStore dispatchStore) $ \_ -> bgroup "process-manager" [dispatchCell withContext traced | withContext <- [False, True], traced <- [False, True]]]+ ]+ where+ options traced = defaultRunCommandOptions & #tracer .~ (if traced then Just tracer else Nothing)+ suffix traced = if traced then "/tracer" else ""+ commandCell distinct traced =+ let label = (if distinct then "append-distinct-streams" else "append-one-stream") <> suffix traced+ action counter = do+ iteration <- nextIteration label 100 counter+ for_ [1 .. 100 :: Int] $ \index -> runStoreChecked store do+ let target = stream ("tracing-" <> Text.pack label <> "-" <> Text.pack (show iteration) <> "-" <> Text.pack (show (if distinct then index else 0)))+ result <- runCommand (options traced) legacyAcceptedOneStream target EmitOne+ case result of+ Right value | value ^. #eventsAppended == 1 -> pure ()+ other -> liftIO (fail (show other))+ in selectedCounterWithCleanup label 100 (prepareCommand store label 100 1000 action) (\n -> workCountProbe store (Text.pack label) ("tracing-" <> Text.pack label <> "-") (n * if distinct then 100 else 1) (n * 100)) $ \counter -> bench label (nfIO (action counter))+ batchCommandCell traced =+ let label = "append-batch" <> suffix traced+ action counter = do+ iteration <- nextBoundedIteration label (commandBudget EmitLarge) counter+ let target = stream ("tracing-" <> Text.pack label <> "-" <> Text.pack (show iteration))+ runStoreChecked store $ do+ result <- runCommand (options traced) legacyAcceptedLargeStream target EmitLarge+ case result of+ Right value | value ^. #eventsAppended == 100 -> pure ()+ other -> liftIO (fail (show other))+ in selectedCounterWithCleanup label 100 (prepareCommand store label 100 (commandBudget EmitLarge) action) (\n -> workCountProbe store (Text.pack label) ("tracing-" <> Text.pack label <> "-") n (n * 100)) $ \counter -> bench label (nfIO (action counter))+ publishCell withContext traced =+ let label = (if withContext then "publish-contexts" else "publish-backlog") <> suffix traced+ action = do+ runStoreChecked outboxStore (Store.runTransaction (Tx.sql "TRUNCATE keiro.keiro_outbox"))+ let messages = scenarioMessages (const Nothing)+ contextual = zipWith (\index (oid, event) -> (oid, event & #traceContext ?~ TraceContext ("00-0af7651916cd43dd8448eb211c80319c-" <> Text.justifyRight 16 '0' (Text.pack (showHex index "")) <> "-01") Nothing)) [1 :: Int ..] messages+ seedOutbox outboxStore (if withContext then contextual else messages)+ runStoreChecked outboxStore $ do+ let drain = do+ backlog <- countOutboxBacklog+ when (backlog > 0) $ do+ void (publishClaimedOutbox (simulatedPublish (BrokerModel 0 0)) (defaultPublishOptions & #tracer .~ (if traced then Just tracer else Nothing)) Nothing)+ drain+ drain+ in selected (warmAction outboxStore "tracing" label action) $ \_ -> bench label (nfIO action)+ dispatchCell withContext traced =+ let label = (if withContext then "dispatch-contexts" else "dispatch") <> suffix traced+ action counter = do+ iteration <- nextIteration label 100 counter+ runResourceStoreChecked runner $+ for_ [1 .. 100 :: Int] $ \index -> do+ let n = iteration * 100 + index+ input = FanoutInput (Text.pack label <> "-" <> Text.pack (show n)) 1+ let plain = fanoutAdapter input+ contextual = case plain of+ Adapter {source = incoming} ->+ plain+ { telemetry = defaultAdapterTelemetry {traceCorrelation = LinkOnly},+ source = fmap (\ingested@Ingested {envelope = envelope} -> ingested {envelope = envelope & #traceContext ?~ [("traceparent", "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01")]}) incoming+ }+ runDomainProcessManagerWorker (options traced) fanoutDomainProcessManager (if withContext then contextual else plain) (\message -> Just (iterationSourceEvent n, message))+ in selectedCounterWithCleanup label 100 (prepareCommand dispatchStore label 100 1000 action) (\n -> workCountProbe dispatchStore (Text.pack label) ("bench-command-domain-process-manager:" <> Text.pack label <> "-") (n * 100) (n * 100)) $ \counter -> bench label (nfIO (action counter))
bench/ProducerIdentityBench.hs view
@@ -35,6 +35,10 @@ sqlBench "legacy-fresh-1000" (run True False False), freshBench True "deterministic-fresh-1000" (run True True False), sqlBench "deterministic-fresh-and-replay-1000" (run True True True),+ sqlBench "trace-policy-legacy-1000" (runTrace PreserveDraftTrace False),+ sqlBench "trace-policy-derived-1000" (runTrace (DeriveTraceAfter (GlobalPosition 0)) False),+ sqlBench "trace-policy-pre-cutover-1000" (runTrace (DeriveTraceAfter (GlobalPosition 1000)) False),+ sqlBench "trace-policy-explicit-1000" (runTrace (DeriveTraceAfter (GlobalPosition 0)) True), sqlBench "legacy-single-tx-per-event-1000" (run False False False), freshBench False "deterministic-single-tx-per-event-1000" (run False True False) ]@@ -44,6 +48,17 @@ freshBench batch name action = selected (checkFreshRatio store sequenceRef batch) $ \_ -> sqlBench name action run = runProducerWorkload store sequenceRef+ runTrace policy explicit = do+ generation <- atomicModifyIORef' sequenceRef (\n -> (n + 1, n + 1))+ result <- Store.runStoreIO store $ do+ Store.runTransaction (Tx.sql "TRUNCATE keiro.keiro_outbox")+ let context = TraceContext "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" Nothing+ configuredProducer = producer & #tracePolicy .~ policy+ tracedDraft = if explicit then draft & #traceContext ?~ context else draft+ actions = [enqueueProducerEventTx configuredProducer (sourceEvent generation i & #metadata ?~ Aeson.object ["traceparent" Aeson..= (context ^. #traceparent)]) 0 tracedDraft | i <- [1 .. 1000]]+ outcomes <- Store.runTransaction (sequence actions)+ unless (all (\case ProducerInserted _ -> True; _ -> False) outcomes) (error "trace-policy benchmark failed to insert")+ either (fail . show) pure result -- The diagnostic pairs call precisely the normal benchmark workload, including -- timed reset, preparation, insert-result checks and transaction boundaries.@@ -91,7 +106,7 @@ isDuplicate _ = False producer :: IntegrationProducer ()-producer = IntegrationProducer "bench-producer" "bench.outbox" "msg" (\_ _ -> Just draft)+producer = IntegrationProducer "bench-producer" "bench.outbox" "msg" PreserveDraftTrace (\_ _ -> Just draft) draft :: IntegrationEventDraft draft =
keiro.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.0 name: keiro-version: 0.24.0.0+version: 0.25.0.0 synopsis: Event sourcing framework and workflow engine description: A library that composes kiroku, keiki, and shibuya into an@@ -177,13 +177,13 @@ hs-opentelemetry-semantic-conventions >=1.40 && <2, keiki >=0.9 && <0.10, keiki-codec-json >=0.9 && <0.10,- keiro-core ^>=0.24.0.0,+ keiro-core ^>=0.25.0.0, kiroku-store ^>=0.13.0.1, lens >=5.2 && <5.4, mmzk-typeid >=0.7 && <0.8, random >=1.2.1 && <1.4, scientific >=0.3 && <0.4,- shibuya-core ^>=0.12.0.0,+ shibuya-core ^>=0.13.0.1, stm >=2.5 && <2.6, streamly >=0.11 && <0.12, streamly-core >=0.3 && <0.4,@@ -205,6 +205,7 @@ CatalogSpec Compatibility.ReadModel011 ConsumerInspectionSpec+ DurableTracingSpec ExternalReadSpec GroupRebuildSpec GuardedCatalogExample@@ -249,11 +250,12 @@ keiki, keiki-codec-json, keiro,- keiro-migrations ^>=0.24.0.0,- keiro-test-support ^>=0.24.0.0,+ keiro-migrations ^>=0.25.0.0,+ keiro-test-support ^>=0.25.0.0, kiroku-store ^>=0.13.0.1, process >=1.6 && <1.7,- shibuya-core ^>=0.12.0.0,+ shibuya-core ^>=0.13.0.1,+ shibuya-kiroku-adapter ^>=0.8.0.0, stm >=2.5 && <2.6, streamly-core >=0.3 && <0.4, text >=2.1 && <2.2,@@ -271,6 +273,8 @@ Retention.Fixture Retention.Legs Retention.Measure+ Retention.SpanOwnership+ Retention.Tracing ghc-options: -threaded@@ -280,20 +284,25 @@ build-depends: aeson >=2.2 && <2.3, base >=4.21 && <5,+ bytestring >=0.11 && <0.13, effectful >=2.6 && <2.8, effectful-core >=2.6 && <2.7 || >=2.7.1.1 && <2.8,+ hs-opentelemetry-api >=1.0 && <1.1,+ hs-opentelemetry-exporter-in-memory >=1.0 && <1.1,+ hs-opentelemetry-sdk >=1.0 && <1.1, hspec >=2.11, keiki >=0.9 && <0.10, keiki-codec-json >=0.9 && <0.10, keiro,- keiro-core ^>=0.24.0.0,- keiro-test-support ^>=0.24.0.0,+ keiro-core ^>=0.25.0.0,+ keiro-test-support ^>=0.25.0.0, kiroku-store ^>=0.13.0.1,- shibuya-core ^>=0.12.0.0,- shibuya-kiroku-adapter ^>=0.7.0.1,+ shibuya-core ^>=0.13.0.1,+ shibuya-kiroku-adapter ^>=0.8.0.0, stm >=2.5 && <2.6, streamly-core >=0.3 && <0.4, text >=2.1 && <2.2,+ time >=1.12 && <1.15, uuid >=1.3 && <1.4, vector >=0.13 && <0.14, @@ -308,6 +317,7 @@ other-modules: BenchSetup CommandProbe+ DurableTraceBench GuardedCatalogBench GuardedCatalogExample GuardedCatalogFixture@@ -323,9 +333,11 @@ build-depends: aeson >=2.2.2 && <2.3,+ async >=2.2 && <2.3, base >=4.21 && <5, bytestring >=0.11 && <0.13, containers >=0.6 && <0.8,+ contravariant-extras >=0.3 && <0.4, deepseq >=1.5 && <1.6, directory >=1.3 && <1.4, effectful >=2.6 && <2.8,@@ -333,14 +345,15 @@ hasql >=1.10 && <1.11, hasql-transaction >=1.1 && <1.3, hs-opentelemetry-api >=1.0 && <1.1,+ hs-opentelemetry-exporter-in-memory >=1.0 && <1.1, hs-opentelemetry-sdk >=1.0 && <1.1, keiki >=0.9 && <0.10, keiro,- keiro-core ^>=0.24.0.0,- keiro-test-support ^>=0.24.0.0,+ keiro-core ^>=0.25.0.0,+ keiro-test-support ^>=0.25.0.0, kiroku-store ^>=0.13.0.1, process >=1.6 && <1.7,- shibuya-core ^>=0.12.0.0,+ shibuya-core ^>=0.13.0.1, streamly-core >=0.3 && <0.4, tasty-bench >=0.4, text >=2.1 && <2.2,
retention/Main.hs view
@@ -8,6 +8,7 @@ import Keiro.Test.Postgres (withFreshResourceStore, withMigratedSuite) import Retention.Legs (Leg (..), allLegs) import Retention.Measure+import Retention.SpanOwnership qualified as SpanOwnership import System.Environment (lookupEnv) import Test.Hspec @@ -16,7 +17,8 @@ config <- gateConfigFromEnvironment selected <- selectLegs withMigratedSuite \fixture ->- hspec $+ hspec $ do+ SpanOwnership.spec fixture describe "Retention legs" $ forM_ selected \leg -> it (Text.unpack leg.legName) $
retention/Retention/Fixture.hs view
@@ -9,6 +9,7 @@ TargetEvent (..), TargetEventStream, retentionTargetStream,+ retentionNoOpTargetStream, LedgerCommand (..), LedgerEvent (..), LedgerRegs,@@ -23,6 +24,7 @@ retentionTarget, signalStreamName, appendSignals,+ appendSignalsWithMetadata, decodeSignal, ackAdapter, sampleOnAck,@@ -81,7 +83,7 @@ import Kiroku.Store.Subscription.Types qualified as Sub import Kiroku.Store.Types (EventData (..), EventId (..), EventType (..), ExpectedVersion (..), GlobalPosition (..), RecordedEvent (..), StreamName (..), StreamVersion (..)) import Numeric.Natural (Natural)-import Shibuya.Adapter (Adapter (..))+import Shibuya.Adapter (Adapter (..), defaultAdapterTelemetry) import Shibuya.Core.Ack (AckDecision (..)) import Shibuya.Core.Ack qualified as Ack import Shibuya.Core.AckHandle (AckHandle (..))@@ -219,6 +221,15 @@ retentionTargetStream :: ValidatedEventStream (HsPred '[] TargetCommand) '[] TargetState TargetCommand TargetEvent retentionTargetStream = mkEventStreamOrThrow "retention-target" targetStreamDef +-- A real silent command for probing typed outcome ownership without writing events.+retentionNoOpTargetStream :: ValidatedEventStream (HsPred '[] TargetCommand) '[] TargetState TargetCommand TargetEvent+retentionNoOpTargetStream =+ mkEventStreamOrThrow+ "retention-no-op"+ targetStreamDef+ { transducer = targetTransducer {edgesOut = \state -> [edge {output = []} | edge <- targetTransducer.edgesOut state]}+ }+ snapshottedTargetStream :: ValidatedEventStream (HsPred '[] TargetCommand) '[] TargetState TargetCommand TargetEvent snapshottedTargetStream = mkEventStreamOrThrow@@ -369,7 +380,10 @@ signalStreamName index = StreamName ("retentionsource-" <> Text.pack (show index)) appendSignals :: StoreRunner -> Int -> IO ()-appendSignals (StoreRunner runStore) count =+appendSignals = appendSignalsWithMetadata (const Nothing)++appendSignalsWithMetadata :: (Int -> Maybe Value) -> StoreRunner -> Int -> IO ()+appendSignalsWithMetadata metadataFor (StoreRunner runStore) count = forM_ [1 .. count] \index -> do let signal = Signal (Text.pack (show index)) (index `mod` 16) event =@@ -377,7 +391,7 @@ { eventId = Nothing, eventType = EventType "RetentionSignal", payload = toJSON signal,- metadata = Nothing,+ metadata = metadataFor index, causationId = Nothing, correlationId = Nothing }@@ -398,7 +412,7 @@ (source, cancel) <- subscriptionAckStream store config validatedBufferSize let ingested = fmap (toIngested cancel) (Streamly.morphInner liftIO source) pure- ( Adapter {adapterName = "retention-ack-bridge", source = ingested, shutdown = liftIO cancel},+ ( Adapter {telemetry = defaultAdapterTelemetry, adapterName = "retention-ack-bridge", source = ingested, shutdown = liftIO cancel}, cancel )
retention/Retention/Legs.hs view
@@ -11,33 +11,45 @@ import Control.Concurrent.STM (atomically, putTMVar) import Control.Exception (evaluate, finally) import Control.Monad (forM_, unless, void, when)-import Data.Aeson (toJSON)+import Data.Aeson (Value, object, toJSON, (.=))+import Data.ByteString qualified as BS import Data.IORef (atomicModifyIORef', newIORef, readIORef) import Data.Text (Text) import Data.Text qualified as Text import Data.Text.IO qualified as Text.IO+import Data.Time (UTCTime (..), secondsToDiffTime)+import Data.Time.Calendar (Day (ModifiedJulianDay)) import Data.UUID qualified as UUID import Data.Vector qualified as Vector import Effectful (Eff, IOE, liftIO, (:>)) import Effectful.Exception qualified as Effectful import Keiro.Command (defaultRunCommandOptions) import Keiro.Command qualified as Command+import Keiro.EventStream (SnapshotPolicy (Every)) import Keiro.EventStream.Validate (unvalidated)+import Keiro.Integration.Event (IntegrationContentType (..), IntegrationEvent (..), TraceContext (..))+import Keiro.Outbox qualified as Outbox import Keiro.ProcessManager (defaultWorkerOptions, runProcessManagerWorkerWith) import Keiro.Projection (AsyncApplyOutcome (..), AsyncProjection (..), applyAsyncProjection) import Keiro.ReadModel.Schema (registerReadModel) import Keiro.Router (runRouterWorkerWith) import Keiro.Snapshot.Schema (SnapshotRow (..), lookupSnapshotRow) import Keiro.Stream qualified as KeiroStream+import Keiro.Telemetry qualified as Telemetry import Keiro.Test.Postgres (StoreRunner (..))+import Keiro.Timer qualified as Timer+import Keiro.Workflow qualified as Workflow import Kiroku.Store qualified as Store import Kiroku.Store.Subscription qualified as Subscription import Kiroku.Store.Subscription.Stream (AckItem (..), defaultStreamBufferSize, subscriptionAckStream) import Kiroku.Store.Subscription.Types (SubscriptionConfig, SubscriptionName (..), SubscriptionResult (..), SubscriptionTarget (..), defaultSubscriptionConfig) import Kiroku.Store.Transaction qualified as StoreTx import Kiroku.Store.Types (CategoryName (..), EventData (..), EventId (..), EventType (..), ExpectedVersion (..), RecordedEvent, StreamName (..), StreamVersion (..))-import Retention.Fixture (LedgerCommand (..), TargetCommand (..), TargetEventStream, ackAdapter, appendSignals, decodeSignal, ledgerTarget, retentionLedger, retentionManager, retentionRouter, retentionTargetStream, sampleOnAck, seedLedger, seedLedgerRaw)+import Numeric (showHex)+import OpenTelemetry.Trace.Core (Tracer)+import Retention.Fixture (LedgerCommand (..), TargetCommand (..), TargetEventStream, ackAdapter, appendSignals, appendSignalsWithMetadata, decodeSignal, ledgerTarget, retentionLedger, retentionManager, retentionRouter, retentionTargetStream, sampleOnAck, seedLedger, seedLedgerRaw) import Retention.Measure (GateConfig (..), HeapSample (..), Verdict (..), measureLeg, measureLegWithSampler, sampleHeap)+import Retention.Tracing (withRecordingTracer) import Shibuya.Adapter (Adapter (..)) import Shibuya.Adapter.Kiroku (defaultKirokuAdapterConfig, kirokuAdapter) import Shibuya.Core.Ack (AckDecision (..))@@ -71,6 +83,13 @@ Leg "hydrate-only" hydrateOnly, Leg "kiroku-read-tail" kirokuReadTail, Leg "command-short-history" commandShortHistory,+ Leg "command-short-history-tracer" (\config handle -> withRecordingTracer (\tracer -> commandShortHistoryWithTracer (Just tracer) config handle)),+ Leg "outbox-publisher" (outboxPublisher Nothing False),+ Leg "outbox-publisher-tracer" (\config handle -> withRecordingTracer (\tracer -> outboxPublisher (Just tracer) False config handle)),+ Leg "outbox-publisher-contexts-tracer" (\config handle -> withRecordingTracer (\tracer -> outboxPublisher (Just tracer) True config handle)),+ Leg "pm-worker-tracer" (\config handle -> withRecordingTracer (\tracer -> pmWorkerWithTracer (Just tracer) config handle)),+ Leg "timer-fire-tracer" (\config handle -> withRecordingTracer (\tracer -> timerFireTracer tracer config handle)),+ Leg "workflow-snapshot-resume-tracer" (\config handle -> withRecordingTracer (\tracer -> workflowSnapshotResumeTracer tracer config handle)), Leg "command-long-history-verify-every" commandLongHistoryVerifyEvery ] @@ -135,16 +154,19 @@ pure measured commandShortHistory :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])-commandShortHistory config (_, StoreRunner runStore) = do- let options = defaultRunCommandOptions {Command.seedVerifySampleRate = 0}- measured <- measureLeg config "command-short-history" \index -> do+commandShortHistory = commandShortHistoryWithTracer Nothing++commandShortHistoryWithTracer :: Maybe Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])+commandShortHistoryWithTracer tracer config (_, StoreRunner runStore) = do+ let options = defaultRunCommandOptions {Command.seedVerifySampleRate = 0, Command.tracer = tracer}+ measured <- measureLeg config (maybe "command-short-history" (const "command-short-history-tracer") tracer) \index -> do let target = KeiroStream.stream ("retentiontarget-" <> Text.pack (show (index `mod` 16))) :: KeiroStream.Stream TargetEventStream expected = StreamVersion (fromIntegral ((index - 1) `div` 16 + 1)) outcome <- runStore (Command.runCommand options retentionTargetStream target (Credit 1)) case outcome of Right (Right result) | result.eventsAppended == 1 && result.streamVersion == expected -> pure () other -> fail ("command-short-history: command " <> show index <> ": " <> show other)- postReturn "command-short-history" (operationCount config)+ postReturn (maybe "command-short-history" (const "command-short-history-tracer") tracer) (operationCount config) pure measured baselineNoStore :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])@@ -298,17 +320,23 @@ when (index < expected) (go (index + 1) rest) pmWorker :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])-pmWorker config (store, runner@(StoreRunner runStore)) = do- appendSignals runner (operationCount config)- measured <- measureLegWithSampler config "pm-worker" \sample -> do+pmWorker = pmWorkerWithTracer Nothing++pmWorkerWithTracer :: Maybe Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])+pmWorkerWithTracer tracer config (store, runner@(StoreRunner runStore)) = do+ let creationMetadata index = Just (object ["traceparent" .= ("00-0af7651916cd43dd8448eb211c80319c-" <> Text.justifyRight 16 '0' (Text.pack (showHex index "")) <> "-01")])+ case tracer of+ Nothing -> appendSignals runner (operationCount config)+ Just _ -> appendSignalsWithMetadata creationMetadata runner (operationCount config)+ measured <- measureLegWithSampler config (maybe "pm-worker" (const "pm-worker-tracer") tracer) \sample -> do outcome <- runStore do adapter <- kirokuAdapter store (defaultKirokuAdapterConfig (SubscriptionName "retention-pm") sourceTarget) instrumented <- sampleOnAck (operationCount config) config.blockSize sample adapter- runProcessManagerWorkerWith defaultWorkerOptions defaultRunCommandOptions retentionManager instrumented decodeSignal+ runProcessManagerWorkerWith defaultWorkerOptions (defaultRunCommandOptions {Command.tracer = tracer}) retentionManager instrumented decodeSignal `Effectful.finally` adapter.shutdown void (expectRight outcome) assertTargets runner (operationCount config) [0 .. 15] (\accountNumber -> length [index | index <- [1 .. operationCount config], index `mod` 16 == accountNumber])- postReturn "pm-worker" (operationCount config)+ postReturn (maybe "pm-worker" (const "pm-worker-tracer") tracer) (operationCount config) pure measured routerWorker :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])@@ -373,3 +401,86 @@ expectRight :: (Show errorType) => Either errorType a -> IO a expectRight = either (fail . show) pure++-- Processor discards finished spans; ownership is checked separately.+outboxPublisher :: Maybe Tracer -> Bool -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])+outboxPublisher tracer withContext config (_, StoreRunner runStore) = do+ let name = if withContext then "outbox-publisher-contexts-tracer" else maybe "outbox-publisher" (const "outbox-publisher-tracer") tracer+ options = Outbox.defaultPublishOptions {Outbox.tracer = tracer, Outbox.batchSize = 1}+ message index =+ IntegrationEvent+ { messageId = Text.pack (show index),+ source = "retention",+ destination = "retention.events",+ key = Nothing,+ eventType = "Retention",+ schemaVersion = 1,+ contentType = ApplicationJson,+ schemaReference = Nothing,+ sourceEventId = Nothing,+ sourceGlobalPosition = Nothing,+ payloadBytes = BS.replicate 1024 65,+ occurredAt = UTCTime (ModifiedJulianDay 61000) (secondsToDiffTime 0),+ causationId = Nothing,+ correlationId = Nothing,+ traceContext = if withContext then Just (TraceContext ("00-0af7651916cd43dd8448eb211c80319c-" <> Text.justifyRight 16 '0' (Text.pack (showHex index "")) <> "-01") Nothing) else Nothing,+ attributes = Nothing+ }+ -- A fixed backlog: enqueue outside the measured draining loop.+ forM_ [1 .. operationCount config] \index ->+ void (expectRight =<< runStore (Store.runTransaction (Outbox.enqueueIntegrationEventTx (Outbox.OutboxId (UUID.fromWords 0 0 0 (fromIntegral index))) (message index))))+ measured <- measureLegWithSampler config name \sample -> do+ count <- newIORef 0+ let drain = do+ summary <- expectRight =<< runStore (Outbox.publishClaimedOutbox (\rows -> pure [(row.outboxId, Outbox.PublishSucceeded) | row <- rows]) options Nothing)+ when (summary.published == 0) (fail "outbox retention made no progress")+ previous <- readIORef count+ let next = previous + summary.published+ -- Single-message passes keep sampling at exact operation counts.+ atomicModifyIORef' count (const (next, ()))+ when (next `mod` config.blockSize == 0) (sample next)+ when (next < operationCount config) drain+ drain+ backlog <- expectRight =<< runStore Outbox.countOutboxBacklog+ unless (backlog == 0) (fail "outbox retention did not drain backlog")+ postReturn name (operationCount config)+ pure measured++-- A fixed backlog, claimed one row per operation. The SDK records and discards+-- completed spans, as in the publisher and worker legs.+timerFireTracer :: Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])+timerFireTracer tracer config (_, StoreRunner runStore) = do+ let now = UTCTime (ModifiedJulianDay 61000) 0+ options = Timer.defaultTimerWorkerOptions {Timer.tracer = Just tracer, Timer.requeueStuckAfter = Nothing}+ request index = Timer.TimerRequest (Timer.TimerId (UUID.fromWords64 0 (fromIntegral index))) "retention" "one" now (object ["body" .= Text.replicate 1024 "x"]) Nothing+ Telemetry.withWorkflowSpan (Just tracer) (Workflow.WorkflowName "timerRetention") (Workflow.WorkflowId "one") Nothing $ \_ -> do+ requests <- traverse (Timer.captureTimerTraceContext (Just tracer) . request) [1 .. operationCount config]+ expectRight =<< runStore (StoreTx.runTransaction (mapM_ Timer.scheduleTimerTx requests))+ measured <- measureLeg config "timer-fire-tracer" $ \_ -> do+ row <- expectRight =<< runStore (Timer.runTimerWorkerWith Nothing options now (const (pure (Just (EventId UUID.nil)))))+ unless (maybe False (const True) row) (fail "timer retention exhausted its backlog")+ backlog <- expectRight =<< runStore (Timer.countDueTimers now)+ unless (backlog == 0) (fail "timer retention left due rows")+ postReturn "timer-fire-tracer" (operationCount config)+ pure measured++workflowSnapshotResumeTracer :: Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])+workflowSnapshotResumeTracer tracer config (_, StoreRunner runStore) = do+ let name = Workflow.WorkflowName "retention"+ wid = Workflow.WorkflowId "one"+ options = Workflow.defaultWorkflowRunOptions {Workflow.tracer = Just tracer, Workflow.snapshotPolicy = Every 1000}+ seeded <- expectRight =<< runStore (Workflow.runWorkflowWith options name wid seedWorkflow)+ unless (seeded == Workflow.Suspended) (fail "workflow retention did not seed")+ measured <- measureLeg config "workflow-snapshot-resume-tracer" $ \_ -> do+ result <- expectRight =<< runStore (Workflow.runWorkflowWith options name wid holdWorkflow)+ unless (result == Workflow.Suspended) (fail "workflow retention did not resume")+ postReturn "workflow-snapshot-resume-tracer" (operationCount config)+ pure measured++seedWorkflow :: (Workflow.Workflow :> es) => Eff es Value+seedWorkflow = do+ forM_ [1 .. 1000 :: Int] $ \index -> void (Workflow.step (Workflow.StepName ("prefix-" <> Text.pack (show index))) (pure index))+ holdWorkflow++holdWorkflow :: (Workflow.Workflow :> es) => Eff es Value+holdWorkflow = Workflow.awaitStep (Workflow.StepName "hold") (pure ())
+ retention/Retention/SpanOwnership.hs view
@@ -0,0 +1,238 @@+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE OverloadedRecordDot #-}+{-# LANGUAGE OverloadedStrings #-}++-- | Finite ownership probe, separate from the steady-state heap slope gate.+module Retention.SpanOwnership (spec) where++import Control.Concurrent (threadDelay)+import Control.Exception (bracket, evaluate)+import Control.Monad (forM, forM_, void)+import Data.Aeson qualified as Aeson+import Data.ByteString qualified as BS+import Data.ByteString.Lazy qualified as Lazy+import Data.IORef (readIORef, writeIORef)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime (..))+import Data.Time.Calendar (Day (ModifiedJulianDay))+import Data.UUID qualified as UUID+import Data.Vector qualified as Vector+import GHC.Exts (fromList)+import Keiro.Command qualified as Command+import Keiro.Integration.Event qualified as Integration+import Keiro.Outbox qualified as Outbox+import Keiro.Stream qualified as Stream+import Keiro.Telemetry qualified as Telemetry+import Keiro.Test.Postgres (Fixture, StoreRunner (..), withFreshResourceStore)+import Keiro.Timer qualified as Timer+import Keiro.Workflow qualified as Workflow+import Kiroku.Store qualified as Store+import OpenTelemetry.Attributes (toAttribute)+import OpenTelemetry.Context.ThreadLocal qualified as Context+import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter)+import OpenTelemetry.Trace.Core+import Retention.Fixture (TargetCommand (..), retentionNoOpTargetStream)+import Shibuya.Adapter (TraceCorrelation (..))+import Shibuya.Core.Types qualified as Shibuya+import System.Mem (performMajorGC)+import System.Mem.Weak (Weak, deRefWeak, mkWeakPtr)+import Test.Hspec++spec :: Fixture -> Spec+spec fixture = describe "Span payload ownership" do+ forM_ [False, True] \finished ->+ it (if finished then "collects payloads with finished spans retained" else "collects payloads with live spans retained") $+ probe finished False+ it "detects a span-owned payload and collects it after releasing spans" $+ probe True True+ it "collects typed no-op payloads while their uninspected spans remain alive" $+ withFreshResourceStore fixture $ \(_, runner) -> do+ (processor, exported) <- inMemoryListExporter+ bracket+ (createTracerProvider [processor] emptyTracerProviderOptions)+ (\provider -> void (shutdownTracerProvider provider Nothing))+ $ \provider -> do+ weak <- produceNoOpPayloads runner (makeTracer provider "keiro-retention" tracerOptions)+ awaitCollection weak `shouldReturn` True+ spans <- readIORef exported+ length spans `shouldBe` 8++ it "collects batch payloads with mixed destinations and more than 128 linked rows" $ do+ (processor, exported) <- inMemoryListExporter+ bracket+ (createTracerProvider [processor] emptyTracerProviderOptions)+ (\provider -> void (shutdownTracerProvider provider Nothing))+ $ \provider -> do+ weak <- produceBatchPayloads (makeTracer provider "keiro-retention" tracerOptions)+ awaitCollection weak `shouldReturn` True+ spans <- readIORef exported+ length spans `shouldBe` 1++ it "collects fetched timer and journal payloads while durable spans remain retained" $+ withFreshResourceStore fixture $ \(_, runner) -> do+ (processor, exported) <- inMemoryListExporter+ bracket+ (createTracerProvider [processor] emptyTracerProviderOptions)+ (\provider -> void (shutdownTracerProvider provider Nothing))+ $ \provider -> do+ weak <- produceDurablePayloads runner (makeTracer provider "durable-ownership" tracerOptions)+ awaitCollection weak `shouldReturn` True+ spans <- readIORef exported+ length spans `shouldBe` 32++ forM_ [ParentAndLink, LinkOnly] $ \policy ->+ it ("collects subscription envelopes and lazy scalar attributes under " <> show policy) $ do+ (processor, exported) <- inMemoryListExporter+ bracket+ (createTracerProvider [processor] emptyTracerProviderOptions)+ (\provider -> void (shutdownTracerProvider provider Nothing))+ $ \provider -> do+ weak <- produceSubscriptionPayloads (makeTracer provider "keiro-retention" tracerOptions) policy+ awaitCollection weak `shouldReturn` True+ spans <- readIORef exported+ length spans `shouldBe` 8++-- Both probes use rows decoded from PostgreSQL, rather than synthetic contexts.+{-# NOINLINE produceDurablePayloads #-}+produceDurablePayloads :: StoreRunner -> Tracer -> IO [Weak Aeson.Value]+produceDurablePayloads (StoreRunner runStore) tracer = fmap concat $ forM [1 .. 8 :: Int] $ \index -> do+ let name = Workflow.WorkflowName "ownership"+ wid = Workflow.WorkflowId (Text.pack (show index))+ now = UTCTime (ModifiedJulianDay 60000) 0+ payload = Aeson.String (Text.replicate (128 * 1024) (Text.pack (show index)))+ request = Timer.TimerRequest (Timer.TimerId (UUID.fromWords 0 0 0 (fromIntegral index))) "ownership" "one" now payload Nothing+ Telemetry.withWorkflowSpan (Just tracer) name wid Nothing $ \_ -> do+ Right () <- runStore (Workflow.appendJournalEntryTraced (Just tracer) name wid (Workflow.StepRecorded "origin" payload now))+ inSpan' tracer "ownership wake" defaultSpanArguments $ \_ -> do+ Right () <- runStore (Workflow.appendJournalEntryTraced (Just tracer) name wid (Workflow.StepRecorded "wake" payload now))+ pure ()+ captured <- Timer.captureTimerTraceContext (Just tracer) request+ Right () <- runStore (Store.runTransaction (Timer.scheduleTimerTx captured))+ pure ()+ Right events <- runStore (Store.readStreamForward (Workflow.workflowGenerationStreamName name wid 0) (Store.StreamVersion 0) 2)+ Right (Just timer) <- runStore (Timer.lookupTimer (Timer.TimerId (UUID.fromWords 0 0 0 (fromIntegral index))))+ let event = Vector.head events+ wake = events Vector.! 1+ eventPayload <- evaluate (event.payload)+ wakePayload <- evaluate (wake.payload)+ timerPayload <- evaluate (timer.payload)+ eventWeak <- mkWeakPtr eventPayload Nothing+ wakeWeak <- mkWeakPtr wakePayload Nothing+ timerWeak <- mkWeakPtr timerPayload Nothing+ Telemetry.withWorkflowSpan (Just tracer) name wid Nothing $ \sp ->+ Telemetry.recordWorkflowResumeLinks sp (Telemetry.traceContextFromMetadata event.metadata) (Telemetry.traceContextFromMetadata wake.metadata)+ Telemetry.withTimerFireSpan (Just tracer) timer.processManagerName (case timer.timerId of Timer.TimerId uuid -> UUID.toText uuid) timer.traceContext (const (pure ()))+ pure [eventWeak, wakeWeak, timerWeak]++{-# NOINLINE produceSubscriptionPayloads #-}+produceSubscriptionPayloads :: Tracer -> TraceCorrelation -> IO [Weak Text]+produceSubscriptionPayloads tracer policy = forM [1 .. 8 :: Int] $ \index -> do+ payload <- evaluate (Text.replicate (128 * 1024) (Text.pack (show index)))+ weak <- mkWeakPtr payload Nothing+ let envelope =+ Shibuya.Envelope+ { Shibuya.messageId = Shibuya.MessageId (Text.pack (show index)),+ Shibuya.cursor = Nothing,+ Shibuya.partition = Nothing,+ Shibuya.enqueuedAt = Nothing,+ Shibuya.traceContext = Just [("traceparent", "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01")],+ Shibuya.headers = Nothing,+ Shibuya.attempt = Nothing,+ Shibuya.attributes = fromList [("payload.size", toAttribute (Text.length payload))],+ Shibuya.payload = payload+ }+ Telemetry.withSubscriptionProcessSpan (Just tracer) policy "ownership" envelope (const (pure ()))+ pure weak++{-# NOINLINE produceBatchPayloads #-}+produceBatchPayloads :: Tracer -> IO [Weak Text]+produceBatchPayloads tracer = do+ prepared <- forM [1 .. 200 :: Int] $ \index -> do+ payload <- evaluate (Text.replicate (128 * 1024) (Text.pack (show index)))+ weak <- mkWeakPtr payload Nothing+ let now = UTCTime (ModifiedJulianDay 60000) 0+ context = Integration.TraceContext "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" Nothing+ event = Integration.IntegrationEvent (Text.pack (show index)) "retention" (if even index then "even" else "odd") Nothing "Ownership" 1 Integration.ApplicationJson Nothing Nothing Nothing (BS.replicate 1024 65) now Nothing Nothing (Just context) (Just (Aeson.object ["payload" Aeson..= payload]))+ row = Outbox.OutboxRow (Outbox.OutboxId UUID.nil) event Outbox.OutboxPending 0 now Nothing Nothing Nothing Nothing now now+ pure (row, weak)+ Telemetry.withProducerBatchSpan (Just tracer) (map fst prepared) (const (pure ()))+ pure (map snd prepared)++{-# NOINLINE produceNoOpPayloads #-}+produceNoOpPayloads :: StoreRunner -> Tracer -> IO [Weak Text]+produceNoOpPayloads (StoreRunner runStore) tracer =+ forM [1 .. 8 :: Int] $ \index -> do+ payload <- evaluate (Text.replicate (128 * 1024) (Text.pack (show index)))+ weak <- mkWeakPtr payload Nothing+ let handler =+ Command.DomainCommandHandler+ { Command.eventStream = retentionNoOpTargetStream,+ Command.classifySilent = const (Command.SilentNoOp payload)+ }+ options = Command.defaultRunCommandOptions {Command.tracer = Just tracer}+ result <- runStore (Command.runDomainCommand options handler (Stream.stream ("ownership-noop-" <> Text.pack (show index))) (Credit 1))+ case result of+ Right (Right _) -> pure weak+ _ -> fail "no-op ownership command failed"++probe :: Bool -> Bool -> IO ()+probe finished retainPayload = do+ (processor, exported) <- inMemoryListExporter+ bracket+ (createTracerProvider [processor] emptyTracerProviderOptions)+ (\provider -> void (shutdownTracerProvider provider Nothing))+ \provider -> do+ let tracer = makeTracer provider "keiro-retention" tracerOptions+ (live, weakPayloads) <- produce tracer finished retainPayload+ collected <- awaitCollection weakPayloads+ collected `shouldBe` not retainPayload+ if finished+ then do+ spans <- readIORef exported+ length spans `shouldBe` 8+ -- Touch identities after GC, keeping the exporter roots live.+ mapM (evaluate . spanContext) spans >>= (\contexts -> length contexts `shouldBe` 8)+ else do+ mapM isRecording live `shouldReturn` replicate 8 True+ mapM_ (\span -> endSpan span Nothing) live+ writeIORef exported []+ awaitCollection weakPayloads `shouldReturn` True++-- A separate, non-inlined scope prevents the driver retaining workload locals.+{-# NOINLINE produce #-}+produce :: Tracer -> Bool -> Bool -> IO ([Span], [Weak Text])+produce tracer finished retainPayload = do+ results <- forM [1 .. 8 :: Int] \index -> do+ payload <- evaluate (Text.replicate (128 * 1024) (Text.pack (show index)))+ weak <- mkWeakPtr payload Nothing+ context <- Context.getContext+ span <-+ if finished && not retainPayload+ then Telemetry.withCommandSpan (Just tracer) ("ownership-" <> Text.pack (show index)) Nothing $ \current -> do+ stamped <- Telemetry.stampTraceContext (Just (Aeson.object ["payload" Aeson..= payload]))+ _ <- evaluate (Lazy.length (Aeson.encode stamped))+ maybe (fail "expected a recording command span") pure current+ else createSpan tracer context "ownership" defaultSpanArguments+ -- Force only the bounded scalar that real helpers put on spans.+ size <- evaluate (Text.length payload)+ addAttribute span "payload.size" size+ if retainPayload then addAttribute span "negative.payload" payload else pure ()+ if finished then endSpan span Nothing else pure ()+ pure (span, weak)+ pure (if finished then [] else map fst results, map snd results)++awaitCollection :: [Weak a] -> IO Bool+awaitCollection weak = loop (20 :: Int)+ where+ loop remaining = do+ performMajorGC+ alive <- traverse deRefWeak weak+ if all absent alive+ then pure True+ else+ if remaining == 0+ then pure False+ else threadDelay 10_000 >> loop (remaining - 1)+ absent Nothing = True+ absent (Just _) = False
+ retention/Retention/Tracing.hs view
@@ -0,0 +1,25 @@+{-# LANGUAGE OverloadedStrings #-}++module Retention.Tracing (withRecordingTracer) where++import Control.Exception (bracket)+import Control.Monad (unless)+import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter)+import OpenTelemetry.Processor.Span (SpanProcessor (..))+import OpenTelemetry.Trace.Core++-- Benchmark SDK work without exporting I/O or accumulating completed spans.+-- Check recording explicitly: processor-free providers can be non-recording.+withRecordingTracer :: (Tracer -> IO a) -> IO a+withRecordingTracer action = bracket acquire (\(provider, _) -> shutdownTracerProvider provider Nothing) (action . snd)+ where+ acquire = do+ (processor, _) <- inMemoryListExporter+ -- Keep the SDK's recording path but discard completed spans immediately.+ let discard = processor {spanProcessorOnEnd = const (pure ())}+ provider <- createTracerProvider [discard] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-bench" tracerOptions+ inSpan' tracer "recording-check" defaultSpanArguments $ \span -> do+ recording <- isRecording span+ unless recording (fail "benchmark tracer is not recording")+ pure (provider, tracer)
src/Keiro/Command.hs view
@@ -153,6 +153,7 @@ recordSnapshotSeedSkipped, recordSnapshotSeedVerificationFailed, recordSnapshotWriteFailures,+ stampTraceContext, withCommandSpan, ) import Kiroku.Store.Append (appendToStream)@@ -901,10 +902,10 @@ ci -> Eff es (Either CommandError (CommandResult (EventStream phi rs s ci co))) runCommand options validatedEventStream targetStream command =- withCommandSpan (options ^. #tracer) (resolvedStreamName eventStream targetStream) Nothing $ \mSpan -> do+ withCommandSpan (options ^. #tracer) (resolvedStreamName eventStream targetStream) Nothing $ \mSpan -> withStampedCommandMetadata options $ \stampedOptions -> do (outcome, attemptNo) <- domainCommandAttempts- options+ stampedOptions (silentNoOpHandler validatedEventStream) targetStream command@@ -930,11 +931,21 @@ ci -> Eff es (Either CommandError (DomainCommandOutcome (EventStream phi rs s ci co) co rejection noOp)) runDomainCommand options handler@DomainCommandHandler {eventStream = validatedEventStream} targetStream command =- withCommandSpan (options ^. #tracer) (resolvedStreamName (unvalidated validatedEventStream) targetStream) Nothing $ \mSpan -> do- (outcome, attemptNo) <- domainCommandAttempts options handler targetStream command mSpan+ withCommandSpan (options ^. #tracer) (resolvedStreamName (unvalidated validatedEventStream) targetStream) Nothing $ \mSpan -> withStampedCommandMetadata options $ \stampedOptions -> do+ (outcome, attemptNo) <- domainCommandAttempts stampedOptions handler targetStream command mSpan recordDomainCommandOutcome options mSpan attemptNo outcome pure outcome +-- Capture once inside the command span, before any optimistic retry. With no+-- configured tracer the original options and metadata pass through unchanged.+withStampedCommandMetadata :: (MonadIO m) => RunCommandOptions -> (RunCommandOptions -> m a) -> m a+withStampedCommandMetadata options body = case options ^. #tracer of+ Nothing -> body options+ Just _ -> do+ metadata <- stampTraceContext (options ^. #metadata)+ body (options & #metadata .~ metadata)+{-# INLINE withStampedCommandMetadata #-}+ -- | The optimistic-concurrency attempt loop shared by the plain and -- transactional domain command runners: hydrate, detect a conflict fixpoint, -- prepare the plan, classify a silent decision, or hand an accepted batch to@@ -1095,10 +1106,10 @@ ([(co, RecordedEvent)] -> AppendResult -> Tx.Transaction (SqlTransactionDecision a)) -> Eff es (Either CommandError (SqlCommandOutcome (EventStream phi rs s ci co) a)) runCommandWithSqlEventsControlled options validatedEventStream targetStream command afterAppend =- withCommandSpan (options ^. #tracer) (resolvedStreamName eventStream targetStream) Nothing $ \mSpan -> do+ withCommandSpan (options ^. #tracer) (resolvedStreamName eventStream targetStream) Nothing $ \mSpan -> withStampedCommandMetadata options $ \stampedOptions -> do (outcome, attemptNo) <- domainSqlCommandAttempts- options+ stampedOptions (silentNoOpHandler validatedEventStream) targetStream command@@ -1170,8 +1181,8 @@ ([(co, RecordedEvent)] -> AppendResult -> Tx.Transaction (SqlTransactionDecision a)) -> Eff es (Either CommandError (DomainSqlCommandOutcome (EventStream phi rs s ci co) co rejection noOp a)) runDomainCommandWithSqlEventsControlled options handler@DomainCommandHandler {eventStream = validatedEventStream} targetStream command afterAppend =- withCommandSpan (options ^. #tracer) (resolvedStreamName (unvalidated validatedEventStream) targetStream) Nothing $ \mSpan -> do- (outcome, attemptNo) <- domainSqlCommandAttempts options handler targetStream command afterAppend mSpan+ withCommandSpan (options ^. #tracer) (resolvedStreamName (unvalidated validatedEventStream) targetStream) Nothing $ \mSpan -> withStampedCommandMetadata options $ \stampedOptions -> do+ (outcome, attemptNo) <- domainSqlCommandAttempts stampedOptions handler targetStream command afterAppend mSpan recordDomainSqlCommandOutcome options mSpan attemptNo outcome pure outcome @@ -1312,11 +1323,15 @@ addAttribute sp (unkey db_system_name) ("postgresql" :: Text) addAttribute sp (unkey keiro_retry_attempt) (Prelude.fromIntegral attemptNo :: Int64) case result of- Right v ->- addAttribute sp (unkey keiro_events_appended) (Prelude.fromIntegral (eventsOf v) :: Int64)+ Right v -> do+ -- Attribute constructors can be lazy: keep the scalar, not its outcome.+ let !count = Prelude.fromIntegral (eventsOf v) :: Int64+ addAttribute sp (unkey keiro_events_appended) count Left err -> do- addAttribute sp (unkey error_type) (commandErrorClass err)- setStatus sp (Error (Text.take 256 (Text.pack (show err))))+ let !errorClass = commandErrorClass err+ !description = Text.pack (Prelude.take 256 (show err))+ addAttribute sp (unkey error_type) errorClass+ setStatus sp (Error description) recordDomainCommandOutcome :: (IOE :> es) =>@@ -1362,8 +1377,9 @@ Eff es () recordDomainDecision options mSpan domainDecision = do let decisionClass = domainDecisionClass domainDecision- for_ mSpan $ \sp ->- addAttribute sp (unkey keiro_command_decision) (commandDecisionClassText decisionClass)+ for_ mSpan $ \sp -> do+ let !decisionText = commandDecisionClassText decisionClass+ addAttribute sp (unkey keiro_command_decision) decisionText recordCommandDecision (options ^. #metrics) decisionClass domainDecisionClass :: DomainDecision co rejection noOp -> CommandDecisionClass
src/Keiro/Outbox.hs view
@@ -53,6 +53,7 @@ -- * Canonical producer-subscription helper IntegrationProducer (..),+ ProducerTracePolicy (..), IntegrationProducerConfigError (..), IntegrationEventDraft (..), mkIntegrationProducer,@@ -84,7 +85,6 @@ TraceContext, ) import Keiro.Outbox.Identity-import Keiro.Outbox.Kafka (outboxRowToKafkaRecord) import Keiro.Outbox.Schema import Keiro.Outbox.Types import Keiro.Prelude@@ -97,19 +97,17 @@ recordOutboxReclaimed, recordOutboxRejected, recordOutboxRetried,- withProducerSpan,+ traceContextFromMetadata,+ withProducerBatchSpan, ) import Kiroku.Store.Effect (Store) import Kiroku.Store.Transaction (runTransaction) import Kiroku.Store.Types (EventId, GlobalPosition, RecordedEvent (..))-import OpenTelemetry.Attributes.Key (AttributeKey (..), unkey)+import OpenTelemetry.Attributes.Key (unkey) import OpenTelemetry.SemanticConventions (error_type) import OpenTelemetry.Trace.Core (SpanStatus (..), addAttribute, setStatus) import "hasql-transaction" Hasql.Transaction qualified as Tx -keiro_outbox_batch_size :: AttributeKey Int64-keiro_outbox_batch_size = AttributeKey "keiro.outbox.batch.size"- -- | Mint a fresh time-ordered UUIDv7 for use as an 'OutboxId'. freshOutboxId :: (IOE :> es) => Eff es OutboxId freshOutboxId = fmap OutboxId (liftIO V7.genUUID)@@ -119,6 +117,9 @@ -- caller supplies a stable 'OutboxId' so retried command attempts coalesce -- on the @(source, message_id)@ unique constraint. --+-- This inline path has no source event. Capture trace context before entering+-- the transaction and supply the same context on replay.+-- -- Ordering caveat: under 'PerKeyHeadOfLine' and 'PerSourceStream', the publisher -- orders rows by @created_at@, which PostgreSQL sets to transaction-start time. -- If two concurrent transactions enqueue the same key/source and commit in the@@ -135,6 +136,13 @@ -- Producer-subscription helper -- --------------------------------------------------------------------------- +-- | Immutable per-producer/store adoption policy. Never recompute or move the+-- cutoff after activation; see the producer guide for quiescence and rollback.+data ProducerTracePolicy+ = PreserveDraftTrace+ | DeriveTraceAfter !GlobalPosition+ deriving stock (Generic, Eq, Show)+ -- | Configuration for the canonical producer subscription. -- -- A service running 'IntegrationProducer' reads its private event stream,@@ -149,6 +157,8 @@ -- * 'messageIdPrefix' — non-empty namespace, validated using TypeID prefix -- syntax. The resulting deterministic message ID is opaque text, not a TypeID. -- Prefer 'mkIntegrationProducer'; direct record construction bypasses validation.+-- * 'tracePolicy' — immutable per-producer/store trace adoption; begin with+-- 'PreserveDraftTrace' and follow the producer guide before setting a cutoff. -- * 'mapEvent' — pure mapper from a private 'RecordedEvent' and its -- decoded payload to an 'IntegrationEventDraft'. Returning 'Nothing' -- skips the event without enqueuing a row.@@ -156,6 +166,7 @@ { name :: !Text, source :: !Text, messageIdPrefix :: !Text,+ tracePolicy :: !ProducerTracePolicy, mapEvent :: !(RecordedEvent -> e -> Maybe IntegrationEventDraft) } deriving stock (Generic)@@ -273,7 +284,15 @@ .~ ((draft ^. #sourceEventId) <|> Just (recorded ^. #eventId)) & #sourceGlobalPosition .~ ((draft ^. #sourceGlobalPosition) <|> Just (recorded ^. #globalPosition))- event = normalizeProducerEvent (draftToEvent (producer ^. #source) (identity ^. #messageId) withProvenance)+ withTrace = case producer ^. #tracePolicy of+ PreserveDraftTrace -> withProvenance+ DeriveTraceAfter cutoff+ | recorded ^. #globalPosition > cutoff -> case draft ^. #traceContext of+ Just _ -> withProvenance+ Nothing -> maybe withProvenance (\context -> withProvenance & #traceContext ?~ context) (traceContextFromMetadata (recorded ^. #metadata))+ | otherwise -> withProvenance+ event = normalizeProducerEvent (draftToEvent (producer ^. #source) (identity ^. #messageId) withTrace)+{-# INLINE enqueueProducerEventTx #-} -- --------------------------------------------------------------------------- -- Publisher worker@@ -362,11 +381,10 @@ publishRows :: [OutboxRow] -> Eff es (Map.Map OutboxId PublishOutcome) publishRows [] = pure Map.empty- publishRows batch@(firstRow : _) =+ publishRows batch = Map.fromList <$> withBatchSpan batch- firstRow ( do attempted <- trySync (publish batch) let normalized =@@ -381,17 +399,11 @@ withBatchSpan :: [OutboxRow] ->- OutboxRow -> Eff es [(OutboxId, PublishOutcome)] -> Eff es [(OutboxId, PublishOutcome)]- withBatchSpan batch firstRow action =- withProducerSpan- (options ^. #tracer)- (firstRow ^. #event)- (outboxRowToKafkaRecord firstRow)- $ \mSpan -> do- for_ mSpan $ \sp ->- addAttribute sp (unkey keiro_outbox_batch_size) (fromIntegral (length batch) :: Int64)+ withBatchSpan batch action =+ withProducerBatchSpan (options ^. #tracer) batch $+ \mSpan -> do outcomes <- action case (mSpan, firstFailure outcomes) of (Just sp, Just errMsg) -> do
src/Keiro/Outbox/Types.hs view
@@ -164,9 +164,9 @@ -- -- The optional 'tracer' field opts the publisher into OpenTelemetry -- instrumentation: when present, the publisher opens a @Producer@-kind--- span around each publish call, attributing the first row's--- 'IntegrationEvent.destination' (topic), 'IntegrationEvent.messageId',--- and Kafka key per the messaging semantic conventions. When 'tracer' is+-- root span around each publish call with at most 128 creation-context links.+-- A common destination is attributed on the span; mixed destinations and message+-- IDs are on links. Only singleton batches have per-message span attributes. When 'tracer' is -- 'Nothing' (the default) the publisher emits no spans. data OutboxPublishOptions = OutboxPublishOptions { batchSize :: !Int,
src/Keiro/ProcessManager.hs view
@@ -169,8 +169,8 @@ import Keiro.ProcessManager.Provenance import Keiro.Projection (InlineProjection, runCommandWithProjections, runDomainCommandWithProjections) import Keiro.Stream (Stream)-import Keiro.Telemetry (KeiroMetrics, recordDispatchDeadLettered, recordDispatchDuplicate, recordDispatchFailed, recordDispatchPoison)-import Keiro.Timer (TimerRequest, scheduleTimerTx)+import Keiro.Telemetry (KeiroMetrics, recordDispatchDeadLettered, recordDispatchDuplicate, recordDispatchFailed, recordDispatchPoison, recordSubscriptionOutcome, withSubscriptionProcessSpan)+import Keiro.Timer (TimerRequest, captureTimerTraceContext, scheduleTimerTx) import Kiroku.Store.Effect (Store) import Kiroku.Store.Effect.Resource (KirokuStoreResource) import Kiroku.Store.Error (StoreError (..))@@ -587,13 +587,16 @@ case existingManagerId of Just matchedId -> finish correlationId (PMStateDuplicate matchedId) action Nothing -> do+ timerRequests <- case options ^. #tracer of+ Nothing -> pure (action ^. #timers)+ configured -> traverse (captureTimerTraceContext configured) (action ^. #timers) managerOutcome <- runCommandWithSql managerOptions (manager ^. #eventStream) managerStream (action ^. #command)- (\_ -> traverse_ scheduleTimerTx (action ^. #timers))+ (\_ -> traverse_ scheduleTimerTx timerRequests) case managerOutcome of Left err -> do benign <- confirmBenignDuplicate managerStreamName managerEventId err@@ -604,7 +607,7 @@ -- No-op manager commands do not execute runCommandWithSql's callback, -- so schedule timer-only reactions explicitly. case scheduledInAppend of- Nothing -> runTransaction (traverse_ scheduleTimerTx (action ^. #timers))+ Nothing -> runTransaction (traverse_ scheduleTimerTx timerRequests) Just () -> pure () finish correlationId (PMStateAppended managerResult) action where@@ -728,13 +731,16 @@ case existingManagerId of Just matchedId -> finish (PMStateDuplicate matchedId) Nothing -> do+ timerRequests <- case options ^. #tracer of+ Nothing -> pure (action ^. #timers)+ configured -> traverse (captureTimerTraceContext configured) (action ^. #timers) managerOutcome <- runCommandWithSql managerOptions (manager ^. #eventStream) managerStream (action ^. #command)- (\_ -> traverse_ scheduleTimerTx (action ^. #timers))+ (\_ -> traverse_ scheduleTimerTx timerRequests) case managerOutcome of Left err -> do benign <- confirmBenignDuplicate managerStreamName managerEventId err@@ -743,7 +749,7 @@ else pure (Left err) Right (managerResult, scheduledInAppend) -> do case scheduledInAppend of- Nothing -> runTransaction (traverse_ scheduleTimerTx (action ^. #timers))+ Nothing -> runTransaction (traverse_ scheduleTimerTx timerRequests) Just () -> pure () finish (PMStateAppended managerResult) @@ -901,10 +907,19 @@ Adapter es msg -> (msg -> Maybe (RecordedEvent, input)) -> Eff es ()-runDomainProcessManagerWorkerWith workerOptions options manager Adapter {source = adapterSource} decodeMessage =+runDomainProcessManagerWorkerWith workerOptions options manager Adapter {source = adapterSource, telemetry = adapterTelemetry} decodeMessage = Streamly.fold Fold.drain- $ Streamly.mapM handleIngested adapterSource+ $ Streamly.mapM handleTraced adapterSource where+ -- Select the original handler without inspecting per-message telemetry.+ handleTraced = case options ^. #tracer of+ Nothing -> handleIngested+ Just _ -> \ingested@Ingested {envelope = traceEnvelope} -> instrument traceEnvelope $ \mSpan -> do+ decision <- handleIngested ingested+ recordSubscriptionOutcome mSpan decision+ pure decision+ instrument = withSubscriptionProcessSpan (options ^. #tracer) (adapterTelemetry ^. #traceCorrelation) (manager ^. #name)+ handleIngested :: Ingested es msg -> Eff es AckDecision handleIngested Ingested {envelope = env@Envelope {payload = message}, ack = AckHandle finalizeAck} = do decision <- case decodeMessage message of@@ -1008,10 +1023,19 @@ Adapter es msg -> (msg -> Maybe (RecordedEvent, input)) -> Eff es ()-runProcessManagerWorkerWith workerOptions options manager Adapter {source = adapterSource} decodeMessage =+runProcessManagerWorkerWith workerOptions options manager Adapter {source = adapterSource, telemetry = adapterTelemetry} decodeMessage = Streamly.fold Fold.drain- $ Streamly.mapM handleIngested adapterSource+ $ Streamly.mapM handleTraced adapterSource where+ -- Select the original handler without inspecting per-message telemetry.+ handleTraced = case options ^. #tracer of+ Nothing -> handleIngested+ Just _ -> \ingested@Ingested {envelope = traceEnvelope} -> instrument traceEnvelope $ \mSpan -> do+ decision <- handleIngested ingested+ recordSubscriptionOutcome mSpan decision+ pure decision+ instrument = withSubscriptionProcessSpan (options ^. #tracer) (adapterTelemetry ^. #traceCorrelation) (manager ^. #name)+ handleIngested :: Ingested es msg -> Eff es AckDecision handleIngested Ingested {envelope = env@Envelope {payload = message}, ack = AckHandle finalizeAck} = do decision <- case decodeMessage message of
src/Keiro/ProcessManager/Reaction.hs view
@@ -71,6 +71,7 @@ import Keiro.DeadLetter (DispatcherKind (..)) import Keiro.EventStream (EventStream) import Keiro.EventStream.Validate (ValidatedEventStream, unvalidated)+import Keiro.Integration.Event (TraceContext) import Keiro.Prelude import Keiro.ProcessManager ( DispatchFailure (..),@@ -87,13 +88,14 @@ ) import Keiro.Projection (InlineProjection, runCommandWithProjections) import Keiro.Stream (Stream)-import Keiro.Telemetry (recordDispatchDuplicate, recordDispatchFailed, recordDispatchPoison)+import Keiro.Telemetry (Tracer, recordDispatchDuplicate, recordDispatchFailed, recordDispatchPoison, recordSubscriptionOutcome, traceContextFromCurrentSpan, withSubscriptionProcessSpan) import Keiro.Timer ( TimerId, TimerRequest, cancelTimerTx, scheduleTimerOnceTx, scheduleTimerTx,+ withTimerTraceContext, ) import Kiroku.Store.Effect (Store) import Kiroku.Store.Effect.Resource (KirokuStoreResource)@@ -281,20 +283,21 @@ runReactiveProcessManagerEngine reducer options manager sourceEvent input = case (manager ^. #react) input of NoAdvance unconditional -> do- timers <- runTimerPhase unconditional+ timers <- runTimerPhase (options ^. #tracer) unconditional finish ReactionNotAdvanced timers unconditional AdvanceReaction sagaCommand unconditional acceptedOnly -> do existing <- firstExistingEventId options sagaStreamName managerProbes case existing of Just matchedId -> recoverAndFinish matchedId (unconditional <> acceptedOnly) Nothing -> do+ creation <- timerPhaseContext (options ^. #tracer) (unconditional <> acceptedOnly) outcome <- runDomainCommandWithSqlEvents managerOptions (manager ^. #sagaHandler) sagaStream sagaCommand- (\_ _ -> runTimerPhaseTx (unconditional <> acceptedOnly))+ (\_ _ -> runTimerPhaseTx creation (unconditional <> acceptedOnly)) case outcome of Left commandError -> do raced <- firstExistingEventId options sagaStreamName managerProbes@@ -310,7 +313,7 @@ case raced of Just matchedId -> recoverAndFinish matchedId (unconditional <> acceptedOnly) Nothing -> do- timers <- runTimerPhase unconditional+ timers <- runTimerPhase (options ^. #tracer) unconditional finish (ReactionEvaluated domainOutcome) timers unconditional Right (_, Just _) -> Prelude.error "runReactiveProcessManagerOnce: silent decision returned a timer callback result"@@ -366,10 +369,19 @@ Adapter es msg -> (msg -> Maybe (RecordedEvent, input)) -> Eff es ()-runReactiveProcessManagerWorkerWith workerOptions options manager Adapter {source = adapterSource} decodeMessage =+runReactiveProcessManagerWorkerWith workerOptions options manager Adapter {source = adapterSource, telemetry = adapterTelemetry} decodeMessage = Streamly.fold Fold.drain- $ Streamly.mapM handleIngested adapterSource+ $ Streamly.mapM handleTraced adapterSource where+ -- Select the original handler without inspecting per-message telemetry.+ handleTraced = case options ^. #tracer of+ Nothing -> handleIngested+ Just _ -> \ingested@Ingested {envelope = traceEnvelope} -> instrument traceEnvelope $ \mSpan -> do+ decision <- handleIngested ingested+ recordSubscriptionOutcome mSpan decision+ pure decision+ instrument = withSubscriptionProcessSpan (options ^. #tracer) (adapterTelemetry ^. #traceCorrelation) (manager ^. #name)+ handleIngested :: Ingested es msg -> Eff es AckDecision handleIngested Ingested {envelope = env@Envelope {payload = message}, ack = AckHandle finalizeAck} = do decision <- case decodeMessage message of@@ -486,12 +498,19 @@ runReactiveProcessManagerWorkerWith defaultWorkerOptions -- | Execute just the timer subsequence in one transaction.-runTimerPhase :: (IOE :> es, Store :> es) => [FollowUp targetCi] -> Eff es ReactionTimerEffects-runTimerPhase followUps+timerPhaseContext :: (MonadIO m) => Maybe Tracer -> [FollowUp targetCi] -> m (Maybe TraceContext)+timerPhaseContext Nothing _ = pure Nothing+timerPhaseContext (Just _) followUps+ | Prelude.any (\case FollowSchedule {} -> True; _ -> False) followUps = traceContextFromCurrentSpan+ | otherwise = pure Nothing++runTimerPhase :: (IOE :> es, Store :> es) => Maybe Tracer -> [FollowUp targetCi] -> Eff es ReactionTimerEffects+runTimerPhase tracer followUps | List.null timerFollowUps = pure zeroTimerEffects | otherwise = do emitTimerPhaseProbe (List.length timerFollowUps)- runTransaction (runTimerPhaseTx followUps)+ creation <- timerPhaseContext tracer followUps+ runTransaction (runTimerPhaseTx creation followUps) where timerFollowUps = List.filter isTimerFollowUp followUps isTimerFollowUp FollowSchedule {} = True@@ -519,16 +538,16 @@ emitTimerPhaseProbe _ = pure () #endif -runTimerPhaseTx :: [FollowUp targetCi] -> Tx.Transaction ReactionTimerEffects-runTimerPhaseTx = foldM step zeroTimerEffects+runTimerPhaseTx :: Maybe TraceContext -> [FollowUp targetCi] -> Tx.Transaction ReactionTimerEffects+runTimerPhaseTx creation = foldM step zeroTimerEffects where step summary = \case FollowDispatch {} -> pure summary FollowSchedule Rearm request -> do- scheduleTimerTx request+ scheduleTimerTx (withTimerTraceContext creation request) pure summary {statementsCommitted = summary ^. #statementsCommitted Prelude.+ 1} FollowSchedule Once request -> do- inserted <- scheduleTimerOnceTx request+ inserted <- scheduleTimerOnceTx (withTimerTraceContext creation request) pure summary { statementsCommitted = summary ^. #statementsCommitted Prelude.+ 1,
src/Keiro/Router.hs view
@@ -90,7 +90,7 @@ import Keiro.Projection (InlineProjection, runCommandWithProjections, runDomainCommandWithProjections) import Keiro.Router.Selection import Keiro.Stream (Stream)-import Keiro.Telemetry (recordDispatchDuplicate, recordDispatchFailed, recordDispatchPoison)+import Keiro.Telemetry (recordDispatchDuplicate, recordDispatchFailed, recordDispatchPoison, recordSubscriptionOutcome, withSubscriptionProcessSpan) import Kiroku.Store.Effect (Store) import Kiroku.Store.Effect.Resource (KirokuStoreResource) import Kiroku.Store.Error (StoreError (..))@@ -551,10 +551,19 @@ Adapter es msg -> (msg -> Maybe (RecordedEvent, input)) -> Eff es ()-runDomainRouterWorkerWith workerOptions options router Adapter {source = adapterSource} decodeMessage =+runDomainRouterWorkerWith workerOptions options router Adapter {source = adapterSource, telemetry = adapterTelemetry} decodeMessage = Streamly.fold Fold.drain- $ Streamly.mapM handleIngested adapterSource+ $ Streamly.mapM handleTraced adapterSource where+ -- Select the original handler without inspecting per-message telemetry.+ handleTraced = case options ^. #tracer of+ Nothing -> handleIngested+ Just _ -> \ingested@Ingested {envelope = traceEnvelope} -> instrument traceEnvelope $ \mSpan -> do+ decision <- handleIngested ingested+ recordSubscriptionOutcome mSpan decision+ pure decision+ instrument = withSubscriptionProcessSpan (options ^. #tracer) (adapterTelemetry ^. #traceCorrelation) (router ^. #name)+ handleIngested :: Ingested es msg -> Eff es AckDecision handleIngested Ingested {envelope = env@Envelope {payload = message}, ack = AckHandle finalizeAck} = do decision <- case decodeMessage message of@@ -670,10 +679,19 @@ Adapter es msg -> (msg -> Maybe (RecordedEvent, input)) -> Eff es ()-runRouterWorkerWith workerOptions options router Adapter {source = adapterSource} decodeMessage =+runRouterWorkerWith workerOptions options router Adapter {source = adapterSource, telemetry = adapterTelemetry} decodeMessage = Streamly.fold Fold.drain- $ Streamly.mapM handleIngested adapterSource+ $ Streamly.mapM handleTraced adapterSource where+ -- Select the original handler without inspecting per-message telemetry.+ handleTraced = case options ^. #tracer of+ Nothing -> handleIngested+ Just _ -> \ingested@Ingested {envelope = traceEnvelope} -> instrument traceEnvelope $ \mSpan -> do+ decision <- handleIngested ingested+ recordSubscriptionOutcome mSpan decision+ pure decision+ instrument = withSubscriptionProcessSpan (options ^. #tracer) (adapterTelemetry ^. #traceCorrelation) (router ^. #name)+ handleIngested :: Ingested es msg -> Eff es AckDecision handleIngested Ingested {envelope = env@Envelope {payload = message}, ack = AckHandle finalizeAck} = do decision <- case decodeMessage message of@@ -779,11 +797,20 @@ Adapter es msg -> (msg -> Maybe (RecordedEvent, input)) -> Eff es ()-runDeclarativeRouterWorkerWith workerOptions options router Adapter {source = adapterSource} decodeMessage =+runDeclarativeRouterWorkerWith workerOptions options router Adapter {source = adapterSource, telemetry = adapterTelemetry} decodeMessage = Streamly.fold Fold.drain- $ Streamly.mapM handleIngested adapterSource+ $ Streamly.mapM handleTraced adapterSource where contract = router ^. #selectionContract++ -- Select the original handler without inspecting per-message telemetry.+ handleTraced = case options ^. #tracer of+ Nothing -> handleIngested+ Just _ -> \ingested@Ingested {envelope = traceEnvelope} -> instrument traceEnvelope $ \mSpan -> do+ decision <- handleIngested ingested+ recordSubscriptionOutcome mSpan decision+ pure decision+ instrument = withSubscriptionProcessSpan (options ^. #tracer) (adapterTelemetry ^. #traceCorrelation) (router ^. #name) handleIngested :: Ingested es msg -> Eff es AckDecision handleIngested Ingested {envelope = env@Envelope {payload = message}, ack = AckHandle finalizeAck} = do
src/Keiro/Telemetry.hs view
@@ -27,12 +27,20 @@ ( -- * Span helpers Tracer, withProducerSpan,+ withProducerBatchSpan, withConsumerSpan,+ withSubscriptionProcessSpan,+ recordSubscriptionOutcome, withCommandSpan, withWorkflowSpan,+ withTimerFireSpan,+ recordWorkflowResumeLinks, -- * W3C TraceContext bridge traceContextFromCurrentSpan,+ traceContextFromMetadata,+ stampTraceContext,+ creationLinkFromTraceContext, traceContextFromHeaders, injectTraceContext, @@ -186,6 +194,10 @@ ) where +import Data.Aeson qualified as Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List qualified as List import Keiro.Inbox.Kafka (KafkaInboundRecord) import Keiro.Integration.Event ( IntegrationEvent,@@ -193,21 +205,20 @@ headerTraceParent, headerTraceState, )-import Keiro.Outbox.Kafka (KafkaProducerRecord)+import Keiro.Outbox.Kafka (KafkaProducerRecord, outboxRowToKafkaRecord)+import Keiro.Outbox.Types (OutboxRow) import Keiro.Prelude import Keiro.Workflow.Types (StepName (..), WorkflowId (..), WorkflowName (..)) import Kiroku.Store.Observability (KirokuEvent (..))+import Kiroku.Store.Types (CategoryName (..), StreamName (..), categoryName) import "base" Control.Exception (bracket)+import "base" GHC.Exts (fromList, toList) import "base" GHC.Stack (HasCallStack) import "bytestring" Data.ByteString qualified as ByteString-import "hs-opentelemetry-api" OpenTelemetry.Attributes- ( addAttributesFromBuilder,- defaultAttributeLimits,- emptyAttributes,- (.@),- )+import "hs-opentelemetry-api" OpenTelemetry.Attributes (Attribute (..), PrimitiveAttribute (..), addAttributesFromBuilder, defaultAttributeLimits, emptyAttributes, toAttribute, (.@)) import "hs-opentelemetry-api" OpenTelemetry.Attributes.Key (AttributeKey (..)) import "hs-opentelemetry-api" OpenTelemetry.Context (insertSpan, lookupSpan)+import "hs-opentelemetry-api" OpenTelemetry.Context qualified as Context import "hs-opentelemetry-api" OpenTelemetry.Context.ThreadLocal ( attachContext, detachContext,@@ -228,13 +239,19 @@ ) import "hs-opentelemetry-api" OpenTelemetry.Trace.Core ( InstrumentationLibrary (..),+ NewLink (..), Span, SpanArguments (..), SpanKind (..),+ SpanStatus (..), Tracer, addAttribute,+ addLink, defaultSpanArguments, inSpan',+ setStatus,+ spanId,+ traceId, wrapSpanContext, ) import "hs-opentelemetry-propagator-w3c" OpenTelemetry.Propagator.W3CTraceContext@@ -257,6 +274,11 @@ messaging_operation_type, messaging_system, )+import "shibuya-core" Shibuya.Adapter (TraceCorrelation)+import "shibuya-core" Shibuya.Core.Ack (AckDecision (..))+import "shibuya-core" Shibuya.Core.Types qualified as Shibuya+import "shibuya-core" Shibuya.Telemetry.Process (withProcessSpanIO)+import "shibuya-core" Shibuya.Telemetry.Semantic (processSpanName) import "text" Data.Text qualified as Text import "text" Data.Text.Encoding qualified as TE import "unliftio-core" Control.Monad.IO.Unlift (MonadUnliftIO, withRunInIO)@@ -346,7 +368,113 @@ name = "send " <> (event ^. #destination) args = defaultSpanArguments {kind = Producer} --- | Run @body@ inside a @Consumer@-kind span named @"process " <> topic@.+-- | Publish a durable batch as a root span with at most 128 creation links.+-- No row, metadata projection or payload is retained by the SDK. The untraced+-- branch deliberately does not inspect the batch.+withProducerBatchSpan ::+ (MonadUnliftIO m, HasCallStack) =>+ Maybe Tracer -> [OutboxRow] -> (Maybe Span -> m a) -> m a+withProducerBatchSpan Nothing _ body = body Nothing+withProducerBatchSpan (Just tracer) rows body = do+ let (!count, !common, !mixed, !valid, !identities) = List.foldl' collect (0 :: Int64, Nothing, False, 0 :: Int64, []) rows+ !name = maybe "send" ("send " <>) (if mixed then Nothing else common)+ !batchLinks = List.foldl' (makeLink mixed) [] identities+ !args = defaultSpanArguments {kind = Producer, links = batchLinks}+ withRunInIO $ \runInIO ->+ bracket (attachContext Context.empty) detachContext $ \_ -> runInIO $+ inSpan' tracer name args $ \sp -> do+ addAttribute sp (unkey messaging_system) ("kafka" :: Text)+ addAttribute sp (unkey messaging_operation_type) ("publish" :: Text)+ addAttribute sp (unkey messaging_operation_name) ("send" :: Text)+ unless mixed $ for_ common (addAttribute sp (unkey messaging_destination_name))+ addAttribute sp "messaging.batch.message_count" count+ addAttribute sp "keiro.outbox.batch.size" count+ when (valid > 128) $ addAttribute sp "keiro.outbox.links.dropped" (valid - 128)+ case rows of+ [row] -> setProducerAttributes sp (row ^. #event) (outboxRowToKafkaRecord row)+ _ -> pure ()+ body (Just sp)+ where+ collect (!count, !common, !mixed, !valid, !saved) row =+ let event = row ^. #event+ !destination = event ^. #destination+ !common' = common <|> Just destination+ !mixed' = mixed || maybe False (/= destination) common+ in case (event ^. #traceContext) >>= creationLinkFromTraceContext of+ Nothing -> (count + 1, common', mixed', valid, saved)+ Just link+ | valid >= 128 -> (count + 1, common', mixed', valid + 1, saved)+ | otherwise ->+ let !messageId = event ^. #messageId+ !identity = (linkContext link, messageId, destination)+ in (count + 1, common', mixed', valid + 1, identity : saved)+ makeLink mixed acc (!context, !messageId, !destination) =+ let !fields =+ fromList $+ [(unkey messaging_message_id, toAttribute messageId)]+ <> [(unkey messaging_destination_name, toAttribute destination) | mixed]+ !link = NewLink context fields+ in link : acc++-- | One subscription delivery, using its source adapter's correlation policy.+-- Partial application shares the full name/policy once per worker. Without a+-- tracer neither the name, policy nor envelope is inspected.+withSubscriptionProcessSpan ::+ (MonadUnliftIO m) =>+ Maybe Tracer -> TraceCorrelation -> Text -> Shibuya.Envelope msg -> (Maybe Span -> m a) -> m a+withSubscriptionProcessSpan Nothing _ _ = \_ body -> body Nothing+withSubscriptionProcessSpan (Just tracer) policy destination =+ let !name = processSpanName destination+ !correlation = policy+ in \envelope body -> withRunInIO $ \runInIO ->+ withProcessSpanIO tracer correlation name (envelope ^. #traceContext) $ \sp -> runInIO $ do+ let !messageId = case envelope ^. #messageId of Shibuya.MessageId value -> value+ for_ (toList (envelope ^. #attributes)) $ \(!key, value) -> do+ let !scalar = forceSpanAttribute value+ addAttribute sp key scalar+ addAttribute sp (unkey messaging_system) ("keiro" :: Text)+ addAttribute sp (unkey messaging_destination_name) destination+ addAttribute sp (unkey messaging_operation_name) ("process" :: Text)+ addAttribute sp (unkey messaging_operation_type) ("process" :: Text)+ addAttribute sp (unkey messaging_message_id) messageId+ body (Just sp)++-- Attribute wrappers are lazy, including values projected from adapter records.+-- Evaluate their primitive values before handing ownership to the SDK.+forceSpanAttribute :: Attribute -> Attribute+forceSpanAttribute (AttributeValue value) = let !scalar = forcePrimitive value in AttributeValue scalar+forceSpanAttribute (AttributeArray values) =+ let !scalars = reverse (List.foldl' (\acc value -> let !scalar = forcePrimitive value in scalar : acc) [] values)+ in AttributeArray scalars++forcePrimitive :: PrimitiveAttribute -> PrimitiveAttribute+forcePrimitive value = case value of+ TextAttribute !text -> TextAttribute text+ BoolAttribute !boolean -> BoolAttribute boolean+ IntAttribute !integer -> IntAttribute integer+ DoubleAttribute !number -> DoubleAttribute number++-- | Observe the worker's existing settled acknowledgement, after finalization.+-- A retry/dead-letter/halt means this delivery did not complete successfully;+-- retain only the existing decision vocabulary, never its business reason.+recordSubscriptionOutcome :: (MonadIO m) => Maybe Span -> AckDecision -> m ()+recordSubscriptionOutcome Nothing _ = pure ()+recordSubscriptionOutcome (Just sp) decision = do+ let !classification = case decision of+ AckOk -> "ack_ok" :: Text+ AckRetry _ -> "ack_retry"+ AckDeadLetter _ -> "ack_dead_letter"+ AckHalt _ -> "ack_halt"+ addAttribute sp "shibuya.ack.decision" classification+ case decision of+ AckOk -> pure ()+ _ -> do+ addAttribute sp "error.type" classification+ setStatus sp (Error classification)++-- | Run @body@ inside a @Consumer@-kind span named @"process " <> topic@,+-- child of and linked to the W3C creation context. Inbox handlers execute inside+-- this transport span; the inbox opens no independent span. -- -- Like 'withProducerSpan', the helper is a pass-through under a 'Nothing' -- tracer. The 'KafkaInboundRecord' is required so the helper can populate@@ -376,7 +504,7 @@ body (Just sp) where name = "process " <> (record ^. #topic)- args = defaultSpanArguments {kind = Consumer}+ args = defaultSpanArguments {kind = Consumer, links = maybe [] pure (traceContextFromHeaders (record ^. #headers) >>= creationLinkFromTraceContext)} -- | Run @body@ with the OpenTelemetry context temporarily augmented by a -- parent span extracted from the supplied header list via the W3C@@ -406,8 +534,8 @@ ts = fmap TE.encodeUtf8 (Prelude.lookup headerTraceState hsList) in decodeSpanContext tp ts --- | Open an @Internal@ span around a command run, named after the resolved--- stream identifier. Attributes capture the stream name and (when the+-- | Open an @Internal@ span named @command <category>@ around a command run.+-- The full resolved stream identifier remains an attribute. Attributes capture the stream name and (when the -- caller supplies it) the retry attempt number. The number of events -- appended is attached after a successful append by the caller via -- 'addAttribute span keiro_events_appended n'.@@ -422,13 +550,14 @@ m a withCommandSpan Nothing _ _ body = body Nothing withCommandSpan (Just tracer) streamName retryAttempt body =- inSpan' tracer streamName args $ \sp -> do+ inSpan' tracer ("command " <> category) args $ \sp -> do addAttribute sp (unkey keiro_stream_name) streamName case retryAttempt of Nothing -> pure () Just n -> addAttribute sp (unkey keiro_retry_attempt) n body (Just sp) where+ CategoryName category = categoryName (StreamName streamName) args = defaultSpanArguments {kind = Internal} -- | Open an @Internal@ span around a workflow run (or a single step/resume when@@ -446,17 +575,54 @@ m a withWorkflowSpan Nothing _ _ _ body = body Nothing withWorkflowSpan (Just tracer) name wid mStep body =- inSpan' tracer spanName args $ \sp -> do- addAttribute sp (unkey keiro_workflow_name) (unWorkflowName name)- addAttribute sp (unkey keiro_workflow_id) (unWorkflowId wid)+ withEmptyContext $ inSpan' tracer spanName args $ \sp -> do+ let !workflowName = unWorkflowName name+ !workflowId = unWorkflowId wid+ addAttribute sp (unkey keiro_workflow_name) workflowName+ addAttribute sp (unkey keiro_workflow_id) workflowId case mStep of Nothing -> pure ()- Just s -> addAttribute sp (unkey keiro_workflow_step) (unStepName s)+ Just (StepName !step) -> addAttribute sp (unkey keiro_workflow_step) step body (Just sp) where spanName = "workflow " <> unWorkflowName name args = defaultSpanArguments {kind = Internal} +-- | Fire delayed work as a root, linked to the stored scheduling context.+withTimerFireSpan ::+ (MonadUnliftIO m, HasCallStack) =>+ Maybe Tracer -> Text -> Text -> Maybe TraceContext -> (Maybe Span -> m a) -> m a+withTimerFireSpan Nothing _ _ _ body = body Nothing+withTimerFireSpan (Just tracer) manager timerId context body =+ let !managerName = manager+ !identifier = timerId+ !creationLinks = maybe [] pure (context >>= creationLinkFromTraceContext)+ !args = defaultSpanArguments {kind = Internal, links = creationLinks}+ in withEmptyContext $ inSpan' tracer ("timer fire " <> managerName) args $ \sp -> do+ addAttribute sp "keiro.timer.id" identifier+ addAttribute sp "keiro.timer.process_manager" managerName+ body (Just sp)++-- | Add bounded, evaluated journal evidence after lookup. Samplers cannot see+-- these late links. Equal span identities prefer the exact wake label.+recordWorkflowResumeLinks :: (MonadIO m) => Maybe Span -> Maybe TraceContext -> Maybe TraceContext -> m ()+recordWorkflowResumeLinks Nothing _ _ = pure ()+recordWorkflowResumeLinks (Just sp) origin wake = do+ let originLink = origin >>= creationLinkFromTraceContext+ wakeLink = wake >>= creationLinkFromTraceContext+ same a b = traceId (linkContext a) == traceId (linkContext b) && spanId (linkContext a) == spanId (linkContext b)+ add role link =+ let !fields = fromList [("keiro.workflow.link", toAttribute (role :: Text))]+ !labelled = link {linkAttributes = fields}+ in addLink sp labelled+ case (originLink, wakeLink) of+ (Just a, Just b) | same a b -> add "wake" b+ _ -> for_ originLink (add "origin") >> for_ wakeLink (add "wake")++withEmptyContext :: (MonadUnliftIO m) => m a -> m a+withEmptyContext body = withRunInIO $ \runInIO ->+ bracket (attachContext Context.empty) detachContext (const (runInIO body))+ -- --------------------------------------------------------------------------- -- W3C TraceContext bridge -- ---------------------------------------------------------------------------@@ -483,6 +649,50 @@ | otherwise = Just (TE.decodeUtf8 tracestateBytes) pure (Just (TraceContext traceparent tracestate)) +-- | Read opaque W3C creation-context strings from event metadata. Validation+-- belongs to the link decoder, so malformed context never fails domain work.+traceContextFromMetadata :: Maybe Value -> Maybe TraceContext+traceContextFromMetadata (Just (Aeson.Object fields)) = do+ Aeson.String parent <- KeyMap.lookup traceParentMetadataKey fields+ state <- case KeyMap.lookup traceStateMetadataKey fields of+ Nothing -> Just Nothing+ Just (Aeson.String value) -> Just (Just value)+ _ -> Nothing+ pure (TraceContext parent state)+traceContextFromMetadata _ = Nothing++traceParentMetadataKey, traceStateMetadataKey :: Key.Key+traceParentMetadataKey = Key.fromText headerTraceParent+traceStateMetadataKey = Key.fromText headerTraceState++-- | Stamp the current creation context without overriding a caller's parent.+-- Non-object metadata is left to the codec's existing validation. An orphaned+-- caller tracestate is replaced with the captured context's matching state.+stampTraceContext :: (MonadIO m) => Maybe Value -> m (Maybe Value)+stampTraceContext original = case original of+ Just (Aeson.Object fields)+ | KeyMap.member traceParentMetadataKey fields -> pure original+ | otherwise -> stamp fields+ Nothing -> stamp KeyMap.empty+ _ -> pure original+ where+ stamp fields = do+ current <- traceContextFromCurrentSpan+ pure $ case current of+ Nothing -> original+ Just (TraceContext parent state) ->+ let withParent = KeyMap.insert traceParentMetadataKey (Aeson.String parent) (KeyMap.delete traceStateMetadataKey fields)+ withState = maybe withParent (\value -> KeyMap.insert traceStateMetadataKey (Aeson.String value) withParent) state+ in Just (Aeson.Object withState)++-- | Decode and evaluate a span identity before it can be retained by a span.+-- The resulting link owns no event, metadata object, or deferred record lookup.+creationLinkFromTraceContext :: TraceContext -> Maybe NewLink+creationLinkFromTraceContext (TraceContext parent state) =+ case decodeSpanContext (Just (TE.encodeUtf8 parent)) (TE.encodeUtf8 <$> state) of+ Nothing -> Nothing+ Just context -> let !link = NewLink context mempty in Just link+ -- | Lift a 'TraceContext' out of a flat @[(Text, Text)]@ header list. This -- is the mirror image of 'integrationHeaders': it does not validate the -- @traceparent@ format (the W3C propagator's parser does that at consume@@ -521,11 +731,13 @@ addAttribute sp (unkey messaging_system) ("kafka" :: Text) addAttribute sp (unkey messaging_operation_type) ("publish" :: Text) addAttribute sp (unkey messaging_operation_name) ("send" :: Text)- addAttribute sp (unkey messaging_destination_name) (event ^. #destination)- addAttribute sp (unkey messaging_message_id) (event ^. #messageId)+ let !destination = event ^. #destination+ !messageId = event ^. #messageId+ addAttribute sp (unkey messaging_destination_name) destination+ addAttribute sp (unkey messaging_message_id) messageId case event ^. #key of Nothing -> pure ()- Just k -> addAttribute sp (unkey messaging_kafka_message_key) k+ Just !k -> addAttribute sp (unkey messaging_kafka_message_key) k setConsumerAttributes :: (MonadIO m) =>@@ -538,18 +750,21 @@ addAttribute sp (unkey messaging_system) ("kafka" :: Text) addAttribute sp (unkey messaging_operation_type) ("process" :: Text) addAttribute sp (unkey messaging_operation_name) ("process" :: Text)- addAttribute sp (unkey messaging_destination_name) (record ^. #topic)- addAttribute sp (unkey messaging_destination_partition_id) (showText (record ^. #partition))- addAttribute sp (unkey messaging_kafka_offset) (record ^. #offset)+ let !topic = record ^. #topic+ !partition = showText (record ^. #partition)+ !offset = record ^. #offset+ addAttribute sp (unkey messaging_destination_name) topic+ addAttribute sp (unkey messaging_destination_partition_id) partition+ addAttribute sp (unkey messaging_kafka_offset) offset case record ^. #key of Nothing -> pure ()- Just k -> addAttribute sp (unkey messaging_kafka_message_key) k+ Just !k -> addAttribute sp (unkey messaging_kafka_message_key) k case consumerGroup of Nothing -> pure ()- Just g -> addAttribute sp (unkey messaging_consumer_group_name) g+ Just !g -> addAttribute sp (unkey messaging_consumer_group_name) g case mEvent of Nothing -> pure ()- Just event -> addAttribute sp (unkey messaging_message_id) (event ^. #messageId)+ Just event -> let !messageId = event ^. #messageId in addAttribute sp (unkey messaging_message_id) messageId showText :: (Show a) => a -> Text showText = Text.pack . show
src/Keiro/Timer.hs view
@@ -24,6 +24,8 @@ ( -- * Timer types TimerId (..), TimerRequest (..),+ withTimerTraceContext,+ captureTimerTraceContext, TimerRow (..), TimerStatus (..), @@ -95,20 +97,25 @@ import Data.Text qualified as Text import Data.Time.Clock (NominalDiffTime, diffUTCTime)+import Data.UUID qualified as UUID import Effectful (Eff, IOE, (:>)) import Keiro.Prelude import Keiro.Telemetry ( KeiroMetrics,+ Tracer, recordTimerAttempts, recordTimerBacklog, recordTimerFireLag, recordTimerRequeued, recordTimerStuck,+ traceContextFromCurrentSpan,+ withTimerFireSpan, ) import Keiro.Timer.Schema import Keiro.Timer.Types import Kiroku.Store.Effect (Store) import Kiroku.Store.Types (EventId)+import OpenTelemetry.Trace.Core (addAttribute) -- | Options controlling 'runTimerWorkerWith'. data TimerWorkerOptions = TimerWorkerOptions@@ -121,9 +128,10 @@ -- runs longer than this timeout may be fired again; timer handlers must be -- idempotent under keiro's at-least-once timer contract. @Nothing@ disables -- automatic ordinary requeue. Expired foreground recovery always runs.- requeueStuckAfter :: !(Maybe NominalDiffTime)+ requeueStuckAfter :: !(Maybe NominalDiffTime),+ tracer :: !(Maybe Tracer) }- deriving stock (Generic, Eq, Show)+ deriving stock (Generic, Show) data TimerWorkerConfigError = InvalidTimerMaxAttempts !Int@@ -132,8 +140,16 @@ -- | The default worker policy: never auto-dead-letter; requeue stale firings after five minutes. defaultTimerWorkerOptions :: TimerWorkerOptions-defaultTimerWorkerOptions = TimerWorkerOptions {maxAttempts = Nothing, requeueStuckAfter = Just 300}+defaultTimerWorkerOptions = TimerWorkerOptions {tracer = Nothing, maxAttempts = Nothing, requeueStuckAfter = Just 300} +-- | Capture creation context in the effect layer, before constructing SQL work.+-- Neither a missing tracer nor an explicit request context reads ambient context.+captureTimerTraceContext :: (MonadIO m) => Maybe Tracer -> TimerRequest -> m TimerRequest+captureTimerTraceContext Nothing request = pure request+captureTimerTraceContext (Just _) request = case request ^. #traceContext of+ Just _ -> pure request+ Nothing -> flip withTimerTraceContext request <$> traceContextFromCurrentSpan+ -- | Validate timer worker options before starting a worker loop. mkTimerWorkerOptions :: TimerWorkerOptions -> Either TimerWorkerConfigError TimerWorkerOptions mkTimerWorkerOptions opts =@@ -227,10 +243,20 @@ (timer ^. #timerId) ("timer exceeded attempt ceiling of " <> Text.pack (show attemptCeiling)) pure (Just timer)- _ -> do- fired <- fire timer- for_ fired (\eventId -> void (markTimerFired (timer ^. #timerId) eventId))- pure (Just timer)+ _ -> case options ^. #tracer of+ Nothing -> fireAndMark timer+ Just tracer ->+ withTimerFireSpan (Just tracer) (timer ^. #processManagerName) (case timer ^. #timerId of TimerId uuid -> UUID.toText uuid) (timer ^. #traceContext) $ \mSpan -> do+ for_ mSpan $ \sp -> do+ let !lag = realToFrac (now `diffUTCTime` (timer ^. #fireAt)) * 1000 :: Double+ addAttribute sp "keiro.timer.fire_lag" lag+ fireAndMark timer+ where+ -- Keep the original fire/mark path when tracing is disabled.+ fireAndMark timer = do+ fired <- fire timer+ for_ fired (\eventId -> void (markTimerFired (timer ^. #timerId) eventId))+ pure (Just timer) -- | Claim and fire up to @limit@ timers due at @now@ in one pass, returning how -- many were processed.
src/Keiro/Timer/Schema.hs view
@@ -76,7 +76,7 @@ ) where -import Contravariant.Extras (contrazip2, contrazip5, contrazip6)+import Contravariant.Extras (contrazip2, contrazip5, contrazip6, contrazip8) import Data.Aeson (object, (.=)) import Data.Int (Int32) import Data.List.NonEmpty (NonEmpty (..))@@ -89,6 +89,7 @@ import Hasql.Encoders qualified as E import Hasql.Statement (Statement, preparable) import Keiro.Inspection.Cursor+import Keiro.Integration.Event (TraceContext (..)) import Keiro.Prelude hiding ((.=)) import Keiro.Timer.Types (TimerId (..), TimerRequest (..)) import Kiroku.Store.Effect (Store)@@ -126,7 +127,8 @@ payload :: !Value, status :: !TimerStatus, attempts :: !Int,- firedEventId :: !(Maybe EventId)+ firedEventId :: !(Maybe EventId),+ traceContext :: !(Maybe TraceContext) } deriving stock (Generic, Eq, Show) @@ -333,7 +335,7 @@ findPendingTimersStmt :: Bool -> Bool -> Bool -> Statement (Maybe Text, Maybe Text, Maybe UTCTime, Maybe UUID, Int64) [TimerInspection] findPendingTimersStmt manager correlation cursor = preparable- ( "SELECT timer_id,process_manager_name,correlation_id,fire_at,payload,status,attempts,fired_event_id,last_error FROM keiro.keiro_timers WHERE status IN ('scheduled','firing')"+ ( "SELECT timer_id,process_manager_name,correlation_id,fire_at,payload,status,attempts,fired_event_id,traceparent,tracestate,last_error FROM keiro.keiro_timers WHERE status IN ('scheduled','firing')" <> (if manager then " AND process_manager_name=$1" else " AND $1::text IS NULL") <> (if correlation then " AND correlation_id=$2" else " AND $2::text IS NULL") <> (if cursor then " AND (fire_at,timer_id)>($3,$4)" else " AND $3::timestamptz IS NULL AND $4::uuid IS NULL")@@ -433,7 +435,7 @@ AND process_manager_name COLLATE "C" = $2 AND last_error COLLATE "C" = $3 AND attempts < $4 RETURNING kt.timer_id, kt.process_manager_name, kt.correlation_id, kt.fire_at,- kt.payload, kt.status, kt.attempts, kt.fired_event_id, kt.resume_claim_token, kt.resume_lease_until+ kt.payload, kt.status, kt.attempts, kt.fired_event_id, kt.traceparent, kt.tracestate, kt.resume_claim_token, kt.resume_lease_until """ ( contrazip6 (E.param (E.nonNullable E.uuid))@@ -558,7 +560,9 @@ request ^. #correlationId, request ^. #fireAt, request ^. #payload,- statusToText Scheduled+ statusToText Scheduled,+ fmap (^. #traceparent) (request ^. #traceContext),+ (request ^. #traceContext) >>= (^. #tracestate) ) scheduleTimerStmt @@ -578,7 +582,9 @@ request ^. #correlationId, request ^. #fireAt, request ^. #payload,- statusToText Scheduled+ statusToText Scheduled,+ fmap (^. #traceparent) (request ^. #traceContext),+ (request ^. #traceContext) >>= (^. #tracestate) ) scheduleTimerOnceStmt @@ -680,50 +686,56 @@ runTransaction $ Tx.statement (timerIdToUuid timerId, reason) deadLetterTimerStmt -scheduleTimerStmt :: Statement (UUID, Text, Text, UTCTime, Value, Text) ()+scheduleTimerStmt :: Statement (UUID, Text, Text, UTCTime, Value, Text, Maybe Text, Maybe Text) () scheduleTimerStmt = preparable """ INSERT INTO keiro.keiro_timers- (timer_id, process_manager_name, correlation_id, fire_at, payload, status)+ (timer_id, process_manager_name, correlation_id, fire_at, payload, status, traceparent, tracestate) VALUES- ($1, $2, $3, $4, $5, $6)+ ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (timer_id) DO UPDATE SET process_manager_name = EXCLUDED.process_manager_name, correlation_id = EXCLUDED.correlation_id, fire_at = EXCLUDED.fire_at, payload = EXCLUDED.payload,+ traceparent = EXCLUDED.traceparent,+ tracestate = EXCLUDED.tracestate, status = EXCLUDED.status, updated_at = now() WHERE keiro_timers.status = 'scheduled' """- ( contrazip6+ ( contrazip8 (E.param (E.nonNullable E.uuid)) (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.timestamptz)) (E.param (E.nonNullable E.jsonb)) (E.param (E.nonNullable E.text))+ (E.param (E.nullable E.text))+ (E.param (E.nullable E.text)) ) D.noResult -scheduleTimerOnceStmt :: Statement (UUID, Text, Text, UTCTime, Value, Text) Bool+scheduleTimerOnceStmt :: Statement (UUID, Text, Text, UTCTime, Value, Text, Maybe Text, Maybe Text) Bool scheduleTimerOnceStmt = preparable """ INSERT INTO keiro.keiro_timers- (timer_id, process_manager_name, correlation_id, fire_at, payload, status)+ (timer_id, process_manager_name, correlation_id, fire_at, payload, status, traceparent, tracestate) VALUES- ($1, $2, $3, $4, $5, $6)+ ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (timer_id) DO NOTHING """- ( contrazip6+ ( contrazip8 (E.param (E.nonNullable E.uuid)) (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.timestamptz)) (E.param (E.nonNullable E.jsonb)) (E.param (E.nonNullable E.text))+ (E.param (E.nullable E.text))+ (E.param (E.nullable E.text)) ) ((> 0) <$> D.rowsAffected) @@ -747,7 +759,7 @@ FROM due WHERE kt.timer_id = due.timer_id RETURNING kt.timer_id, kt.process_manager_name, kt.correlation_id, kt.fire_at,- kt.payload, kt.status, kt.attempts, kt.fired_event_id+ kt.payload, kt.status, kt.attempts, kt.fired_event_id, kt.traceparent, kt.tracestate """ (E.param (E.nonNullable E.timestamptz)) (D.rowMaybe timerRowDecoder)@@ -757,7 +769,7 @@ preparable """ SELECT timer_id, process_manager_name, correlation_id, fire_at,- payload, status, attempts, fired_event_id+ payload, status, attempts, fired_event_id, traceparent, tracestate FROM keiro.keiro_timers WHERE timer_id = $1 """@@ -769,7 +781,7 @@ preparable """ SELECT timer_id, process_manager_name, correlation_id, fire_at,- payload, status, attempts, fired_event_id, last_error+ payload, status, attempts, fired_event_id, traceparent, tracestate, last_error FROM keiro.keiro_timers WHERE timer_id = $1 """@@ -781,7 +793,7 @@ preparable """ SELECT timer_id, process_manager_name, correlation_id, fire_at,- payload, status, attempts, fired_event_id, last_error+ payload, status, attempts, fired_event_id, traceparent, tracestate, last_error FROM keiro.keiro_timers WHERE status = 'dead' AND ($1::text IS NULL OR process_manager_name COLLATE "C" = $1)@@ -859,7 +871,7 @@ preparable """ SELECT timer_id, process_manager_name, correlation_id, fire_at,- payload, status, attempts, fired_event_id+ payload, status, attempts, fired_event_id, traceparent, tracestate FROM keiro.keiro_timers WHERE status = 'firing' AND ($1::timestamptz IS NULL OR updated_at <= $1)@@ -943,6 +955,7 @@ <*> D.column (D.nonNullable (D.refine statusFromText D.text)) <*> (fromIntegral <$> D.column (D.nonNullable D.int8)) <*> (fmap EventId <$> D.column (D.nullable D.uuid))+ <*> ((\parent state -> fmap (\value -> TraceContext value state) parent) <$> D.column (D.nullable D.text) <*> D.column (D.nullable D.text)) statusToText :: TimerStatus -> Text statusToText = \case
src/Keiro/Timer/Types.hs view
@@ -4,10 +4,12 @@ module Keiro.Timer.Types ( TimerId (..), TimerRequest (..),+ withTimerTraceContext, ) where import Data.UUID (UUID)+import Keiro.Integration.Event (TraceContext) import Keiro.Prelude -- | A timer's stable identifier. A caller-chosen id makes scheduling@@ -27,6 +29,15 @@ processManagerName :: !Text, correlationId :: !Text, fireAt :: !UTCTime,- payload :: !Value+ payload :: !Value,+ traceContext :: !(Maybe TraceContext) } deriving stock (Generic, Eq, Show)++-- | Fill missing context from a value captured before the transaction. Explicit+-- caller context wins; absence preserves the original request without inspection.+withTimerTraceContext :: Maybe TraceContext -> TimerRequest -> TimerRequest+withTimerTraceContext Nothing request = request+withTimerTraceContext context request = case request ^. #traceContext of+ Just _ -> request+ Nothing -> request & #traceContext .~ context
src/Keiro/Workflow.hs view
@@ -120,6 +120,7 @@ step, awaitStep, currentWorkflow,+ currentWorkflowTracer, currentRunGeneration, freshOrdinal, continueAsNew,@@ -136,6 +137,9 @@ -- * Journal append helpers (used by wake-source plans) JournalAppendOutcome (..), prepareJournalAppend,+ prepareJournalAppendWithMetadata,+ appendJournalEntryTraced,+ appendJournalEntryReturningIdTraced, appendJournalEntry, appendJournalEntryReturningId, deterministicJournalId,@@ -198,8 +202,10 @@ recordSnapshotWriteFailures, recordWorkflowActive, recordWorkflowJournalLength,+ recordWorkflowResumeLinks, recordWorkflowStepExecuted, recordWorkflowStepReplayed,+ traceContextFromMetadata, withWorkflowSpan, ) import Keiro.Workflow.Instance@@ -211,23 +217,28 @@ defaultWorkflowInstanceFilter, forceReleaseInstanceLease, listWorkflowInstances,- markInstanceSuspendedAwaiting,+ markInstanceSuspendedAwaitingTraced, renewInstanceLease, ) import Keiro.Workflow.Journal ( JournalAppendOutcome (..),- appendJournal, appendJournalEntry, appendJournalEntryReturningId,+ appendJournalEntryReturningIdTraced,+ appendJournalEntryTraced,+ appendJournalTraced, deterministicJournalId,+ journalTraceMetadata, prepareJournalAppend,+ prepareJournalAppendWithMetadata,+ readWorkflowTraceResume, ) import Keiro.Workflow.Schema (WorkflowStepRow (..), clearWorkflowWakeAfterTx, currentGeneration, findUnfinishedWorkflowIds, loadStepIndex, lookupStepResult, recordStepTx, setWorkflowWakeAfterTx, stepExists, terminalMarkers) import Keiro.Workflow.Snapshot (lookupWorkflowSnapshot, writeWorkflowSnapshot) import Keiro.Workflow.Types import Kiroku.Store.Effect (Store) import Kiroku.Store.Error (StoreError)-import Kiroku.Store.Read (readStreamForwardStream)+import Kiroku.Store.Read (getEvent, readStreamForwardStream) import Kiroku.Store.Transaction (runTransaction) import Kiroku.Store.Types (StreamId, StreamVersion (..)) import Streamly.Data.Fold qualified as Fold@@ -252,6 +263,8 @@ CurrentWorkflow :: Workflow m (WorkflowName, WorkflowId) -- | The journal generation this run is operating on. CurrentRunGeneration :: Workflow m Int+ -- | Explicit tracing policy of the current run, for durable wake sources.+ CurrentWorkflowTracer :: Workflow m (Maybe Tracer) -- | A per-run, per-namespace counter for deterministic ordinal step names. FreshOrdinal :: Text -> Workflow m Int -- | EP-48: snapshot the carried seed, rotate onto a fresh journal generation,@@ -297,6 +310,10 @@ currentWorkflow :: (Workflow :> es) => Eff es (WorkflowName, WorkflowId) currentWorkflow = send CurrentWorkflow +-- | Tracing policy used by this invocation; ambient context does not enable it.+currentWorkflowTracer :: (Workflow :> es) => Eff es (Maybe Tracer)+currentWorkflowTracer = send CurrentWorkflowTracer+ -- | The journal generation this run is operating on. Wake sources include it -- in their durable identities so a generation opened by 'continueAsNew' never -- collides with prior-generation rows.@@ -544,12 +561,14 @@ WorkflowId -> Eff (Workflow : es) a -> Eff es (WorkflowOutcome a)-runWorkflowWith options name wid action = do+runWorkflowWith options name wid action = withWorkflowSpan (options ^. #tracer) name wid Nothing $ \sp -> do -- EP-48: resolve the CURRENT (highest) generation once per run and operate -- only on it. A never-rotating workflow stays at generation 0, so naming, -- load, and append are byte-for-byte as before. A rotated workflow resolves -- to its newest generation, so discovery/resume transparently continue there.- gen <- currentGeneration name wid+ (gen, traceHint) <- case sp of+ Nothing -> (,Nothing) <$> currentGeneration name wid+ Just _ -> readWorkflowTraceResume name wid -- Terminal short-circuit (EP-43): a workflow whose journal carries a -- WorkflowCancelled or WorkflowFailed marker makes no further progress. Both -- markers are index rows on the current generation, so one query answers for@@ -559,7 +578,7 @@ case (cancelledStepName `elem` markers, failedStepName `elem` markers) of (True, _) -> pure Cancelled (_, True) -> pure Failed- _ -> runActive gen+ _ -> runActive gen sp traceHint where -- EP-44 telemetry handles, pulled from the run options once. Both default -- to 'Nothing' (see 'defaultWorkflowRunOptions'), so a default-options run@@ -567,16 +586,34 @@ mMetrics = options ^. #metrics mTracer = options ^. #tracer fireAppendWitness = for_ (options ^. #onJournalAppend) liftIO- runActive :: Int -> Eff es (WorkflowOutcome a)- runActive gen =- -- EP-44: maintain the process-wide live-run count and sample the- -- @keiro.workflow.active@ gauge on both entry and exit, and open the- -- whole-run @workflow \<name\>@ span (step 'Nothing'). The body is- -- unchanged from EP-41 except for the journal-length recording below.+ -- Bounded reference discovery plus two normal event reads. The hint is+ -- captured before interpretation; snapshots never participate in attribution.+ recoverResumeLinks Nothing _ _ = pure ()+ recoverResumeLinks sp gen hint = do+ for_ hint $ \(awaited, originReference) -> do+ origin <- traverse (\(eid, _, _) -> getEvent eid) originReference+ wake <- getEvent (deterministicJournalId name wid gen awaited)+ let originContext = do+ (_, expectedStream, version) <- originReference+ guard (version == StreamVersion 1)+ row <- fromMaybe Nothing origin+ guard (row ^. #originalStreamId == expectedStream && row ^. #originalVersion == version)+ traceContextFromMetadata (row ^. #metadata)+ wakeContext = do+ (_, expectedStream, _) <- originReference+ row <- wake+ guard (row ^. #originalStreamId == expectedStream)+ case decodeRecorded workflowJournalCodec row of+ Right (StepRecorded key _ _) | key == awaited -> traceContextFromMetadata (row ^. #metadata)+ _ -> Nothing+ recordWorkflowResumeLinks sp originContext wakeContext+ runActive gen sp traceHint =+ -- Maintain the live-run count around active work. The root span already+ -- covers generation/hint discovery and terminal checks at run entry. bracket_ (liftIO (atomicModifyIORef' activeCountRef (\n -> (n + 1, ()))) >> sampleActive) (liftIO (atomicModifyIORef' activeCountRef (\n -> (n - 1, ()))) >> sampleActive)- (withWorkflowSpan mTracer name wid Nothing (\_sp -> interpreted))+ (recoverResumeLinks sp gen traceHint >> interpreted) where sampleActive = liftIO (readIORef activeCountRef) >>= recordWorkflowActive mMetrics interpreted = do@@ -605,7 +642,7 @@ -- advisory lock, so a wake landing in this gap cannot be masked by a -- 'suspended' status that exact discovery would never return. RunSuspendedOn awaitedStep -> do- markInstanceSuspendedAwaiting name wid gen awaitedStep+ markInstanceSuspendedAwaitingTraced (isJust mTracer) name wid gen awaitedStep pure Suspended RunOutcome outcome -> case outcome of Completed result -> do@@ -616,7 +653,7 @@ -- workflow this is 'JournalAlreadyPresent' and no terminal snapshot is -- taken (one was already taken on the original completing run, if the -- policy fired).- appendCompletion name wid gen now >>= \case+ appendJournalTraced mTracer name wid gen (WorkflowCompleted now) >>= \case JournalAppended appendResult -> do fireAppendWitness when@@ -661,7 +698,7 @@ then do let encoded = Aeson.toJSON (map unPatchId (Set.toList patches)) now <- liftIO getCurrentTime- appendJournal name wid runGen (StepRecorded patchSetStepName encoded now) >>= \case+ appendJournalTraced mTracer name wid runGen (StepRecorded patchSetStepName encoded now) >>= \case JournalAppended {} -> do fireAppendWitness pure (Map.insert patchSetStepName encoded initial)@@ -702,7 +739,7 @@ a <- localSeqUnlift env (\unlift -> unlift act) let encoded = Aeson.toJSON a now <- liftIO getCurrentTime- appendOutcome <- appendJournal name wid gen (StepRecorded key encoded now)+ appendOutcome <- appendJournalTraced mTracer name wid gen (StepRecorded key encoded now) case appendOutcome of JournalAppended appendResult -> do -- Miss: @act@ ran and was journaled — a fresh execution.@@ -767,6 +804,7 @@ throwIO (WorkflowSuspend key) CurrentWorkflow -> pure (name, wid) CurrentRunGeneration -> pure gen+ CurrentWorkflowTracer -> pure mTracer FreshOrdinal namespace -> liftIO . atomicModifyIORef' ordinalRef $ \counters -> let n = Map.findWithDefault 0 namespace counters@@ -793,7 +831,7 @@ let decision = unPatchId pid `elem` (recordedSet :: [Text]) encoded = Aeson.toJSON decision now <- liftIO getCurrentTime- appendOutcome <- appendJournal name wid gen (StepRecorded key encoded now)+ appendOutcome <- appendJournalTraced mTracer name wid gen (StepRecorded key encoded now) case appendOutcome of JournalAppended {} -> do fireAppendWitness@@ -897,12 +935,6 @@ -- Journal append helpers -- --------------------------------------------------------------------------- --- | Attempt the completion marker append. The caller maps a lifecycle refusal--- to the winning workflow outcome instead of reporting a spurious append error.-appendCompletion :: (IOE :> es, Store :> es) => WorkflowName -> WorkflowId -> Int -> UTCTime -> Eff es JournalAppendOutcome-appendCompletion name wid gen now =- appendJournal name wid gen (WorkflowCompleted now)- -- | Perform a continue-as-new rotation (EP-48): close generation @gen@ and open -- generation @gen + 1@, seeded with @seedJson@ and the deployed patch set. Returns -- 'ContinuedAsNew'.@@ -937,8 +969,10 @@ patchEvent = StepRecorded patchSetStepName encodedPatches now <- liftIO getCurrentTime+ metadata <- journalTraceMetadata (options ^. #tracer) seedTx <-- prepareJournalAppend+ prepareJournalAppendWithMetadata+ metadata name wid nextGen@@ -948,13 +982,15 @@ then pure Nothing else Just- <$> prepareJournalAppend+ <$> prepareJournalAppendWithMetadata+ metadata name wid nextGen (patchEvent now) rotationTx <-- prepareJournalAppend+ prepareJournalAppendWithMetadata+ metadata name wid gen
src/Keiro/Workflow/Awakeable.hs view
@@ -61,7 +61,9 @@ -- * External completion (outside a workflow) signalAwakeable,+ signalAwakeableTraced, signalAwakeableFrom,+ signalAwakeableFromTraced, cancelAwakeable, -- * Errors@@ -78,6 +80,7 @@ import Effectful (Eff, IOE, (:>)) import Effectful.Exception (throwIO) import Keiro.Prelude+import Keiro.Telemetry (Tracer) import Keiro.Workflow ( JournalAppendOutcome (..), StepName (..),@@ -86,15 +89,15 @@ WorkflowId (..), WorkflowJournalEvent (..), WorkflowName (..),- appendJournalEntry,+ appendJournalEntryTraced, awaitStep, awakeableAllocStepPrefix, awakeableStepPrefix, currentGeneration, currentRunGeneration, currentWorkflow,+ currentWorkflowTracer, freshOrdinal,- prepareJournalAppend, step, ) import Keiro.Workflow.Awakeable.Internal.Identity (generation0AwakeableUuidProbes)@@ -108,6 +111,7 @@ registerAwakeableTx, ) import Keiro.Workflow.Instance (WorkflowStatus (..), upsertInstanceTx)+import Keiro.Workflow.Journal (journalTraceMetadata, prepareJournalAppendWithMetadata) import Keiro.Workflow.Schema (lockWorkflowStepTx, workflowStepLockKey) import Kiroku.Store.Effect (Store) import Kiroku.Store.Transaction (runTransaction)@@ -249,7 +253,9 @@ | row ^. #status == Completed, Just payload <- row ^. #payload -> do now <- liftIO getCurrentTime- appendJournalEntry+ tracer <- currentWorkflowTracer+ appendJournalEntryTraced+ tracer name wid StepRecorded@@ -287,10 +293,14 @@ -- A 'False' return therefore does not mean "nothing happened": the journal may -- still have been repaired. Returns 'False' for an unknown id. signalAwakeable :: (IOE :> es, Store :> es, ToJSON r) => AwakeableId -> r -> Eff es Bool-signalAwakeable aid result =+signalAwakeable = signalAwakeableTraced Nothing++-- | Stamp the wake journal only when explicitly enabled.+signalAwakeableTraced :: (IOE :> es, Store :> es, ToJSON r) => Maybe Tracer -> AwakeableId -> r -> Eff es Bool+signalAwakeableTraced tracer aid result = lookupAwakeable (awakeableIdToUuid aid) >>= \case Nothing -> pure False- Just row -> signalAwakeableFrom row result+ Just row -> signalAwakeableFromTraced tracer row result -- | The transaction-decision core of 'signalAwakeable', exposed so race -- contracts can deterministically interpose between the initial row read and the@@ -305,7 +315,10 @@ AwakeableRow -> r -> Eff es Bool-signalAwakeableFrom row result+signalAwakeableFrom = signalAwakeableFromTraced Nothing++signalAwakeableFromTraced :: (IOE :> es, Store :> es, ToJSON r) => Maybe Tracer -> AwakeableRow -> r -> Eff es Bool+signalAwakeableFromTraced tracer row result | row ^. #status == Cancelled = pure False | otherwise = do now <- liftIO getCurrentTime@@ -320,8 +333,10 @@ let ownerName = WorkflowName (row ^. #ownerWorkflowName) ownerId = WorkflowId (row ^. #ownerWorkflowId) gen <- currentGeneration ownerName ownerId+ metadata <- journalTraceMetadata tracer appendTx <-- prepareJournalAppend+ prepareJournalAppendWithMetadata+ metadata ownerName ownerId gen
src/Keiro/Workflow/Child.hs view
@@ -72,6 +72,7 @@ -- * Driving a child to completion (used by the resume worker) runChildWorkflow, childCompletionHook,+ childCompletionHookTraced, -- * Errors WorkflowChildCancelled (..),@@ -82,11 +83,13 @@ import Control.Exception (Exception) import Data.Aeson qualified as Aeson import Data.Aeson.KeyMap qualified as KeyMap+import Data.IORef (newIORef, readIORef, writeIORef) import Data.Text qualified as Text import Effectful (Eff, IOE, (:>)) import Effectful.Error.Static (Error) import Effectful.Exception (throwIO) import Keiro.Prelude+import Keiro.Telemetry (Tracer) import Keiro.Workflow ( JournalAppendOutcome (..), StepName (..),@@ -97,12 +100,12 @@ WorkflowName (..), WorkflowOutcome (..), WorkflowRunOptions,- appendJournalEntry,+ appendJournalEntryTraced, awaitStep, childStepPrefix, currentGeneration, currentWorkflow,- prepareJournalAppend,+ currentWorkflowTracer, runWorkflowWith, step, )@@ -114,6 +117,7 @@ registerChildTx, ) import Keiro.Workflow.Instance (WorkflowStatus (..), lookupInstance, upsertInstanceTx)+import Keiro.Workflow.Journal (appendJournalWithMetadata, journalTraceMetadata, prepareJournalAppendWithMetadata) import Kiroku.Store.Effect (Store) import Kiroku.Store.Error (StoreError) import Kiroku.Store.Transaction (runTransaction)@@ -246,8 +250,10 @@ (mInstance >>= (^. #lastError)) throwIO (WorkflowChildFailed childNm childWid reason) | (row ^. #status) == ChildCompleted,- Just resultValue <- row ^. #result ->- appendJournalEntry+ Just resultValue <- row ^. #result -> do+ tracer <- currentWorkflowTracer+ appendJournalEntryTraced+ tracer (WorkflowName (row ^. #parentName)) (WorkflowId (row ^. #parentId)) StepRecorded@@ -315,10 +321,21 @@ pure Cancelled Just ChildFailed -> pure Failed _ -> do- outcome <- runWorkflowWith opts childNm childWid action+ -- Capture inside the child's run: its span has ended when propagation+ -- executes. The untraced path allocates no metadata cell.+ (outcome, metadata) <- case opts ^. #tracer of+ Nothing -> (,Nothing) <$> runWorkflowWith opts childNm childWid action+ tracer -> do+ captured <- liftIO (newIORef Nothing)+ result <- runWorkflowWith opts childNm childWid $ do+ value <- action+ md <- journalTraceMetadata tracer+ liftIO (writeIORef captured md)+ pure value+ (result,) <$> liftIO (readIORef captured) case outcome of Completed result -> do- childCompletionHook childNm childWid (toJSON result)+ childCompletionHookWithMetadata metadata childNm childWid (toJSON result) pure (Completed result) other -> pure other @@ -336,7 +353,15 @@ WorkflowId -> Aeson.Value -> Eff es ()-childCompletionHook childNm childWid resultValue = do+childCompletionHook = childCompletionHookWithMetadata Nothing++childCompletionHookTraced :: (IOE :> es, Store :> es) => Maybe Tracer -> WorkflowName -> WorkflowId -> Aeson.Value -> Eff es ()+childCompletionHookTraced tracer name wid value = do+ metadata <- journalTraceMetadata tracer+ childCompletionHookWithMetadata metadata name wid value++childCompletionHookWithMetadata :: (IOE :> es, Store :> es) => Maybe Aeson.Value -> WorkflowName -> WorkflowId -> Aeson.Value -> Eff es ()+childCompletionHookWithMetadata metadata childNm childWid resultValue = do mrow <- lookupChild (unWorkflowId childWid) (unWorkflowName childNm) for_ mrow $ \row -> case row ^. #status of Running -> do@@ -345,7 +370,8 @@ parentId = WorkflowId (row ^. #parentId) gen <- currentGeneration parentName parentId appendTx <-- prepareJournalAppend+ prepareJournalAppendWithMetadata+ metadata parentName parentId gen@@ -366,7 +392,7 @@ throwOnAppendConflict appendOutcome ChildCompleted -> for_ (row ^. #result) $ \stored ->- appendJournalEntry+ appendJournalEntryWithMetadata (WorkflowName (row ^. #parentName)) (WorkflowId (row ^. #parentId)) StepRecorded@@ -376,6 +402,10 @@ } ChildCancelled -> pure () ChildFailed -> pure ()+ where+ appendJournalEntryWithMetadata name wid event = do+ gen <- currentGeneration name wid+ appendJournalWithMetadata metadata name wid gen event >>= throwOnAppendConflict -- A refusal is deliberately not condemned: the parent is terminal, so its -- await-step sentinel must not land, but the child row transition (completed or
src/Keiro/Workflow/Instance.hs view
@@ -17,7 +17,10 @@ statusToText, statusFromText, upsertInstanceTx,+ upsertSuspendedInstanceTx, markInstanceSuspendedAwaiting,+ markInstanceSuspendedAwaitingTraced,+ lookupTraceAwaitedStep, lookupInstance, lookupInstanceSession, WorkflowHeaderKey,@@ -80,6 +83,7 @@ statusFromText, statusToText, upsertInstanceTx,+ upsertSuspendedInstanceTx, ) import Keiro.Workflow.Journal ( JournalAppendOutcome (..),@@ -213,7 +217,11 @@ -- | The step name the run parked on. Text -> Eff es ()-markInstanceSuspendedAwaiting (WorkflowName nameText) (WorkflowId widText) gen awaitedStep =+markInstanceSuspendedAwaiting = markInstanceSuspendedAwaitingTraced False++-- | The hint is attribution only, including when a wake won the step lock.+markInstanceSuspendedAwaitingTraced :: (Store :> es) => Bool -> WorkflowName -> WorkflowId -> Int -> Text -> Eff es ()+markInstanceSuspendedAwaitingTraced traced (WorkflowName nameText) (WorkflowId widText) gen awaitedStep = runTransaction $ do lockWorkflowStepTx (workflowStepLockKey widText nameText gen awaitedStep) resolved <- lookupStepResultTx widText nameText gen awaitedStep@@ -223,8 +231,19 @@ Nothing -> pure False Just aid -> maybe False (/= Pending) <$> lookupAwakeableStatusTx aid let status = if isJust resolved || abandoned then WfRunning else WfSuspended- upsertInstanceTx widText nameText (fromIntegral gen) status Nothing+ upsertSuspendedInstanceTx widText nameText (fromIntegral gen) status (if traced then Just awaitedStep else Nothing) +-- | Read a generation-qualified attribution hint without loading the instance.+lookupTraceAwaitedStep :: (Store :> es) => WorkflowName -> WorkflowId -> Int -> Eff es (Maybe Text)+lookupTraceAwaitedStep (WorkflowName name) (WorkflowId wid) gen =+ fromMaybe Nothing <$> runTransaction (Tx.statement (wid, name, fromIntegral gen) statement)+ where+ statement =+ preparable+ "SELECT trace_awaited_step FROM keiro.keiro_workflows WHERE workflow_id = $1 AND workflow_name = $2 AND generation = $3"+ (contrazip3 (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.int4)))+ (D.rowMaybe (D.column (D.nullable D.text)))+ awakeableUuidFromStep :: Text -> Maybe UUID awakeableUuidFromStep stepName = UUID.fromText =<< Text.stripPrefix awakeableStepPrefix stepName@@ -816,6 +835,7 @@ leased_by = NULL, lease_expires_at = NULL, completed_at = NULL,+ trace_awaited_step = NULL, updated_at = now() WHERE workflow_id = $1 AND workflow_name = $2
src/Keiro/Workflow/Instance/Schema.hs view
@@ -9,10 +9,11 @@ statusToText, statusFromText, upsertInstanceTx,+ upsertSuspendedInstanceTx, ) where -import Contravariant.Extras (contrazip5)+import Contravariant.Extras (contrazip5, contrazip6) import Data.Int (Int32) import Hasql.Decoders qualified as D import Hasql.Encoders qualified as E@@ -49,31 +50,59 @@ upsertInstanceTx wid name gen status mLastError = Tx.statement (wid, name, gen, statusToText status, mLastError) upsertInstanceStmt +-- | Suspension holds the awaited step lock before setting this advisory hint.+upsertSuspendedInstanceTx :: Text -> Text -> Int32 -> WorkflowStatus -> Maybe Text -> Tx.Transaction ()+upsertSuspendedInstanceTx wid name gen status hint =+ Tx.statement (wid, name, gen, statusToText status, Nothing, hint) $+ upsertStatement True $+ contrazip6 textParam textParam intParam textParam optionalText optionalText+ upsertInstanceStmt :: Statement (Text, Text, Int32, Text, Maybe Text) ()-upsertInstanceStmt =+upsertInstanceStmt = upsertStatement False (contrazip5 textParam textParam intParam textParam optionalText)++textParam :: E.Params Text+textParam = E.param (E.nonNullable E.text)++intParam :: E.Params Int32+intParam = E.param (E.nonNullable E.int4)++optionalText :: E.Params (Maybe Text)+optionalText = E.param (E.nullable E.text)++upsertStatement :: Bool -> E.Params a -> Statement a ()+upsertStatement suspension encoder = preparable- """- INSERT INTO keiro.keiro_workflows- (workflow_id, workflow_name, generation, status, last_error, completed_at)- VALUES ($1, $2, $3, $4, $5,- CASE WHEN $4 IN ('completed', 'cancelled', 'failed') THEN now() ELSE NULL END)- ON CONFLICT (workflow_id, workflow_name) DO UPDATE- SET generation = GREATEST(keiro_workflows.generation, EXCLUDED.generation),- status = EXCLUDED.status,- last_error = EXCLUDED.last_error,- updated_at = now(),- completed_at = CASE- WHEN EXCLUDED.status IN ('completed', 'cancelled', 'failed')- THEN COALESCE(keiro_workflows.completed_at, now())- ELSE keiro_workflows.completed_at- END- WHERE keiro_workflows.status NOT IN ('completed', 'cancelled', 'failed')- """- ( contrazip5- (E.param (E.nonNullable E.text))- (E.param (E.nonNullable E.text))- (E.param (E.nonNullable E.int4))- (E.param (E.nonNullable E.text))- (E.param (E.nullable E.text))+ ( """+ INSERT INTO keiro.keiro_workflows+ (workflow_id, workflow_name, generation, status, last_error, completed_at, trace_awaited_step)+ VALUES ($1, $2, $3, $4, $5,+ CASE WHEN $4 IN ('completed', 'cancelled', 'failed') THEN now() ELSE NULL END,+ """+ <> " "+ <> (if suspension then "$6)" else "NULL)")+ <> " "+ <> """+ ON CONFLICT (workflow_id, workflow_name) DO UPDATE+ SET generation = GREATEST(keiro_workflows.generation, EXCLUDED.generation),+ status = EXCLUDED.status,+ last_error = EXCLUDED.last_error,+ trace_awaited_step = CASE+ WHEN EXCLUDED.generation > keiro_workflows.generation+ OR EXCLUDED.status IN ('completed', 'cancelled', 'failed') THEN NULL+ """+ <> " "+ <> (if suspension then "WHEN EXCLUDED.generation = keiro_workflows.generation THEN EXCLUDED.trace_awaited_step " else "")+ <> " "+ <> """+ ELSE keiro_workflows.trace_awaited_step END,+ updated_at = now(),+ completed_at = CASE+ WHEN EXCLUDED.status IN ('completed', 'cancelled', 'failed')+ THEN COALESCE(keiro_workflows.completed_at, now())+ ELSE keiro_workflows.completed_at+ END+ WHERE keiro_workflows.status NOT IN ('completed', 'cancelled', 'failed')+ """ )+ encoder D.noResult
src/Keiro/Workflow/Journal.hs view
@@ -3,6 +3,13 @@ module Keiro.Workflow.Journal ( JournalAppendOutcome (..), prepareJournalAppend,+ prepareJournalAppendWithMetadata,+ appendJournalWithMetadata,+ appendJournalTraced,+ appendJournalEntryTraced,+ appendJournalEntryReturningIdTraced,+ journalTraceMetadata,+ readWorkflowTraceResume, appendJournal, appendJournalEntry, appendJournalEntryReturningId,@@ -10,19 +17,26 @@ ) where +import Contravariant.Extras (contrazip4) import Data.Aeson qualified as Aeson+import Data.Int (Int32) import Data.Maybe (listToMaybe) import Data.Text qualified as Text import Data.UUID.V5 qualified as UUID.V5 import Effectful (Eff, IOE, (:>)) import Effectful.Exception (throwIO)+import Hasql.Decoders qualified as D+import Hasql.Encoders qualified as E+import Hasql.Statement (preparable) import Keiro.Codec (encodeForAppendWithMetadata) import Keiro.DeterministicId (identitySeedBytes) import Keiro.Prelude+import Keiro.Telemetry (Tracer, stampTraceContext) import Keiro.Workflow.Instance.Schema (WorkflowStatus (..), upsertInstanceTx) import Keiro.Workflow.Schema ( WorkflowStepRow (..), currentGeneration,+ currentGenerationTx, lockWorkflowStepTx, lookupStepResultTx, recordStepTx,@@ -33,7 +47,7 @@ import Keiro.Workflow.Types import Kiroku.Store.Effect (Store) import Kiroku.Store.Transaction (AppendConflict, appendToStreamTx, prepareEventsIO, runTransaction)-import Kiroku.Store.Types (AppendResult, EventData, EventId (..), ExpectedVersion (..))+import Kiroku.Store.Types (AppendResult, EventData, EventId (..), ExpectedVersion (..), StreamId (..), StreamName (..), StreamVersion (..)) import "hasql-transaction" Hasql.Transaction qualified as Tx -- | What a journal append did.@@ -65,7 +79,11 @@ Int -> WorkflowJournalEvent -> Eff es (Tx.Transaction JournalAppendOutcome)-prepareJournalAppend name wid gen event = do+prepareJournalAppend = prepareJournalAppendWithMetadata Nothing++-- | Metadata is captured in the effect layer, before constructing SQL.+prepareJournalAppendWithMetadata :: (IOE :> es) => Maybe Aeson.Value -> WorkflowName -> WorkflowId -> Int -> WorkflowJournalEvent -> Eff es (Tx.Transaction JournalAppendOutcome)+prepareJournalAppendWithMetadata metadata name wid gen event = do let key = journalKey event entryId = deterministicJournalId name wid gen key requestedEntryId = case event of@@ -88,7 +106,7 @@ _ -> listToMaybe <$> workflowLifecycleMarkersTx (unWorkflowId wid) (unWorkflowName name) gen- base <- case encodeForAppendWithMetadata workflowJournalCodec Nothing event of+ base <- case encodeForAppendWithMetadata workflowJournalCodec metadata event of Right encoded -> pure encoded Left err -> throwIO (WorkflowJournalEncodeError (Text.pack (show err))) let entry = base & #eventId .~ requestedEntryId :: EventData@@ -127,9 +145,26 @@ Int -> WorkflowJournalEvent -> Eff es JournalAppendOutcome-appendJournal name wid gen event =- prepareJournalAppend name wid gen event >>= runTransaction+appendJournal = appendJournalWithMetadata Nothing +appendJournalWithMetadata :: (IOE :> es, Store :> es) => Maybe Aeson.Value -> WorkflowName -> WorkflowId -> Int -> WorkflowJournalEvent -> Eff es JournalAppendOutcome+appendJournalWithMetadata metadata name wid gen event =+ prepareJournalAppendWithMetadata metadata name wid gen event >>= runTransaction++-- | Explicit enablement; ambient context alone never enables journal tracing.+journalTraceMetadata :: (MonadIO m) => Maybe Tracer -> m (Maybe Aeson.Value)+journalTraceMetadata Nothing = pure Nothing+journalTraceMetadata (Just _) = stampTraceContext Nothing++appendJournalTraced :: (IOE :> es, Store :> es) => Maybe Tracer -> WorkflowName -> WorkflowId -> Int -> WorkflowJournalEvent -> Eff es JournalAppendOutcome+appendJournalTraced tracer name wid gen event = do+ metadata <- journalTraceMetadata tracer+ appendJournalWithMetadata metadata name wid gen event++appendJournalEntryTraced :: (IOE :> es, Store :> es) => Maybe Tracer -> WorkflowName -> WorkflowId -> WorkflowJournalEvent -> Eff es ()+appendJournalEntryTraced tracer name wid event =+ void (appendJournalEntryReturningIdTraced tracer name wid event)+ -- | Append one event on the current generation, idempotently. -- -- A late wake-source step aimed at a cancelled or failed workflow is a quiet@@ -151,11 +186,14 @@ WorkflowId -> WorkflowJournalEvent -> Eff es EventId-appendJournalEntryReturningId name wid event = do+appendJournalEntryReturningId = appendJournalEntryReturningIdTraced Nothing++appendJournalEntryReturningIdTraced :: (IOE :> es, Store :> es) => Maybe Tracer -> WorkflowName -> WorkflowId -> WorkflowJournalEvent -> Eff es EventId+appendJournalEntryReturningIdTraced tracer name wid event = do gen <- currentGeneration name wid let key = journalKey event entryId = deterministicJournalId name wid gen key- appendJournal name wid gen event >>= \case+ appendJournalTraced tracer name wid gen event >>= \case JournalAppended {} -> pure entryId JournalAlreadyPresent {} -> pure entryId JournalRefusedTerminal {} -> pure entryId@@ -205,3 +243,38 @@ UUID.V5.generateNamed UUID.V5.namespaceURL $ identitySeedBytes $ Text.intercalate ":" ["keiro", "workflow", name, wid, Text.pack (show gen), key]++-- | The attribution hint and an indexed first-event reference. Select the+-- reference before joining payloads: the generic forward read can otherwise+-- sort an entire journal before applying LIMIT. Payloads still go through+-- Store.getEvent so configured read decoders remain authoritative.+readWorkflowTraceResume :: (Store :> es) => WorkflowName -> WorkflowId -> Eff es (Int, Maybe (Text, Maybe (EventId, StreamId, StreamVersion)))+readWorkflowTraceResume name wid =+ runTransaction $ do+ gen <- currentGenerationTx name wid+ let StreamName journalName = workflowGenerationStreamName name wid gen+ hint <- Tx.statement (unWorkflowId wid, unWorkflowName name, fromIntegral gen :: Int32, journalName) statement+ pure (gen, hint)+ where+ statement =+ preparable+ """+ SELECT w.trace_awaited_step, first.event_id, s.stream_id, first.stream_version+ FROM keiro.keiro_workflows w+ LEFT JOIN kiroku.streams s ON s.stream_name = $4 AND s.deleted_at IS NULL+ LEFT JOIN LATERAL (+ SELECT se.event_id, se.stream_version FROM kiroku.stream_events se+ WHERE se.stream_id = s.stream_id AND se.stream_version > 0+ AND se.stream_version >= s.truncate_before+ ORDER BY se.stream_version ASC LIMIT 1+ ) first ON true+ WHERE w.workflow_id = $1 AND w.workflow_name = $2 AND w.generation = $3+ AND w.trace_awaited_step IS NOT NULL+ """+ (contrazip4 (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.text)) (E.param (E.nonNullable E.int4)) (E.param (E.nonNullable E.text)))+ (D.rowMaybe ((,) <$> D.column (D.nonNullable D.text) <*> origin))+ origin =+ (\eid sid version -> (,,) <$> (EventId <$> eid) <*> (StreamId <$> sid) <*> (StreamVersion <$> version))+ <$> D.column (D.nullable D.uuid)+ <*> D.column (D.nullable D.int8)+ <*> D.column (D.nullable D.int8)
src/Keiro/Workflow/Resume.hs view
@@ -128,11 +128,9 @@ WorkflowName (..), WorkflowOutcome (..), WorkflowRunOptions,- appendJournalEntry, currentGeneration, defaultWorkflowRunOptions, findUnfinishedWorkflowIds,- prepareJournalAppend, runWorkflowWith, ) import Keiro.Workflow.Awakeable.Schema (countPendingAwakeables)@@ -146,6 +144,7 @@ recordCrashTx, releaseInstance, )+import Keiro.Workflow.Journal (appendJournalEntryTraced, journalTraceMetadata, prepareJournalAppendWithMetadata) import Kiroku.Store.Connection (KirokuStore) import Kiroku.Store.Effect (Store, runStoreIO) import Kiroku.Store.Error (StoreError)@@ -553,9 +552,10 @@ mChild <- lookupChild widText wnameText case mChild of Nothing ->- appendJournalEntry name wid (WorkflowFailed rendered now)- Just childRow ->- appendFailedChildAndWakeParent name wid rendered now childRow+ appendJournalEntryTraced (runOptions opts ^. #tracer) name wid (WorkflowFailed rendered now)+ Just childRow -> do+ metadata <- journalTraceMetadata (runOptions opts ^. #tracer)+ appendFailedChildAndWakeParent metadata name wid rendered now childRow liftIO $ logEvent opts (ResumeWorkflowMarkedFailed wnameText widText rendered) recordWorkflowFailed mMetrics 1 pure@@ -576,20 +576,22 @@ appendFailedChildAndWakeParent :: (IOE :> es, Store :> es) =>+ Maybe Aeson.Value -> WorkflowName -> WorkflowId -> Text -> UTCTime -> ChildRow -> Eff es ()-appendFailedChildAndWakeParent childNm childWid reason now childRow = do+appendFailedChildAndWakeParent metadata childNm childWid reason now childRow = do childGen <- currentGeneration childNm childWid let parentNm = WorkflowName (childRow ^. #parentName) parentWid = WorkflowId (childRow ^. #parentId) parentGen <- currentGeneration parentNm parentWid- childFailTx <- prepareJournalAppend childNm childWid childGen (WorkflowFailed reason now)+ childFailTx <- prepareJournalAppendWithMetadata metadata childNm childWid childGen (WorkflowFailed reason now) parentWakeTx <-- prepareJournalAppend+ prepareJournalAppendWithMetadata+ metadata parentNm parentWid parentGen
src/Keiro/Workflow/Schema.hs view
@@ -33,6 +33,7 @@ stepExists, terminalMarkers, currentGeneration,+ currentGenerationTx, findUnfinishedWorkflowIds, ) where@@ -218,8 +219,12 @@ -- the same logical-id key space, so @MAX(generation)@ is unambiguously the -- current generation. currentGeneration :: (Store :> es) => WorkflowName -> WorkflowId -> Eff es Int-currentGeneration (WorkflowName name) (WorkflowId wid) =- fromIntegral <$> runTransaction (Tx.statement (wid, name) currentGenerationStmt)+currentGeneration name wid = runTransaction (currentGenerationTx name wid)++-- | The same generation lookup, composable with advisory entry reads.+currentGenerationTx :: WorkflowName -> WorkflowId -> Tx.Transaction Int+currentGenerationTx (WorkflowName name) (WorkflowId wid) =+ fromIntegral <$> Tx.statement (wid, name) currentGenerationStmt -- | Return the @(workflow_id, workflow_name)@ of every workflow instance that -- has progress to make right now. Discovery is /exact/: an instance is returned
src/Keiro/Workflow/Sleep.hs view
@@ -80,8 +80,11 @@ -- * Firing and worker wiring workflowSleepFireAction,+ workflowSleepFireActionTraced, runWorkflowTimerWorker,+ runWorkflowTimerWorkerWith, drainWorkflowSleepTimers,+ drainWorkflowSleepTimersWith, -- * Timer id, payload, and step-name helpers sleepTimerId,@@ -104,14 +107,17 @@ import Effectful.Exception (throwIO) import Keiro.DeterministicId (identitySeedBytes) import Keiro.Prelude-import Keiro.Telemetry (KeiroMetrics)+import Keiro.Telemetry (KeiroMetrics, Tracer) import Keiro.Timer ( TimerId (..), TimerRequest (..), TimerRow,+ TimerWorkerOptions (..), cancelTimer,- drainDueTimers,- runTimerWorker,+ captureTimerTraceContext,+ defaultTimerWorkerOptions,+ drainDueTimersWith,+ runTimerWorkerWith, scheduleTimerOnceTx, ) import Keiro.Workflow@@ -127,13 +133,14 @@ currentGeneration, currentRunGeneration, currentWorkflow,+ currentWorkflowTracer, deterministicJournalId, freshOrdinal,- prepareJournalAppend, setWorkflowWakeAfterTx, sleepStepPrefix, ) import Keiro.Workflow.Instance qualified as Instance+import Keiro.Workflow.Journal (journalTraceMetadata, prepareJournalAppendWithMetadata) import Kiroku.Store.Effect (Store) import Kiroku.Store.Transaction (runTransaction) import Kiroku.Store.Types (EventId)@@ -276,7 +283,8 @@ now <- liftIO getCurrentTime let request = TimerRequest- { timerId = sleepTimerId name wid gen full,+ { traceContext = Nothing,+ timerId = sleepTimerId name wid gen full, processManagerName = unWorkflowName name, correlationId = unWorkflowId wid, fireAt = addUTCTime delta now,@@ -286,8 +294,10 @@ -- leave both fire_at and wake_after untouched, so re-entering a due or -- already-fired sleep cannot postpone discovery. Firing clears the -- hint in the same transaction as its journal append.+ tracer <- currentWorkflowTracer+ tracedRequest <- captureTimerTraceContext tracer request runTransaction $ do- inserted <- scheduleTimerOnceTx request+ inserted <- scheduleTimerOnceTx tracedRequest when inserted (setWorkflowWakeAfterTx name wid (request ^. #fireAt)) -- | Durably pause the workflow for @delta@ under an ordinal name (the @N@th@@ -318,7 +328,11 @@ -- exactly-once journaling even after the workflow has rotated. workflowSleepFireAction :: (Store :> es, IOE :> es) => TimerRow -> Eff es (Maybe EventId)-workflowSleepFireAction row =+workflowSleepFireAction = workflowSleepFireActionTraced Nothing++-- | Explicitly trace the wake append using the timer fire span.+workflowSleepFireActionTraced :: (Store :> es, IOE :> es) => Maybe Tracer -> TimerRow -> Eff es (Maybe EventId)+workflowSleepFireActionTraced tracer row = case parseSleepPayload (row ^. #payload) of Nothing -> pure Nothing Just (full, payloadGen) -> do@@ -341,8 +355,10 @@ currentGen (matchSleepTimerGeneration name wid currentGen full (row ^. #timerId)) now <- liftIO getCurrentTime+ metadata <- journalTraceMetadata tracer appendTx <-- prepareJournalAppend+ prepareJournalAppendWithMetadata+ metadata name wid targetGen@@ -394,9 +410,12 @@ -- | Fallback fire action for non-sleep (process-manager) timers. (TimerRow -> Eff es (Maybe EventId)) -> Eff es (Maybe TimerRow)-runWorkflowTimerWorker metrics now pmFire =- runTimerWorker metrics now (workflowSleepOrPmFire pmFire)+runWorkflowTimerWorker = runWorkflowTimerWorkerWith defaultTimerWorkerOptions +runWorkflowTimerWorkerWith :: (IOE :> es, Store :> es) => TimerWorkerOptions -> Maybe KeiroMetrics -> UTCTime -> (TimerRow -> Eff es (Maybe EventId)) -> Eff es (Maybe TimerRow)+runWorkflowTimerWorkerWith options metrics now pmFire =+ runTimerWorkerWith metrics options now (workflowSleepOrPmFire (options ^. #tracer) pmFire)+ -- | 'runWorkflowTimerWorker' over a whole batch: drain up to @limit@ due timers -- in one pass, routing each to the workflow-sleep fire action or the supplied -- process-manager fallback exactly as the single-claim worker does.@@ -416,18 +435,22 @@ -- | Fallback fire action for non-sleep (process-manager) timers. (TimerRow -> Eff es (Maybe EventId)) -> Eff es Int-drainWorkflowSleepTimers metrics now limit pmFire =- drainDueTimers metrics now limit (workflowSleepOrPmFire pmFire)+drainWorkflowSleepTimers = drainWorkflowSleepTimersWith defaultTimerWorkerOptions +drainWorkflowSleepTimersWith :: (IOE :> es, Store :> es) => TimerWorkerOptions -> Maybe KeiroMetrics -> UTCTime -> Int -> (TimerRow -> Eff es (Maybe EventId)) -> Eff es Int+drainWorkflowSleepTimersWith options metrics now limit pmFire =+ drainDueTimersWith metrics options now limit (workflowSleepOrPmFire (options ^. #tracer) pmFire)+ -- | Route one claimed timer: a workflow sleep wakes its workflow, anything else -- falls through to the caller's process-manager action. workflowSleepOrPmFire :: (IOE :> es, Store :> es) =>+ Maybe Tracer -> (TimerRow -> Eff es (Maybe EventId)) -> TimerRow -> Eff es (Maybe EventId)-workflowSleepOrPmFire pmFire row = do- handled <- workflowSleepFireAction row+workflowSleepOrPmFire tracer pmFire row = do+ handled <- workflowSleepFireActionTraced tracer row case handled of Just eid -> pure (Just eid) Nothing -> pmFire row
+ test/DurableTracingSpec.hs view
@@ -0,0 +1,335 @@+{-# LANGUAGE BlockArguments #-}+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE TypeOperators #-}++module DurableTracingSpec (spec) where++import Control.Exception (bracket, try)+import Control.Monad (forM_, void)+import Data.Aeson qualified as Aeson+import Data.IORef (IORef, readIORef)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Time (getCurrentTime)+import Data.UUID qualified as UUID+import Data.Vector qualified as Vector+import Effectful (Eff, IOE, (:>))+import Effectful.Exception qualified as Exception+import Keiro.Codec qualified+import Keiro.Command (defaultRunCommandOptions, runCommand)+import Keiro.Integration.Event (TraceContext)+import Keiro.Prelude (liftIO, (&), (.~), (^.))+import Keiro.ProcessManager (PMCommand (..))+import Keiro.ProcessManager.Reaction qualified as Reaction+import Keiro.Stream (stream)+import Keiro.Telemetry qualified as Telemetry+import Keiro.Test.Postgres (Fixture, StoreRunner (..), withFreshResourceStore)+import Keiro.Timer qualified as Timer+import Keiro.Workflow+import Keiro.Workflow.Awakeable qualified as Awakeable+import Keiro.Workflow.Child qualified as Child+import Keiro.Workflow.Instance qualified as Instance+import Keiro.Workflow.Sleep qualified as Sleep+import Keiro.Workflow.Snapshot (writeWorkflowSnapshot)+import Kiroku.Store qualified as Store+import Kiroku.Store.Subscription.Types (SubscriptionName (..), SubscriptionTarget (..))+import Kiroku.Store.Types (CategoryName (..), ExpectedVersion (..), RecordedEvent, StreamName (..), StreamVersion (..))+import OpenTelemetry.Attributes (Attribute (..), PrimitiveAttribute (..), lookupAttribute)+import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter)+import OpenTelemetry.Trace.Core+import OpenTelemetry.Util (appendOnlyBoundedCollectionValues)+import ReactionExample qualified as Example+import Shibuya.Adapter (Adapter (..))+import Shibuya.Adapter.Kiroku (defaultKirokuAdapterConfig, kirokuAdapter)+import Streamly.Data.Stream qualified as Streamly+import Test.Hspec++spec :: Fixture -> Spec+spec fixture = describe "durable tracing" $ around (withFreshResourceStore fixture) do+ it "links exact sleep wake and origin beyond a snapshot, ignoring a later event and preserving replay" $ \(store, StoreRunner runStore) ->+ withRecorder $ \tracer exported -> do+ let name = WorkflowName "traceSleep"+ wid = WorkflowId "one"+ options = defaultWorkflowRunOptions & #tracer .~ Just tracer+ flow = do+ value <- step (StepName "origin") (pure (7 :: Int))+ _ <- awaitValue (StepName "sleep:pause")+ _ <- awaitValue (StepName "hold")+ pure value+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended+ runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right (Just "sleep:pause")+ [first] <- readIORef exported+ let firstContext = spanContext first+ now <- getCurrentTime+ let commandOptions = defaultRunCommandOptions & #tracer .~ Just tracer+ timerRequest = Timer.TimerRequest (Sleep.sleepTimerId name wid 0 "sleep:pause") "traceSleep" "one" now (Sleep.sleepTimerPayload 0 "sleep:pause") Nothing+ manager =+ (Example.exampleReactionManager now)+ { Reaction.name = "trace-reaction",+ Reaction.react =+ const+ ( Reaction.NoAdvance+ [ Reaction.FollowSchedule Reaction.Once timerRequest,+ Reaction.FollowDispatch (PMCommand (stream "tracetarget-one") (Example.SendAlert "one"))+ ]+ )+ }+ Right (Right _) <- runStore $ runCommand commandOptions Example.exampleTargetEventStream (stream "tracesource-one") (Example.SendAlert "source")+ Right () <- runStore do+ adapter@Adapter {source = incoming, shutdown = stop} <- kirokuAdapter store (defaultKirokuAdapterConfig (SubscriptionName "durable-chain") (Category (CategoryName "tracesource")))+ Reaction.runReactiveProcessManagerWorker+ commandOptions+ manager+ (adapter {source = Streamly.take 1 incoming})+ (\event -> Just (event, Example.IncidentReported "one" Example.Urgent))+ `Exception.finally` stop+ processing <- named "process trace-reaction" =<< readIORef exported+ sourceSpan <- named "command tracesource" =<< readIORef exported+ assertRoot processing+ assertLinks processing [(Nothing, spanContext sourceSpan)]+ dispatched <- named "command tracetarget" =<< readIORef exported+ case spanParent dispatched of+ Just parent -> getSpanContext parent >>= (\ctx -> sameContext ctx (spanContext processing))+ _ -> expectationFailure "dispatched command lost processing parent"+ Right (Just fired) <-+ runStore $+ Sleep.runWorkflowTimerWorkerWith+ (Timer.defaultTimerWorkerOptions & #tracer .~ Just tracer)+ Nothing+ now+ (const (pure Nothing))+ case fired ^. #traceContext >>= Telemetry.creationLinkFromTraceContext of+ Just link -> sameContext (linkContext link) (spanContext processing)+ Nothing -> expectationFailure "reaction did not capture processing creation context"+ spansAfterFire <- readIORef exported+ fire <- named "timer fire traceSleep" spansAfterFire+ assertRoot fire+ assertLinks fire [(Nothing, spanContext processing)]+ -- This later append must never replace the exact awaited sleep event.+ inSpan' tracer "unrelated" defaultSpanArguments $ \_ -> do+ Right _ <- runStore $ appendJournalEntryTraced (Just tracer) name wid (StepRecorded "unrelated" Aeson.Null now)+ pure ()+ Right events <- runStore $ Store.readStreamForward (workflowGenerationStreamName name wid 0) (StreamVersion 0) 20+ let lastEvent = Vector.last events+ state = Map.fromList [("origin", Aeson.toJSON (7 :: Int)), ("sleep:pause", Aeson.Null), ("unrelated", Aeson.Null)]+ runStore (writeWorkflowSnapshot (lastEvent ^. #originalStreamId) (lastEvent ^. #originalVersion) state) `shouldReturn` Right ()+ -- A failed resume restores ambient state and leaves attribution available+ -- for the next attempt; both origin and wake are before the snapshot.+ inSpan' tracer "ambient" defaultSpanArguments $ \ambient -> do+ let failing = do+ _ <- step (StepName "origin") (pure (7 :: Int))+ Sleep.sleepNamed (StepName "pause") (-1)+ Exception.throwIO (userError "retry")+ failed <- try (runStore $ runWorkflowWith options name wid failing) :: IO (Either IOError (Either Store.StoreError (WorkflowOutcome Int)))+ failed `shouldSatisfy` either (const True) (const False)+ assertTraceContext (spanContextOf ambient) =<< Telemetry.traceContextFromCurrentSpan+ inSpan' tracer "ambient-success" defaultSpanArguments $ \ambient -> do+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended+ assertTraceContext (spanContextOf ambient) =<< Telemetry.traceContextFromCurrentSpan+ spans <- readIORef exported+ resumes <- filterMName "workflow traceSleep" spans+ length resumes `shouldBe` 3+ forM_ (take 2 resumes) $ \resume -> do+ assertRoot resume+ assertLinks resume [(Just "origin", firstContext), (Just "wake", spanContext fire)]+ -- Replay observes the same map with tracing disabled; metadata is not folded.+ runStore (loadStepIndex name wid 0) `shouldReturn` Right state+ runStore (runWorkflow name wid flow) `shouldReturn` Right Suspended+ runStore (loadStepIndex name wid 0) `shouldReturn` Right state+ runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right Nothing++ it "deduplicates first-event sleep as wake and leaves legacy timers unlinked" $ \(_, StoreRunner runStore) ->+ withRecorder $ \tracer exported -> do+ let name = WorkflowName "firstSleep"+ wid = WorkflowId "one"+ options = defaultWorkflowRunOptions & #tracer .~ Just tracer+ flow = Sleep.sleepNamed (StepName "pause") (-1)+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended+ now <- getCurrentTime+ Right (Just _) <-+ runStore $+ Sleep.runWorkflowTimerWorkerWith+ (Timer.defaultTimerWorkerOptions & #tracer .~ Just tracer)+ Nothing+ now+ (const (pure Nothing))+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right (Completed ())+ spans <- readIORef exported+ fire <- named "timer fire firstSleep" spans+ resumed <- named "workflow firstSleep" spans+ assertLinks resumed [(Just "wake", spanContext fire)]+ runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right Nothing+ let request = Timer.TimerRequest (Timer.TimerId UUID.nil) "legacy" "one" now Aeson.Null Nothing+ runStore (Store.runTransaction (Timer.scheduleTimerTx request)) `shouldReturn` Right ()+ Right (Just _) <-+ runStore $+ Timer.runTimerWorkerWith+ Nothing+ (Timer.defaultTimerWorkerOptions & #tracer .~ Just tracer)+ now+ (const (pure Nothing))+ legacy <- named "timer fire legacy" =<< readIORef exported+ assertRoot legacy+ assertLinks legacy []++ it "stamps explicitly traced signals, preserves first metadata on retry, and ignores ambient-only enablement" $ \(_, StoreRunner runStore) ->+ withRecorder $ \tracer exported -> do+ let name = WorkflowName "signalTrace"+ wid = WorkflowId "one"+ options = defaultWorkflowRunOptions & #tracer .~ Just tracer+ flow = do+ (_, wait) <- Awakeable.awakeableNamed (StepName "reply")+ asInt wait+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended+ Right rows <- runStore $ Store.readStreamForward (workflowGenerationStreamName name wid 0) (StreamVersion 0) 10+ aid <- case decodeJournal (Vector.head rows) of+ Right (StepRecorded _ value _) -> case Aeson.fromJSON value of+ Aeson.Success aid -> pure aid+ _ -> fail "awakeable allocation"+ _ -> fail "allocation journal missing"+ inSpan' tracer "signal" defaultSpanArguments $ \_ ->+ runStore (Awakeable.signalAwakeableTraced (Just tracer) aid (42 :: Int)) `shouldReturn` Right True+ Right before <- runStore $ Store.getEvent (deterministicJournalId name wid 0 (awakeableStepPrefix <> Awakeable.awakeableIdText aid))+ inSpan' tracer "retry" defaultSpanArguments $ \_ ->+ runStore (Awakeable.signalAwakeableTraced (Just tracer) aid (999 :: Int)) `shouldReturn` Right False+ Right after <- runStore $ Store.getEvent (deterministicJournalId name wid 0 (awakeableStepPrefix <> Awakeable.awakeableIdText aid))+ fmap (^. #metadata) after `shouldBe` fmap (^. #metadata) before+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right (Completed 42)+ signal <- named "signal" =<< readIORef exported+ resumed <- named "workflow signalTrace" =<< readIORef exported+ links <- linksOf resumed+ [spanId (frozenLinkContext link) | link <- links, roleOf link == Just "wake"] `shouldBe` [spanId (spanContext signal)]+ inSpan' tracer "ambient-only" defaultSpanArguments $ \_ ->+ runStore (runWorkflow (WorkflowName "plain") wid (step (StepName "x") (pure (1 :: Int)))) `shouldReturn` Right (Completed 1)+ Right plain <- runStore $ Store.readStreamForward (workflowGenerationStreamName (WorkflowName "plain") wid 0) (StreamVersion 0) 10+ map (Telemetry.traceContextFromMetadata . (^. #metadata)) (Vector.toList plain) `shouldBe` [Nothing, Nothing]++ it "propagates the finished child context to the parent's exact wake" $ \(_, StoreRunner runStore) ->+ withRecorder $ \tracer exported -> do+ let parentName = WorkflowName "parentTrace"+ parentId = WorkflowId "one"+ childName = WorkflowName "childTrace"+ childId = WorkflowId "two"+ options = defaultWorkflowRunOptions & #tracer .~ Just tracer+ parent = Child.spawnChild childName childId traceChild >>= Child.awaitChild+ runStore (runWorkflowWith options parentName parentId parent) `shouldReturn` Right Suspended+ runStore (Child.runChildWorkflow options childName childId traceChild) `shouldReturn` Right (Completed 9)+ runStore (runWorkflowWith options parentName parentId parent) `shouldReturn` Right (Completed 9)+ spans <- readIORef exported+ childSpan <- named "workflow childTrace" spans+ parentSpan <- named "workflow parentTrace" spans+ links <- linksOf parentSpan+ [spanId (frozenLinkContext link) | link <- links, roleOf link == Just "wake"] `shouldBe` [spanId (spanContext childSpan)]++ it "records wake-before-suspend hints, clears on rotation, and omits missing or legacy evidence" $ \(_, StoreRunner runStore) ->+ withRecorder $ \tracer exported -> do+ let name = WorkflowName "hintTrace"+ wid = WorkflowId "one"+ options = defaultWorkflowRunOptions & #tracer .~ Just tracer+ now <- getCurrentTime+ -- A real wake commits before suspension takes the same per-step lock.+ runStore (appendJournalEntry name wid (StepRecorded "gate" Aeson.Null now)) `shouldReturn` Right ()+ runStore (Instance.markInstanceSuspendedAwaitingTraced True name wid 0 "gate") `shouldReturn` Right ()+ Right (Just row) <- runStore (Instance.lookupInstance name wid)+ row ^. #status `shouldBe` Instance.WfRunning+ runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right (Just "gate")+ runStore (runWorkflowWith options name wid (awaitValue (StepName "next"))) `shouldReturn` Right Suspended+ resumed <- named "workflow hintTrace" =<< readIORef exported+ assertLinks resumed [] -- legacy metadata has no context+ runStore (Instance.markInstanceSuspendedAwaitingTraced True name wid 0 "missing") `shouldReturn` Right ()+ runStore (runWorkflowWith options name wid (continueAsNew (1 :: Int))) `shouldReturn` Right (ContinuedAsNew :: WorkflowOutcome ())+ runStore (Instance.lookupTraceAwaitedStep name wid 1) `shouldReturn` Right Nothing+ -- No first event means a cancellation-only wake cannot invent a link.+ let emptyName = WorkflowName "missingTrace"+ runStore (Instance.markInstanceSuspendedAwaitingTraced True emptyName wid 0 "awk:missing") `shouldReturn` Right ()+ runStore (runWorkflowWith options emptyName wid (awaitValue (StepName "hold"))) `shouldReturn` Right Suspended+ emptySpan <- named "workflow missingTrace" =<< readIORef exported+ assertLinks emptySpan []++ it "omits malformed, foreign-stream, and mismatched-step wake evidence" $ \(_, StoreRunner runStore) ->+ withRecorder $ \tracer exported ->+ forM_ ["malformed", "wrong-stream", "wrong-step"] $ \scenario -> do+ let name = WorkflowName scenario+ wid = WorkflowId "one"+ options = defaultWorkflowRunOptions & #tracer .~ Just tracer+ flow = do+ _ <- step (StepName "origin") (pure (7 :: Int))+ _ <- awaitValue (StepName "gate")+ awaitValue (StepName "hold")+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended+ now <- getCurrentTime+ let metadata = Just (Aeson.object ["traceparent" Aeson..= (if scenario == "malformed" then "bad" else "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" :: Text)])+ event = StepRecorded (if scenario == "wrong-step" then "other" else "gate") Aeson.Null now+ target = if scenario == "wrong-stream" then StreamName "foreign-one" else workflowGenerationStreamName name wid 0+ entry <- either (fail . show) pure (Keiro.Codec.encodeForAppendWithMetadata workflowJournalCodec metadata event)+ Right _ <- runStore (Store.appendToStream target AnyVersion [entry & #eventId .~ Just (deterministicJournalId name wid 0 "gate")])+ runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended+ resume <- named ("workflow " <> scenario) =<< readIORef exported+ links <- linksOf resume+ map roleOf links `shouldBe` [Just "origin"]++withRecorder :: (Tracer -> IORef [ImmutableSpan] -> IO a) -> IO a+withRecorder action = do+ (processor, exported) <- inMemoryListExporter+ bracket+ (createTracerProvider [processor] emptyTracerProviderOptions)+ (\provider -> void (shutdownTracerProvider provider Nothing))+ (\provider -> action (makeTracer provider "durable-test" tracerOptions) exported)++named :: Text -> [ImmutableSpan] -> IO ImmutableSpan+named wanted spans = do+ matches <- filterMName wanted spans+ case matches of+ first : _ -> pure first+ _ -> fail ("missing span " <> show wanted)++filterMName :: Text -> [ImmutableSpan] -> IO [ImmutableSpan]+filterMName wanted spans = do+ pairs <- traverse (\sp -> (sp,) . hotName <$> readIORef (spanHot sp)) spans+ pure [sp | (sp, name) <- pairs, name == wanted]++linksOf :: ImmutableSpan -> IO [Link]+linksOf sp = Vector.toList . appendOnlyBoundedCollectionValues . hotLinks <$> readIORef (spanHot sp)++roleOf :: Link -> Maybe Text+roleOf link = case lookupAttribute (frozenLinkAttributes link) "keiro.workflow.link" of+ Just (AttributeValue (TextAttribute role)) -> Just role+ _ -> Nothing++assertRoot :: ImmutableSpan -> Expectation+assertRoot sp = case spanParent sp of+ Nothing -> pure ()+ Just _ -> expectationFailure "durable span inherited a parent"++assertLinks :: ImmutableSpan -> [(Maybe Text, SpanContext)] -> Expectation+assertLinks sp expected = do+ actual <- linksOf sp+ [(roleOf link, traceId (frozenLinkContext link), spanId (frozenLinkContext link)) | link <- actual]+ `shouldMatchList` [(role, traceId ctx, spanId ctx) | (role, ctx) <- expected]++sameContext :: SpanContext -> SpanContext -> Expectation+sameContext a b = (traceId a, spanId a) `shouldBe` (traceId b, spanId b)++assertTraceContext :: IO SpanContext -> Maybe TraceContext -> Expectation+assertTraceContext expected context = do+ ctx <- expected+ case context >>= Telemetry.creationLinkFromTraceContext of+ Nothing -> expectationFailure "ambient context lost"+ Just link -> sameContext ctx (linkContext link)++spanContextOf :: Span -> IO SpanContext+spanContextOf = getSpanContext++decodeJournal :: RecordedEvent -> Either String WorkflowJournalEvent+decodeJournal row = case Keiro.Codec.decodeRecorded workflowJournalCodec row of+ Left err -> Left (show err)+ Right event -> Right event++awaitValue :: (Workflow :> es) => StepName -> Eff es Aeson.Value+awaitValue key = awaitStep key (pure ())++asInt :: Eff es Int -> Eff es Int+asInt = id++traceChild :: (Workflow :> es) => Eff es Int+traceChild = step (StepName "result") (pure 9)
test/Main.hs view
@@ -12,7 +12,7 @@ import Control.Concurrent.MVar (MVar, modifyMVar, newEmptyMVar, newMVar, putMVar, readMVar, takeMVar, tryPutMVar) import Control.Concurrent.STM (atomically, putTMVar) import Control.Exception (AsyncException (..), ErrorCall, Exception, SomeException, displayException, evaluate, finally, throwIO, try)-import Control.Monad (forM, forM_)+import Control.Monad (forM, forM_, replicateM_) import Data.Aeson (object, withObject, (.:), (.:?)) import Data.Aeson qualified as Aeson import Data.Aeson.KeyMap qualified as KeyMap@@ -40,6 +40,7 @@ import Data.Vector qualified as Vector import Data.Version (showVersion) import Data.Word (Word64)+import DurableTracingSpec qualified import Effectful (Eff, IOE, runEff, (:>)) import Effectful.Error.Static (Error, throwError) import Effectful.Exception qualified as EffException@@ -422,12 +423,17 @@ ) import OpenTelemetry.Trace.Core ( ImmutableSpan (..),+ Link (..),+ NewLink (..), Span, SpanContext (..), SpanHot (..), SpanKind,+ defaultSpanArguments, getSpanContext,+ inSpan', )+import OpenTelemetry.Util (appendOnlyBoundedCollectionValues) import Paths_keiro qualified as Package import PreCanonicalRecoverySpec qualified import PreimageSpec qualified@@ -436,7 +442,8 @@ import ReactionExample qualified import ReadModelSpec qualified import ReplayCompatibilitySpec qualified-import Shibuya.Adapter (Adapter (..))+import Shibuya.Adapter (Adapter (..), AdapterTelemetry (traceCorrelation), TraceCorrelation (..), defaultAdapterTelemetry)+import Shibuya.Adapter.Kiroku (defaultKirokuAdapterConfig, kirokuAdapter) import Shibuya.Core.Ack (AckDecision (..), DeadLetterReason (..), HaltReason (..), RetryDelay (..), deadLetterCodeText, deadLetterReasonCode, deadLetterReasonDetail, renderDeadLetterReason) import Shibuya.Core.AckHandle (AckHandle (..)) import Shibuya.Core.Ingested (DeliveryStatus (..), Ingested (..))@@ -454,6 +461,7 @@ main :: IO () main = withMigratedSuite $ \fixture -> hspec $ do+ DurableTracingSpec.spec fixture WorkflowInspectionSpec.spec fixture ConsumerInspectionSpec.spec CatalogSpec.spec@@ -1852,6 +1860,9 @@ fmap (^. #eventId) (Vector.toList recorded) `shouldBe` [supplied] it "retries an optimistic conflict after rehydrating the winning event" $ \storeHandle -> do+ (processor, spansRef) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-test" tracerOptions conflictInserted <- newIORef False let target = stream "counter-command-conflict" :: Stream CounterEventStream conflictStreamName = StreamName "counter-command-conflict"@@ -1868,7 +1879,7 @@ case outcome of Right _ -> pure () Left err -> expectationFailure ("failed to insert conflict event: " <> show err)- options = defaultRunCommandOptions & #beforeAppend .~ insertConflict+ options = defaultRunCommandOptions & #beforeAppend .~ insertConflict & #tracer ?~ tracer result <- Store.runStoreIO storeHandle $ runCommand options counterEventStream target (Add 2)@@ -1883,6 +1894,12 @@ traverse (decodeRecorded counterCodec) (Vector.toList recorded) `shouldBe` Right [CounterAdded 10, CounterAdded 2] + captured <- traverse captureSpan =<< readIORef spansRef+ length captured `shouldBe` 1+ forM_ captured $ \span -> forM_ (drop 1 (Vector.toList recorded)) $ \event ->+ assertCreationContext (csContext span) (event ^. #metadata)+ void (shutdownTracerProvider provider Nothing)+ it "reports true retry attempts and command conflict metrics when the retry budget is exhausted" $ \storeHandle -> do (exporter, metricsRef) <- inMemoryMetricExporter (provider, _env) <-@@ -2404,7 +2421,7 @@ let sp = case spans of (s : _) -> s [] -> error "no command span captured"- csName sp `shouldBe` "counter-command-otel"+ csName sp `shouldBe` "command counter" show (csKind sp) `shouldBe` "Internal" textAttr (csAttributes sp) "keiro.stream.name" `shouldBe` Just "counter-command-otel" textAttr (csAttributes sp) "db.system.name" `shouldBe` Just "postgresql"@@ -2417,6 +2434,44 @@ Ok -> pure () other -> expectationFailure ("expected Unset/Ok, got " <> show other) + describe "Keiro.Command creation context" $ around (withFreshResourceStore fixture) $ do+ forM_ ["plain", "domain", "sql", "domain-sql"] $ \flavor ->+ it ("stamps every event of one batch in the " <> flavor <> " entry point") $ \(_storeHandle, StoreRunner runner) -> do+ (processor, spansRef) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-test" tracerOptions+ target = stream ("creation-" <> Text.pack flavor) :: Stream CounterEventStream+ options = defaultRunCommandOptions & #tracer ?~ tracer & #metadata ?~ object ["audit" Aeson..= True]+ Right (Right ()) <- runner $ case flavor of+ "plain" -> void <$> runCommand options multiCounterEventStream target (Add 9)+ "domain" -> void <$> runDomainCommand options multiCounterDomainHandler target (Add 9)+ "sql" -> void <$> runCommandWithSqlEvents options multiCounterEventStream target (Add 9) (\_ _ -> pure ())+ _ -> void <$> runDomainCommandWithSqlEvents options multiCounterDomainHandler target (Add 9) (\_ _ -> pure ())+ Right recorded <- runner (Store.readStreamForward (streamName target) (StreamVersion 0) 10)+ Vector.length recorded `shouldBe` 2+ spans <- traverse captureSpan =<< readIORef spansRef+ length spans `shouldBe` 1+ forM_ spans $ \captured -> do+ csName captured `shouldBe` "command creation"+ forM_ recorded $ \event -> do+ assertCreationContext (csContext captured) (event ^. #metadata)+ case event ^. #metadata of+ Just (Aeson.Object fields) -> KeyMap.lookup "audit" fields `shouldBe` Just (Aeson.Bool True)+ other -> expectationFailure (show other)+ void (shutdownTracerProvider provider Nothing)++ it "does not capture an ambient span when the command tracer is absent" $ \(_storeHandle, StoreRunner runner) -> do+ (processor, _) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-test" tracerOptions+ target = stream "creation-no-tracer" :: Stream CounterEventStream+ inSpan' tracer "ambient" defaultSpanArguments $ \_ -> do+ Right (Right _) <- runner (runCommand defaultRunCommandOptions counterEventStream target (Add 1))+ pure ()+ Right recorded <- runner (Store.readStreamForward (streamName target) (StreamVersion 0) 10)+ fmap (Telemetry.traceContextFromMetadata . (^. #metadata)) (Vector.toList recorded) `shouldBe` [Nothing]+ void (shutdownTracerProvider provider Nothing)+ describe "Keiro.Command enrichment parity" $ do let addMarker eventData = pure (eventData & #metadata %~ injectMarker) injectMarker = \case@@ -5350,6 +5405,84 @@ pure (Vector.length events) Prelude.sum persistedCounts `shouldBe` fanOut + describe "subscription processing spans" $ around (withFreshResourceStore fixture) $ do+ for_ [("pm", "counter-pm"), ("domain-pm", "domain-pm"), ("router", "failing-router"), ("domain-router", "domain-router"), ("declarative", "declarative-test-router"), ("reaction", "counter-reaction"), ("poison", "counter-pm"), ("no-context", "counter-pm")] $ \(scenario, workerName) ->+ it ("wraps and settles " <> scenario <> " with one processing span per delivery") $ \(_, StoreRunner runner) -> do+ (processor, exported) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ decisions <- newIORef []+ contract <- testSelectionContract EmptyAck FailureRetry 2+ let tracer = makeTracer provider "keiro-test" tracerOptions+ options = defaultRunCommandOptions & #tracer ?~ tracer+ recorded = recordedFromEventId (EventId sampleUuid) (CounterAdded 1)+ policy = if scenario == "router" then ParentAndLink else LinkOnly+ context = if scenario == "no-context" then Nothing else sampleIntegrationEnvelope ^. #traceContext+ adapter :: (IOE :> es) => [message] -> Adapter es message+ adapter messages = withAdapterCreation policy context (inMemoryAdapter decisions messages)+ domainInput = DomainDispatchInput "trace" [CoordinatorAccept 1]+ selection = selectionRouter contract (pure (Right [PMCommand (stream "selection-trace") (Add 1)]))+ reaction = Reaction.AdvanceReaction (Add 1) [] [Reaction.FollowDispatch (PMCommand (stream "reaction-trace") (Add 1))]+ Right () <- runner $ case scenario of+ "pm" -> runProcessManagerWorker options counterProcessManager (adapter [(recorded, CounterAdded 1), (recorded, CounterAdded 1)]) Just+ "domain-pm" -> runDomainProcessManagerWorkerWith defaultWorkerOptions options domainProcessManager (adapter [(recorded, domainInput)]) Just+ "router" -> runRouterWorker options failingRouter (adapter [(recorded, RouteGroup "trace")]) Just+ "domain-router" -> runDomainRouterWorkerWith defaultWorkerOptions options domainRouter (adapter [(recorded, domainInput)]) Just+ "declarative" -> runDeclarativeRouterWorkerWith defaultWorkerOptions options selection (adapter [(recorded, RouteGroup "trace")]) Just+ "reaction" -> Reaction.runReactiveProcessManagerWorker options counterReactionManager (adapter [(recorded, ("trace", reaction))]) Just+ "poison" -> runProcessManagerWorker options counterProcessManager (adapter [recorded]) (const Nothing)+ _ -> runProcessManagerWorker options counterProcessManager (adapter [(recorded, CounterAdded 1)]) Just+ _ <- shutdownTracerProvider provider Nothing+ spans <- traverse captureSpan =<< readIORef exported+ let processing = filter ((== "process " <> Text.pack workerName) . csName) spans+ length processing `shouldBe` (if scenario == "pm" then 2 else 1)+ for_ processing $ \span -> do+ textAttr (csAttributes span) "messaging.system" `shouldBe` Just "keiro"+ textAttr (csAttributes span) "messaging.operation.name" `shouldBe` Just "process"+ textAttr (csAttributes span) "messaging.message.id" `shouldBe` Just "router-test-message"+ case context >>= Telemetry.creationLinkFromTraceContext of+ Nothing -> csLinks span `shouldSatisfy` null+ Just expected -> do+ map (spanId . frozenLinkContext) (csLinks span) `shouldBe` [spanId (linkContext expected)]+ case policy of+ LinkOnly -> do+ csParent span `shouldSatisfy` isNothing+ traceId (csContext span) `shouldNotBe` traceId (linkContext expected)+ ParentAndLink -> do+ parent <- traverse getSpanContext (csParent span)+ fmap spanId parent `shouldBe` Just (spanId (linkContext expected))+ if scenario `elem` ["router", "poison"]+ then csStatus span `shouldBe` Error "ack_halt"+ else textAttr (csAttributes span) "error.type" `shouldBe` Nothing+ length <$> readIORef decisions `shouldReturn` length processing++ it "chain: command, event, Kiroku processing span, command, event is link-followable" $ \(storeHandle, StoreRunner runner) -> do+ (processor, exported) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-test" tracerOptions+ options = defaultRunCommandOptions & #tracer ?~ tracer+ source = stream "traceorigin-1" :: Stream CounterEventStream+ decode event = case decodeRecorded counterCodec event of+ Right input -> Just (event, input)+ Left _ -> Nothing+ Right (Right _) <- runner (runCommand options counterEventStream source (Add 4))+ Right () <- runner $ do+ adapter@Adapter {source = incoming, shutdown = stop} <- kirokuAdapter storeHandle (defaultKirokuAdapterConfig (SubscriptionName "trace-chain") (Category (CategoryName "traceorigin")))+ runProcessManagerWorker options counterProcessManager (adapter {source = Streamly.take 1 incoming}) decode `EffException.finally` stop+ Right origin <- runner $ Store.readStreamForward (StreamName "traceorigin-1") (StreamVersion 0) 10+ Right target <- runner $ Store.readStreamForward (StreamName "counter-target-order-1") (StreamVersion 0) 10+ _ <- shutdownTracerProvider provider Nothing+ spans <- traverse captureSpan =<< readIORef exported+ let [created] = filter ((== Just "traceorigin-1") . (\sp -> textAttr (csAttributes sp) "keiro.stream.name")) spans+ [processing] = filter ((== "process counter-pm") . csName) spans+ [dispatched] = filter ((== Just "counter-target-order-1") . (\sp -> textAttr (csAttributes sp) "keiro.stream.name")) spans+ assertCreationContext (csContext created) (Vector.head origin ^. #metadata)+ map (spanId . frozenLinkContext) (csLinks processing) `shouldBe` [spanId (csContext created)]+ csParent processing `shouldSatisfy` isNothing+ traceId (csContext processing) `shouldNotBe` traceId (csContext created)+ parent <- traverse getSpanContext (csParent dispatched)+ fmap spanId parent `shouldBe` Just (spanId (csContext processing))+ assertCreationContext (csContext dispatched) (Vector.head target ^. #metadata)+ describe "Keiro.ProcessManager" $ around (withFreshResourceStore fixture) $ do it "discovers sorted lookup names without evaluating instance callbacks" $ \_ -> do let unused = error "discovery invoked an instance callback"@@ -7324,6 +7457,37 @@ Store.readStreamForward (StreamName "timer-target") (StreamVersion 0) 10 fmap (^. #eventId) (Vector.toList targetEvents) `shouldBe` [firedEventId] + it "links a timer scheduled inside a command and parents the fired command" $ \storeHandle -> do+ (processor, exported) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ let tracer = makeTracer provider "timer-chain" tracerOptions+ options = defaultRunCommandOptions & #tracer ?~ tracer+ schedule = do+ request <- captureTimerTraceContext (Just tracer) counterTimerRequest+ Store.runStoreIO storeHandle (Store.runTransaction (scheduleTimerTx request)) `shouldReturn` Right ()+ Right (Right _) <-+ Store.runStoreIO storeHandle $+ runCommand (options & #beforeAppend .~ schedule) counterEventStream (stream "schedule-source") (Add 1)+ Right (Just _) <- Store.runStoreIO storeHandle $+ runTimerWorkerWith Nothing (defaultTimerWorkerOptions & #tracer ?~ tracer) dueTimerTime $ \_ -> do+ result <- runCommand (options & #eventIds .~ [EventId sampleUuid2]) counterEventStream (stream "timer-child") (Add 2)+ liftIO (shouldBeRight_ result)+ pure (Just (EventId sampleUuid2))+ spans <- traverse captureSpan =<< readIORef exported+ let [source] = filter ((== "command schedule") . csName) spans+ [fire] = filter ((== "timer fire counter-pm") . csName) spans+ [child] = filter ((== "command timer") . csName) spans+ fmap (spanId . frozenLinkContext) (csLinks fire) `shouldBe` [spanId (csContext source)]+ case csParent fire of+ Nothing -> pure ()+ _ -> expectationFailure "timer fire must be a root"+ case csParent child of+ Just parent -> (spanId <$> getSpanContext parent) `shouldReturn` spanId (csContext fire)+ _ -> expectationFailure "fired command must be a child of the fire span"+ Right events <- Store.runStoreIO storeHandle $ Store.readStreamForward (StreamName "timer-child") (StreamVersion 0) 10+ forM_ events (assertCreationContext (csContext child) . (^. #metadata))+ void (shutdownTracerProvider provider Nothing)+ it "records timer backlog, fire lag, attempts, and stuck count" $ \storeHandle -> do (exporter, metricsRef) <- inMemoryMetricExporter (provider, _env) <-@@ -7644,7 +7808,47 @@ Store.withStore (Store.defaultConnectionSettings conn) $ \storeHandle -> do runDraft sampleDraft storeHandle `shouldReturn` Right (ProducerOutbox.ProducerDuplicateIdentical identity) rows storeHandle `shouldReturn` Right [before]+ for_ [9, 10, 11] $ \position ->+ it ("preserves immutable trace cutover across restart, rollback and GC at " <> show position) $+ withFreshDatabase fixture $ \conn -> do+ let context = TraceContext "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" Nothing+ event = recorded & #globalPosition .~ GlobalPosition position & #metadata ?~ object ["traceparent" Aeson..= (context ^. #traceparent)]+ policy = sampleProducer & #tracePolicy .~ ProducerOutbox.DeriveTraceAfter (GlobalPosition 10)+ firstPolicy = if position <= 10 then sampleProducer else policy+ -- Overridden provenance must not determine policy eligibility.+ draft = sampleDraft & #sourceGlobalPosition ?~ GlobalPosition 999+ expected = if position > 10 then Just context else Nothing+ Store.withStore (Store.defaultConnectionSettings conn) $ \storeHandle -> do+ enqueue firstPolicy event 0 draft storeHandle `shouldReturn` Right (ProducerOutbox.ProducerInserted identity)+ Store.withStore (Store.defaultConnectionSettings conn) $ \storeHandle -> do+ -- Restart and supported rollback both retain the same policy.+ replicateM_ 2 $ enqueue policy event 0 draft storeHandle `shouldReturn` Right (ProducerOutbox.ProducerDuplicateIdentical identity)+ Right [row] <- rows storeHandle+ (row ^. #event) ^. #traceContext `shouldBe` expected+ Prelude.lookup headerTraceParent (integrationHeaders (row ^. #event)) `shouldBe` fmap (^. #traceparent) expected+ let explicit = TraceContext "explicit" Nothing+ enqueue policy event 0 (draft & #traceContext ?~ explicit) storeHandle+ `shouldReturn` Right (ProducerOutbox.ProducerIdentityConflict identity (ProducerOutbox.TraceField NonEmpty.:| []))+ Right _ <- Store.runStoreIO storeHandle $ publishClaimedOutbox (perRow (const (pure PublishSucceeded))) defaultPublishOptions Nothing+ now <- getCurrentTime+ Store.runStoreIO storeHandle (garbageCollectSent 0 (addUTCTime 1 now)) `shouldReturn` Right 1+ enqueue policy event 0 draft storeHandle `shouldReturn` Right (ProducerOutbox.ProducerInserted identity)+ Right [after] <- rows storeHandle+ after ^. #event `shouldBe` row ^. #event around (withFreshStore fixture) $ do+ it "keeps explicit context and absent source metadata under derived policy" $ \storeHandle -> do+ let policy = sampleProducer & #tracePolicy .~ ProducerOutbox.DeriveTraceAfter (GlobalPosition 0)+ explicit = TraceContext "explicit" Nothing+ enqueue policy recorded 0 sampleDraft storeHandle `shouldReturn` Right (ProducerOutbox.ProducerInserted identity)+ Right [plain] <- rows storeHandle+ (plain ^. #event) ^. #traceContext `shouldBe` Nothing+ let event = recorded & #metadata ?~ object ["traceparent" Aeson..= ("source" :: Text)]+ draft = sampleDraft & #traceContext ?~ explicit+ result <- enqueue policy event 1 draft storeHandle+ shouldBeRight_ result+ Right allRows <- rows storeHandle+ map ((^. #traceContext) . (^. #event)) allRows `shouldContain` [Just explicit]+ around (withFreshStore fixture) $ do it "returns inserted, identical retry, and defaults recorded provenance" $ \storeHandle -> do runDraft sampleDraft storeHandle `shouldReturn` Right (ProducerOutbox.ProducerInserted identity) Right [before] <- rows storeHandle@@ -8873,7 +9077,7 @@ textAttr (csAttributes batchSpan) "messaging.destination.name" `shouldBe` Just (okEvent ^. #destination) textAttr (csAttributes batchSpan) "messaging.kafka.message.key"- `shouldBe` (okEvent ^. #key)+ `shouldBe` Nothing intAttr (csAttributes batchSpan) "keiro.outbox.batch.size" `shouldBe` Just 2 textAttr (csAttributes batchSpan) "error.type" `shouldBe` Just "publish_failed" case csStatus batchSpan of@@ -10036,7 +10240,7 @@ InboxKafka.integrationEventFromKafka record `shouldBe` Left (InboxKafka.MissingHeader "keiro-message-id") - it "withConsumerSpan parents the consumer span under an upstream producer span via W3C headers" $ do+ it "withConsumerSpan parents and links the consumer to the W3C creation context" $ do (processor, spansRef) <- inMemoryListExporter provider <- createTracerProvider [processor] emptyTracerProviderOptions let tracer = makeTracer provider "keiro-test" tracerOptions@@ -10081,6 +10285,7 @@ Just parent -> do parentCtx <- getSpanContext parent spanId parentCtx `shouldBe` spanId (csContext producerSp)+ map (spanId . frozenLinkContext) (csLinks consumerSp) `shouldBe` [spanId (csContext producerSp)] -- Consumer span carries the expected attributes. show (csKind consumerSp) `shouldBe` "Consumer" textAttr (csAttributes consumerSp) "messaging.system" `shouldBe` Just "kafka"@@ -10329,6 +10534,66 @@ snd result `shouldBe` "ok" fst result `shouldSatisfy` isNothing + it "does not inspect batch rows without a tracer" $ do+ Telemetry.withProducerBatchSpan Nothing (error "batch inspected") (const (pure (7 :: Int))) `shouldReturn` 7++ for_ [2, 200] $ \count ->+ it ("opens a root mixed send span with bounded creation links for " <> show count <> " rows") $ do+ (processor, spansRef) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-test" tracerOptions+ inSpan' tracer "ambient" defaultSpanArguments $ \outer -> do+ contexts <- forM [1 .. count] $ \_ -> inSpan' tracer "creation" defaultSpanArguments $ \_ ->+ maybe (fail "missing creation context") pure =<< Telemetry.traceContextFromCurrentSpan+ let rows = zipWith (\index context -> telemetryOutboxRow (sampleIntegrationEnvelope & #messageId .~ Text.pack (show index) & #destination .~ (if even index then "even" else "odd") & #traceContext ?~ context)) [1 :: Int ..] contexts+ Telemetry.withProducerBatchSpan (Just tracer) rows $ \_ -> pure ()+ restored <- Telemetry.traceContextFromCurrentSpan+ outerContext <- getSpanContext outer+ restoredLink <- maybe (fail "missing restored parent") pure (restored >>= Telemetry.creationLinkFromTraceContext)+ spanId (linkContext restoredLink) `shouldBe` spanId outerContext+ exported <- readIORef spansRef+ captured <- traverse captureSpan exported+ let [batch] = filter ((== "send") . csName) captured+ csParent batch `shouldSatisfy` isNothing+ intAttr (csAttributes batch) "messaging.batch.message_count" `shouldBe` Just (fromIntegral count)+ textAttr (csAttributes batch) "messaging.destination.name" `shouldBe` Nothing+ textAttr (csAttributes batch) "messaging.message.id" `shouldBe` Nothing+ intAttr (csAttributes batch) "keiro.outbox.links.dropped" `shouldBe` (if count > 128 then Just 72 else Nothing)+ length (csLinks batch) `shouldBe` min 128 count+ for_ (zip [1 :: Int ..] (csLinks batch)) $ \(index, link) -> do+ textAttr (frozenLinkAttributes link) "messaging.message.id" `shouldBe` Just (Text.pack (show index))+ textAttr (frozenLinkAttributes link) "messaging.destination.name" `shouldBe` Just (if even index then "even" else "odd")+ let Just expected = Telemetry.creationLinkFromTraceContext (contexts Prelude.!! (index - 1))+ spanId (frozenLinkContext link) `shouldBe` spanId (linkContext expected)+ _ <- shutdownTracerProvider provider Nothing+ pure ()++ it "does not inspect subscription name, policy or envelope without a tracer" $ do+ Telemetry.withSubscriptionProcessSpan Nothing (error "policy inspected") (error "name inspected") (error "envelope inspected") (const (pure (7 :: Int))) `shouldReturn` 7++ it "keeps unstamped, non-object and caller-parent metadata unchanged" $ do+ let caller = Just (object ["traceparent" Aeson..= ("caller" :: Text), "audit" Aeson..= True])+ ordinary = Just (object ["audit" Aeson..= True])+ Telemetry.stampTraceContext Nothing `shouldReturn` Nothing+ Telemetry.stampTraceContext ordinary `shouldReturn` ordinary+ Telemetry.stampTraceContext (Just (Aeson.String "invalid metadata")) `shouldReturn` Just (Aeson.String "invalid metadata")+ Telemetry.stampTraceContext caller `shouldReturn` caller+ Telemetry.traceContextFromMetadata (Just (object ["traceparent" Aeson..= (7 :: Int)])) `shouldBe` Nothing+ Telemetry.creationLinkFromTraceContext (TraceContext "malformed" Nothing) `shouldSatisfy` isNothing++ it "round-trips the current creation context and preserves an explicit caller parent" $ do+ (processor, _) <- inMemoryListExporter+ provider <- createTracerProvider [processor] emptyTracerProviderOptions+ let tracer = makeTracer provider "keiro-test" tracerOptions+ caller = Just (object ["traceparent" Aeson..= ("caller" :: Text)])+ inSpan' tracer "metadata" defaultSpanArguments $ \span -> do+ expected <- getSpanContext span+ stamped <- Telemetry.stampTraceContext (Just (object ["tracestate" Aeson..= ("orphan=value" :: Text)]))+ assertCreationContext expected stamped+ Telemetry.traceContextFromMetadata stamped `shouldSatisfy` maybe False (isNothing . (^. #tracestate))+ Telemetry.stampTraceContext caller `shouldReturn` caller+ void (shutdownTracerProvider provider Nothing)+ it "re-exports AttributeKeys whose textual payload matches the spec name" $ do attrKeyText Telemetry.messaging_operation_type `shouldBe` "messaging.operation.type" attrKeyText Telemetry.messaging_operation_name `shouldBe` "messaging.operation.name"@@ -14845,7 +15110,8 @@ void $ scheduleTimerOnceTx TimerRequest- { timerId,+ { traceContext = Nothing,+ timerId, processManagerName = "gc-terminal-fire", correlationId = "gtf-1", fireAt = now,@@ -15530,12 +15796,21 @@ -- @spanHot :: IORef SpanHot@ field rather than directly on 'ImmutableSpan', -- so the tests read that reference once after the span ends and assert on -- this flat record.+assertCreationContext :: SpanContext -> Maybe Aeson.Value -> Expectation+assertCreationContext expected metadata =+ case Telemetry.traceContextFromMetadata metadata >>= Telemetry.creationLinkFromTraceContext of+ Nothing -> expectationFailure "expected a valid creation context"+ Just link -> do+ traceId (linkContext link) `shouldBe` traceId expected+ spanId (linkContext link) `shouldBe` spanId expected+ data CapturedSpan = CapturedSpan { csName :: Text, csKind :: SpanKind, csAttributes :: Attributes, csStatus :: SpanStatus, csContext :: SpanContext,+ csLinks :: [Link], csParent :: Maybe Span } @@ -15549,6 +15824,7 @@ csAttributes = hotAttributes hot, csStatus = hotStatus hot, csContext = spanContext sp,+ csLinks = Vector.toList (appendOnlyBoundedCollectionValues (hotLinks hot)), csParent = spanParent sp } @@ -15757,12 +16033,18 @@ E.noParams (D.singleRow (fromIntegral <$> D.column (D.nonNullable D.int8))) +telemetryOutboxRow :: IntegrationEvent -> OutboxRow+telemetryOutboxRow event =+ let now = UTCTime (ModifiedJulianDay 60000) (secondsToDiffTime 0)+ in OutboxRow (OutboxId outboxUuid1) event OutboxPending 0 now Nothing Nothing Nothing Nothing now now+ sampleProducer :: IntegrationProducer () sampleProducer = IntegrationProducer { name = "ordering-integration-producer", source = "ordering", messageIdPrefix = "msg",+ tracePolicy = ProducerOutbox.PreserveDraftTrace, mapEvent = \_recorded () -> Just sampleDraft } @@ -17787,7 +18069,8 @@ counterTimerRequest :: TimerRequest counterTimerRequest = TimerRequest- { timerId = TimerId sampleUuid,+ { traceContext = Nothing,+ timerId = TimerId sampleUuid, processManagerName = "counter-pm", correlationId = "order-1", fireAt = dueTimerTime,@@ -17819,7 +18102,8 @@ plainTimerRequest :: Int -> TimerRequest plainTimerRequest i = TimerRequest- { timerId = TimerId (fromWords64 0xd7a10 (fromIntegral i)),+ { traceContext = Nothing,+ timerId = TimerId (fromWords64 0xd7a10 (fromIntegral i)), processManagerName = "drain-pm", correlationId = "drain-" <> Text.pack (show i), fireAt = dueTimerTime,@@ -18891,6 +19175,7 @@ inMemoryAdapter decisionsRef messages = Adapter { adapterName = "router-test-adapter",+ telemetry = defaultAdapterTelemetry, source = Streamly.fromList (fmap ingest messages), shutdown = pure () }@@ -18902,6 +19187,17 @@ lease = Nothing, deliveryStatus = pure DeliveryCurrent }++withAdapterCreation :: TraceCorrelation -> Maybe TraceContext -> Adapter es msg -> Adapter es msg+withAdapterCreation policy context adapter@Adapter {source = messages} =+ adapter+ { telemetry = defaultAdapterTelemetry {traceCorrelation = policy},+ source = fmap install messages+ }+ where+ install ingested@Ingested {envelope = env} =+ ingested {envelope = env & #traceContext .~ fmap headers context}+ headers context = [(TE.encodeUtf8 headerTraceParent, TE.encodeUtf8 (context ^. #traceparent))] <> maybe [] (\state -> [(TE.encodeUtf8 IntegrationEvent.headerTraceState, TE.encodeUtf8 state)]) (context ^. #tracestate) routerTestEnvelope :: msg -> Envelope msg routerTestEnvelope message =
test/ReactionExample.hs view
@@ -152,7 +152,8 @@ where reminderTimer correlationId = TimerRequest- { timerId = exampleReminderTimerId,+ { traceContext = Nothing,+ timerId = exampleReminderTimerId, processManagerName = "incident-reaction-example-reminder", correlationId, fireAt = addUTCTime 300 injectedNow,@@ -160,7 +161,8 @@ } escalationTimer correlationId = TimerRequest- { timerId = exampleEscalationTimerId,+ { traceContext = Nothing,+ timerId = exampleEscalationTimerId, processManagerName = "incident-reaction-example-escalation", correlationId, fireAt = addUTCTime 900 injectedNow,