packages feed

keiro-0.25.0.0: retention/Retention/Legs.hs

{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE OverloadedRecordDot #-}

module Retention.Legs
  ( Leg (..),
    allLegs,
  )
where

import Control.Concurrent (threadDelay)
import Control.Concurrent.STM (atomically, putTMVar)
import Control.Exception (evaluate, finally)
import Control.Monad (forM_, unless, void, when)
import Data.Aeson (Value, object, toJSON, (.=))
import Data.ByteString qualified as BS
import Data.IORef (atomicModifyIORef', newIORef, readIORef)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text.IO
import Data.Time (UTCTime (..), secondsToDiffTime)
import Data.Time.Calendar (Day (ModifiedJulianDay))
import Data.UUID qualified as UUID
import Data.Vector qualified as Vector
import Effectful (Eff, IOE, liftIO, (:>))
import Effectful.Exception qualified as Effectful
import Keiro.Command (defaultRunCommandOptions)
import Keiro.Command qualified as Command
import Keiro.EventStream (SnapshotPolicy (Every))
import Keiro.EventStream.Validate (unvalidated)
import Keiro.Integration.Event (IntegrationContentType (..), IntegrationEvent (..), TraceContext (..))
import Keiro.Outbox qualified as Outbox
import Keiro.ProcessManager (defaultWorkerOptions, runProcessManagerWorkerWith)
import Keiro.Projection (AsyncApplyOutcome (..), AsyncProjection (..), applyAsyncProjection)
import Keiro.ReadModel.Schema (registerReadModel)
import Keiro.Router (runRouterWorkerWith)
import Keiro.Snapshot.Schema (SnapshotRow (..), lookupSnapshotRow)
import Keiro.Stream qualified as KeiroStream
import Keiro.Telemetry qualified as Telemetry
import Keiro.Test.Postgres (StoreRunner (..))
import Keiro.Timer qualified as Timer
import Keiro.Workflow qualified as Workflow
import Kiroku.Store qualified as Store
import Kiroku.Store.Subscription qualified as Subscription
import Kiroku.Store.Subscription.Stream (AckItem (..), defaultStreamBufferSize, subscriptionAckStream)
import Kiroku.Store.Subscription.Types (SubscriptionConfig, SubscriptionName (..), SubscriptionResult (..), SubscriptionTarget (..), defaultSubscriptionConfig)
import Kiroku.Store.Transaction qualified as StoreTx
import Kiroku.Store.Types (CategoryName (..), EventData (..), EventId (..), EventType (..), ExpectedVersion (..), RecordedEvent, StreamName (..), StreamVersion (..))
import Numeric (showHex)
import OpenTelemetry.Trace.Core (Tracer)
import Retention.Fixture (LedgerCommand (..), TargetCommand (..), TargetEventStream, ackAdapter, appendSignals, appendSignalsWithMetadata, decodeSignal, ledgerTarget, retentionLedger, retentionManager, retentionRouter, retentionTargetStream, sampleOnAck, seedLedger, seedLedgerRaw)
import Retention.Measure (GateConfig (..), HeapSample (..), Verdict (..), measureLeg, measureLegWithSampler, sampleHeap)
import Retention.Tracing (withRecordingTracer)
import Shibuya.Adapter (Adapter (..))
import Shibuya.Adapter.Kiroku (defaultKirokuAdapterConfig, kirokuAdapter)
import Shibuya.Core.Ack (AckDecision (..))
import Shibuya.Core.AckHandle (AckHandle (..))
import Shibuya.Core.Ingested (Ingested (..))
import Streamly.Data.Stream (Stream)
import Streamly.Data.Stream qualified as Streamly
import System.Environment (lookupEnv)
import System.Timeout (timeout)
import Text.Read (readMaybe)

data Leg = Leg
  { legName :: !Text,
    run :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
  }

allLegs :: [Leg]
allLegs =
  [ Leg "baseline-no-store" baselineNoStore,
    Leg "kiroku-append-probe" kirokuAppendProbe,
    Leg "kiroku-append-only" kirokuAppendOnly,
    Leg "kiroku-append-tx" kirokuAppendTx,
    Leg "kiroku-probe-only" kirokuProbeOnly,
    Leg "kiroku-subscribe-ack" kirokuSubscribeAck,
    Leg "hand-bridge-ack" handBridgeAck,
    Leg "shibuya-adapter-ack" shibuyaAdapterAck,
    Leg "pm-worker" pmWorker,
    Leg "router-worker" routerWorker,
    Leg "projection-apply" projectionApply,
    Leg "command-long-history" commandLongHistory,
    Leg "hydrate-only" hydrateOnly,
    Leg "kiroku-read-tail" kirokuReadTail,
    Leg "command-short-history" commandShortHistory,
    Leg "command-short-history-tracer" (\config handle -> withRecordingTracer (\tracer -> commandShortHistoryWithTracer (Just tracer) config handle)),
    Leg "outbox-publisher" (outboxPublisher Nothing False),
    Leg "outbox-publisher-tracer" (\config handle -> withRecordingTracer (\tracer -> outboxPublisher (Just tracer) False config handle)),
    Leg "outbox-publisher-contexts-tracer" (\config handle -> withRecordingTracer (\tracer -> outboxPublisher (Just tracer) True config handle)),
    Leg "pm-worker-tracer" (\config handle -> withRecordingTracer (\tracer -> pmWorkerWithTracer (Just tracer) config handle)),
    Leg "timer-fire-tracer" (\config handle -> withRecordingTracer (\tracer -> timerFireTracer tracer config handle)),
    Leg "workflow-snapshot-resume-tracer" (\config handle -> withRecordingTracer (\tracer -> workflowSnapshotResumeTracer tracer config handle)),
    Leg "command-long-history-verify-every" commandLongHistoryVerifyEvery
  ]

operationCount :: GateConfig -> Int
operationCount config = config.blocks * config.blockSize

commandLongHistory :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
commandLongHistory = commandLongHistoryWithRate "command-long-history" 0

commandLongHistoryVerifyEvery :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
commandLongHistoryVerifyEvery = commandLongHistoryWithRate "command-long-history-verify-every" 1

commandLongHistoryWithRate :: Text -> Int -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
commandLongHistoryWithRate name rate config (_, runner@(StoreRunner runStore)) = do
  seedLedger runner ledgerTarget 10_000
  let options = defaultRunCommandOptions {Command.seedVerifySampleRate = rate}
  measured <- measureLeg config name \index -> do
    outcome <- runStore (Command.runCommand options retentionLedger ledgerTarget (Deposit 1))
    case outcome of
      Right (Right result)
        | result.eventsAppended == 1
            && result.streamVersion == StreamVersion (fromIntegral (10_000 + index)) ->
            pure ()
      other -> fail (Text.unpack name <> ": command " <> show index <> ": " <> show other)
  snapshot <- runStore do
    maybeId <- Store.lookupStreamId (KeiroStream.streamName ledgerTarget)
    traverse lookupSnapshotRow maybeId
  let expectedSnapshot = StreamVersion (fromIntegral (((10_000 + operationCount config) `div` 100) * 100))
  case snapshot of
    Right (Just (Just row)) | row.streamVersion == expectedSnapshot -> pure ()
    other -> fail (Text.unpack name <> ": final snapshot: " <> show other)
  postReturn name (operationCount config)
  pure measured

hydrateOnly :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
hydrateOnly config (_, runner@(StoreRunner runStore)) = do
  seedLedger runner ledgerTarget 10_000
  let options = defaultRunCommandOptions {Command.seedVerifySampleRate = 0}
  measured <- measureLeg config "hydrate-only" \_ -> do
    outcome <- runStore (Command.hydrate options (unvalidated retentionLedger) ledgerTarget)
    case outcome of
      Right (Right hydrated) | hydrated.streamVersion == StreamVersion 10_000 -> pure ()
      Right (Right hydrated) -> fail ("hydrate-only: unexpected version " <> show hydrated.streamVersion)
      Right (Left failure) -> fail ("hydrate-only: " <> show failure)
      Left failure -> fail ("hydrate-only: " <> show failure)
  postReturn "hydrate-only" (operationCount config)
  pure measured

kirokuReadTail :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
kirokuReadTail config (_, runner@(StoreRunner runStore)) = do
  seedLedgerRaw runner ledgerTarget 10_000
  let name = KeiroStream.streamName ledgerTarget
  measured <- measureLeg config "kiroku-read-tail" \_ -> do
    outcome <- runStore do
      maybeId <- Store.lookupStreamId name
      events <- Store.readStreamForward name (StreamVersion 9_900) 256
      pure (maybeId, events)
    case outcome of
      Right (Just _, events) | Vector.length events == 100 -> pure ()
      other -> fail ("kiroku-read-tail: " <> show (fmap (\(streamId, events) -> (streamId, Vector.length events)) other))
  postReturn "kiroku-read-tail" (operationCount config)
  pure measured

commandShortHistory :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
commandShortHistory = commandShortHistoryWithTracer Nothing

commandShortHistoryWithTracer :: Maybe Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
commandShortHistoryWithTracer tracer config (_, StoreRunner runStore) = do
  let options = defaultRunCommandOptions {Command.seedVerifySampleRate = 0, Command.tracer = tracer}
  measured <- measureLeg config (maybe "command-short-history" (const "command-short-history-tracer") tracer) \index -> do
    let target = KeiroStream.stream ("retentiontarget-" <> Text.pack (show (index `mod` 16))) :: KeiroStream.Stream TargetEventStream
        expected = StreamVersion (fromIntegral ((index - 1) `div` 16 + 1))
    outcome <- runStore (Command.runCommand options retentionTargetStream target (Credit 1))
    case outcome of
      Right (Right result) | result.eventsAppended == 1 && result.streamVersion == expected -> pure ()
      other -> fail ("command-short-history: command " <> show index <> ": " <> show other)
  postReturn (maybe "command-short-history" (const "command-short-history-tracer") tracer) (operationCount config)
  pure measured

baselineNoStore :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
baselineNoStore config _ =
  measureLeg config "baseline-no-store" \index ->
    void (evaluate (sum (replicate 100 (index `mod` 17))))

kirokuAppendProbe :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
kirokuAppendProbe config (_, StoreRunner runStore) = do
  measured <- measureLegWithSampler config "kiroku-append-probe" \sample -> do
    outcome <- runStore $ forM_ [1 .. operationCount config] \index -> do
      let (streamName, eventId, event) = plainEvent index
      _ <- Store.appendToStream streamName AnyVersion [event]
      exists <- Store.eventExistsInStream streamName eventId
      unless exists $ liftIO $ fail ("kiroku-append-probe: missing event " <> show index)
      when (index `mod` config.blockSize == 0) $ liftIO (sample index)
    void (expectRight outcome)
  postReturn "kiroku-append-probe" (operationCount config)
  settleSeconds <- lookupEnv "KEIRO_RETENTION_SETTLE_SECONDS"
  forM_ settleSeconds \rawSeconds -> case readMaybe rawSeconds of
    Just seconds | seconds > 0 -> do
      threadDelay (seconds * 1_000_000)
      settled <- sampleHeap (operationCount config)
      Text.IO.putStrLn ("settled leg=kiroku-append-probe after_seconds=" <> Text.pack (show seconds) <> " live_bytes=" <> Text.pack (show settled.liveBytes) <> " large_objects_bytes=" <> Text.pack (show settled.largeObjectBytes))
    _ -> fail "KEIRO_RETENTION_SETTLE_SECONDS must be a positive integer"
  pure measured

kirokuAppendOnly :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
kirokuAppendOnly config (_, StoreRunner runStore) = do
  requestedRate <- lookupEnv "KEIRO_RETENTION_APPEND_RATE"
  delayMicros <- case requestedRate of
    Nothing -> pure 0
    Just rawRate -> case readMaybe rawRate of
      Just rate | rate > 0 -> pure (1_000_000 `div` rate)
      _ -> fail "KEIRO_RETENTION_APPEND_RATE must be a positive integer"
  measured <- measureLegWithSampler config "kiroku-append-only" \sample -> do
    outcome <- runStore $ forM_ [1 .. operationCount config] \index -> do
      let (streamName, _, event) = plainEvent index
      _ <- Store.appendToStream streamName AnyVersion [event]
      when (delayMicros > 0) $ liftIO (threadDelay delayMicros)
      when (index `mod` config.blockSize == 0) $ liftIO (sample index)
    void (expectRight outcome)
  postReturn "kiroku-append-only" (operationCount config)
  pure measured

kirokuAppendTx :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
kirokuAppendTx config (_, StoreRunner runStore) = do
  measured <- measureLegWithSampler config "kiroku-append-tx" \sample -> do
    outcome <- runStore $ forM_ [1 .. operationCount config] \index -> do
      let (streamName, _, event) = plainEvent index
      result <- StoreTx.runTransactionAppendingResource streamName AnyVersion [event] (const (pure ()))
      case result of
        Left failure -> liftIO $ fail ("kiroku-append-tx: " <> show failure)
        Right () -> pure ()
      when (index `mod` config.blockSize == 0) $ liftIO (sample index)
    void (expectRight outcome)
  postReturn "kiroku-append-tx" (operationCount config)
  pure measured

kirokuProbeOnly :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
kirokuProbeOnly config (_, StoreRunner runStore) = do
  setup <- runStore $ forM_ [1 .. operationCount config] \index -> do
    let (streamName, _, event) = plainEvent index
    void (Store.appendToStream streamName AnyVersion [event])
  void (expectRight setup)
  measured <- measureLegWithSampler config "kiroku-probe-only" \sample -> do
    outcome <- runStore $ forM_ [1 .. operationCount config] \index -> do
      let (streamName, eventId, _) = plainEvent index
      exists <- Store.eventExistsInStream streamName eventId
      unless exists $ liftIO $ fail ("kiroku-probe-only: missing event " <> show index)
      when (index `mod` config.blockSize == 0) $ liftIO (sample index)
    void (expectRight outcome)
  postReturn "kiroku-probe-only" (operationCount config)
  pure measured

plainEvent :: Int -> (StreamName, EventId, EventData)
plainEvent index =
  let streamName = StreamName ("retentionplain-" <> Text.pack (show (index `mod` 16)))
      eventId = EventId (UUID.fromWords 0 0 0 (fromIntegral index))
      event =
        EventData
          { eventId = Just eventId,
            eventType = EventType "RetentionPlain",
            payload = toJSON index,
            metadata = Nothing,
            causationId = Nothing,
            correlationId = Nothing
          }
   in (streamName, eventId, event)

sourceTarget :: SubscriptionTarget
sourceTarget = Category (CategoryName "retentionsource")

sourceConfig :: Text -> SubscriptionConfig
sourceConfig name = defaultSubscriptionConfig (SubscriptionName name) sourceTarget (\_ -> pure Continue)

kirokuSubscribeAck :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
kirokuSubscribeAck config (store, runner) = do
  appendSignals runner (operationCount config)
  measured <- measureLegWithSampler config "kiroku-subscribe-ack" \sample -> do
    (source, cancel) <- subscriptionAckStream store (sourceConfig "retention-kiroku-ack") defaultStreamBufferSize
    consumeAck (operationCount config) config.blockSize sample source `finally` cancel
  postReturn "kiroku-subscribe-ack" (operationCount config)
  pure measured

consumeAck :: Int -> Int -> (Int -> IO ()) -> Stream IO AckItem -> IO ()
consumeAck expected blockSize sample = go 1
  where
    go index source = do
      next <- timeout 30_000_000 (Streamly.uncons source)
      case next of
        Nothing -> fail ("kiroku-subscribe-ack: timed out after " <> show (index - 1) <> " events")
        Just Nothing -> fail ("kiroku-subscribe-ack: ended after " <> show (index - 1) <> " events")
        Just (Just (item, rest)) -> do
          atomically (putTMVar item.ackReply Continue)
          when (index `mod` blockSize == 0) (sample index)
          when (index < expected) (go (index + 1) rest)

handBridgeAck :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
handBridgeAck config (store, runner@(StoreRunner runStore)) = do
  appendSignals runner (operationCount config)
  measured <- measureLegWithSampler config "hand-bridge-ack" \sample -> do
    outcome <- runStore do
      (adapter, _) <- liftIO $ ackAdapter store (sourceConfig "retention-hand-bridge-ack") 256
      consumeAdapter (operationCount config) config.blockSize sample adapter `Effectful.finally` adapter.shutdown
    void (expectRight outcome)
  postReturn "hand-bridge-ack" (operationCount config)
  pure measured

shibuyaAdapterAck :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
shibuyaAdapterAck config (store, runner@(StoreRunner runStore)) = do
  appendSignals runner (operationCount config)
  measured <- measureLegWithSampler config "shibuya-adapter-ack" \sample -> do
    outcome <- runStore do
      adapter <- kirokuAdapter store (defaultKirokuAdapterConfig (SubscriptionName "retention-shibuya-ack") sourceTarget)
      consumeAdapter (operationCount config) config.blockSize sample adapter `Effectful.finally` adapter.shutdown
    void (expectRight outcome)
  postReturn "shibuya-adapter-ack" (operationCount config)
  pure measured

consumeAdapter :: (IOE :> es) => Int -> Int -> (Int -> IO ()) -> Adapter es RecordedEvent -> Eff es ()
consumeAdapter expected blockSize sample adapter = go 1 adapter.source
  where
    go index source = do
      next <- Streamly.uncons source
      case next of
        Nothing -> liftIO $ fail ("shibuya-adapter-ack: ended after " <> show (index - 1) <> " events")
        Just (item, rest) -> do
          item.ack.finalize AckOk
          when (index `mod` blockSize == 0) $ liftIO (sample index)
          when (index < expected) (go (index + 1) rest)

pmWorker :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
pmWorker = pmWorkerWithTracer Nothing

pmWorkerWithTracer :: Maybe Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
pmWorkerWithTracer tracer config (store, runner@(StoreRunner runStore)) = do
  let creationMetadata index = Just (object ["traceparent" .= ("00-0af7651916cd43dd8448eb211c80319c-" <> Text.justifyRight 16 '0' (Text.pack (showHex index "")) <> "-01")])
  case tracer of
    Nothing -> appendSignals runner (operationCount config)
    Just _ -> appendSignalsWithMetadata creationMetadata runner (operationCount config)
  measured <- measureLegWithSampler config (maybe "pm-worker" (const "pm-worker-tracer") tracer) \sample -> do
    outcome <- runStore do
      adapter <- kirokuAdapter store (defaultKirokuAdapterConfig (SubscriptionName "retention-pm") sourceTarget)
      instrumented <- sampleOnAck (operationCount config) config.blockSize sample adapter
      runProcessManagerWorkerWith defaultWorkerOptions (defaultRunCommandOptions {Command.tracer = tracer}) retentionManager instrumented decodeSignal
        `Effectful.finally` adapter.shutdown
    void (expectRight outcome)
  assertTargets runner (operationCount config) [0 .. 15] (\accountNumber -> length [index | index <- [1 .. operationCount config], index `mod` 16 == accountNumber])
  postReturn (maybe "pm-worker" (const "pm-worker-tracer") tracer) (operationCount config)
  pure measured

routerWorker :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
routerWorker config (store, runner@(StoreRunner runStore)) = do
  let snapshotTargets = operationCount config > 1500
  Text.IO.putStrLn $ if snapshotTargets then "router-worker target snapshots: Every 100" else "router-worker target snapshots: Never"
  appendSignals runner (operationCount config)
  measured <- measureLegWithSampler config "router-worker" \sample -> do
    outcome <- runStore do
      adapter <- kirokuAdapter store (defaultKirokuAdapterConfig (SubscriptionName "retention-router") sourceTarget)
      instrumented <- sampleOnAck (operationCount config) config.blockSize sample adapter
      runRouterWorkerWith defaultWorkerOptions defaultRunCommandOptions (retentionRouter snapshotTargets) instrumented decodeSignal
        `Effectful.finally` adapter.shutdown
    void (expectRight outcome)
  assertTargets runner (operationCount config) [0 .. 3] (const (operationCount config))
  postReturn "router-worker" (operationCount config)
  pure measured

assertTargets :: StoreRunner -> Int -> [Int] -> (Int -> Int) -> IO ()
assertTargets (StoreRunner runStore) total accounts expectedFor =
  forM_ accounts \accountNumber -> do
    let name = StreamName ("retentiontarget-" <> Text.pack (show accountNumber))
    events <- expectRight =<< runStore (Store.readStreamForward name (StreamVersion 0) (fromIntegral (total + 1)))
    unless (Vector.length events == expectedFor accountNumber) $
      fail ("target " <> show name <> ": expected " <> show (expectedFor accountNumber) <> " events, got " <> show (Vector.length events))

projectionApply :: GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
projectionApply config (store, runner@(StoreRunner runStore)) = do
  void (expectRight =<< runStore (registerReadModel "retention-activity" 1 "retention-activity-v1"))
  appendSignals runner (operationCount config)
  measured <- measureLegWithSampler config "projection-apply" \sample -> do
    countRef <- newIORef 0
    let projection =
          AsyncProjection
            { name = "retention-activity",
              readModelName = "retention-activity",
              subscriptionName = "retention-projection",
              applyRecorded = \_ -> pure (),
              idempotencyKey = (.eventId)
            }
        handler event = do
          outcome <- expectRight =<< runStore (Store.runTransaction (applyAsyncProjection projection event))
          unless (outcome == AsyncApplied) $ fail ("projection-apply: " <> show outcome)
          count <- atomicModifyIORef' countRef (\previous -> let next = previous + 1 in (next, next))
          when (count `mod` config.blockSize == 0) (sample count)
          pure if count == operationCount config then Stop else Continue
        config' = defaultSubscriptionConfig (SubscriptionName "retention-projection") sourceTarget handler
    completed <- timeout 120_000_000 (Subscription.withSubscription store config' Subscription.wait)
    case completed of
      Nothing -> fail "projection-apply: subscription timed out"
      Just (Left failure) -> fail ("projection-apply: " <> show failure)
      Just (Right ()) -> pure ()
    count <- readIORef countRef
    unless (count == operationCount config) $ fail ("projection-apply: processed " <> show count <> " events")
  postReturn "projection-apply" (operationCount config)
  pure measured

postReturn :: Text -> Int -> IO ()
postReturn name count = do
  sample <- sampleHeap count
  Text.IO.putStrLn ("post-return leg=" <> name <> " live_bytes=" <> Text.pack (show sample.liveBytes) <> " large_objects_bytes=" <> Text.pack (show sample.largeObjectBytes))

expectRight :: (Show errorType) => Either errorType a -> IO a
expectRight = either (fail . show) pure

-- Processor discards finished spans; ownership is checked separately.
outboxPublisher :: Maybe Tracer -> Bool -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
outboxPublisher tracer withContext config (_, StoreRunner runStore) = do
  let name = if withContext then "outbox-publisher-contexts-tracer" else maybe "outbox-publisher" (const "outbox-publisher-tracer") tracer
      options = Outbox.defaultPublishOptions {Outbox.tracer = tracer, Outbox.batchSize = 1}
      message index =
        IntegrationEvent
          { messageId = Text.pack (show index),
            source = "retention",
            destination = "retention.events",
            key = Nothing,
            eventType = "Retention",
            schemaVersion = 1,
            contentType = ApplicationJson,
            schemaReference = Nothing,
            sourceEventId = Nothing,
            sourceGlobalPosition = Nothing,
            payloadBytes = BS.replicate 1024 65,
            occurredAt = UTCTime (ModifiedJulianDay 61000) (secondsToDiffTime 0),
            causationId = Nothing,
            correlationId = Nothing,
            traceContext = if withContext then Just (TraceContext ("00-0af7651916cd43dd8448eb211c80319c-" <> Text.justifyRight 16 '0' (Text.pack (showHex index "")) <> "-01") Nothing) else Nothing,
            attributes = Nothing
          }
  -- A fixed backlog: enqueue outside the measured draining loop.
  forM_ [1 .. operationCount config] \index ->
    void (expectRight =<< runStore (Store.runTransaction (Outbox.enqueueIntegrationEventTx (Outbox.OutboxId (UUID.fromWords 0 0 0 (fromIntegral index))) (message index))))
  measured <- measureLegWithSampler config name \sample -> do
    count <- newIORef 0
    let drain = do
          summary <- expectRight =<< runStore (Outbox.publishClaimedOutbox (\rows -> pure [(row.outboxId, Outbox.PublishSucceeded) | row <- rows]) options Nothing)
          when (summary.published == 0) (fail "outbox retention made no progress")
          previous <- readIORef count
          let next = previous + summary.published
          -- Single-message passes keep sampling at exact operation counts.
          atomicModifyIORef' count (const (next, ()))
          when (next `mod` config.blockSize == 0) (sample next)
          when (next < operationCount config) drain
    drain
  backlog <- expectRight =<< runStore Outbox.countOutboxBacklog
  unless (backlog == 0) (fail "outbox retention did not drain backlog")
  postReturn name (operationCount config)
  pure measured

-- A fixed backlog, claimed one row per operation. The SDK records and discards
-- completed spans, as in the publisher and worker legs.
timerFireTracer :: Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
timerFireTracer tracer config (_, StoreRunner runStore) = do
  let now = UTCTime (ModifiedJulianDay 61000) 0
      options = Timer.defaultTimerWorkerOptions {Timer.tracer = Just tracer, Timer.requeueStuckAfter = Nothing}
      request index = Timer.TimerRequest (Timer.TimerId (UUID.fromWords64 0 (fromIntegral index))) "retention" "one" now (object ["body" .= Text.replicate 1024 "x"]) Nothing
  Telemetry.withWorkflowSpan (Just tracer) (Workflow.WorkflowName "timerRetention") (Workflow.WorkflowId "one") Nothing $ \_ -> do
    requests <- traverse (Timer.captureTimerTraceContext (Just tracer) . request) [1 .. operationCount config]
    expectRight =<< runStore (StoreTx.runTransaction (mapM_ Timer.scheduleTimerTx requests))
  measured <- measureLeg config "timer-fire-tracer" $ \_ -> do
    row <- expectRight =<< runStore (Timer.runTimerWorkerWith Nothing options now (const (pure (Just (EventId UUID.nil)))))
    unless (maybe False (const True) row) (fail "timer retention exhausted its backlog")
  backlog <- expectRight =<< runStore (Timer.countDueTimers now)
  unless (backlog == 0) (fail "timer retention left due rows")
  postReturn "timer-fire-tracer" (operationCount config)
  pure measured

workflowSnapshotResumeTracer :: Tracer -> GateConfig -> (Store.KirokuStore, StoreRunner) -> IO (Verdict, [HeapSample])
workflowSnapshotResumeTracer tracer config (_, StoreRunner runStore) = do
  let name = Workflow.WorkflowName "retention"
      wid = Workflow.WorkflowId "one"
      options = Workflow.defaultWorkflowRunOptions {Workflow.tracer = Just tracer, Workflow.snapshotPolicy = Every 1000}
  seeded <- expectRight =<< runStore (Workflow.runWorkflowWith options name wid seedWorkflow)
  unless (seeded == Workflow.Suspended) (fail "workflow retention did not seed")
  measured <- measureLeg config "workflow-snapshot-resume-tracer" $ \_ -> do
    result <- expectRight =<< runStore (Workflow.runWorkflowWith options name wid holdWorkflow)
    unless (result == Workflow.Suspended) (fail "workflow retention did not resume")
  postReturn "workflow-snapshot-resume-tracer" (operationCount config)
  pure measured

seedWorkflow :: (Workflow.Workflow :> es) => Eff es Value
seedWorkflow = do
  forM_ [1 .. 1000 :: Int] $ \index -> void (Workflow.step (Workflow.StepName ("prefix-" <> Text.pack (show index))) (pure index))
  holdWorkflow

holdWorkflow :: (Workflow.Workflow :> es) => Eff es Value
holdWorkflow = Workflow.awaitStep (Workflow.StepName "hold") (pure ())