packages feed

keiro-0.25.0.0: test/DurableTracingSpec.hs

{-# LANGUAGE BlockArguments #-}
{-# LANGUAGE DataKinds #-}
{-# LANGUAGE TypeOperators #-}

module DurableTracingSpec (spec) where

import Control.Exception (bracket, try)
import Control.Monad (forM_, void)
import Data.Aeson qualified as Aeson
import Data.IORef (IORef, readIORef)
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Time (getCurrentTime)
import Data.UUID qualified as UUID
import Data.Vector qualified as Vector
import Effectful (Eff, IOE, (:>))
import Effectful.Exception qualified as Exception
import Keiro.Codec qualified
import Keiro.Command (defaultRunCommandOptions, runCommand)
import Keiro.Integration.Event (TraceContext)
import Keiro.Prelude (liftIO, (&), (.~), (^.))
import Keiro.ProcessManager (PMCommand (..))
import Keiro.ProcessManager.Reaction qualified as Reaction
import Keiro.Stream (stream)
import Keiro.Telemetry qualified as Telemetry
import Keiro.Test.Postgres (Fixture, StoreRunner (..), withFreshResourceStore)
import Keiro.Timer qualified as Timer
import Keiro.Workflow
import Keiro.Workflow.Awakeable qualified as Awakeable
import Keiro.Workflow.Child qualified as Child
import Keiro.Workflow.Instance qualified as Instance
import Keiro.Workflow.Sleep qualified as Sleep
import Keiro.Workflow.Snapshot (writeWorkflowSnapshot)
import Kiroku.Store qualified as Store
import Kiroku.Store.Subscription.Types (SubscriptionName (..), SubscriptionTarget (..))
import Kiroku.Store.Types (CategoryName (..), ExpectedVersion (..), RecordedEvent, StreamName (..), StreamVersion (..))
import OpenTelemetry.Attributes (Attribute (..), PrimitiveAttribute (..), lookupAttribute)
import OpenTelemetry.Exporter.InMemory.Span (inMemoryListExporter)
import OpenTelemetry.Trace.Core
import OpenTelemetry.Util (appendOnlyBoundedCollectionValues)
import ReactionExample qualified as Example
import Shibuya.Adapter (Adapter (..))
import Shibuya.Adapter.Kiroku (defaultKirokuAdapterConfig, kirokuAdapter)
import Streamly.Data.Stream qualified as Streamly
import Test.Hspec

spec :: Fixture -> Spec
spec fixture = describe "durable tracing" $ around (withFreshResourceStore fixture) do
  it "links exact sleep wake and origin beyond a snapshot, ignoring a later event and preserving replay" $ \(store, StoreRunner runStore) ->
    withRecorder $ \tracer exported -> do
      let name = WorkflowName "traceSleep"
          wid = WorkflowId "one"
          options = defaultWorkflowRunOptions & #tracer .~ Just tracer
          flow = do
            value <- step (StepName "origin") (pure (7 :: Int))
            _ <- awaitValue (StepName "sleep:pause")
            _ <- awaitValue (StepName "hold")
            pure value
      runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended
      runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right (Just "sleep:pause")
      [first] <- readIORef exported
      let firstContext = spanContext first
      now <- getCurrentTime
      let commandOptions = defaultRunCommandOptions & #tracer .~ Just tracer
          timerRequest = Timer.TimerRequest (Sleep.sleepTimerId name wid 0 "sleep:pause") "traceSleep" "one" now (Sleep.sleepTimerPayload 0 "sleep:pause") Nothing
          manager =
            (Example.exampleReactionManager now)
              { Reaction.name = "trace-reaction",
                Reaction.react =
                  const
                    ( Reaction.NoAdvance
                        [ Reaction.FollowSchedule Reaction.Once timerRequest,
                          Reaction.FollowDispatch (PMCommand (stream "tracetarget-one") (Example.SendAlert "one"))
                        ]
                    )
              }
      Right (Right _) <- runStore $ runCommand commandOptions Example.exampleTargetEventStream (stream "tracesource-one") (Example.SendAlert "source")
      Right () <- runStore do
        adapter@Adapter {source = incoming, shutdown = stop} <- kirokuAdapter store (defaultKirokuAdapterConfig (SubscriptionName "durable-chain") (Category (CategoryName "tracesource")))
        Reaction.runReactiveProcessManagerWorker
          commandOptions
          manager
          (adapter {source = Streamly.take 1 incoming})
          (\event -> Just (event, Example.IncidentReported "one" Example.Urgent))
          `Exception.finally` stop
      processing <- named "process trace-reaction" =<< readIORef exported
      sourceSpan <- named "command tracesource" =<< readIORef exported
      assertRoot processing
      assertLinks processing [(Nothing, spanContext sourceSpan)]
      dispatched <- named "command tracetarget" =<< readIORef exported
      case spanParent dispatched of
        Just parent -> getSpanContext parent >>= (\ctx -> sameContext ctx (spanContext processing))
        _ -> expectationFailure "dispatched command lost processing parent"
      Right (Just fired) <-
        runStore $
          Sleep.runWorkflowTimerWorkerWith
            (Timer.defaultTimerWorkerOptions & #tracer .~ Just tracer)
            Nothing
            now
            (const (pure Nothing))
      case fired ^. #traceContext >>= Telemetry.creationLinkFromTraceContext of
        Just link -> sameContext (linkContext link) (spanContext processing)
        Nothing -> expectationFailure "reaction did not capture processing creation context"
      spansAfterFire <- readIORef exported
      fire <- named "timer fire traceSleep" spansAfterFire
      assertRoot fire
      assertLinks fire [(Nothing, spanContext processing)]
      -- This later append must never replace the exact awaited sleep event.
      inSpan' tracer "unrelated" defaultSpanArguments $ \_ -> do
        Right _ <- runStore $ appendJournalEntryTraced (Just tracer) name wid (StepRecorded "unrelated" Aeson.Null now)
        pure ()
      Right events <- runStore $ Store.readStreamForward (workflowGenerationStreamName name wid 0) (StreamVersion 0) 20
      let lastEvent = Vector.last events
          state = Map.fromList [("origin", Aeson.toJSON (7 :: Int)), ("sleep:pause", Aeson.Null), ("unrelated", Aeson.Null)]
      runStore (writeWorkflowSnapshot (lastEvent ^. #originalStreamId) (lastEvent ^. #originalVersion) state) `shouldReturn` Right ()
      -- A failed resume restores ambient state and leaves attribution available
      -- for the next attempt; both origin and wake are before the snapshot.
      inSpan' tracer "ambient" defaultSpanArguments $ \ambient -> do
        let failing = do
              _ <- step (StepName "origin") (pure (7 :: Int))
              Sleep.sleepNamed (StepName "pause") (-1)
              Exception.throwIO (userError "retry")
        failed <- try (runStore $ runWorkflowWith options name wid failing) :: IO (Either IOError (Either Store.StoreError (WorkflowOutcome Int)))
        failed `shouldSatisfy` either (const True) (const False)
        assertTraceContext (spanContextOf ambient) =<< Telemetry.traceContextFromCurrentSpan
      inSpan' tracer "ambient-success" defaultSpanArguments $ \ambient -> do
        runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended
        assertTraceContext (spanContextOf ambient) =<< Telemetry.traceContextFromCurrentSpan
      spans <- readIORef exported
      resumes <- filterMName "workflow traceSleep" spans
      length resumes `shouldBe` 3
      forM_ (take 2 resumes) $ \resume -> do
        assertRoot resume
        assertLinks resume [(Just "origin", firstContext), (Just "wake", spanContext fire)]
      -- Replay observes the same map with tracing disabled; metadata is not folded.
      runStore (loadStepIndex name wid 0) `shouldReturn` Right state
      runStore (runWorkflow name wid flow) `shouldReturn` Right Suspended
      runStore (loadStepIndex name wid 0) `shouldReturn` Right state
      runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right Nothing

  it "deduplicates first-event sleep as wake and leaves legacy timers unlinked" $ \(_, StoreRunner runStore) ->
    withRecorder $ \tracer exported -> do
      let name = WorkflowName "firstSleep"
          wid = WorkflowId "one"
          options = defaultWorkflowRunOptions & #tracer .~ Just tracer
          flow = Sleep.sleepNamed (StepName "pause") (-1)
      runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended
      now <- getCurrentTime
      Right (Just _) <-
        runStore $
          Sleep.runWorkflowTimerWorkerWith
            (Timer.defaultTimerWorkerOptions & #tracer .~ Just tracer)
            Nothing
            now
            (const (pure Nothing))
      runStore (runWorkflowWith options name wid flow) `shouldReturn` Right (Completed ())
      spans <- readIORef exported
      fire <- named "timer fire firstSleep" spans
      resumed <- named "workflow firstSleep" spans
      assertLinks resumed [(Just "wake", spanContext fire)]
      runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right Nothing
      let request = Timer.TimerRequest (Timer.TimerId UUID.nil) "legacy" "one" now Aeson.Null Nothing
      runStore (Store.runTransaction (Timer.scheduleTimerTx request)) `shouldReturn` Right ()
      Right (Just _) <-
        runStore $
          Timer.runTimerWorkerWith
            Nothing
            (Timer.defaultTimerWorkerOptions & #tracer .~ Just tracer)
            now
            (const (pure Nothing))
      legacy <- named "timer fire legacy" =<< readIORef exported
      assertRoot legacy
      assertLinks legacy []

  it "stamps explicitly traced signals, preserves first metadata on retry, and ignores ambient-only enablement" $ \(_, StoreRunner runStore) ->
    withRecorder $ \tracer exported -> do
      let name = WorkflowName "signalTrace"
          wid = WorkflowId "one"
          options = defaultWorkflowRunOptions & #tracer .~ Just tracer
          flow = do
            (_, wait) <- Awakeable.awakeableNamed (StepName "reply")
            asInt wait
      runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended
      Right rows <- runStore $ Store.readStreamForward (workflowGenerationStreamName name wid 0) (StreamVersion 0) 10
      aid <- case decodeJournal (Vector.head rows) of
        Right (StepRecorded _ value _) -> case Aeson.fromJSON value of
          Aeson.Success aid -> pure aid
          _ -> fail "awakeable allocation"
        _ -> fail "allocation journal missing"
      inSpan' tracer "signal" defaultSpanArguments $ \_ ->
        runStore (Awakeable.signalAwakeableTraced (Just tracer) aid (42 :: Int)) `shouldReturn` Right True
      Right before <- runStore $ Store.getEvent (deterministicJournalId name wid 0 (awakeableStepPrefix <> Awakeable.awakeableIdText aid))
      inSpan' tracer "retry" defaultSpanArguments $ \_ ->
        runStore (Awakeable.signalAwakeableTraced (Just tracer) aid (999 :: Int)) `shouldReturn` Right False
      Right after <- runStore $ Store.getEvent (deterministicJournalId name wid 0 (awakeableStepPrefix <> Awakeable.awakeableIdText aid))
      fmap (^. #metadata) after `shouldBe` fmap (^. #metadata) before
      runStore (runWorkflowWith options name wid flow) `shouldReturn` Right (Completed 42)
      signal <- named "signal" =<< readIORef exported
      resumed <- named "workflow signalTrace" =<< readIORef exported
      links <- linksOf resumed
      [spanId (frozenLinkContext link) | link <- links, roleOf link == Just "wake"] `shouldBe` [spanId (spanContext signal)]
      inSpan' tracer "ambient-only" defaultSpanArguments $ \_ ->
        runStore (runWorkflow (WorkflowName "plain") wid (step (StepName "x") (pure (1 :: Int)))) `shouldReturn` Right (Completed 1)
      Right plain <- runStore $ Store.readStreamForward (workflowGenerationStreamName (WorkflowName "plain") wid 0) (StreamVersion 0) 10
      map (Telemetry.traceContextFromMetadata . (^. #metadata)) (Vector.toList plain) `shouldBe` [Nothing, Nothing]

  it "propagates the finished child context to the parent's exact wake" $ \(_, StoreRunner runStore) ->
    withRecorder $ \tracer exported -> do
      let parentName = WorkflowName "parentTrace"
          parentId = WorkflowId "one"
          childName = WorkflowName "childTrace"
          childId = WorkflowId "two"
          options = defaultWorkflowRunOptions & #tracer .~ Just tracer
          parent = Child.spawnChild childName childId traceChild >>= Child.awaitChild
      runStore (runWorkflowWith options parentName parentId parent) `shouldReturn` Right Suspended
      runStore (Child.runChildWorkflow options childName childId traceChild) `shouldReturn` Right (Completed 9)
      runStore (runWorkflowWith options parentName parentId parent) `shouldReturn` Right (Completed 9)
      spans <- readIORef exported
      childSpan <- named "workflow childTrace" spans
      parentSpan <- named "workflow parentTrace" spans
      links <- linksOf parentSpan
      [spanId (frozenLinkContext link) | link <- links, roleOf link == Just "wake"] `shouldBe` [spanId (spanContext childSpan)]

  it "records wake-before-suspend hints, clears on rotation, and omits missing or legacy evidence" $ \(_, StoreRunner runStore) ->
    withRecorder $ \tracer exported -> do
      let name = WorkflowName "hintTrace"
          wid = WorkflowId "one"
          options = defaultWorkflowRunOptions & #tracer .~ Just tracer
      now <- getCurrentTime
      -- A real wake commits before suspension takes the same per-step lock.
      runStore (appendJournalEntry name wid (StepRecorded "gate" Aeson.Null now)) `shouldReturn` Right ()
      runStore (Instance.markInstanceSuspendedAwaitingTraced True name wid 0 "gate") `shouldReturn` Right ()
      Right (Just row) <- runStore (Instance.lookupInstance name wid)
      row ^. #status `shouldBe` Instance.WfRunning
      runStore (Instance.lookupTraceAwaitedStep name wid 0) `shouldReturn` Right (Just "gate")
      runStore (runWorkflowWith options name wid (awaitValue (StepName "next"))) `shouldReturn` Right Suspended
      resumed <- named "workflow hintTrace" =<< readIORef exported
      assertLinks resumed [] -- legacy metadata has no context
      runStore (Instance.markInstanceSuspendedAwaitingTraced True name wid 0 "missing") `shouldReturn` Right ()
      runStore (runWorkflowWith options name wid (continueAsNew (1 :: Int))) `shouldReturn` Right (ContinuedAsNew :: WorkflowOutcome ())
      runStore (Instance.lookupTraceAwaitedStep name wid 1) `shouldReturn` Right Nothing
      -- No first event means a cancellation-only wake cannot invent a link.
      let emptyName = WorkflowName "missingTrace"
      runStore (Instance.markInstanceSuspendedAwaitingTraced True emptyName wid 0 "awk:missing") `shouldReturn` Right ()
      runStore (runWorkflowWith options emptyName wid (awaitValue (StepName "hold"))) `shouldReturn` Right Suspended
      emptySpan <- named "workflow missingTrace" =<< readIORef exported
      assertLinks emptySpan []

  it "omits malformed, foreign-stream, and mismatched-step wake evidence" $ \(_, StoreRunner runStore) ->
    withRecorder $ \tracer exported ->
      forM_ ["malformed", "wrong-stream", "wrong-step"] $ \scenario -> do
        let name = WorkflowName scenario
            wid = WorkflowId "one"
            options = defaultWorkflowRunOptions & #tracer .~ Just tracer
            flow = do
              _ <- step (StepName "origin") (pure (7 :: Int))
              _ <- awaitValue (StepName "gate")
              awaitValue (StepName "hold")
        runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended
        now <- getCurrentTime
        let metadata = Just (Aeson.object ["traceparent" Aeson..= (if scenario == "malformed" then "bad" else "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" :: Text)])
            event = StepRecorded (if scenario == "wrong-step" then "other" else "gate") Aeson.Null now
            target = if scenario == "wrong-stream" then StreamName "foreign-one" else workflowGenerationStreamName name wid 0
        entry <- either (fail . show) pure (Keiro.Codec.encodeForAppendWithMetadata workflowJournalCodec metadata event)
        Right _ <- runStore (Store.appendToStream target AnyVersion [entry & #eventId .~ Just (deterministicJournalId name wid 0 "gate")])
        runStore (runWorkflowWith options name wid flow) `shouldReturn` Right Suspended
        resume <- named ("workflow " <> scenario) =<< readIORef exported
        links <- linksOf resume
        map roleOf links `shouldBe` [Just "origin"]

withRecorder :: (Tracer -> IORef [ImmutableSpan] -> IO a) -> IO a
withRecorder action = do
  (processor, exported) <- inMemoryListExporter
  bracket
    (createTracerProvider [processor] emptyTracerProviderOptions)
    (\provider -> void (shutdownTracerProvider provider Nothing))
    (\provider -> action (makeTracer provider "durable-test" tracerOptions) exported)

named :: Text -> [ImmutableSpan] -> IO ImmutableSpan
named wanted spans = do
  matches <- filterMName wanted spans
  case matches of
    first : _ -> pure first
    _ -> fail ("missing span " <> show wanted)

filterMName :: Text -> [ImmutableSpan] -> IO [ImmutableSpan]
filterMName wanted spans = do
  pairs <- traverse (\sp -> (sp,) . hotName <$> readIORef (spanHot sp)) spans
  pure [sp | (sp, name) <- pairs, name == wanted]

linksOf :: ImmutableSpan -> IO [Link]
linksOf sp = Vector.toList . appendOnlyBoundedCollectionValues . hotLinks <$> readIORef (spanHot sp)

roleOf :: Link -> Maybe Text
roleOf link = case lookupAttribute (frozenLinkAttributes link) "keiro.workflow.link" of
  Just (AttributeValue (TextAttribute role)) -> Just role
  _ -> Nothing

assertRoot :: ImmutableSpan -> Expectation
assertRoot sp = case spanParent sp of
  Nothing -> pure ()
  Just _ -> expectationFailure "durable span inherited a parent"

assertLinks :: ImmutableSpan -> [(Maybe Text, SpanContext)] -> Expectation
assertLinks sp expected = do
  actual <- linksOf sp
  [(roleOf link, traceId (frozenLinkContext link), spanId (frozenLinkContext link)) | link <- actual]
    `shouldMatchList` [(role, traceId ctx, spanId ctx) | (role, ctx) <- expected]

sameContext :: SpanContext -> SpanContext -> Expectation
sameContext a b = (traceId a, spanId a) `shouldBe` (traceId b, spanId b)

assertTraceContext :: IO SpanContext -> Maybe TraceContext -> Expectation
assertTraceContext expected context = do
  ctx <- expected
  case context >>= Telemetry.creationLinkFromTraceContext of
    Nothing -> expectationFailure "ambient context lost"
    Just link -> sameContext ctx (linkContext link)

spanContextOf :: Span -> IO SpanContext
spanContextOf = getSpanContext

decodeJournal :: RecordedEvent -> Either String WorkflowJournalEvent
decodeJournal row = case Keiro.Codec.decodeRecorded workflowJournalCodec row of
  Left err -> Left (show err)
  Right event -> Right event

awaitValue :: (Workflow :> es) => StepName -> Eff es Aeson.Value
awaitValue key = awaitStep key (pure ())

asInt :: Eff es Int -> Eff es Int
asInt = id

traceChild :: (Workflow :> es) => Eff es Int
traceChild = step (StepName "result") (pure 9)