packages feed

keiro-pgmq 0.24.0.0 → 0.25.0.0

raw patch · 4 files changed

+187/−42 lines, 4 filesdep ~keiro-coredep ~keiro-test-supportdep ~shibuya-corePVP ok

version bump matches the API change (PVP)

Dependency ranges changed: keiro-core, keiro-test-support, shibuya-core, shibuya-pgmq-adapter

API changes (from Hackage documentation)

+ Keiro.PGMQ.Job: LinkOnly :: TraceCorrelation
+ Keiro.PGMQ.Job: ParentAndLink :: TraceCorrelation
+ Keiro.PGMQ.Job: [telemetry] :: JobTuning -> !AdapterTelemetry
+ Keiro.PGMQ.Job: data AdapterTelemetry
+ Keiro.PGMQ.Job: data TraceCorrelation
+ Keiro.PGMQ.Job: defaultAdapterTelemetry :: AdapterTelemetry
- Keiro.PGMQ.Job: JobTuning :: !Int32 -> !Int32 -> !JobPolling -> !JobOrdering -> JobTuning
+ Keiro.PGMQ.Job: JobTuning :: !Int32 -> !Int32 -> !JobPolling -> !JobOrdering -> !AdapterTelemetry -> JobTuning

Files

CHANGELOG.md view
@@ -6,6 +6,18 @@  ## [Unreleased] +## 0.25.0.0 — 2026-11-14++### Breaking Changes++- process spans are now named `process <jobName>` (previously `<jobName> process`).+  Both execution paths use Shibuya's shared helper and link stored creation context.+- `JobTuning` requires `telemetry :: AdapterTelemetry`; `mkJobTuning` and+  `defaultJobTuning` select `ParentAndLink`. Record-update to `LinkOnly` for independent+  delayed/retried traces. These policy types and defaults are re-exported.+- Require shibuya-core 0.13.0.1 and shibuya-pgmq-adapter 0.19.0.0. Consumer extraction+  remains W3C-only; common attributes and acknowledgement behavior agree on both paths.+ ## 0.24.0.0 — 2026-11-05  No changes this release.
keiro-pgmq.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.0 name: keiro-pgmq-version: 0.24.0.0+version: 0.25.0.0 synopsis: PostgreSQL job-queue (PGMQ) integration for Keiro description:   A typed background-job queue for Keiro applications on top of PGMQ (the@@ -65,13 +65,13 @@     hasql >=1.10 && <1.11,     hasql-pool >=1.2 && <1.5,     hs-opentelemetry-api >=1.0 && <1.1,-    keiro-core ^>=0.24.0.0,+    keiro-core ^>=0.25.0.0,     pgmq-config >=0.7.0.0 && <0.8,     pgmq-core >=0.7.0.0 && <0.8,     pgmq-effectful >=0.7.0.0 && <0.8,     pgmq-hasql >=0.7.0.0 && <0.8,-    shibuya-core ^>=0.12.0.0,-    shibuya-pgmq-adapter ^>=0.18.0.0,+    shibuya-core ^>=0.13.0.1,+    shibuya-pgmq-adapter ^>=0.19.0.0,     streamly-core >=0.3 && <0.4,     text >=2.1 && <2.2,     time >=1.12 && <1.15,@@ -97,13 +97,13 @@     hs-opentelemetry-propagator-w3c >=1.0 && <1.1,     hs-opentelemetry-sdk >=1.0 && <1.1,     hspec >=2.11,-    keiro-core ^>=0.24.0.0,+    keiro-core ^>=0.25.0.0,     keiro-pgmq,-    keiro-test-support ^>=0.24.0.0,+    keiro-test-support ^>=0.25.0.0,     pgmq-config >=0.7.0.0 && <0.8,     pgmq-core >=0.7.0.0 && <0.8,     pgmq-effectful >=0.7.0.0 && <0.8,     pgmq-migration >=0.7.0.0 && <0.8,-    shibuya-core ^>=0.12.0.0,-    shibuya-pgmq-adapter ^>=0.18.0.0,+    shibuya-core ^>=0.13.0.1,+    shibuya-pgmq-adapter ^>=0.19.0.0,     text >=2.1 && <2.2,
src/Keiro/PGMQ/Job.hs view
@@ -48,11 +48,12 @@ -- Both execution shapes propagate W3C trace context and emit the same common -- per-message span. A message enqueued with 'enqueueTraced' carries @traceparent@ -- (and optional @tracestate@) in PGMQ's JSONB @headers@ column; at consumption--- time that context is extracted and installed as the parent, so the handler's--- span continues the producer's trace even across processes. The span is--- Consumer-kind, named @\<jobName\> process@, and carries+-- time the W3C context is extracted as a creation link. 'ParentAndLink' also+-- continues that trace; 'LinkOnly' opens an independent root when context decodes.+-- Both paths call Shibuya's shared process-span helper. The span is+-- Consumer-kind, named @process \<jobName\>@, and carries -- @messaging.system=shibuya@, @messaging.destination.name=\<jobName\>@,--- @messaging.operation.type=process@, @messaging.message.id@,+-- @messaging.operation.type=process@, @messaging.operation.name=process@, @messaging.message.id@, -- @shibuya.partition@ for FIFO deliveries, and @shibuya.ack.decision@ once the -- message has actually been finalized. @AckOk@ and @AckRetry@ end the span @OK@; -- dead-lettering and halting end it @ERROR@ with the reason.@@ -80,6 +81,9 @@     JobPolling (..),     JobOrdering (..),     JobTuning (..),+    AdapterTelemetry (traceCorrelation),+    TraceCorrelation (..),+    defaultAdapterTelemetry,     JobTuningConfigError (..),     JobConsumptionConfigError (..),     mkJobTuning,@@ -162,6 +166,7 @@ import "pgmq-effectful" Pgmq.Effectful qualified as Pgmq import "pgmq-effectful" Pgmq.Effectful.Effect (readGrouped, readGroupedHead, readGroupedRoundRobin) import "pgmq-hasql" Pgmq.Hasql.Statements.Types (ReadGrouped (..))+import "shibuya-core" Shibuya.Adapter (AdapterTelemetry (traceCorrelation), TraceCorrelation (..), defaultAdapterTelemetry) import "shibuya-core" Shibuya.App   ( AppConfig (..),     AppError,@@ -183,10 +188,9 @@ import "shibuya-core" Shibuya.Core.Ingested qualified as Shibuya import "shibuya-core" Shibuya.Core.Lease (Lease (..)) import "shibuya-core" Shibuya.Core.Types (Attempt (..), Envelope (..))---- Qualified only for 'unMessageId': shibuya's @MessageId@ type name would--- otherwise collide with @Pgmq.Effectful@'s, which the producer signatures use. import "shibuya-core" Shibuya.Core.Types qualified as ShibuyaTypes+import "shibuya-core" Shibuya.Telemetry.Process (withProcessSpan)+ import "shibuya-core" Shibuya.Telemetry.Effect   ( Span,     SpanStatus (..),@@ -196,18 +200,15 @@     recordException,     setStatus,     toAttribute,-    withExtractedContext,-    withSpan',   )-import "shibuya-core" Shibuya.Telemetry.Propagation (extractTraceContext) import "shibuya-core" Shibuya.Telemetry.Semantic   ( attrMessagingDestinationName,     attrMessagingMessageId,     attrMessagingOperation,+    attrMessagingOperationName,     attrMessagingSystem,     attrShibuyaAckDecision,     attrShibuyaPartition,-    consumerSpanArgs,     eventHandlerCompleted,     eventHandlerStarted,     mkEvent,@@ -329,7 +330,8 @@   { visibilityTimeout :: !Int32,     batchSize :: !Int32,     polling :: !JobPolling,-    ordering :: !JobOrdering+    ordering :: !JobOrdering,+    telemetry :: !AdapterTelemetry   }   deriving stock (Eq, Show) @@ -340,7 +342,8 @@     { visibilityTimeout = 30,       batchSize = 1,       polling = PollEvery 1,-      ordering = Unordered+      ordering = Unordered,+      telemetry = defaultAdapterTelemetry     }  data JobTuningConfigError@@ -354,7 +357,7 @@   | visibilityTimeout < 1 = Left (NonPositiveVisibilityTimeout visibilityTimeout)   | batchSize < 1 = Left (NonPositiveBatchSize batchSize)   | not (validPolling polling) = Left NonPositivePollInterval-  | otherwise = Right JobTuning {visibilityTimeout, batchSize, polling, ordering = Unordered}+  | otherwise = Right JobTuning {visibilityTimeout, batchSize, polling, ordering = Unordered, telemetry = defaultAdapterTelemetry}  -- | Set the FIFO read strategy on an existing tuning, e.g. -- @withOrdering FifoThroughput defaultJobTuning@. Every 'JobOrdering' value is@@ -801,6 +804,7 @@   (defaultConfig job.jobQueue.physicalName)     { visibilityTimeout = tuning.visibilityTimeout,       batchSize = tuning.batchSize,+      telemetry = tuning.telemetry,       polling = toPollingConfig tuning.polling,       fifoConfig = toFifoConfig tuning.ordering,       maxRetries = job.jobPolicy.maxRetries,@@ -911,27 +915,29 @@ -- and still get exactly one span. -- -- The attribute set is deliberately the subset the two execution shapes agree on:--- the OTel @messaging.*@ quartet plus @shibuya.partition@ for FIFO deliveries. The+-- the common OTel @messaging.*@ attributes plus @shibuya.partition@ for FIFO deliveries. The -- @shibuya.inflight.*@ gauges are omitted because the direct drain has no shibuya -- inbox and no concurrency meter to report. withOneShotProcessSpan ::   (IOE :> es, Tracing :> es) =>+  TraceCorrelation ->   Job p ->   Envelope Value ->   (Span -> Eff es a) ->   Eff es a-withOneShotProcessSpan job envelope act =-  withExtractedContext (envelope.traceContext >>= extractTraceContext) $-    withSpan' (processSpanName job.jobName) consumerSpanArgs $ \traceSpan -> do-      let ShibuyaTypes.MessageId messageIdText = envelope.messageId-      addAttribute traceSpan attrMessagingSystem ("shibuya" :: Text)-      addAttribute traceSpan attrMessagingDestinationName job.jobName-      addAttribute traceSpan attrMessagingOperation ("process" :: Text)-      addAttribute traceSpan attrMessagingMessageId messageIdText-      case envelope.partition of-        Just partition -> addAttribute traceSpan attrShibuyaPartition partition-        Nothing -> pure ()-      act traceSpan+withOneShotProcessSpan policy job envelope act =+  withProcessSpan policy (processSpanName job.jobName) envelope.traceContext $ \traceSpan -> do+    let !messageIdText = case envelope.messageId of ShibuyaTypes.MessageId text -> text+        !destination = job.jobName+    addAttribute traceSpan attrMessagingSystem ("shibuya" :: Text)+    addAttribute traceSpan attrMessagingDestinationName destination+    addAttribute traceSpan attrMessagingOperation ("process" :: Text)+    addAttribute traceSpan attrMessagingOperationName ("process" :: Text)+    addAttribute traceSpan attrMessagingMessageId messageIdText+    case envelope.partition of+      Just partition -> addAttribute traceSpan attrShibuyaPartition partition+      Nothing -> pure ()+    act traceSpan  -- | Record a finalization that already succeeded on the process span, using the -- same decision text and status mapping as shibuya's continuous runner. Call this@@ -1042,7 +1048,7 @@     step count message = do       let envelope = pgmqMessageToEnvelope message       disposed <--        withOneShotProcessSpan job envelope (processMessage message envelope)+        withOneShotProcessSpan tuning.telemetry.traceCorrelation job envelope (processMessage message envelope)       pure $         if disposed           then count + 1
test/Main.hs view
@@ -18,7 +18,7 @@ module Main (main) where  import Control.Concurrent (threadDelay)-import Control.Exception (bracket, throwIO, try)+import Control.Exception (bracket, evaluate, throwIO, try) import Data.Aeson (FromJSON, ToJSON, Value (..), object, parseJSON, toJSON, (.=)) import Data.Aeson.Key qualified as Key import Data.Aeson.KeyMap qualified as KeyMap@@ -66,6 +66,8 @@ import Shibuya.Adapter.Pgmq (PgmqAdapterEnv) import Shibuya.App (AppHandle, ShutdownConfig (..), SupervisionStrategy (IgnoreFailures), stopAppGracefully) import Shibuya.Telemetry.Effect (Tracing)+import System.Mem (performMajorGC)+import System.Mem.Weak (Weak, deRefWeak, mkWeakPtr) import System.Timeout (timeout) import Test.Hspec @@ -250,6 +252,7 @@     csStatus :: OTel.SpanStatus,     csContext :: OTel.SpanContext,     csParent :: Maybe OTel.Span,+    csLinks :: [OTel.Link],     csEventNames :: [Text]   } @@ -264,6 +267,7 @@         csStatus = hotStatus hot,         csContext = spanContext sp,         csParent = spanParent sp,+        csLinks = toList (appendOnlyBoundedCollectionValues (hotLinks hot)),         csEventNames =           map eventName (toList (appendOnlyBoundedCollectionValues (hotEvents hot)))       }@@ -281,18 +285,18 @@ parentSpanContext :: CapturedSpan -> IO (Maybe OTel.SpanContext) parentSpanContext = traverse OTel.getSpanContext . csParent --- | The one @\<jobName\> process@ span a single one-shot delivery must produce.+-- | The one @process \<jobName\>@ span a single one-shot delivery must produce. -- Anything other than exactly one is a failure that names every captured span, so -- a duplicate wrapper is diagnosed rather than silently accepted by taking the -- head of the list. theProcessSpan :: Text -> [CapturedSpan] -> IO CapturedSpan theProcessSpan jobName spans =-  case spansNamed (jobName <> " process") spans of+  case spansNamed ("process " <> jobName) spans of     [only] -> pure only     other ->       fail         ( "expected exactly one "-            <> show (jobName <> " process")+            <> show ("process " <> jobName)             <> " span, got "             <> show (length other)             <> "; all captured spans: "@@ -323,6 +327,53 @@         then pure True         else threadDelay 100_000 >> loop +-- Keep the actual decoded payload's weak key, then leave every handler/runtime+-- scope before checking collection with the uninspected exporter still alive.+{-# NOINLINE produceCorrelatedJobs #-}+produceCorrelatedJobs :: Text -> OTel.TracerProvider -> OTel.Tracer -> TraceCorrelation -> Job Ping -> IO (OTel.SpanContext, [Weak Text])+produceCorrelatedJobs connStr provider tracer policy job = do+  weakRef <- newIORef []+  producer <- OTel.createSpan tracer Ctxt.empty "enqueue" OTel.defaultSpanArguments+  producerContext <- OTel.getSpanContext producer+  let tuning = defaultJobTuning {polling = PollEvery 0.01, telemetry = defaultAdapterTelemetry {traceCorrelation = policy}}+      handler _ payload = do+        weak <- liftIO $ evaluate payload.message >>= \text -> mkWeakPtr text Nothing+        liftIO $ modifyIORef' weakRef (weak :)+        pure Done+      enqueuePayload = bracket (CtxtLocal.attachContext (Ctxt.insertSpan producer Ctxt.empty)) CtxtLocal.detachContext $ \_ ->+        runDbTraced connStr tracer $ do+          ensureJobQueue job+          _ <- enqueueTraced provider job (MessageHeaders (object [])) (Ping (Text.replicate (128 * 1024) "x") 1)+          pure ()+  enqueuePayload+  runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job handler) `shouldReturn` 1+  enqueuePayload+  processed <- runDbTraced connStr tracer $ do+    result <- runJobWorkers IgnoreFailures 16 [jobProcessorWithContext tuning job handler]+    case result of+      Left err -> liftIO $ fail (show err)+      Right app -> do+        ready <- liftIO $ waitUntil ((== 2) . length <$> readIORef weakRef)+        stopAppQuickly app+        pure ready+  processed `shouldBe` True+  OTel.endSpan producer Nothing+  weak <- readIORef weakRef+  pure (producerContext, weak)++collectWeakTexts :: [Weak Text] -> IO Bool+collectWeakTexts keys = loop (20 :: Int)+  where+    loop remaining = do+      performMajorGC+      values <- traverse deRefWeak keys+      if all (maybe True (const False)) values+        then pure True+        else+          if remaining == 0+            then pure False+            else threadDelay 10_000 >> loop (remaining - 1)+ spec :: SpecWith Text spec = do   it "round-trips a payload through aesonJobCodec" $ \_connStr -> do@@ -1045,6 +1096,80 @@     map csName spans `shouldSatisfy` elem ("publish " <> queue)     map csName spans `shouldSatisfy` elem ("receive " <> queue) +  mapM_+    ( \policy ->+        it ("both paths share process correlation and release payloads under " <> show policy) $ \connStr -> do+          (provider, spansRef) <- setupCapturingProvider+          let tracer = OTel.makeTracer provider "keiro-pgmq-test" OTel.tracerOptions+              job = mkJob "keiro_pgmq_test.correlation"+          (producerContext, weak) <- produceCorrelatedJobs connStr provider tracer policy job+          collectWeakTexts weak `shouldReturn` True+          -- The exporter remains a strong root throughout the collection assertion.+          exported <- readIORef spansRef+          length exported `shouldSatisfy` (> 0)+          spans <- capturedSpans provider spansRef+          let processSpans = spansNamed ("process " <> job.jobName) spans+          length processSpans `shouldBe` 2+          mapM_+            ( \processSpan -> do+                csKind processSpan `shouldBe` OTel.Consumer+                textAttr (csAttributes processSpan) "messaging.operation.name" `shouldBe` Just "process"+                textAttr (csAttributes processSpan) "messaging.operation.type" `shouldBe` Just "process"+                textAttr (csAttributes processSpan) "messaging.destination.name" `shouldBe` Just job.jobName+                textAttr (csAttributes processSpan) "shibuya.ack.decision" `shouldBe` Just "ack_ok"+                csStatus processSpan `shouldBe` OTel.Ok+                case csLinks processSpan of+                  [link] -> do+                    OTel.traceId (OTel.frozenLinkContext link) `shouldBe` OTel.traceId producerContext+                    OTel.spanId (OTel.frozenLinkContext link) `shouldBe` OTel.spanId producerContext+                  _ -> expectationFailure "expected one creation link"+                parent <- parentSpanContext processSpan+                case policy of+                  ParentAndLink -> do+                    OTel.traceId (csContext processSpan) `shouldBe` OTel.traceId producerContext+                    fmap OTel.spanId parent `shouldBe` Just (OTel.spanId producerContext)+                  LinkOnly -> do+                    OTel.traceId (csContext processSpan) `shouldNotBe` OTel.traceId producerContext+                    fmap OTel.spanId parent `shouldBe` Nothing+            )+            processSpans+          case (policy, processSpans) of+            (LinkOnly, [a, b]) -> OTel.traceId (csContext a) `shouldNotBe` OTel.traceId (csContext b)+            _ -> pure ()+    )+    [ParentAndLink, LinkOnly]++  it "delayed and retried LinkOnly deliveries remain distinct roots linked to enqueue" $ \connStr -> do+    (provider, spansRef) <- setupCapturingProvider+    let tracer = OTel.makeTracer provider "keiro-pgmq-test" OTel.tracerOptions+        job = mkJob "keiro_pgmq_test.delayed_correlation"+        tuning = defaultJobTuning {telemetry = defaultAdapterTelemetry {traceCorrelation = LinkOnly}}+    producer <- OTel.createSpan tracer Ctxt.empty "enqueue" OTel.defaultSpanArguments+    producerContext <- OTel.getSpanContext producer+    bracket (CtxtLocal.attachContext (Ctxt.insertSpan producer Ctxt.empty)) CtxtLocal.detachContext $ \_ ->+      runDbTraced connStr tracer $ do+        ensureJobQueue job+        _ <- enqueueTracedWithDelay provider job 1 (MessageHeaders (object [])) (Ping "delayed" 1)+        pure ()+    OTel.endSpan producer Nothing+    runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job (\_ _ -> pure Done)) `shouldReturn` 0+    threadDelay 1_100_000+    runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job (\_ _ -> pure (Retry (RetryDelay 0)))) `shouldReturn` 1+    runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job (\_ _ -> pure Done)) `shouldReturn` 1+    spans <- capturedSpans provider spansRef+    let deliveries = spansNamed ("process " <> job.jobName) spans+    length deliveries `shouldBe` 2+    mapM_+      ( \delivery -> do+          parentSpanContext delivery >>= (\parent -> fmap OTel.spanId parent `shouldBe` Nothing)+          OTel.traceId (csContext delivery) `shouldNotBe` OTel.traceId producerContext+          map (OTel.spanId . OTel.frozenLinkContext) (csLinks delivery) `shouldBe` [OTel.spanId producerContext]+      )+      deliveries+    case deliveries of+      [a, b] -> OTel.traceId (csContext a) `shouldNotBe` OTel.traceId (csContext b)+      _ -> pure ()+   -- EP-111 M2: the central proof — the one-shot process span continues the   -- producer's trace using only what the PGMQ headers carry.   it "one-shot process span continues the enqueued W3C parent" $ \connStr -> do@@ -1294,7 +1419,8 @@             { visibilityTimeout = 30,               batchSize = 0,               polling = PollEvery 1,-              ordering = Unordered+              ordering = Unordered,+              telemetry = defaultAdapterTelemetry             }         mismatchedTuning = defaultJobTuning         unsafeTuning =@@ -1356,7 +1482,8 @@             { visibilityTimeout = 0,               batchSize = 1,               polling = PollEvery 1,-              ordering = FifoHeads+              ordering = FifoHeads,+              telemetry = defaultAdapterTelemetry             }         build tuning =           runDb connStr $