keiro-pgmq 0.24.0.0 → 0.25.0.0
raw patch · 4 files changed
+187/−42 lines, 4 filesdep ~keiro-coredep ~keiro-test-supportdep ~shibuya-corePVP ok
version bump matches the API change (PVP)
Dependency ranges changed: keiro-core, keiro-test-support, shibuya-core, shibuya-pgmq-adapter
API changes (from Hackage documentation)
+ Keiro.PGMQ.Job: LinkOnly :: TraceCorrelation
+ Keiro.PGMQ.Job: ParentAndLink :: TraceCorrelation
+ Keiro.PGMQ.Job: [telemetry] :: JobTuning -> !AdapterTelemetry
+ Keiro.PGMQ.Job: data AdapterTelemetry
+ Keiro.PGMQ.Job: data TraceCorrelation
+ Keiro.PGMQ.Job: defaultAdapterTelemetry :: AdapterTelemetry
- Keiro.PGMQ.Job: JobTuning :: !Int32 -> !Int32 -> !JobPolling -> !JobOrdering -> JobTuning
+ Keiro.PGMQ.Job: JobTuning :: !Int32 -> !Int32 -> !JobPolling -> !JobOrdering -> !AdapterTelemetry -> JobTuning
Files
- CHANGELOG.md +12/−0
- keiro-pgmq.cabal +8/−8
- src/Keiro/PGMQ/Job.hs +34/−28
- test/Main.hs +133/−6
CHANGELOG.md view
@@ -6,6 +6,18 @@ ## [Unreleased] +## 0.25.0.0 — 2026-11-14++### Breaking Changes++- process spans are now named `process <jobName>` (previously `<jobName> process`).+ Both execution paths use Shibuya's shared helper and link stored creation context.+- `JobTuning` requires `telemetry :: AdapterTelemetry`; `mkJobTuning` and+ `defaultJobTuning` select `ParentAndLink`. Record-update to `LinkOnly` for independent+ delayed/retried traces. These policy types and defaults are re-exported.+- Require shibuya-core 0.13.0.1 and shibuya-pgmq-adapter 0.19.0.0. Consumer extraction+ remains W3C-only; common attributes and acknowledgement behavior agree on both paths.+ ## 0.24.0.0 — 2026-11-05 No changes this release.
keiro-pgmq.cabal view
@@ -1,6 +1,6 @@ cabal-version: 3.0 name: keiro-pgmq-version: 0.24.0.0+version: 0.25.0.0 synopsis: PostgreSQL job-queue (PGMQ) integration for Keiro description: A typed background-job queue for Keiro applications on top of PGMQ (the@@ -65,13 +65,13 @@ hasql >=1.10 && <1.11, hasql-pool >=1.2 && <1.5, hs-opentelemetry-api >=1.0 && <1.1,- keiro-core ^>=0.24.0.0,+ keiro-core ^>=0.25.0.0, pgmq-config >=0.7.0.0 && <0.8, pgmq-core >=0.7.0.0 && <0.8, pgmq-effectful >=0.7.0.0 && <0.8, pgmq-hasql >=0.7.0.0 && <0.8,- shibuya-core ^>=0.12.0.0,- shibuya-pgmq-adapter ^>=0.18.0.0,+ shibuya-core ^>=0.13.0.1,+ shibuya-pgmq-adapter ^>=0.19.0.0, streamly-core >=0.3 && <0.4, text >=2.1 && <2.2, time >=1.12 && <1.15,@@ -97,13 +97,13 @@ hs-opentelemetry-propagator-w3c >=1.0 && <1.1, hs-opentelemetry-sdk >=1.0 && <1.1, hspec >=2.11,- keiro-core ^>=0.24.0.0,+ keiro-core ^>=0.25.0.0, keiro-pgmq,- keiro-test-support ^>=0.24.0.0,+ keiro-test-support ^>=0.25.0.0, pgmq-config >=0.7.0.0 && <0.8, pgmq-core >=0.7.0.0 && <0.8, pgmq-effectful >=0.7.0.0 && <0.8, pgmq-migration >=0.7.0.0 && <0.8,- shibuya-core ^>=0.12.0.0,- shibuya-pgmq-adapter ^>=0.18.0.0,+ shibuya-core ^>=0.13.0.1,+ shibuya-pgmq-adapter ^>=0.19.0.0, text >=2.1 && <2.2,
src/Keiro/PGMQ/Job.hs view
@@ -48,11 +48,12 @@ -- Both execution shapes propagate W3C trace context and emit the same common -- per-message span. A message enqueued with 'enqueueTraced' carries @traceparent@ -- (and optional @tracestate@) in PGMQ's JSONB @headers@ column; at consumption--- time that context is extracted and installed as the parent, so the handler's--- span continues the producer's trace even across processes. The span is--- Consumer-kind, named @\<jobName\> process@, and carries+-- time the W3C context is extracted as a creation link. 'ParentAndLink' also+-- continues that trace; 'LinkOnly' opens an independent root when context decodes.+-- Both paths call Shibuya's shared process-span helper. The span is+-- Consumer-kind, named @process \<jobName\>@, and carries -- @messaging.system=shibuya@, @messaging.destination.name=\<jobName\>@,--- @messaging.operation.type=process@, @messaging.message.id@,+-- @messaging.operation.type=process@, @messaging.operation.name=process@, @messaging.message.id@, -- @shibuya.partition@ for FIFO deliveries, and @shibuya.ack.decision@ once the -- message has actually been finalized. @AckOk@ and @AckRetry@ end the span @OK@; -- dead-lettering and halting end it @ERROR@ with the reason.@@ -80,6 +81,9 @@ JobPolling (..), JobOrdering (..), JobTuning (..),+ AdapterTelemetry (traceCorrelation),+ TraceCorrelation (..),+ defaultAdapterTelemetry, JobTuningConfigError (..), JobConsumptionConfigError (..), mkJobTuning,@@ -162,6 +166,7 @@ import "pgmq-effectful" Pgmq.Effectful qualified as Pgmq import "pgmq-effectful" Pgmq.Effectful.Effect (readGrouped, readGroupedHead, readGroupedRoundRobin) import "pgmq-hasql" Pgmq.Hasql.Statements.Types (ReadGrouped (..))+import "shibuya-core" Shibuya.Adapter (AdapterTelemetry (traceCorrelation), TraceCorrelation (..), defaultAdapterTelemetry) import "shibuya-core" Shibuya.App ( AppConfig (..), AppError,@@ -183,10 +188,9 @@ import "shibuya-core" Shibuya.Core.Ingested qualified as Shibuya import "shibuya-core" Shibuya.Core.Lease (Lease (..)) import "shibuya-core" Shibuya.Core.Types (Attempt (..), Envelope (..))---- Qualified only for 'unMessageId': shibuya's @MessageId@ type name would--- otherwise collide with @Pgmq.Effectful@'s, which the producer signatures use. import "shibuya-core" Shibuya.Core.Types qualified as ShibuyaTypes+import "shibuya-core" Shibuya.Telemetry.Process (withProcessSpan)+ import "shibuya-core" Shibuya.Telemetry.Effect ( Span, SpanStatus (..),@@ -196,18 +200,15 @@ recordException, setStatus, toAttribute,- withExtractedContext,- withSpan', )-import "shibuya-core" Shibuya.Telemetry.Propagation (extractTraceContext) import "shibuya-core" Shibuya.Telemetry.Semantic ( attrMessagingDestinationName, attrMessagingMessageId, attrMessagingOperation,+ attrMessagingOperationName, attrMessagingSystem, attrShibuyaAckDecision, attrShibuyaPartition,- consumerSpanArgs, eventHandlerCompleted, eventHandlerStarted, mkEvent,@@ -329,7 +330,8 @@ { visibilityTimeout :: !Int32, batchSize :: !Int32, polling :: !JobPolling,- ordering :: !JobOrdering+ ordering :: !JobOrdering,+ telemetry :: !AdapterTelemetry } deriving stock (Eq, Show) @@ -340,7 +342,8 @@ { visibilityTimeout = 30, batchSize = 1, polling = PollEvery 1,- ordering = Unordered+ ordering = Unordered,+ telemetry = defaultAdapterTelemetry } data JobTuningConfigError@@ -354,7 +357,7 @@ | visibilityTimeout < 1 = Left (NonPositiveVisibilityTimeout visibilityTimeout) | batchSize < 1 = Left (NonPositiveBatchSize batchSize) | not (validPolling polling) = Left NonPositivePollInterval- | otherwise = Right JobTuning {visibilityTimeout, batchSize, polling, ordering = Unordered}+ | otherwise = Right JobTuning {visibilityTimeout, batchSize, polling, ordering = Unordered, telemetry = defaultAdapterTelemetry} -- | Set the FIFO read strategy on an existing tuning, e.g. -- @withOrdering FifoThroughput defaultJobTuning@. Every 'JobOrdering' value is@@ -801,6 +804,7 @@ (defaultConfig job.jobQueue.physicalName) { visibilityTimeout = tuning.visibilityTimeout, batchSize = tuning.batchSize,+ telemetry = tuning.telemetry, polling = toPollingConfig tuning.polling, fifoConfig = toFifoConfig tuning.ordering, maxRetries = job.jobPolicy.maxRetries,@@ -911,27 +915,29 @@ -- and still get exactly one span. -- -- The attribute set is deliberately the subset the two execution shapes agree on:--- the OTel @messaging.*@ quartet plus @shibuya.partition@ for FIFO deliveries. The+-- the common OTel @messaging.*@ attributes plus @shibuya.partition@ for FIFO deliveries. The -- @shibuya.inflight.*@ gauges are omitted because the direct drain has no shibuya -- inbox and no concurrency meter to report. withOneShotProcessSpan :: (IOE :> es, Tracing :> es) =>+ TraceCorrelation -> Job p -> Envelope Value -> (Span -> Eff es a) -> Eff es a-withOneShotProcessSpan job envelope act =- withExtractedContext (envelope.traceContext >>= extractTraceContext) $- withSpan' (processSpanName job.jobName) consumerSpanArgs $ \traceSpan -> do- let ShibuyaTypes.MessageId messageIdText = envelope.messageId- addAttribute traceSpan attrMessagingSystem ("shibuya" :: Text)- addAttribute traceSpan attrMessagingDestinationName job.jobName- addAttribute traceSpan attrMessagingOperation ("process" :: Text)- addAttribute traceSpan attrMessagingMessageId messageIdText- case envelope.partition of- Just partition -> addAttribute traceSpan attrShibuyaPartition partition- Nothing -> pure ()- act traceSpan+withOneShotProcessSpan policy job envelope act =+ withProcessSpan policy (processSpanName job.jobName) envelope.traceContext $ \traceSpan -> do+ let !messageIdText = case envelope.messageId of ShibuyaTypes.MessageId text -> text+ !destination = job.jobName+ addAttribute traceSpan attrMessagingSystem ("shibuya" :: Text)+ addAttribute traceSpan attrMessagingDestinationName destination+ addAttribute traceSpan attrMessagingOperation ("process" :: Text)+ addAttribute traceSpan attrMessagingOperationName ("process" :: Text)+ addAttribute traceSpan attrMessagingMessageId messageIdText+ case envelope.partition of+ Just partition -> addAttribute traceSpan attrShibuyaPartition partition+ Nothing -> pure ()+ act traceSpan -- | Record a finalization that already succeeded on the process span, using the -- same decision text and status mapping as shibuya's continuous runner. Call this@@ -1042,7 +1048,7 @@ step count message = do let envelope = pgmqMessageToEnvelope message disposed <-- withOneShotProcessSpan job envelope (processMessage message envelope)+ withOneShotProcessSpan tuning.telemetry.traceCorrelation job envelope (processMessage message envelope) pure $ if disposed then count + 1
test/Main.hs view
@@ -18,7 +18,7 @@ module Main (main) where import Control.Concurrent (threadDelay)-import Control.Exception (bracket, throwIO, try)+import Control.Exception (bracket, evaluate, throwIO, try) import Data.Aeson (FromJSON, ToJSON, Value (..), object, parseJSON, toJSON, (.=)) import Data.Aeson.Key qualified as Key import Data.Aeson.KeyMap qualified as KeyMap@@ -66,6 +66,8 @@ import Shibuya.Adapter.Pgmq (PgmqAdapterEnv) import Shibuya.App (AppHandle, ShutdownConfig (..), SupervisionStrategy (IgnoreFailures), stopAppGracefully) import Shibuya.Telemetry.Effect (Tracing)+import System.Mem (performMajorGC)+import System.Mem.Weak (Weak, deRefWeak, mkWeakPtr) import System.Timeout (timeout) import Test.Hspec @@ -250,6 +252,7 @@ csStatus :: OTel.SpanStatus, csContext :: OTel.SpanContext, csParent :: Maybe OTel.Span,+ csLinks :: [OTel.Link], csEventNames :: [Text] } @@ -264,6 +267,7 @@ csStatus = hotStatus hot, csContext = spanContext sp, csParent = spanParent sp,+ csLinks = toList (appendOnlyBoundedCollectionValues (hotLinks hot)), csEventNames = map eventName (toList (appendOnlyBoundedCollectionValues (hotEvents hot))) }@@ -281,18 +285,18 @@ parentSpanContext :: CapturedSpan -> IO (Maybe OTel.SpanContext) parentSpanContext = traverse OTel.getSpanContext . csParent --- | The one @\<jobName\> process@ span a single one-shot delivery must produce.+-- | The one @process \<jobName\>@ span a single one-shot delivery must produce. -- Anything other than exactly one is a failure that names every captured span, so -- a duplicate wrapper is diagnosed rather than silently accepted by taking the -- head of the list. theProcessSpan :: Text -> [CapturedSpan] -> IO CapturedSpan theProcessSpan jobName spans =- case spansNamed (jobName <> " process") spans of+ case spansNamed ("process " <> jobName) spans of [only] -> pure only other -> fail ( "expected exactly one "- <> show (jobName <> " process")+ <> show ("process " <> jobName) <> " span, got " <> show (length other) <> "; all captured spans: "@@ -323,6 +327,53 @@ then pure True else threadDelay 100_000 >> loop +-- Keep the actual decoded payload's weak key, then leave every handler/runtime+-- scope before checking collection with the uninspected exporter still alive.+{-# NOINLINE produceCorrelatedJobs #-}+produceCorrelatedJobs :: Text -> OTel.TracerProvider -> OTel.Tracer -> TraceCorrelation -> Job Ping -> IO (OTel.SpanContext, [Weak Text])+produceCorrelatedJobs connStr provider tracer policy job = do+ weakRef <- newIORef []+ producer <- OTel.createSpan tracer Ctxt.empty "enqueue" OTel.defaultSpanArguments+ producerContext <- OTel.getSpanContext producer+ let tuning = defaultJobTuning {polling = PollEvery 0.01, telemetry = defaultAdapterTelemetry {traceCorrelation = policy}}+ handler _ payload = do+ weak <- liftIO $ evaluate payload.message >>= \text -> mkWeakPtr text Nothing+ liftIO $ modifyIORef' weakRef (weak :)+ pure Done+ enqueuePayload = bracket (CtxtLocal.attachContext (Ctxt.insertSpan producer Ctxt.empty)) CtxtLocal.detachContext $ \_ ->+ runDbTraced connStr tracer $ do+ ensureJobQueue job+ _ <- enqueueTraced provider job (MessageHeaders (object [])) (Ping (Text.replicate (128 * 1024) "x") 1)+ pure ()+ enqueuePayload+ runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job handler) `shouldReturn` 1+ enqueuePayload+ processed <- runDbTraced connStr tracer $ do+ result <- runJobWorkers IgnoreFailures 16 [jobProcessorWithContext tuning job handler]+ case result of+ Left err -> liftIO $ fail (show err)+ Right app -> do+ ready <- liftIO $ waitUntil ((== 2) . length <$> readIORef weakRef)+ stopAppQuickly app+ pure ready+ processed `shouldBe` True+ OTel.endSpan producer Nothing+ weak <- readIORef weakRef+ pure (producerContext, weak)++collectWeakTexts :: [Weak Text] -> IO Bool+collectWeakTexts keys = loop (20 :: Int)+ where+ loop remaining = do+ performMajorGC+ values <- traverse deRefWeak keys+ if all (maybe True (const False)) values+ then pure True+ else+ if remaining == 0+ then pure False+ else threadDelay 10_000 >> loop (remaining - 1)+ spec :: SpecWith Text spec = do it "round-trips a payload through aesonJobCodec" $ \_connStr -> do@@ -1045,6 +1096,80 @@ map csName spans `shouldSatisfy` elem ("publish " <> queue) map csName spans `shouldSatisfy` elem ("receive " <> queue) + mapM_+ ( \policy ->+ it ("both paths share process correlation and release payloads under " <> show policy) $ \connStr -> do+ (provider, spansRef) <- setupCapturingProvider+ let tracer = OTel.makeTracer provider "keiro-pgmq-test" OTel.tracerOptions+ job = mkJob "keiro_pgmq_test.correlation"+ (producerContext, weak) <- produceCorrelatedJobs connStr provider tracer policy job+ collectWeakTexts weak `shouldReturn` True+ -- The exporter remains a strong root throughout the collection assertion.+ exported <- readIORef spansRef+ length exported `shouldSatisfy` (> 0)+ spans <- capturedSpans provider spansRef+ let processSpans = spansNamed ("process " <> job.jobName) spans+ length processSpans `shouldBe` 2+ mapM_+ ( \processSpan -> do+ csKind processSpan `shouldBe` OTel.Consumer+ textAttr (csAttributes processSpan) "messaging.operation.name" `shouldBe` Just "process"+ textAttr (csAttributes processSpan) "messaging.operation.type" `shouldBe` Just "process"+ textAttr (csAttributes processSpan) "messaging.destination.name" `shouldBe` Just job.jobName+ textAttr (csAttributes processSpan) "shibuya.ack.decision" `shouldBe` Just "ack_ok"+ csStatus processSpan `shouldBe` OTel.Ok+ case csLinks processSpan of+ [link] -> do+ OTel.traceId (OTel.frozenLinkContext link) `shouldBe` OTel.traceId producerContext+ OTel.spanId (OTel.frozenLinkContext link) `shouldBe` OTel.spanId producerContext+ _ -> expectationFailure "expected one creation link"+ parent <- parentSpanContext processSpan+ case policy of+ ParentAndLink -> do+ OTel.traceId (csContext processSpan) `shouldBe` OTel.traceId producerContext+ fmap OTel.spanId parent `shouldBe` Just (OTel.spanId producerContext)+ LinkOnly -> do+ OTel.traceId (csContext processSpan) `shouldNotBe` OTel.traceId producerContext+ fmap OTel.spanId parent `shouldBe` Nothing+ )+ processSpans+ case (policy, processSpans) of+ (LinkOnly, [a, b]) -> OTel.traceId (csContext a) `shouldNotBe` OTel.traceId (csContext b)+ _ -> pure ()+ )+ [ParentAndLink, LinkOnly]++ it "delayed and retried LinkOnly deliveries remain distinct roots linked to enqueue" $ \connStr -> do+ (provider, spansRef) <- setupCapturingProvider+ let tracer = OTel.makeTracer provider "keiro-pgmq-test" OTel.tracerOptions+ job = mkJob "keiro_pgmq_test.delayed_correlation"+ tuning = defaultJobTuning {telemetry = defaultAdapterTelemetry {traceCorrelation = LinkOnly}}+ producer <- OTel.createSpan tracer Ctxt.empty "enqueue" OTel.defaultSpanArguments+ producerContext <- OTel.getSpanContext producer+ bracket (CtxtLocal.attachContext (Ctxt.insertSpan producer Ctxt.empty)) CtxtLocal.detachContext $ \_ ->+ runDbTraced connStr tracer $ do+ ensureJobQueue job+ _ <- enqueueTracedWithDelay provider job 1 (MessageHeaders (object [])) (Ping "delayed" 1)+ pure ()+ OTel.endSpan producer Nothing+ runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job (\_ _ -> pure Done)) `shouldReturn` 0+ threadDelay 1_100_000+ runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job (\_ _ -> pure (Retry (RetryDelay 0)))) `shouldReturn` 1+ runDbTraced connStr tracer (runJobOnceWithContext tuning 1 job (\_ _ -> pure Done)) `shouldReturn` 1+ spans <- capturedSpans provider spansRef+ let deliveries = spansNamed ("process " <> job.jobName) spans+ length deliveries `shouldBe` 2+ mapM_+ ( \delivery -> do+ parentSpanContext delivery >>= (\parent -> fmap OTel.spanId parent `shouldBe` Nothing)+ OTel.traceId (csContext delivery) `shouldNotBe` OTel.traceId producerContext+ map (OTel.spanId . OTel.frozenLinkContext) (csLinks delivery) `shouldBe` [OTel.spanId producerContext]+ )+ deliveries+ case deliveries of+ [a, b] -> OTel.traceId (csContext a) `shouldNotBe` OTel.traceId (csContext b)+ _ -> pure ()+ -- EP-111 M2: the central proof — the one-shot process span continues the -- producer's trace using only what the PGMQ headers carry. it "one-shot process span continues the enqueued W3C parent" $ \connStr -> do@@ -1294,7 +1419,8 @@ { visibilityTimeout = 30, batchSize = 0, polling = PollEvery 1,- ordering = Unordered+ ordering = Unordered,+ telemetry = defaultAdapterTelemetry } mismatchedTuning = defaultJobTuning unsafeTuning =@@ -1356,7 +1482,8 @@ { visibilityTimeout = 0, batchSize = 1, polling = PollEvery 1,- ordering = FifoHeads+ ordering = FifoHeads,+ telemetry = defaultAdapterTelemetry } build tuning = runDb connStr $