pqi-conformance 1.0.15.0 → 1.0.16.0

raw patch · 4 files changed

+101/−1 lines, 4 filesPVP ok

version bump matches the API change (PVP)

API changes (from Hackage documentation)

Files

• CHANGELOG.md +6/−0
• pqi-conformance.cabal +2/−1
• src/library/Pqi/Conformance/Operation/ExecParams.hs +2/−0
• src/library/Pqi/Conformance/Operation/ExecParams/LargeField.hs +91/−0

@@ -1,3 +1,9 @@+# v1.0.16.0++## Non-breaking++- Added an `execParams` spec covering a single 64 MiB binary `bytea` field, generated by the server and read back with `getvalue`. Besides the length and contents, it observes whether the calling thread's Haskell allocation (`getAllocationCounter`) over the query and the read stays within 4 times the field size, since the divergence is in cost rather than in any returned value. `libpq` allocates 1.0 times the field size. Found `pqi-native` allocating quadratically in the frame size: `Transport.fillTo` appended every received chunk to a strict `ByteString`, copying the accumulated frame each time, and asked `recv` for a buffer as large as the rest of the frame. That came to ~36 times the field size over the suite's TCP link and ~8000 times over a Unix-domain socket. Found via `pqi-native` issue #13.+ # v1.0.15.0  ## Non-breaking

@@ -1,6 +1,6 @@ cabal-version: 3.0 name: pqi-conformance-version: 1.0.15.0+version: 1.0.16.0 category: Database, PostgreSQL, Testing synopsis: Differential conformance tests for pqi adapters description:@@ -120,6 +120,7 @@     Pqi.Conformance.Operation.Exec.ConnectionLostMidQuery     Pqi.Conformance.Operation.Exec.LostConnection     Pqi.Conformance.Operation.ExecParams+    Pqi.Conformance.Operation.ExecParams.LargeField     Pqi.Conformance.Operation.ExecParams.LostConnection     Pqi.Conformance.Operation.ExecPrepared     Pqi.Conformance.Operation.ExitPipelineMode

@@ -9,6 +9,7 @@ import qualified Pqi as Lq import Pqi.Conformance.Harness import Pqi.Conformance.Observation+import qualified Pqi.Conformance.Operation.ExecParams.LargeField as LargeField import qualified Pqi.Conformance.Operation.ExecParams.LostConnection as LostConnection import Pqi.Conformance.Prelude import Pqi.Conformance.Scenario@@ -16,6 +17,7 @@  spec :: Lq.Adapter -> SpecWith ByteString spec adapter = do+  LargeField.spec adapter   LostConnection.spec adapter    describe "execParams" do

@@ -0,0 +1,91 @@+-- | Coverage for receiving a single large field: one row with one binary+-- @bytea@ cell of 64 MiB, generated by the server, fetched through+-- 'Pqi.execParams' and read back with 'Pqi.getvalue'.+--+-- The divergence: @pqi-native@'s 'Pqi.Native.Transport.fillTo' grew its read+-- buffer by appending every received chunk to a strict @ByteString@+-- (@modifyIORef' buffer (<> chunk)@), copying the whole accumulated frame on+-- each receive, and asked @recv@ for a buffer as large as the rest of the+-- frame, allocating that much again on each receive. Receiving a frame thus+-- allocated quadratically in its size: over a Unix-domain socket, ~32 GiB+-- for a 16 MiB field and ~512 GiB for a 64 MiB one, where @libpq@ (through the reference adapter)+-- allocates about one copy of the value on the Haskell heap. The value+-- itself came back intact, only slowly.+--+-- Where it surfaced: @pqi-native@ issue #13.+--+-- Why this observation: the divergence is in cost, not in any returned+-- value, and cost is not directly comparable across adapters. Wall-clock+-- time depends on the machine and the container link, so the observation is+-- the calling thread's Haskell allocation over the query and the read of the+-- value ('getAllocationCounter', which counts per thread and needs no RTS+-- flags), reduced to whether it stays within 'allocationFactor' times the+-- field size. Any receive path that copies the data a bounded number of+-- times passes. How far the quadratic one overshoots depends on how the+-- transport chunks the stream: over a Unix-domain socket the issue measured+-- ~8000 times the field size, and over this suite's Docker-forwarded TCP+-- link, whose receives are far larger, ~36 times, which still clears the+-- bound ninefold. The returned length and contents are observed as well, so+-- that a fix cannot pass by truncating the value.+module Pqi.Conformance.Operation.ExecParams.LargeField+  ( spec,+  )+where++import Control.Exception (evaluate)+import qualified Data.ByteString as ByteString+import qualified Data.ByteString.Char8 as ByteString.Char8+import qualified Pqi+import Pqi.Conformance.Harness+import Pqi.Conformance.Prelude+import System.Mem (getAllocationCounter)+import Test.Hspec++-- | 64 MiB: large enough for quadratic growth to dwarf any linear constant+-- even over the large receives of a forwarded TCP link (at 16 MiB the+-- unfixed candidate allocated only ~14 times the field size there), small+-- enough to keep the reference fast.+fieldSize :: Int+fieldSize = 64 * 1024 * 1024++-- | How many times over the field size the calling thread may allocate while+-- receiving and reading it. The reference allocates about once (measured+-- 1.0 times); the bound leaves room for a receive path that copies a few+-- times more.+allocationFactor :: Int+allocationFactor = 4++spec :: Pqi.Adapter -> SpecWith ByteString+spec adapter =+  describe "execParams" do+    describe "large field" do+      it "receives a 64 MiB binary bytea with allocation linear in its size" \conninfo ->+        differential adapter conninfo scenario++-- | Fetch the field, then report the result status, the value's length,+-- whether every byte is the expected one, and whether the allocation stayed+-- within the bound. The allocation is taken across 'Pqi.execParams' and+-- 'Pqi.getvalue' only. The length is forced inside that span, since a lazy+-- adapter could otherwise defer the work past it.+scenario :: Pqi.Connection -> IO (Maybe (Pqi.ExecStatus, Maybe (Int, Bool), Bool))+scenario connection = do+  before <- getAllocationCounter+  result <-+    Pqi.execParams+      connection+      "select decode(repeat('ab', $1 :: int4), 'hex')"+      [Just (23, ByteString.Char8.pack (show fieldSize), Pqi.Text)]+      Pqi.Binary+  for result \r -> do+    status <- Pqi.resultStatus r+    value <- Pqi.getvalue r 0 0+    for_ value \v -> evaluate (ByteString.length v)+    after <- getAllocationCounter+    -- The counter counts down as the thread allocates.+    let allocated = fromIntegral (before - after) :: Int+        withinBound = allocated <= allocationFactor * fieldSize+    pure+      ( status,+        fmap (\v -> (ByteString.length v, ByteString.all (== 0xab) v)) value,+        withinBound+      )