kenshou-synthetic-0.1.0.0: src/Kenshou/Suite/Synthetic/Concurrency.hs
-- | @synthetic/table/concurrency/competing-writers-hold-disjoint-leases@: contender
-- processes repeatedly take a lease on a key under a transaction-scoped advisory
-- lock, append one item while they hold it, and release it. Their facts, ordered by
-- the database clock, must show that no owner acted on a key another owner held.
module Kenshou.Suite.Synthetic.Concurrency
( contenderRole,
competingWriters,
leaseLabel,
)
where
import Control.Concurrent (threadDelay)
import Control.Concurrent.STM (atomically)
import Control.Monad (forM, forM_, when)
import Data.Aeson (FromJSON (..), object, withObject, (.:), (.=))
import Data.Aeson.Types (parseMaybe)
import Data.Functor.Contravariant ((>$<))
import Data.Int (Int64)
import Data.List (sortOn)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import Hasql.Decoders qualified as Decoders
import Hasql.Encoders qualified as Encoders
import Hasql.Statement qualified as Statement
import Kenshou.Check.Fact (FactKind (..), ProcId (..))
import Kenshou.Check.Invariant
import Kenshou.Check.Ledger (LedgerConfig, LedgerWriter, defaultLedgerConfig, record, sealLedger, withLedger)
import Kenshou.Check.Ledger.Read (discoverLedgers)
import Kenshou.Check.Process
import Kenshou.Check.Scenario
import Kenshou.Check.Verdict (InvariantClass (..))
import Kenshou.Core.Context (RunContext (..))
import Kenshou.Core.Contribution (AdvisoryKeySpec (..), AdvisoryLabel (..))
import Kenshou.Core.Dimension
import Kenshou.Core.Env (PostgresRequirement (..), noEnvironment)
import Kenshou.Core.Env qualified as Env
import Kenshou.Core.Id (parseScenarioId, renderRunId)
import Kenshou.Core.Knob
import Kenshou.Core.Phase (zeroPhases)
import Kenshou.Core.Role
import Kenshou.Core.Scenario
import Kenshou.Suite.Synthetic.Knobs (intKnob, knob)
import Kenshou.Suite.Synthetic.Roles (awaitStart, awaitStop, intArg, roleName)
import Kenshou.Suite.Synthetic.Schema (syntheticSchema)
import Kenshou.Suite.Synthetic.Table (run, withConnection)
-- | A lease step a contender reports, at a database clock time in microseconds.
data Step = Step {kind :: !Text, key :: !Text, at :: !Int64}
instance FromJSON Step where
parseJSON = withObject "lease step" \value -> Step <$> value .: "kind" <*> value .: "key" <*> value .: "at"
-- | The label a stall capture gives the advisory lock a contender takes on a key.
leaseLabel :: Text -> AdvisoryLabel
leaseLabel key = AdvisoryLabel ("synthetic lease " <> key) (HashText ("synthetic/lease/" <> key))
-- | Takes the lease on each of @keys@ keys in turn, @iterations@ times, appending one
-- item under each lease; every step reports the database clock time.
contenderRole :: WorkerRole
contenderRole = WorkerRole (roleName "synthetic/contender") "Takes leases on keys and appends an item while holding each." \context -> do
connection <- maybe (fail "the contender requires PostgreSQL") (pure . (.connectionString)) context.init.postgres
let iterations = intArg context "iterations"
keys = intArg context "keys"
owner = context.init.instanceName
step kind key at = object ["kind" .= (kind :: Text), "key" .= key, "at" .= at]
withConnection connection ("synthetic-contender-" <> owner) \database -> do
context.send WrkReady
started <- awaitStart context
when started do
forM_ [1 .. iterations] \iteration -> do
let key = "lease-" <> Text.pack (show (iteration `mod` keys))
run database () (unit "begin")
run database key takeLock
acquiredAt <- run database (key, owner) acquireLease
actedAt <- run database key appendUnderLease
releasedAt <- run database (key, owner) releaseLease
run database () (unit "commit")
context.send (WrkFacts [step "acquired" key acquiredAt, step "acted" key actedAt, step "released" key releasedAt])
now <- getCurrentTime
context.send (WrkProgress iteration now)
awaitStop context
competingWriters :: Scenario
competingWriters =
Scenario
{ id = either (error . show) id (parseScenarioId "synthetic/table/concurrency/competing-writers-hold-disjoint-leases"),
revision = 1,
summary = "Runs contending writers that take per-key leases and checks no owner acts on another's lease.",
tier = TierSmoke,
placement = PlaceEither,
knobs =
[ intKnob "contention.contenders" "Contending writer processes" 4 2 32,
intKnob "contender.iterations" "Leases each contender takes" 100 1 100000,
intKnob "lease.keys" "Keys the contenders share" 3 1 100
],
dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) noDimensions,
phases = zeroPhases,
requires = noEnvironment {Env.postgres = Just (PostgresRequirement [syntheticSchema] [] False)},
knownDefect = Nothing,
run = runContention
}
runContention :: RunContext -> IO ScenarioReport
runContention context = withCheck context \environment -> withSupervisor environment \supervisor -> do
let contenders = fromIntegral (knobInt context.knobs (knob "contention.contenders")) :: Int
iterations = knobInt context.knobs (knob "contender.iterations")
keys = knobInt context.knobs (knob "lease.keys")
children <- forM [0 .. contenders - 1] \index -> do
spec <- roleProcess environment "synthetic/contender" index (object ["iterations" .= iterations, "keys" .= keys])
child <- spawn supervisor spec
awaitReady child 10000
pure child
forM_ children \child -> sendCommand child CtlStart
finished <- and <$> traverse (\child -> waitFor 60000 (atomically ((>= iterations) . (.count) <$> progress child))) children
forM_ children \child -> stopGracefully supervisor child 2000
steps <- forM (zip [0 ..] children) \(index, child) -> do
messages <- readChildMessages child
pure [(index, step) | WrkFacts values <- messages, Just step <- fmap (parseMaybe parseJSON) values :: [Maybe Step]]
-- Each contender records its own steps in its own ledger scope, in the order the
-- database clock put them, so wall-clock order across owners is the real order.
withLedgers [defaultLedgerConfig environment.ledgerDirectory (ProcId "synthetic-contender" index 0) (renderRunId context.runId) | index <- [0 .. contenders - 1]] \writers -> do
forM_ (sortOn ((.at) . snd) (concat steps)) \(index, step) ->
record (writers !! index) (stepKind step.kind) step.key step.at step.key mempty
mapM_ sealLedger writers
sealLedger environment.ledger
ledgerSet <- discoverLedgers environment.ledgerDirectory
verdicts <- runCheckers environment ledgerSet [disjointOwnership "disjoint-ownership" Contract]
if finished && length (concat steps) == contenders * fromIntegral iterations * 3
then finishWithVerdicts environment verdicts
else pure (failedWith ["contenders-incomplete"] "not every contender finished its leases")
where
stepKind :: Text -> FactKind
stepKind = \case
"acquired" -> Acquired
"acted" -> Acted
_ -> Released
withLedgers :: [LedgerConfig] -> ([LedgerWriter] -> IO a) -> IO a
withLedgers configs action = go configs []
where
go [] opened = action (reverse opened)
go (config : rest) opened = withLedger config \writer -> go rest (writer : opened)
waitFor :: Int -> IO Bool -> IO Bool
waitFor millis condition = loop millis
where
loop left = do
satisfied <- condition
if satisfied || left <= 0 then pure satisfied else threadDelay 20000 >> loop (left - 20)
unit :: Text -> Statement.Statement () ()
unit sql = Statement.unpreparable sql Encoders.noParams Decoders.noResult
clockMicros :: Decoders.Result Int64
clockMicros = Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.int8))
takeLock :: Statement.Statement Text ()
takeLock = Statement.unpreparable "select pg_advisory_xact_lock(hashtext('synthetic/lease/' || $1))" (Encoders.param (Encoders.nonNullable Encoders.text)) Decoders.noResult
acquireLease :: Statement.Statement (Text, Text) Int64
acquireLease =
Statement.unpreparable
"insert into synthetic.leases (key, owner, acquired_at, expires_at) values ($1, $2, clock_timestamp(), clock_timestamp() + interval '5 seconds') on conflict (key) do update set owner = excluded.owner, acquired_at = excluded.acquired_at, expires_at = excluded.expires_at returning (extract(epoch from clock_timestamp()) * 1000000)::bigint"
pairParams
clockMicros
appendUnderLease :: Statement.Statement Text Int64
appendUnderLease =
Statement.unpreparable
"insert into synthetic.items (key, seq, item, writer) select $1, coalesce(max(seq), 0) + 1, $1 || '/' || (coalesce(max(seq), 0) + 1), 'contender' from synthetic.items where key = $1 returning (extract(epoch from clock_timestamp()) * 1000000)::bigint"
(Encoders.param (Encoders.nonNullable Encoders.text))
clockMicros
releaseLease :: Statement.Statement (Text, Text) Int64
releaseLease =
Statement.unpreparable
"with released as (delete from synthetic.leases where key = $1 and owner = $2) select (extract(epoch from clock_timestamp()) * 1000000)::bigint"
pairParams
clockMicros
pairParams :: Encoders.Params (Text, Text)
pairParams = (fst >$< Encoders.param (Encoders.nonNullable Encoders.text)) <> (snd >$< Encoders.param (Encoders.nonNullable Encoders.text))