packages feed

kenshou-synthetic-0.1.0.0: src/Kenshou/Suite/Synthetic/Correctness.hs

-- | @synthetic/table/correctness/append-survives-kill-restart@: a writer process
-- appends items while it is killed and restarted, a reader process observes them,
-- and the run's ledger must show no loss, per-key order, duplicates only within the
-- crash windows, quiescence, and agreement with the table.
module Kenshou.Suite.Synthetic.Correctness (appendSurvivesKillRestart) where

import Control.Concurrent (threadDelay)
import Control.Concurrent.STM (atomically)
import Control.Monad (forM_)
import Data.Aeson (object, (.=))
import Data.Int (Int64)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Text (Text)
import Kenshou.Check.Fact (FactKind (..), ProcId (..))
import Kenshou.Check.Invariant
import Kenshou.Check.Ledger (defaultLedgerConfig, record, recordDurable, sealLedger, withLedger)
import Kenshou.Check.Ledger.Read (discoverLedgers)
import Kenshou.Check.Oracle (OracleResult (..), reconcile, runOracle)
import Kenshou.Check.Process
import Kenshou.Check.Scenario
import Kenshou.Check.Verdict (InvariantClass (..))
import Kenshou.Core.Context (ArtifactDir (..), RunContext (..), artifactPath, requirePostgres)
import Kenshou.Core.Dimension
import Kenshou.Core.Env (PostgresRequirement (..), noEnvironment)
import Kenshou.Core.Env qualified as Env
import Kenshou.Core.Id (parseScenarioId, renderRunId)
import Kenshou.Core.Knob
import Kenshou.Core.Phase (zeroPhases)
import Kenshou.Core.Role (ControlMessage (..))
import Kenshou.Core.Scenario
import Kenshou.Suite.Synthetic.Knobs (intKnob, knob)
import Kenshou.Suite.Synthetic.Oracle (itemsQuery)
import Kenshou.Suite.Synthetic.Roles (Fact (..), decodeFacts)
import Kenshou.Suite.Synthetic.Schema (syntheticSchema)

appendSurvivesKillRestart :: Scenario
appendSurvivesKillRestart =
  Scenario
    { id = either (error . show) id (parseScenarioId "synthetic/table/correctness/append-survives-kill-restart"),
      revision = 1,
      summary = "Appends items through a writer that is killed and restarted, and checks a reader sees each exactly once.",
      tier = TierSmoke,
      placement = PlaceEither,
      knobs =
        [ intKnob "writer.count" "Items per key" 500 1 100000,
          intKnob "writer.keys" "Keys written in turn" 8 1 1000,
          intKnob "kill.count" "Times the writer is killed and restarted" 2 0 20
        ],
      dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) noDimensions,
      phases = zeroPhases,
      requires = noEnvironment {Env.postgres = Just (PostgresRequirement [syntheticSchema] [] False)},
      knownDefect = Nothing,
      run = runAppend
    }

runAppend :: RunContext -> IO ScenarioReport
runAppend context = withCheck context \environment -> withSupervisor environment \supervisor -> do
  let count = knobInt context.knobs (knob "writer.count")
      keys = knobInt context.knobs (knob "writer.keys")
      kills = fromIntegral (knobInt context.knobs (knob "kill.count")) :: Int
      total = count * keys
  readerSpec <- roleProcess environment "synthetic/reader" 0 (object ["total" .= total])
  writerSpec <- roleProcess environment "synthetic/writer" 0 (object ["count" .= count, "keys" .= keys])
  reader <- start supervisor readerSpec
  firstWriter <- start supervisor writerSpec
  -- Kill the writer after it has written a share of its items, then restart it.
  writers <- killAndRestart supervisor (max 1 (total `div` fromIntegral (kills + 1))) kills firstWriter
  finished <- waitFor 120000 (atomically ((>= total) . (.count) <$> progress reader))
  _ <- traverse (\writer -> stopGracefully supervisor writer 2000) (lastOf writers)
  _ <- stopGracefully supervisor reader 2000
  writerFacts <- decodeFacts . concat <$> traverse readChildMessages writers
  readerFacts <- decodeFacts <$> readChildMessages reader
  forM_ writerFacts \fact -> case fact.kind of
    "intent" -> record environment.ledger Intent fact.key fact.seq fact.item mempty
    "produced" -> recordDurable environment.ledger Produced fact.key fact.seq fact.item mempty
    _ -> pure ()
  -- The reader's observations go in their own ledger scope, as a consumer process
  -- would write them, so per-key order is judged over the consumer's arrivals.
  withLedger (defaultLedgerConfig environment.ledgerDirectory (ProcId "synthetic-reader" 0 0) (renderRunId context.runId)) \readerLedger -> do
    forM_ readerFacts \fact -> do
      recordDurable readerLedger Observed fact.key fact.seq fact.item mempty
      record readerLedger Terminal fact.key fact.seq fact.item mempty
    sealLedger readerLedger
  sealLedger environment.ledger
  ledgerSet <- discoverLedgers environment.ledgerDirectory
  windows <- crashWindows supervisor
  verdictDirectory <- artifactPath context VerdictsDir ""
  durable <- runOracle verdictDirectory (requirePostgres context) itemsQuery >>= either (fail . show) pure
  verdicts <-
    runCheckers
      environment
      ledgerSet
      [ noLoss "no-loss" Contract readerScope,
        perKeyOrder "per-key-order" Contract readerScope,
        duplicatesWithin "duplicates-within-windows" Contract readerScope (DuplicateBudget (Just 0) Nothing) windows,
        eventualQuiescence "eventual-quiescence" Contract (Deadline 120000000),
        reconcile "ledger-versus-oracle" Contract durable.rows
      ]
  if finished && fromIntegral (length readerFacts) == total
    then finishWithVerdicts environment verdicts
    else pure (failedWith ["reader-incomplete"] "the reader did not observe every item before the deadline")
  where
    lastOf values = case reverse values of
      final : _ -> [final]
      [] -> []

readerScope :: Text
readerScope = "synthetic-reader/0"

start :: Supervisor -> ProcessSpec -> IO Child
start supervisor spec = do
  child <- spawn supervisor spec
  awaitReady child 10000
  sendCommand child CtlStart
  pure child

-- | Kill the writer each time it has written another @step@ items, restarting it,
-- and return every incarnation in order.
killAndRestart :: Supervisor -> Int64 -> Int -> Child -> IO [Child]
killAndRestart supervisor step remaining child
  | remaining <= 0 = pure [child]
  | otherwise = do
      _ <- waitFor 60000 (atomically ((>= step) . (.count) <$> progress child))
      killChild supervisor child
      replacement <- restartChild supervisor child
      awaitReady replacement 10000
      sendCommand replacement CtlStart
      (child :) <$> killAndRestart supervisor step (remaining - 1) replacement

-- | Poll a condition every 20 ms for at most the given milliseconds.
waitFor :: Int -> IO Bool -> IO Bool
waitFor millis condition = loop millis
  where
    loop left = do
      satisfied <- condition
      if satisfied || left <= 0 then pure satisfied else threadDelay 20000 >> loop (left - 20)