kenshou-check-0.1.0.0: src/Kenshou/Check/Selftest/BackendKill.hs
module Kenshou.Check.Selftest.BackendKill (backendKillScenario) where
import Control.Concurrent (threadDelay)
import Control.Concurrent.Async
import Data.Aeson (object, (.=))
import Data.List.NonEmpty (NonEmpty (..))
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Time (getCurrentTime)
import Kenshou.Check.Fault (Fault (..), FaultHandle (..))
import Kenshou.Check.Fault.Postgres
import Kenshou.Check.Scenario
import Kenshou.Check.Verdict
import Kenshou.Core.Context (RunContext (..), requirePostgres)
import Kenshou.Core.Dimension
import Kenshou.Core.Env (EnvRequirements (..), PostgresRequirement (..))
import Kenshou.Core.Env.Postgres (PostgresEnv (..), ServerControl (..), StopMode (..))
import Kenshou.Core.Id (parseScenarioId)
import Kenshou.Core.Phase (zeroPhases)
import Kenshou.Core.Scenario
import System.Environment (getEnvironment)
import System.Exit (ExitCode (..))
import System.Process
backendKillScenario :: Scenario
backendKillScenario =
Scenario
{ id = either (error . show) id (parseScenarioId "selftest/check/concurrency/postgres-backend-kill"),
revision = 1,
summary = "Terminates a named PostgreSQL backend and proves acknowledged rows remain durable.",
tier = TierStandard,
placement = PlaceEither,
knobs = [],
dimensions = postgresDimensions (PgFsyncOff :| [PgDurable]) (Pg18 :| [Pg17]) telemetryOff,
phases = zeroPhases,
requires = EnvRequirements (Just (PostgresRequirement [] [] True)) [] [],
knownDefect = Nothing,
run = runBackendKill
}
runBackendKill :: RunContext -> IO ScenarioReport
runBackendKill context = withCheck context \environment -> do
let postgres = requirePostgres context
sql postgres "create schema if not exists kenshou_selftest; create table if not exists kenshou_selftest.items(id text primary key); insert into kenshou_selftest.items values ('before') on conflict do nothing"
lockHandle <- (holdLock postgres (TableLock "kenshou_selftest" "items")).inject
blockedInsert <- async (query postgres "insert into kenshou_selftest.items values ('locked') on conflict do nothing")
threadDelay 200000
lockBlocked <- isNothing <$> poll blockedInsert
lockHandle.heal
lockReleased <- either (const False) (const True) <$> waitCatch blockedInsert
sleeper <- spawnSleeper postgres
threadDelay 300000
before <- listBackends postgres
let fault = terminateOneBackend postgres (ByApplicationName "kenshou-selftest-writer")
handle <- fault.inject
handle.heal
_ <- waitForProcess sleeper
threadDelay 100000
after <- listBackends postgres
sql postgres "insert into kenshou_selftest.items values ('after') on conflict do nothing"
let shouldCrash = context.dimensions.pgDurability == Just PgDurable
crashRecovered <-
if shouldCrash
then case postgres.control of
Nothing -> pure False
Just control -> do
control.stopServer StopImmediate
control.startServer
(== "1") <$> query postgres "select 1"
else pure True
survived <- query postgres "select count(*) from kenshou_selftest.items where id in ('before','locked','after')"
now <- getCurrentTime
let found = any ((== "kenshou-selftest-writer") . (.applicationName)) before
gone = not (any ((== "kenshou-selftest-writer") . (.applicationName)) after)
survivedHeld = survived == "3" && crashRecovered
backendHeld = found && gone
makeVerdict checker invariant held summary parameters = Verdict checker invariant Contract (if held then Held else Violated) Nothing summary (Map.fromList [("examined", 1), ("violations", if held then 0 else 1)]) parameters [] False [] Nothing now 0
verdicts =
[ makeVerdict "acknowledged-writes-survive" "no-loss" survivedHeld "Committed rows survive backend and optional postmaster crashes." (object ["survivingRows" .= survived, "postmasterCrashAttempted" .= shouldCrash, "postmasterRecovered" .= crashRecovered]),
makeVerdict "writers-recover" "backend-recovery" backendHeld "A client backend was terminated and the database accepted a later write." (object ["victimFound" .= found, "victimGone" .= gone]),
makeVerdict "lock-blocks-and-releases" "lock-release" (lockBlocked && lockReleased) "An access-exclusive lock blocked a writer and healing released it." (object ["blocked" .= lockBlocked, "released" .= lockReleased]),
makeVerdict "backends-terminated" "backend-termination" backendHeld "The selected backend disappeared from pg_stat_activity." (object ["victimFound" .= found, "victimGone" .= gone])
]
finishWithVerdicts environment verdicts
spawnSleeper :: PostgresEnv -> IO ProcessHandle
spawnSleeper postgres = do
inherited <- getEnvironment
let environment = ("PGAPPNAME", "kenshou-selftest-writer") : filter ((/= "PGAPPNAME") . fst) inherited
(_, _, _, handle) <- createProcess (proc "psql" ["-d", Text.unpack postgres.connectionString, "-Atqc", "select pg_sleep(30)"]) {env = Just environment, std_out = NoStream, std_err = NoStream}
pure handle
sql :: PostgresEnv -> Text -> IO ()
sql postgres statement = query postgres statement >> pure ()
query :: PostgresEnv -> Text -> IO Text
query postgres statement = do
(code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack postgres.connectionString, "-Atqc", Text.unpack statement] ""
case code of
ExitSuccess -> pure (Text.strip (Text.pack output))
ExitFailure _ -> ioError (userError err)
isNothing :: Maybe value -> Bool
isNothing Nothing = True
isNothing (Just _) = False
telemetryOff :: DimensionSupport
telemetryOff = DimensionSupport (Supported (Support (TracingOff :| []) TracingOff)) (Supported (Support (MetricsOff :| []) MetricsOff)) NotApplicable NotApplicable