kenshou-diagnose-0.1.0.0: test/Kenshou/Diagnose/StallSpec.hs
module Kenshou.Diagnose.StallSpec (spec) where
import Control.Concurrent (threadDelay)
import Control.Concurrent.Async (wait, withAsync)
import Control.Exception (bracket, finally, try)
import Control.Monad ((>=>))
import Data.Aeson (eitherDecodeFileStrict')
import Data.Int (Int64)
import Data.List (sort)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Word (Word32, Word64)
import Hasql.Connection qualified as Connection
import Hasql.Connection.Settings qualified as Settings
import Hasql.Decoders qualified as Decoders
import Hasql.Encoders qualified as Encoders
import Hasql.Session qualified as Session
import Hasql.Statement qualified as Statement
import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)
import Kenshou.Core.Context (RunContext (..), newRunState)
import Kenshou.Core.Dimension (Dimensions (..), PgDurability (..), PgVersion (..))
import Kenshou.Core.Env (PostgresRequirement (..))
import Kenshou.Core.Env.Postgres (PostgresEnv (..), withPostgresEnv)
import Kenshou.Core.Id (parseRunId, parseScenarioId, selftestLayer)
import Kenshou.Core.Log (nullLogger)
import Kenshou.Core.RunSpec (PostgresSpec (..))
import Kenshou.Diagnose.LockGraph (GraphEdge (..), LockGraph (..), buildGraph)
import Kenshou.Diagnose.Postgres
import Kenshou.Diagnose.Stall qualified as Stall
import Kenshou.Diagnose.Stall.Classify (classify)
import Kenshou.Diagnose.Stall.Types
import System.Directory (doesFileExist)
import System.Environment (lookupEnv)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec
spec :: Spec
spec = describe "Kenshou.Diagnose.Stall" do
it "names a bundle's advisory locks and the migration ledger lock" do
let label = AdvisoryLabel "queue creation" (HashText "synthetic.queue_jobs")
config = Stall.watchdogConfigFor (mkBundle selftestLayer [] []) {advisoryLabels = [label]}
config.advisoryLabels `shouldBe` [label, pgMigrateLedgerLock]
mapM_
fixtureCase
[ ("deadlock", Deadlock),
("lock-wait", LockWait),
("pool-starvation", PoolStarvation),
("blocked-indefinitely", BlockedIndefinitely),
("idle-spin", IdleSpin),
("unknown", Unknown),
("postgres-unavailable", Unknown)
]
it "finds a three-session wait cycle as one cycle" do
let row pid blockers = ActivityRow pid "test" "active" (Just "Lock") (Just "transactionid") (Just 1) (Just 1) blockers "UPDATE test"
graph = buildGraph [row 10 [11], row 11 [12], row 12 [10]] []
graph.cycles `shouldBe` [[10, 11, 12]]
it "reconstructs signed advisory lock keys" do
let original = -2 :: Int64
bits = fromIntegral original :: Word64
high = fromIntegral (bits `div` 0x100000000) :: Word32
low = fromIntegral bits :: Word32
reconstructAdvisoryKey high low `shouldBe` original
describe "PostgreSQL capture" do
it "finds a real row-lock edge on PostgreSQL 18" (lockCaptureCase Pg18)
it "finds a real row-lock edge on PostgreSQL 17" do
available <- lookupEnv "KENSHOU_PG17_BIN"
maybe (pendingWith "KENSHOU_PG17_BIN is not set") (const (lockCaptureCase Pg17)) available
it "captures and aborts a silent scenario at the deadline" $ withSystemTempDirectory "kenshou-watchdog" \directory -> do
runId <- either (expectationFailure . show >=> const (fail "invalid run id")) pure (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")
scenario <- either (expectationFailure . show >=> const (fail "invalid scenario id")) pure (parseScenarioId "selftest/diagnose/concurrency/watchdog")
state <- newRunState
let runContext = RunContext runId scenario undefined undefined undefined undefined undefined undefined undefined directory undefined state
config = Stall.defaultWatchdogConfig {Stall.deadlineSeconds = 0.2, Stall.pollIntervalSeconds = 0.05, Stall.captureStacks = False}
result <- try @Stall.StallDetected $ Stall.withWatchdog runContext config \watchdog -> do
_ <- Stall.newProgress watchdog "work" True
threadDelay 2_000_000
result `shouldSatisfy` either (const True) (const False)
doesFileExist (directory </> "diagnosis" </> "stall-1.json") `shouldReturn` True
fixtureCase :: (FilePath, StallClass) -> Spec
fixtureCase (name, expected) = it ("classifies the " <> name <> " snapshot") do
decoded <- eitherDecodeFileStrict' ("test/fixtures/snapshots/" <> name <> ".json")
case decoded of
Left err -> expectationFailure err
Right snapshot -> do
let (actual, secondary, _) = classify snapshot
actual `shouldBe` expected
sort secondary `shouldSatisfy` (if expected == Unknown then null else elem expected)
lockCaptureCase :: PgVersion -> IO ()
lockCaptureCase version = withSystemTempDirectory "kenshou-diagnose-lock" \directory -> do
runId <- either (expectationFailure . show >=> const (fail "invalid run id")) pure (parseRunId "01997f3a-5b7c-7e21-8a44-0d6c2f9b1e55")
let requirement = PostgresRequirement [] [] False
dimensions = Dimensions Nothing Nothing (Just PgFsyncOff) (Just version)
result <- withPostgresEnv mempty nullLogger directory runId requirement (PostgresEphemeral []) dimensions exerciseLockCapture
case result of
Left err -> expectationFailure (show err)
Right () -> pure ()
exerciseLockCapture :: PostgresEnv -> IO ()
exerciseLockCapture environment =
withConnection environment.connectionString "kenshou-diagnose-holder" \holder ->
withConnection environment.connectionString "kenshou-diagnose-waiter" \waiter ->
withConnection environment.connectionString "kenshou-diagnose-capture" \capture -> do
expectSession holder (sql "CREATE TABLE diagnose_capture (id int primary key)")
expectSession holder (sql "INSERT INTO diagnose_capture VALUES (1)")
holderPid <- connectionPid holder
waiterPid <- connectionPid waiter
expectSession holder (sql "BEGIN")
expectSession holder (sql "UPDATE diagnose_capture SET id=id WHERE id=1")
expectSession waiter (sql "BEGIN")
withAsync (expectSession waiter (sql "UPDATE diagnose_capture SET id=id WHERE id=1")) \waiterUpdate -> do
finally
( do
threadDelay 500_000
activity <- captureActivity capture >>= either (expectationFailure . Text.unpack >=> const (fail "activity capture failed")) pure
locks <- captureLocks capture >>= either (expectationFailure . Text.unpack >=> const (fail "lock capture failed")) pure
let graph = buildGraph activity locks
graph.edges `shouldSatisfy` any (\edge -> edge.waiter == waiterPid && edge.blocker == holderPid)
)
(expectSession holder (sql "ROLLBACK"))
wait waiterUpdate
_ <- Connection.use waiter (sql "ROLLBACK")
pure ()
withConnection :: Text -> Text -> (Connection.Connection -> IO value) -> IO value
withConnection connectionString applicationName = bracket acquire Connection.release
where
acquire = do
result <- Connection.acquire (Settings.connectionString connectionString <> Settings.applicationName applicationName)
either (ioError . userError . show) pure result
expectSession :: Connection.Connection -> Session.Session value -> IO value
expectSession connection session = Connection.use connection session >>= either (ioError . userError . show) pure
connectionPid :: Connection.Connection -> IO Int
connectionPid connection = do
raw <- expectSession connection (Session.statement () pidStatement)
case reads (Text.unpack raw) of
[(pid, "")] -> pure pid
_ -> expectationFailure ("invalid backend pid " <> Text.unpack raw) >> fail "invalid backend pid"
pidStatement :: Statement.Statement () Text
pidStatement = Statement.unpreparable "SELECT pg_backend_pid()::text" Encoders.noParams (Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.text)))
sql :: Text -> Session.Session ()
sql statement = Session.statement () (Statement.unpreparable statement Encoders.noParams Decoders.noResult)