packages feed

kenshou-check-0.1.0.0: src/Kenshou/Check/Fault/Postgres.hs

module Kenshou.Check.Fault.Postgres
  ( Backend (..),
    BackendSelector (..),
    LockTarget (..),
    CrashMode (..),
    listBackends,
    terminateBackends,
    terminateOneBackend,
    holdLock,
    hogConnections,
    withApplicationName,
    withStatementTimeout,
    crashPostmaster,
  )
where

import Control.Concurrent (threadDelay)
import Control.Monad (replicateM)
import Data.Aeson (object, (.=))
import Data.Int (Int32, Int64)
import Data.Text (Text)
import Data.Text qualified as Text
import Kenshou.Check.Fault
import Kenshou.Check.Process (ProcessSpec (..))
import Kenshou.Core.Env.Postgres
import System.Exit (ExitCode (..))
import System.Posix.Signals (sigKILL, signalProcess)
import System.Process

data Backend = Backend
  { pid :: !Int32,
    applicationName :: !Text,
    state :: !Text,
    waitEventType :: !(Maybe Text),
    waitEvent :: !(Maybe Text),
    query :: !Text
  }
  deriving stock (Eq, Show)

data BackendSelector = ByApplicationName Text | ByQueryPattern Text | ByPid Int32 | AllOtherBackends
  deriving stock (Eq, Show)

data LockTarget = TableLock Text Text | RowLock Text Text | AdvisoryLock Int64
  deriving stock (Eq, Show)

data CrashMode = ImmediateShutdown | FastShutdown | KillPostmaster
  deriving stock (Eq, Show)

listBackends :: PostgresEnv -> IO [Backend]
listBackends postgres = do
  output <- psql postgres "SELECT pid, coalesce(application_name,''), coalesce(state,''), coalesce(wait_event_type,''), coalesce(wait_event,''), replace(left(query,200), E'\\t', ' ') FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND backend_type='client backend' ORDER BY pid"
  pure [Backend pid app state (nonEmpty waitType) (nonEmpty waitEvent) query | line <- Text.lines output, [pidText, app, state, waitType, waitEvent, query] <- [Text.splitOn "\x1f" line], Just pid <- [readText pidText]]

terminateBackends :: PostgresEnv -> BackendSelector -> Fault
terminateBackends postgres selector =
  Fault
    { name = "postgres-terminate-backends",
      target = "postgres",
      availability = pure Available,
      inject = do
        victims <- filter (matches selector) <$> listBackends postgres
        mapM_ (\backend -> voidPsql postgres ("SELECT pg_terminate_backend(" <> Text.pack (show backend.pid) <> ")")) victims
        pure (FaultHandle (pure ()) (object ["victims" .= fmap (.pid) victims]))
    }

terminateOneBackend :: PostgresEnv -> BackendSelector -> Fault
terminateOneBackend postgres selector =
  (terminateBackends postgres selector)
    { inject = do
        victims <- take 1 . filter (matches selector) <$> listBackends postgres
        mapM_ (\backend -> voidPsql postgres ("SELECT pg_terminate_backend(" <> Text.pack (show backend.pid) <> ")")) victims
        pure (FaultHandle (pure ()) (object ["victims" .= fmap (.pid) victims]))
    }

holdLock :: PostgresEnv -> LockTarget -> Fault
holdLock postgres target =
  Fault
    { name = "postgres-hold-lock",
      target = "postgres",
      availability = pure Available,
      inject = do
        let sql = "SELECT set_config('application_name','kenshou-fault-lock',false); BEGIN; " <> lockSql target <> "; SELECT pg_sleep(86400)"
        (_, _, _, processHandle) <- createProcess (proc "psql" ["-d", Text.unpack postgres.connectionString, "-v", "ON_ERROR_STOP=1", "-c", Text.unpack sql]) {std_out = NoStream, std_err = NoStream}
        threadDelay 100000
        let heal = voidPsql postgres "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname=current_database() AND application_name='kenshou-fault-lock'" >> void (waitForProcess processHandle)
        pure (FaultHandle heal (object ["target" .= show target]))
    }

hogConnections :: PostgresEnv -> Int -> Fault
hogConnections postgres count =
  Fault
    { name = "postgres-hog-connections",
      target = "postgres",
      availability = pure Available,
      inject = do
        handles <- replicateM count (spawnSleeper postgres)
        pure (FaultHandle (mapM_ killProcess handles) (object ["connections" .= count]))
    }

withStatementTimeout :: Int -> ProcessSpec -> ProcessSpec
withStatementTimeout millis spec = spec {env = upsert "PGOPTIONS" ("-c statement_timeout=" <> show millis) spec.env}

withApplicationName :: Text -> ProcessSpec -> ProcessSpec
withApplicationName applicationName spec = spec {env = upsert "PGAPPNAME" (Text.unpack applicationName) spec.env}

crashPostmaster :: PostgresEnv -> CrashMode -> Fault
crashPostmaster postgres mode =
  Fault
    { name = "postgres-crash-postmaster",
      target = "postgres-postmaster",
      availability = pure (maybe (Unavailable "PostgreSQL environment has no server control") (const Available) postgres.control),
      inject = case postgres.control of
        Nothing -> ioError (userError "PostgreSQL server control unavailable")
        Just control -> do
          control.stopServer (case mode of FastShutdown -> StopFast; _ -> StopImmediate)
          pure (FaultHandle control.startServer (object ["mode" .= show mode]))
    }

matches :: BackendSelector -> Backend -> Bool
matches selector backend = case selector of
  ByApplicationName pattern -> like pattern backend.applicationName
  ByQueryPattern pattern -> like pattern backend.query
  ByPid pid -> backend.pid == pid
  AllOtherBackends -> backend.applicationName /= "kenshou-fault"

like :: Text -> Text -> Bool
like pattern value
  | "%" `Text.isSuffixOf` pattern = Text.dropEnd 1 pattern `Text.isPrefixOf` value
  | otherwise = value == pattern

lockSql :: LockTarget -> Text
lockSql (TableLock schema table) = "LOCK TABLE " <> quote schema <> "." <> quote table <> " IN ACCESS EXCLUSIVE MODE"
lockSql (RowLock schema table) = "SELECT 1 FROM " <> quote schema <> "." <> quote table <> " LIMIT 1 FOR UPDATE"
lockSql (AdvisoryLock key) = "SELECT pg_advisory_xact_lock(" <> Text.pack (show key) <> ")"

quote :: Text -> Text
quote value = "\"" <> Text.replace "\"" "\"\"" value <> "\""

spawnSleeper :: PostgresEnv -> IO ProcessHandle
spawnSleeper postgres = do
  (_, _, _, handle) <- createProcess (proc "psql" ["-d", Text.unpack postgres.connectionString, "-c", "SELECT pg_sleep(86400)"]) {std_out = NoStream, std_err = NoStream}
  pure handle

psql :: PostgresEnv -> Text -> IO Text
psql postgres sql = do
  (code, output, err) <- readProcessWithExitCode "psql" ["-d", Text.unpack postgres.connectionString, "-At", "-F", "\x1f", "-v", "ON_ERROR_STOP=1", "-c", Text.unpack sql] ""
  case code of ExitSuccess -> pure (Text.stripEnd (Text.pack output)); ExitFailure _ -> ioError (userError err)

voidPsql :: PostgresEnv -> Text -> IO ()
voidPsql postgres sql = psql postgres sql >> pure ()

readText :: (Read value) => Text -> Maybe value
readText value = case reads (Text.unpack value) of [(parsed, "")] -> Just parsed; _ -> Nothing

nonEmpty :: Text -> Maybe Text
nonEmpty "" = Nothing
nonEmpty value = Just value

upsert :: (Eq key) => key -> value -> [(key, value)] -> [(key, value)]
upsert key value pairs = (key, value) : filter ((/= key) . fst) pairs

void :: IO value -> IO ()
void action = action >> pure ()

killProcess :: ProcessHandle -> IO ()
killProcess handle = do
  pid <- getPid handle
  maybe (terminateProcess handle) (signalProcess sigKILL) pid
  void (waitForProcess handle)