packages feed

kenshou-measure-0.1.0.0: src/Kenshou/Measure/Sampler/Postgres.hs

module Kenshou.Measure.Sampler.Postgres
  ( PgStatementsMode (..),
    PgSamplerConfig (..),
    PostgresSampler,
    openPostgresSampler,
    samplePostgres,
    closePostgresSampler,
    postgresHasStatements,
    postgresPeriodic,
    postgresArtifactNames,
  )
where

import Control.Exception (finally)
import Control.Monad (forM_, when)
import Data.Text (Text)
import Data.Text qualified as Text
import Hasql.Connection qualified as Connection
import Hasql.Connection.Settings qualified as Settings
import Hasql.Decoders qualified as Decoders
import Hasql.Encoders qualified as Encoders
import Hasql.Session qualified as Session
import Hasql.Statement qualified as Statement
import Kenshou.Measure.Sampler.Csv

data PgStatementsMode = PgStatementsOff | PgStatementsSnapshots | PgStatementsPeriodic
  deriving stock (Eq, Show)

data PgSamplerConfig = PgSamplerConfig
  { connectionString :: Text,
    relations :: [Text],
    statements :: PgStatementsMode,
    serverVersionNum :: Int
  }
  deriving stock (Eq, Show)

data PostgresSampler = PostgresSampler
  { connection :: Connection.Connection,
    activity :: CsvWriter,
    checkpointer :: CsvWriter,
    wal :: CsvWriter,
    database :: CsvWriter,
    relations :: CsvWriter,
    statements :: Maybe CsvWriter,
    watchedRelations :: [Text],
    versionNumber :: Int,
    statementsMode :: PgStatementsMode,
    logLine :: Text -> IO ()
  }

postgresArtifactNames :: Bool -> [FilePath]
postgresArtifactNames hasStatements =
  ["pg-activity.csv", "pg-checkpointer.csv", "pg-wal.csv", "pg-database.csv", "pg-relations.csv"]
    <> ["pg-statements.csv" | hasStatements]

openPostgresSampler :: FilePath -> PgSamplerConfig -> (Text -> IO ()) -> IO (Either Text PostgresSampler)
openPostgresSampler seriesDir config logger = do
  acquired <- Connection.acquire (Settings.connectionString config.connectionString <> Settings.applicationName "kenshou-sampler")
  case acquired of
    Left err -> pure (Left (Text.pack (show err)))
    Right connection -> do
      activity <- openCsv (seriesDir <> "/pg-activity.csv") (base <> ["application_name", "state", "wait_event_type", "wait_event", "connections"])
      checkpointer <- openCsv (seriesDir <> "/pg-checkpointer.csv") (base <> ["num_timed", "num_requested", "write_time_ms", "sync_time_ms", "buffers_written", "checkpoint_lsn", "checkpoint_time_unix", "num_done"])
      wal <- openCsv (seriesDir <> "/pg-wal.csv") (base <> ["current_wal_bytes", "wal_records", "wal_fpi", "wal_bytes", "wal_buffers_full"])
      database <- openCsv (seriesDir <> "/pg-database.csv") (base <> ["xact_commit", "xact_rollback", "blks_read", "blks_hit", "tup_inserted", "tup_updated", "tup_deleted", "deadlocks", "temp_bytes"])
      relations <- openCsv (seriesDir <> "/pg-relations.csv") (base <> ["relation", "relation_bytes", "indexes_bytes", "total_bytes", "n_live_tup", "n_dead_tup", "n_tup_ins", "n_tup_upd", "n_tup_del", "autovacuum_count"])
      statements <-
        if config.statements == PgStatementsOff
          then pure Nothing
          else do
            created <- Connection.use connection (Session.statement () createStatements)
            probed <- case created of Left err -> pure (Left err); Right () -> Connection.use connection (Session.statement () probeStatements)
            case probed of
              Left err -> logger ("pg_stat_statements unavailable: " <> Text.pack (show err)) >> pure Nothing
              Right () -> Just <$> openCsv (seriesDir <> "/pg-statements.csv") (base <> ["queryid", "calls", "total_exec_time_ms", "rows", "shared_blks_hit", "shared_blks_read", "wal_bytes"])
      pure (Right PostgresSampler {connection, activity, checkpointer, wal, database, relations, statements, watchedRelations = config.relations, versionNumber = config.serverVersionNum, statementsMode = config.statements, logLine = logger})
  where
    base = ["t_mono_ns", "t_wall_ms", "phase"]

samplePostgres :: PostgresSampler -> [Text] -> Bool -> IO ()
samplePostgres sampler prefix includeStatements = do
  writeRows sampler sampler.activity prefix activityQuery
  writeRows sampler sampler.checkpointer prefix (if sampler.versionNumber >= 180000 then checkpointer18Query else checkpointer17Query)
  writeRows sampler sampler.wal prefix walQuery
  writeRows sampler sampler.database prefix databaseQuery
  forM_ sampler.watchedRelations \relation -> writeParamRows sampler sampler.relations prefix relation relationQuery
  when includeStatements $ forM_ sampler.statements \writer -> writeRows sampler writer prefix statementsQuery

closePostgresSampler :: PostgresSampler -> IO ()
closePostgresSampler sampler =
  let writers = [sampler.activity, sampler.checkpointer, sampler.wal, sampler.database, sampler.relations] <> maybe [] pure sampler.statements
   in mapM_ closeCsv writers `finally` Connection.release sampler.connection

postgresHasStatements :: PostgresSampler -> Bool
postgresHasStatements = maybe False (const True) . (.statements)

postgresPeriodic :: PostgresSampler -> Bool
postgresPeriodic sampler = sampler.statementsMode == PgStatementsPeriodic

writeRows :: PostgresSampler -> CsvWriter -> [Text] -> Statement.Statement () [Text] -> IO ()
writeRows sampler writer prefix statement = do
  result <- Connection.use sampler.connection (Session.statement () statement)
  case result of
    Left err -> sampler.logLine ("PostgreSQL sampler query failed: " <> Text.pack (show err))
    Right rows -> forM_ rows (appendCsv writer . (prefix <>) . Text.splitOn separator)

writeParamRows :: PostgresSampler -> CsvWriter -> [Text] -> Text -> Statement.Statement Text [Text] -> IO ()
writeParamRows sampler writer prefix parameter statement = do
  result <- Connection.use sampler.connection (Session.statement parameter statement)
  case result of
    Left err -> sampler.logLine ("PostgreSQL relation sampler query failed: " <> Text.pack (show err))
    Right rows -> forM_ rows (appendCsv writer . (prefix <>) . Text.splitOn separator)

separator :: Text
separator = Text.singleton '\x1f'

textRows :: Text -> Statement.Statement () [Text]
textRows sql = Statement.unpreparable sql Encoders.noParams (Decoders.rowList (Decoders.column (Decoders.nonNullable Decoders.text)))

createStatements, probeStatements :: Statement.Statement () ()
createStatements = Statement.unpreparable "CREATE EXTENSION IF NOT EXISTS pg_stat_statements" Encoders.noParams Decoders.noResult
probeStatements = Statement.unpreparable "SELECT 1 FROM pg_stat_statements(false) LIMIT 0" Encoders.noParams Decoders.noResult

activityQuery, checkpointer17Query, checkpointer18Query, walQuery, databaseQuery, statementsQuery :: Statement.Statement () [Text]
activityQuery = textRows "SELECT concat_ws(chr(31), coalesce(application_name,''), coalesce(state,''), coalesce(wait_event_type,''), coalesce(wait_event,''), count(*)::text) FROM pg_stat_activity WHERE backend_type='client backend' AND application_name <> 'kenshou-sampler' GROUP BY application_name,state,wait_event_type,wait_event ORDER BY application_name,state,wait_event_type,wait_event"
checkpointer17Query = textRows "SELECT concat_ws(chr(31), c.num_timed::text, c.num_requested::text, c.write_time::text, c.sync_time::text, c.buffers_written::text, (pg_control_checkpoint()).checkpoint_lsn::text, extract(epoch FROM (pg_control_checkpoint()).checkpoint_time)::bigint::text, '') FROM pg_stat_checkpointer c"
checkpointer18Query = textRows "SELECT concat_ws(chr(31), c.num_timed::text, c.num_requested::text, c.write_time::text, c.sync_time::text, c.buffers_written::text, (pg_control_checkpoint()).checkpoint_lsn::text, extract(epoch FROM (pg_control_checkpoint()).checkpoint_time)::bigint::text, c.num_done::text) FROM pg_stat_checkpointer c"
walQuery = textRows "SELECT concat_ws(chr(31), pg_wal_lsn_diff(pg_current_wal_lsn(),'0/0')::bigint::text, w.wal_records::text, w.wal_fpi::text, w.wal_bytes::bigint::text, w.wal_buffers_full::text) FROM pg_stat_wal w"
databaseQuery = textRows "SELECT concat_ws(chr(31), xact_commit::text, xact_rollback::text, blks_read::text, blks_hit::text, tup_inserted::text, tup_updated::text, tup_deleted::text, deadlocks::text, temp_bytes::text) FROM pg_stat_database WHERE datname=current_database()"
statementsQuery = textRows "SELECT concat_ws(chr(31), queryid::text, calls::text, total_exec_time::text, rows::text, shared_blks_hit::text, shared_blks_read::text, wal_bytes::bigint::text) FROM pg_stat_statements(false) WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database()) ORDER BY queryid"

relationQuery :: Statement.Statement Text [Text]
relationQuery =
  Statement.unpreparable
    "SELECT concat_ws(chr(31), $1::text, pg_relation_size($1::regclass)::text, pg_indexes_size($1::regclass)::text, pg_total_relation_size($1::regclass)::text, s.n_live_tup::text, s.n_dead_tup::text, s.n_tup_ins::text, s.n_tup_upd::text, s.n_tup_del::text, s.autovacuum_count::text) FROM pg_stat_all_tables s WHERE s.relid=$1::regclass"
    (Encoders.param (Encoders.nonNullable Encoders.text))
    (Decoders.rowList (Decoders.column (Decoders.nonNullable Decoders.text)))