kenshou-measure-0.1.0.0: src/Kenshou/Measure/Summary.hs
module Kenshou.Measure.Summary
( SummaryError (..),
SummaryWindow (..),
MeasurementSummary (..),
summarizeRunDir,
summarizeRunDirWithHealth,
)
where
import Control.Exception (SomeException, try)
import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.List (isSuffixOf)
import Data.Map.Strict (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (catMaybes, mapMaybe)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text
import Data.Word (Word64)
import Kenshou.Measure.Health
import Kenshou.Measure.Histogram qualified as Histogram
import Kenshou.Measure.Histogram.Codec (decodeHistogram)
import Kenshou.Measure.Metrics
import Kenshou.Measure.Samples
import Kenshou.Measure.Stats (arithmeticMean)
import System.Directory (doesDirectoryExist, doesFileExist, listDirectory)
import System.FilePath ((</>))
import Text.Read (readMaybe)
newtype SummaryError = SummaryError Text deriving stock (Eq, Show)
data SummaryWindow = SummaryWindow
{ steadyStartMonoNs :: Word64,
steadyEndMonoNs :: Word64,
steadySeconds :: Double
}
deriving stock (Eq, Show)
data MeasurementSummary = MeasurementSummary
{ grade :: Text,
gradeReasons :: [Text],
window :: SummaryWindow,
ops :: Map Text Value,
metrics :: Map Text Metric,
health :: [HealthObservation]
}
deriving stock (Eq, Show)
instance ToJSON SummaryWindow where
toJSON value = object ["steadyStartMonoNs" .= value.steadyStartMonoNs, "steadyEndMonoNs" .= value.steadyEndMonoNs, "steadySeconds" .= value.steadySeconds]
instance FromJSON SummaryWindow where
parseJSON = withObject "SummaryWindow" \value -> SummaryWindow <$> value .: "steadyStartMonoNs" <*> value .: "steadyEndMonoNs" <*> value .: "steadySeconds"
instance ToJSON MeasurementSummary where
toJSON summary =
object
[ "schema" .= ("kenshou.measurements/v1" :: Text),
"algorithm" .= object ["name" .= ("kenshou-summary" :: Text), "version" .= (1 :: Int)],
"grade" .= summary.grade,
"gradeReasons" .= summary.gradeReasons,
"window" .= summary.window,
"ops" .= summary.ops,
"metrics" .= summary.metrics,
"health" .= summary.health
]
instance FromJSON MeasurementSummary where
parseJSON = withObject "MeasurementSummary" \value -> do
schema <- value .: "schema"
if schema /= ("kenshou.measurements/v1" :: Text) then fail "unsupported measurement summary" else pure ()
MeasurementSummary <$> value .: "grade" <*> value .:? "gradeReasons" .!= [] <*> value .: "window" <*> value .: "ops" <*> value .: "metrics" <*> value .:? "health" .!= []
data Counts = Counts {successes :: Word64, failures :: Word64, units :: Word64}
data SampleMeta = SampleMeta {operation :: Text, latencyBasis :: Text, rawFull :: Bool, steadyCounts :: Counts, errorCauses :: Map Text Word64}
instance FromJSON Counts where
parseJSON = withObject "Counts" \value -> Counts <$> value .:? "successes" .!= 0 <*> value .:? "failures" .!= 0 <*> value .:? "units" .!= 0
instance FromJSON SampleMeta where
parseJSON = withObject "SampleMeta" \value -> do
phaseCounts <- value .: "phaseCounts"
steady <- withObject "phaseCounts" (\counts -> counts .:? "steady" .!= Counts 0 0 0) phaseCounts
raw <- value .: "rawSamples"
SampleMeta <$> value .: "operation" <*> value .:? "latencyBasis" .!= "intended-start" <*> pure (raw == String "full") <*> pure steady <*> value .:? "errorCauses" .!= Map.empty
summarizeRunDir :: FilePath -> IO (Either SummaryError MeasurementSummary)
summarizeRunDir = summarizeRunDirWithHealth defaultHealthConfig
summarizeRunDirWithHealth :: HealthConfig -> FilePath -> IO (Either SummaryError MeasurementSummary)
summarizeRunDirWithHealth healthConfig runDir = do
exists <- doesDirectoryExist runDir
if not exists
then pure (Left (SummaryError "run directory does not exist"))
else do
result <- try (summarize healthConfig runDir)
pure (either (Left . SummaryError . Text.pack . show) id (result :: Either SomeException (Either SummaryError MeasurementSummary)))
summarize :: HealthConfig -> FilePath -> IO (Either SummaryError MeasurementSummary)
summarize healthConfig runDir = do
windowResult <- readWindow (runDir </> "series" </> "load.csv")
case windowResult of
Left err -> pure (Left err)
Right window -> do
files <- listDirectory (runDir </> "samples")
operationResults <- traverse (summarizeOperation runDir window) [file | file <- files, ".meta.json" `isSuffixOf` file]
case sequence operationResults of
Left err -> pure (Left err)
Right operations -> do
let operationMap = Map.fromList [(name, value) | (name, value, _, _) <- operations]
operationMetrics = Map.unions [values | (_, _, values, _) <- operations]
totalOps = sum [metric.value | (name, metric) <- Map.toList operationMetrics, ".throughput" `Text.isSuffixOf` name] * window.steadySeconds
runtimeMetrics <- summarizeSeries runDir window totalOps
reasons <- gradeReasonsFromRun runDir (all (\(_, _, _, full) -> full) operations)
health <- evaluateHealth healthConfig runDir
let healthReasons = ["health:" <> item.gate | item <- health, item.severity /= Info]
allReasons = reasons <> healthReasons
pure (Right (MeasurementSummary (if null allReasons then "benchmark" else "exploratory") allReasons window operationMap (operationMetrics <> runtimeMetrics) health))
summarizeOperation :: FilePath -> SummaryWindow -> FilePath -> IO (Either SummaryError (Text, Value, Map Text Metric, Bool))
summarizeOperation runDir window metaFile = do
decoded <- eitherDecodeFileStrict' (runDir </> "samples" </> metaFile) :: IO (Either String SampleMeta)
case decoded of
Left message -> pure (Left (SummaryError (Text.pack message)))
Right meta -> do
let base = Text.unpack meta.operation
histogramPath = runDir </> "samples" </> base <> ".hist"
servicePath = runDir </> "samples" </> base <> ".service.hist"
rawPath = runDir </> "samples" </> base <> ".raw"
latency <- decodeHistogram <$> ByteString.readFile histogramPath
service <- decodeHistogram <$> ByteString.readFile servicePath
rawOk <-
if meta.rawFull
then do
rawExists <- doesFileExist rawPath
if not rawExists then pure False else (== 0) . (.ignoredBytes) <$> readSamples rawPath (const (pure ()))
else pure True
case (latency, service, rawOk) of
(_, _, False) -> pure (Left (SummaryError ("raw sample file is missing or truncated for " <> meta.operation)))
(Left message, _, _) -> pure (Left (SummaryError message))
(_, Left message, _) -> pure (Left (SummaryError message))
(Right latencyHistogram, Right serviceHistogram, _) -> do
let count = Histogram.totalCount latencyHistogram
failures = meta.steadyCounts.failures
throughput = fromIntegral count / window.steadySeconds
unitsPerSecond = fromIntegral meta.steadyCounts.units / window.steadySeconds
errorRate = if count + failures == 0 then 0 else fromIntegral failures / fromIntegral (count + failures)
quantile histogram q = fromIntegral (Histogram.valueAtQuantile histogram q) :: Double
latencyValues = metricSet ("op." <> meta.operation <> ".latency") latencyHistogram
serviceP99 = quantile serviceHistogram 0.99
metrics =
Map.fromList
[ ("op." <> meta.operation <> ".throughput", Metric throughput "ops/s"),
("op." <> meta.operation <> ".units-throughput", Metric unitsPerSecond "units/s"),
("op." <> meta.operation <> ".service.p99", Metric serviceP99 "ns"),
("op." <> meta.operation <> ".error-rate", Metric errorRate "ratio")
]
<> latencyValues
value =
object
[ "latencyBasis" .= meta.latencyBasis,
"coordinatedOmissionRisk" .= (meta.latencyBasis == "actual-start"),
"count" .= count,
"reportedSuccesses" .= meta.steadyCounts.successes,
"failures" .= failures,
"errors" .= meta.errorCauses,
"throughput" .= throughput,
"unitsPerSecond" .= unitsPerSecond,
"latencyNs" .= distribution latencyHistogram,
"serviceNs" .= distribution serviceHistogram
]
pure (Right (meta.operation, value, metrics, meta.rawFull))
where
distribution histogram =
object
[ "p50" .= Histogram.valueAtQuantile histogram 0.50,
"p90" .= Histogram.valueAtQuantile histogram 0.90,
"p99" .= Histogram.valueAtQuantile histogram 0.99,
"p999" .= Histogram.valueAtQuantile histogram 0.999,
"max" .= Histogram.maxValue histogram,
"mean" .= Histogram.meanValue histogram
]
metricSet prefix histogram =
Map.fromList
[ (prefix <> ".p50", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.50)) "ns"),
(prefix <> ".p90", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.90)) "ns"),
(prefix <> ".p95", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.95)) "ns"),
(prefix <> ".p99", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.99)) "ns"),
(prefix <> ".p999", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.999)) "ns"),
(prefix <> ".max", Metric (fromIntegral (Histogram.maxValue histogram)) "ns"),
(prefix <> ".mean", Metric (Histogram.meanValue histogram) "ns")
]
readWindow :: FilePath -> IO (Either SummaryError SummaryWindow)
readWindow path = do
rows <- readCsv path
let steady = [readWord (row !! 0) | row <- rows, length row > 2, row !! 2 == "steady"]
drain = [readWord (row !! 0) | row <- rows, length row > 2, row !! 2 == "drain"]
pure case (catMaybes steady, catMaybes drain) of
(start : _, end : _) | end > start -> Right (SummaryWindow start end (fromIntegral (end - start) / 1e9))
_ -> Left (SummaryError "load.csv has no complete steady window")
summarizeSeries :: FilePath -> SummaryWindow -> Double -> IO (Map Text Metric)
summarizeSeries runDir window totalOps = do
rts <- numericRows (runDir </> "series" </> "rts.csv") "phase" "steady"
proc <- numericRows (runDir </> "series" </> "proc.csv") "phase" "steady"
wal <- numericRows (runDir </> "series" </> "pg-wal.csv") "phase" "steady"
checkpointer <- numericRows (runDir </> "series" </> "pg-checkpointer.csv") "phase" "steady"
let delta column rows = (-) <$> (Map.lookup column =<< lastMay rows) <*> (Map.lookup column =<< headMay rows)
maximumOf column rows = maximumMaybe (mapMaybe (Map.lookup column) rows)
meanOf column rows = case mapMaybe (Map.lookup column) rows of [] -> Nothing; values -> Just (arithmeticMean values)
elapsed = fromIntegral (window.steadyEndMonoNs - window.steadyStartMonoNs)
capabilities = max 1 (maybe 1 id (Map.lookup "capabilities" =<< lastMay rts))
metrics =
[ metric "rts.alloc-rate" "bytes/s" ((/ window.steadySeconds) <$> delta "allocated_bytes" rts),
metric "rts.alloc-bytes-per-op" "bytes/op" ((/ max 1 totalOps) <$> delta "allocated_bytes" rts),
metric "rts.gc-productivity" "ratio" ((\gc total -> 1 - gc / max 1 total) <$> delta "gc_elapsed_ns" rts <*> delta "elapsed_ns" rts),
metric "rts.max-live-bytes" "bytes" (maximumOf "max_live_bytes" rts),
metric "rts.live-bytes-major-mean" "bytes" (meanOf "live_bytes_major_mean" rts),
metric "proc.cpu-utilisation" "ratio" ((/ (max 1 elapsed * capabilities)) <$> delta "cpu_total_ns" proc),
metric "proc.rss-max" "bytes" (maximumOf "rss_bytes" proc),
metric "pg.wal-bytes-per-op" "bytes/op" ((/ max 1 totalOps) <$> delta "wal_bytes" wal),
metric "pg.checkpoints-in-window" "count" ((+) <$> delta "num_timed" checkpointer <*> delta "num_requested" checkpointer)
]
pure (Map.fromList (catMaybes metrics))
where
metric name unit value = (name,) . (`Metric` unit) <$> value
gradeReasonsFromRun :: FilePath -> Bool -> IO [Text]
gradeReasonsFromRun runDir rawFull = do
decoded <- eitherDecodeFileStrict' (runDir </> "run-spec.json") :: IO (Either String Value)
let durability = case decoded of
Right (Object root) -> do
Object dimensions <- KeyMap.lookup "dimensions" root
String value <- KeyMap.lookup "pg.durability" dimensions
pure value
_ -> Nothing
pure (["raw samples are not retained in full" | not rawFull] <> ["pg.durability=fsync-off" | durability == Just "fsync-off"])
readCsv :: FilePath -> IO [[Text]]
readCsv path = do
exists <- doesFileExist path
if not exists then pure [] else fmap (Text.splitOn ",") . drop 1 . Text.lines <$> Text.readFile path
numericRows :: FilePath -> Text -> Text -> IO [Map Text Double]
numericRows path filterColumn filterValue = do
exists <- doesFileExist path
if not exists
then pure []
else do
linesValue <- Text.lines <$> Text.readFile path
case linesValue of
[] -> pure []
header : rows -> do
let columns = Text.splitOn "," header
decoded = fmap (Map.fromList . zip columns . fmap readDouble . Text.splitOn ",") rows
pure [Map.mapMaybe id row | (raw, row) <- zip rows decoded, field columns filterColumn raw == Just filterValue]
field :: [Text] -> Text -> Text -> Maybe Text
field columns wanted row = do
index <- lookupIndex wanted columns
atMay (Text.splitOn "," row) index
lookupIndex :: (Eq value) => value -> [value] -> Maybe Int
lookupIndex wanted = go 0
where
go _ [] = Nothing; go index (value : rest) = if value == wanted then Just index else go (index + 1) rest
readDouble :: Text -> Maybe Double
readDouble value | Text.null value = Nothing
readDouble value = readMaybe (Text.unpack value)
readWord :: Text -> Maybe Word64
readWord = readMaybe . Text.unpack
headMay, lastMay :: [value] -> Maybe value
headMay [] = Nothing; headMay (value : _) = Just value
lastMay [] = Nothing; lastMay values = Just (last values)
maximumMaybe :: (Ord value) => [value] -> Maybe value
maximumMaybe [] = Nothing; maximumMaybe values = Just (maximum values)
atMay :: [value] -> Int -> Maybe value
atMay values index = case drop index values of value : _ -> Just value; [] -> Nothing