packages feed

kenshou-measure (empty) → 0.1.0.0

raw patch · 57 files changed

+5199/−0 lines, 57 filesdep +aesondep +asyncdep +base

Dependencies added: aeson, async, base, bytestring, containers, directory, filepath, hasql, hedgehog, hspec, hspec-hedgehog, kenshou-core, kenshou-measure, optparse-applicative, primitive, process, splitmix, stm, temporary, text, time, unix

Files

+ CHANGELOG.md view
@@ -0,0 +1,12 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of the measurement toolkit: load models, latency recording, samplers, statistics, health gates, and paired comparison.+- `poolMethodologyFor` reads a layer's recommended pool size from its bundle instead of a built-in constant.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+   list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+   this list of conditions and the following disclaimer in the documentation+   and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+   may be used to endorse or promote products derived from this software without+   specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ cbits/kenshou_proc_darwin.c view
@@ -0,0 +1,12 @@+#include <libproc.h>+#include <stdint.h>+#include <unistd.h>++int kenshou_proc_taskinfo(uint64_t *resident_size, int *thread_count) {+  struct proc_taskinfo info;+  int bytes = proc_pidinfo(getpid(), PROC_PIDTASKINFO, 0, &info, sizeof(info));+  if (bytes != sizeof(info)) return 0;+  *resident_size = info.pti_resident_size;+  *thread_count = info.pti_threadnum;+  return 1;+}
+ kenshou-measure.cabal view
@@ -0,0 +1,143 @@+cabal-version: 3.0+name: kenshou-measure+version: 0.1.0.0+synopsis: Measurement and comparison toolkit for Kenshou scenarios+description: Load generation, latency recording, sampling, summaries, health gates, and paired comparisons.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+  test/fixtures/health-notice.json+  test/fixtures/proc/host-stat.txt+  test/fixtures/proc/loadavg.txt+  test/fixtures/proc/meminfo.txt+  test/fixtures/proc/stat.txt+  test/fixtures/proc/status.txt+  test/golden/histogram-v1.hex++common warnings+  ghc-options:+    -Wall+    -Wcompat+    -Widentities+    -Wincomplete-record-updates+    -Wincomplete-uni-patterns+    -Wpartial-fields+    -Wredundant-constraints++  default-language: GHC2024+  default-extensions:+    BlockArguments+    DuplicateRecordFields+    ImportQualifiedPost+    NoFieldSelectors+    OverloadedLabels+    OverloadedRecordDot+    OverloadedStrings++library+  import: warnings+  hs-source-dirs: src+  exposed-modules:+    Kenshou.Measure+    Kenshou.Measure.Cli+    Kenshou.Measure.Clock+    Kenshou.Measure.Compare+    Kenshou.Measure.Compare.Compatibility+    Kenshou.Measure.Compare.Ordering+    Kenshou.Measure.Compare.Policy+    Kenshou.Measure.Health+    Kenshou.Measure.Histogram+    Kenshou.Measure.Histogram.Codec+    Kenshou.Measure.Knobs+    Kenshou.Measure.Load+    Kenshou.Measure.Load.Arrival+    Kenshou.Measure.Load.Closed+    Kenshou.Measure.Load.Open+    Kenshou.Measure.Load.Series+    Kenshou.Measure.Load.Types+    Kenshou.Measure.Methodology+    Kenshou.Measure.Metrics+    Kenshou.Measure.Perf+    Kenshou.Measure.Phase+    Kenshou.Measure.Recorder+    Kenshou.Measure.Sampler+    Kenshou.Measure.Sampler.Csv+    Kenshou.Measure.Sampler.Host+    Kenshou.Measure.Sampler.Postgres+    Kenshou.Measure.Sampler.Process+    Kenshou.Measure.Sampler.Rts+    Kenshou.Measure.Samples+    Kenshou.Measure.Selftest+    Kenshou.Measure.Selftest.PgInsert+    Kenshou.Measure.Selftest.RegressionInjected+    Kenshou.Measure.Selftest.SleepService+    Kenshou.Measure.Session+    Kenshou.Measure.Stats+    Kenshou.Measure.Summary++  build-depends:+    aeson >=2.2 && <2.3,+    async >=2.2 && <2.3,+    base >=4.21 && <4.22,+    bytestring >=0.12 && <0.13,+    containers >=0.7 && <0.8,+    directory >=1.3 && <1.4,+    filepath >=1.5 && <1.6,+    hasql >=1.10 && <1.11,+    kenshou-core ^>=0.1.0.0,+    optparse-applicative >=0.19 && <0.20,+    primitive >=0.9 && <0.10,+    process >=1.6 && <1.7,+    splitmix >=0.1 && <0.2,+    stm >=2.5 && <2.6,+    text >=2.1 && <2.2,+    time >=1.14 && <1.15,+    unix >=2.8 && <2.9,++  if os(osx)+    c-sources: cbits/kenshou_proc_darwin.c++test-suite kenshou-measure-test+  import: warnings+  type: exitcode-stdio-1.0+  hs-source-dirs: test+  main-is: Main.hs+  other-modules:+    Kenshou.Measure.ClockSpec+    Kenshou.Measure.CompareSpec+    Kenshou.Measure.HistogramSpec+    Kenshou.Measure.LoadSpec+    Kenshou.Measure.MethodologySpec+    Kenshou.Measure.PerfSpec+    Kenshou.Measure.RecorderSpec+    Kenshou.Measure.SamplerSpec+    Kenshou.Measure.SamplesSpec++  build-depends:+    aeson,+    base >=4.21 && <4.22,+    bytestring,+    containers,+    directory >=1.3 && <1.4,+    filepath,+    hedgehog >=1.4 && <1.8,+    hspec >=2.11 && <2.12,+    hspec-hedgehog >=0.0 && <0.4,+    kenshou-core ^>=0.1.0.0,+    kenshou-measure,+    splitmix >=0.1 && <0.2,+    temporary,+    text,+    time >=1.14 && <1.15,++source-repository head+  type: git+  location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Measure.hs view
@@ -0,0 +1,19 @@+-- | Measurement, load generation, sampling, and comparison support for Kenshou scenarios.+module Kenshou.Measure+  ( module Kenshou.Measure.Clock,+    module Kenshou.Measure.Histogram,+    module Kenshou.Measure.Knobs,+    module Kenshou.Measure.Load,+    module Kenshou.Measure.Phase,+    module Kenshou.Measure.Recorder,+    module Kenshou.Measure.Session,+  )+where++import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session
+ src/Kenshou/Measure/Cli.hs view
@@ -0,0 +1,129 @@+module Kenshou.Measure.Cli+  ( compareCommand,+    summarizeCommand,+  )+where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Kenshou.Core.Cli+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility+import Kenshou.Measure.Compare.Policy+import Kenshou.Measure.Metrics+import Kenshou.Measure.Stats+import Kenshou.Measure.Summary+import Options.Applicative+import System.Exit (ExitCode (..))+import System.IO (stderr)++data SummarizeOptions = SummarizeOptions {runDir :: FilePath, json :: Bool, verify :: Bool}++data CompareOptions = CompareOptions+  { baselines :: [FilePath],+    candidates :: [FilePath],+    policySource :: InputSource,+    vary :: [Text],+    output :: Maybe FilePath,+    json :: Bool+  }++summarizeCommand :: CliCommand+summarizeCommand = CliCommand "summarize" "Recompute and verify run measurements" Analysis False (runSummarize <$> summarizeParser)++compareCommand :: CliCommand+compareCommand = CliCommand "compare" "Compare paired baseline and candidate runs" Analysis False (runCompare <$> compareParser)++summarizeParser :: Parser SummarizeOptions+summarizeParser =+  SummarizeOptions+    <$> strArgument (metavar "RUN_DIR" <> help "Completed run directory")+    <*> switch (long "json" <> help "Write JSON to standard output")+    <*> switch (long "verify" <> help "Compare recomputed figures with run-result.json")++compareParser :: Parser CompareOptions+compareParser =+  CompareOptions+    <$> some (strOption (long "baseline" <> metavar "DIR" <> help "Baseline run directory; repeat once per pair"))+    <*> some (strOption (long "candidate" <> metavar "DIR" <> help "Candidate run directory; repeat once per pair"))+    <*> (parseInputSource <$> strOption (long "policy" <> metavar "FILE" <> help "Comparison policy file, or - for stdin"))+    <*> many (Text.pack <$> strOption (long "vary" <> metavar "AXIS" <> help "Allowed varying axis: cohort, dim:NAME, or knob:NAME"))+    <*> optional (strOption (long "out" <> metavar "FILE" <> help "Write the comparison JSON document"))+    <*> switch (long "json" <> help "Write JSON to standard output")++runSummarize :: SummarizeOptions -> CliEnv -> IO ExitCode+runSummarize options _ = do+  result <- summarizeRunDir options.runDir+  case result of+    Left (SummaryError message) -> Text.hPutStrLn stderr ("kenshou summarize: " <> message) >> pure (ExitFailure 4)+    Right summary -> do+      let document = toJSON summary+      if options.json then LazyByteString.putStr (encode document <> "\n") else renderSummary summary+      if not options.verify then pure ExitSuccess else verifyStored options.runDir document++verifyStored :: FilePath -> Value -> IO ExitCode+verifyStored runDir recomputed = do+  decoded <- eitherDecodeFileStrict' (runDir <> "/run-result.json") :: IO (Either String Value)+  case decoded of+    Left message -> Text.hPutStrLn stderr ("kenshou summarize: " <> Text.pack message) >> pure (ExitFailure 4)+    Right result -> case storedMeasurements result of+      Nothing -> Text.hPutStrLn stderr "kenshou summarize: run-result.json has no stored measurements" >> pure (ExitFailure 1)+      Just stored | stored == recomputed -> pure ExitSuccess+      Just _ -> Text.hPutStrLn stderr "kenshou summarize: stored measurements differ from recomputed measurements" >> pure (ExitFailure 1)++storedMeasurements :: Value -> Maybe Value+storedMeasurements (Object root) = do+  Object summaries <- KeyMap.lookup "summaries" root+  Object measurements <- KeyMap.lookup "measurements" summaries+  KeyMap.lookup "measurements" measurements+storedMeasurements _ = Nothing++runCompare :: CompareOptions -> CliEnv -> IO ExitCode+runCompare options _ = do+  policyBytes <- readInputSource options.policySource+  case decodePolicy policyBytes of+    Left message -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)+    Right policy -> case traverse parseVaryingAxis (if null options.vary then ["cohort"] else options.vary) of+      Left message -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)+      Right [] -> pure (ExitFailure 2)+      Right (axis : axes) -> do+        result <- compareRuns policy (axis :| axes) options.baselines options.candidates+        case result of+          Left (CompareError message) -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)+          Right comparison -> do+            let bytes = encode comparison+            maybe (pure ()) (\path -> LazyByteString.writeFile path (bytes <> "\n")) options.output+            if options.json then LazyByteString.putStr (bytes <> "\n") else renderComparison comparison+            pure case verdictExitCode comparison.verdict of 0 -> ExitSuccess; code -> ExitFailure code++renderSummary :: MeasurementSummary -> IO ()+renderSummary summary = do+  Text.putStrLn ("grade: " <> summary.grade)+  mapM_ (\(name, metric) -> Text.putStrLn (name <> ": " <> Text.pack (show metric.value) <> " " <> metric.unit)) (toList summary.metrics)++renderComparison :: Comparison -> IO ()+renderComparison comparison = do+  Text.putStrLn ("comparison " <> comparison.comparisonId <> "  verdict: " <> verdictName comparison.verdict)+  mapM_ (\(_, metric) -> Text.putStrLn (metric.metric <> "  " <> statusName metric.status <> "  ratio=" <> Text.pack (show metric.ratio.estimate))) (toList comparison.metrics)+  mapM_ (Text.putStrLn . ("reason: " <>)) comparison.reasons++verdictName :: Verdict -> Text+verdictName VerdictPass = "pass"+verdictName VerdictRegression = "regression"+verdictName VerdictInconclusive = "inconclusive"+verdictName VerdictInfrastructureFailure = "infrastructure-failure"++statusName :: MetricStatus -> Text+statusName MetricPass = "pass"+statusName MetricRegression = "regression"+statusName MetricInconclusive = "inconclusive"++toList :: Map key value -> [(key, value)]+toList = Map.toList
+ src/Kenshou/Measure/Clock.hs view
@@ -0,0 +1,42 @@+module Kenshou.Measure.Clock+  ( Nanos (..),+    Origin (..),+    nowNs,+    captureOrigin,+    sleepUntilNs,+  )+where++import Control.Concurrent (threadDelay)+import Data.Int (Int64)+import Data.Time.Clock.POSIX (getPOSIXTime)+import Data.Word (Word64)+import GHC.Clock (getMonotonicTimeNSec)++newtype Nanos = Nanos Word64+  deriving stock (Eq, Ord, Show)++data Origin = Origin+  { monoNs :: !Word64,+    wallUnixNs :: !Int64+  }+  deriving stock (Eq, Show)++nowNs :: IO Word64+nowNs = getMonotonicTimeNSec++captureOrigin :: IO Origin+captureOrigin = do+  monotonic <- nowNs+  wall <- getPOSIXTime+  pure (Origin monotonic (floor (wall * 1_000_000_000)))++sleepUntilNs :: Word64 -> IO ()+sleepUntilNs deadline = do+  current <- nowNs+  if current >= deadline+    then pure ()+    else do+      let remainingMicros = max 1 ((deadline - current) `div` 1_000)+      threadDelay (fromIntegral (min remainingMicros 1_000_000))+      sleepUntilNs deadline
+ src/Kenshou/Measure/Compare.hs view
@@ -0,0 +1,323 @@+module Kenshou.Measure.Compare+  ( Verdict (..),+    MetricStatus (..),+    MetricComparison (..),+    Comparison (..),+    CompareError (..),+    compareRuns,+    compareMetricPairs,+    decideVerdict,+    verdictExitCode,+  )+where++import Data.Aeson+import Data.Aeson.Key (fromText)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List (find, nub)+import Data.List.NonEmpty (NonEmpty)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, fromMaybe, mapMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import Data.Time.Format (defaultTimeLocale, parseTimeM)+import Kenshou.Core.Fingerprint (HostFingerprint (..), collectHostFingerprint)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Measure.Compare.Compatibility+import Kenshou.Measure.Compare.Ordering+import Kenshou.Measure.Compare.Policy+import Kenshou.Measure.Health+import Kenshou.Measure.Metrics+import Kenshou.Measure.Stats+import Kenshou.Measure.Summary++data Verdict = VerdictPass | VerdictRegression | VerdictInconclusive | VerdictInfrastructureFailure deriving stock (Eq, Ord, Show)++data MetricStatus = MetricPass | MetricRegression | MetricInconclusive deriving stock (Eq, Ord, Show)++data MetricComparison = MetricComparison+  { metric :: Text,+    unit :: Text,+    pairs :: [(Double, Double)],+    ratio :: Interval,+    delta :: Interval,+    relativeLimit :: Double,+    absoluteFloor :: Double,+    baselineCv :: Double,+    status :: MetricStatus,+    message :: Text+  }+  deriving stock (Eq, Show)++data Comparison = Comparison+  { comparisonId :: Text,+    startedAt :: UTCTime,+    finishedAt :: UTCTime,+    harnessRevision :: Maybe Text,+    harnessDirty :: Maybe Bool,+    design :: Text,+    baselineRuns :: [Text],+    candidateRuns :: [Text],+    policy :: Policy,+    variedFactors :: [VaryingAxis],+    pairCount :: Int,+    metrics :: Map Text MetricComparison,+    reasons :: [Text],+    verdict :: Verdict+  }+  deriving stock (Eq, Show)++newtype CompareError = CompareError Text deriving stock (Eq, Show)++instance ToJSON Verdict where toJSON = String . verdictText++instance ToJSON MetricStatus where toJSON = String . statusText++instance ToJSON MetricComparison where+  toJSON value =+    object+      [ "metric" .= value.metric,+        "unit" .= value.unit,+        "pairs" .= [object ["baseline" .= baseline, "candidate" .= candidate] | (baseline, candidate) <- value.pairs],+        "ratio" .= value.ratio,+        "delta" .= value.delta,+        "relativeLimit" .= value.relativeLimit,+        "absoluteFloor" .= value.absoluteFloor,+        "baselineCv" .= value.baselineCv,+        "status" .= value.status,+        "message" .= value.message+      ]++instance ToJSON Comparison where+  toJSON value =+    object+      [ "schema" .= ("kenshou.comparison/v1" :: Text),+        "comparisonId" .= value.comparisonId,+        "startedAt" .= value.startedAt,+        "finishedAt" .= value.finishedAt,+        "harnessRevision" .= value.harnessRevision,+        "harnessDirty" .= value.harnessDirty,+        "design" .= value.design,+        "baselineRuns" .= value.baselineRuns,+        "candidateRuns" .= value.candidateRuns,+        "algorithm" .= object ["name" .= ("paired-bootstrap-t-envelope" :: Text), "version" .= (1 :: Int), "generator" .= ("splitmix" :: Text), "iterations" .= value.policy.bootstrapIterations, "seed" .= value.policy.resamplingSeed, "confidenceLevel" .= value.policy.confidenceLevel],+        "policy" .= value.policy,+        "variedFactors" .= value.variedFactors,+        "pairCount" .= value.pairCount,+        "metrics" .= value.metrics,+        "reasons" .= value.reasons,+        "verdict" .= value.verdict,+        "exitCode" .= verdictExitCode value.verdict+      ]++data RunData = RunData+  { runId :: Text,+    result :: Value,+    inputs :: Value,+    fingerprint :: Value,+    summary :: MeasurementSummary,+    outcome :: Text,+    startedAt :: Maybe UTCTime+  }++compareRuns :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison)+compareRuns policy axes baselineDirs candidateDirs+  | null baselineDirs || length baselineDirs /= length candidateDirs = pure (Left (CompareError "baseline and candidate runs must form the same non-zero number of pairs"))+  | otherwise = do+      baselineResults <- traverse loadRun baselineDirs+      candidateResults <- traverse loadRun candidateDirs+      case (sequence baselineResults, sequence candidateResults) of+        (Left err, _) -> pure (Left err)+        (_, Left err) -> pure (Left err)+        (Right baselines, Right candidates) -> compareLoaded baselines candidates+  where+    compareLoaded baselines candidates = case compatibilityProblem axes (baselines <> candidates) of+      Just message -> pure (Left (CompareError message))+      Nothing -> case varyingProblem axes baselines candidates of+        Just message -> pure (Left (CompareError message))+        Nothing -> do+          startedAt <- getCurrentTime+          identifier <- renderRunId <$> newRunId+          fingerprint <- collectHostFingerprint+          let referenceFingerprint = maybe Null (.fingerprint) (headMay baselines)+              environmentChanged = any ((/= referenceFingerprint) . (.fingerprint)) (tailSafe baselines <> candidates)+              badOutcomes = any ((`elem` ["errored", "infrastructure-failure"]) . (.outcome)) (baselines <> candidates)+              hardHealth = any (hasSeverity Hard . (.summary)) (baselines <> candidates)+              softHealth = any (hasSeverity Soft . (.summary)) (baselines <> candidates)+              checkpointAsymmetry = any (uncurry (checkpointDiffers policy.maxCheckpointAsymmetry)) (zip baselines candidates)+              metricComparisons = compareMetrics policy baselines candidates+              interleaving = validateInterleaving (catMaybes (concat (zipWith trialRows [0 ..] (zip baselines candidates))))+              lowGrade = any ((/= policy.requireGrade) . (.grade) . (.summary)) (baselines <> candidates)+              reasons =+                ["machine profiles differ" | environmentChanged]+                  <> ["an input run has an infrastructure outcome" | badOutcomes]+                  <> ["an input run has a hard health observation" | hardHealth]+                  <> ["fewer than the required number of pairs" | length baselines < policy.minimumPairs]+                  <> ["runs are not interleaved" | policy.requireInterleaving && either (const True) (const False) interleaving]+                  <> ["an input run is below the required evidence grade" | lowGrade]+                  <> ["an input run has a soft health observation" | softHealth]+                  <> ["checkpoint overlap differs too much within a pair" | checkpointAsymmetry]+              statuses = fmap (.status) (Map.elems metricComparisons)+              verdict = decideVerdict (environmentChanged || badOutcomes || hardHealth) reasons statuses+          finishedAt <- getCurrentTime+          pure (Right (Comparison identifier startedAt finishedAt fingerprint.revision fingerprint.dirty (comparisonDesign baselines candidates) (map (.runId) baselines) (map (.runId) candidates) policy (toList axes) (length baselines) metricComparisons reasons verdict))++comparisonDesign :: [RunData] -> [RunData] -> Text+comparisonDesign baselines candidates = case traverse firstArm (zip baselines candidates) of+  Just arms+    | arms == take (length arms) (cycle [Baseline, Candidate]) -> "abba"+    | arms == take (length arms) (cycle [Candidate, Baseline]) -> "baab"+  _ -> "sequential"+  where+    firstArm (baseline, candidate) = do+      baselineStart <- baseline.startedAt+      candidateStart <- candidate.startedAt+      if baselineStart < candidateStart then Just Baseline else if candidateStart < baselineStart then Just Candidate else Nothing++loadRun :: FilePath -> IO (Either CompareError RunData)+loadRun directory = do+  decoded <- eitherDecodeFileStrict' (directory <> "/run-result.json") :: IO (Either String Value)+  case decoded of+    Left message -> pure (Left (CompareError (Text.pack message)))+    Right result -> do+      summarized <- case jsonAt ["summaries", "measurements", "measurements"] result of+        Just value -> pure case fromJSON value of Error message -> Left (SummaryError (Text.pack message)); Success summary -> Right summary+        Nothing -> summarizeRunDir directory+      pure do+        summary <- either (Left . CompareError . (\(SummaryError message) -> message)) Right summarized+        inputs <- maybe (Left (CompareError "run result has no compatibility inputs")) Right (compatibilityInputs result)+        fingerprint <- maybe (Left (CompareError "run result has no environment fingerprint")) Right (environmentFingerprint result)+        runId <- maybe (Left (CompareError "run result has no run ID")) Right (textAt ["runId"] result)+        let outcome = fromMaybe "unknown" (textAt ["outcome"] result)+            started = textAt ["timings", "startedAt"] result >>= parseUtc+        Right RunData {runId, result, inputs, fingerprint, summary, outcome, startedAt = started}++compatibilityProblem :: NonEmpty VaryingAxis -> [RunData] -> Maybe Text+compatibilityProblem _ [] = Just "no runs"+compatibilityProblem axes (first : rest) = case [message | run <- rest, Left message <- [compatibleExcept axes first.inputs run.inputs]] of+  message : _ -> Just message+  [] -> Nothing++varyingProblem :: NonEmpty VaryingAxis -> [RunData] -> [RunData] -> Maybe Text+varyingProblem axes baselines candidates = findProblem (toList axes)+  where+    findProblem [] = Nothing+    findProblem (VaryControl : rest) =+      let baselineValues = nub (fmap (.inputs) baselines)+          candidateValues = nub (fmap (.inputs) candidates)+       in if length baselineValues /= 1 || baselineValues /= candidateValues+            then Just "control arms must have identical compatibility inputs"+            else findProblem rest+    findProblem (axis : rest) =+      let baselineValues = nub (fmap (varyingValue axis . (.inputs)) baselines)+          candidateValues = nub (fmap (varyingValue axis . (.inputs)) candidates)+       in if length baselineValues /= 1 || length candidateValues /= 1 || baselineValues == candidateValues+            then Just "each varying axis must have one distinct value per arm"+            else findProblem rest++compareMetrics :: Policy -> [RunData] -> [RunData] -> Map Text MetricComparison+compareMetrics policy baselines candidates = Map.fromList (mapMaybe compareOne commonNames)+  where+    allMetricMaps = fmap ((.metrics) . (.summary)) (baselines <> candidates)+    commonNames = case allMetricMaps of [] -> []; first : rest -> filter (\name -> all (Map.member name) rest) (Map.keys first)+    compareOne name = do+      rule <- find (\candidate -> matchesRule candidate.match name) policy.metricRules+      pairedValues <- traverse (pairMetric name) (zip baselines candidates)+      let unit = fromMaybe "" do+            baseline <- headMay baselines+            (.unit) <$> Map.lookup name baseline.summary.metrics+      pure (name, compareMetricPairs policy name unit rule pairedValues)+    pairMetric name (baseline, candidate) = do+      baselineMetric <- Map.lookup name baseline.summary.metrics+      candidateMetric <- Map.lookup name candidate.summary.metrics+      pure (baselineMetric.value, candidateMetric.value)++compareMetricPairs :: Policy -> Text -> Text -> MetricRule -> [(Double, Double)] -> MetricComparison+compareMetricPairs policy name unit rule pairedValues =+  let baselineValues = fmap fst pairedValues+      adverseRatios = fmap (adverseRatio rule.direction) pairedValues+      adverseDeltas = fmap (adverseDelta rule.direction) pairedValues+      ratioBootstrap = bootstrapInterval policy.resamplingSeed policy.bootstrapIterations policy.confidenceLevel geometricMean adverseRatios+      logT = studentTInterval policy.confidenceLevel (fmap log adverseRatios)+      ratioT = Interval (exp logT.low) (exp logT.estimate) (exp logT.high)+      ratioInterval = intervalEnvelope ratioBootstrap ratioT+      deltaInterval = intervalEnvelope (bootstrapInterval (policy.resamplingSeed + 1) policy.bootstrapIterations policy.confidenceLevel arithmeticMean adverseDeltas) (studentTInterval policy.confidenceLevel adverseDeltas)+      tooWide = ratioInterval.low <= 0 || ratioInterval.high / ratioInterval.low > 1 + policy.maxCiRelativeWidth+      comparisonStatus+        | tooWide = MetricInconclusive+        | ratioInterval.low > 1 + rule.relativeLimit && deltaInterval.low > rule.absoluteFloor = MetricRegression+        | ratioInterval.high <= 1 + rule.relativeLimit || deltaInterval.high <= rule.absoluteFloor = MetricPass+        | otherwise = MetricInconclusive+      comparisonMessage = case comparisonStatus of MetricPass -> "within policy limits"; MetricRegression -> "both relative and absolute regression limits exceeded"; MetricInconclusive -> "confidence interval crosses a policy limit or is too wide"+   in MetricComparison name unit pairedValues ratioInterval deltaInterval rule.relativeLimit rule.absoluteFloor (coefficientOfVariation baselineValues) comparisonStatus comparisonMessage+  where+    adverseRatio LowerIsBetter (baseline, candidate) = candidate / max 1e-12 baseline+    adverseRatio HigherIsBetter (baseline, candidate) = baseline / max 1e-12 candidate+    adverseDelta LowerIsBetter (baseline, candidate) = candidate - baseline+    adverseDelta HigherIsBetter (baseline, candidate) = baseline - candidate++hasSeverity :: Severity -> MeasurementSummary -> Bool+hasSeverity wanted summary = any ((== wanted) . (.severity)) summary.health++checkpointDiffers :: Double -> RunData -> RunData -> Bool+checkpointDiffers limit baseline candidate = abs (checkpointOverlap baseline.summary - checkpointOverlap candidate.summary) > limit++checkpointOverlap :: MeasurementSummary -> Double+checkpointOverlap summary = fromMaybe 0 do+  health <- find ((== "checkpoint-in-window") . (.gate)) summary.health+  case health.evidence of+    Object value -> case KeyMap.lookup "overlapFraction" value of Just (Number number) -> Just (realToFrac number); _ -> Nothing+    _ -> Nothing++decideVerdict :: Bool -> [Text] -> [MetricStatus] -> Verdict+decideVerdict infrastructureIssue reasons statuses+  | infrastructureIssue = VerdictInfrastructureFailure+  | not (null reasons) || null statuses = VerdictInconclusive+  | MetricRegression `elem` statuses = VerdictRegression+  | MetricInconclusive `elem` statuses = VerdictInconclusive+  | otherwise = VerdictPass++trialRows :: Int -> (RunData, RunData) -> [Maybe (Arm, Int, UTCTime)]+trialRows pair (baseline, candidate) = [fmap (Baseline,pair,) baseline.startedAt, fmap (Candidate,pair,) candidate.startedAt]++tailSafe :: [value] -> [value]+tailSafe [] = []; tailSafe (_ : rest) = rest++textAt :: [Text] -> Value -> Maybe Text+textAt [] (String value) = Just value+textAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= textAt rest+textAt _ _ = Nothing++jsonAt :: [Text] -> Value -> Maybe Value+jsonAt [] value = Just value+jsonAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= jsonAt rest+jsonAt _ _ = Nothing++headMay :: [value] -> Maybe value+headMay [] = Nothing+headMay (value : _) = Just value++parseUtc :: Text -> Maybe UTCTime+parseUtc = parseTimeM True defaultTimeLocale "%Y-%m-%dT%H:%M:%S%QZ" . Text.unpack++verdictExitCode :: Verdict -> Int+verdictExitCode VerdictPass = 0+verdictExitCode VerdictRegression = 1+verdictExitCode VerdictInconclusive = 3+verdictExitCode VerdictInfrastructureFailure = 4++verdictText :: Verdict -> Text+verdictText VerdictPass = "pass"+verdictText VerdictRegression = "regression"+verdictText VerdictInconclusive = "inconclusive"+verdictText VerdictInfrastructureFailure = "infrastructure-failure"++statusText :: MetricStatus -> Text+statusText MetricPass = "pass"+statusText MetricRegression = "regression"+statusText MetricInconclusive = "inconclusive"++toList :: NonEmpty value -> [value]+toList = foldr (:) []
+ src/Kenshou/Measure/Compare/Compatibility.hs view
@@ -0,0 +1,74 @@+module Kenshou.Measure.Compare.Compatibility+  ( VaryingAxis (..),+    parseVaryingAxis,+    compatibilityInputs,+    environmentFingerprint,+    compatibleExcept,+    varyingValue,+  )+where++import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List.NonEmpty (NonEmpty)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text++data VaryingAxis = VaryControl | VaryCohort | VaryDimension Text | VaryKnob Text deriving stock (Eq, Ord, Show)++instance ToJSON VaryingAxis where+  toJSON VaryControl = String "control"+  toJSON VaryCohort = String "cohort"+  toJSON (VaryDimension name) = String ("dim:" <> name)+  toJSON (VaryKnob name) = String ("knob:" <> name)++parseVaryingAxis :: Text -> Either Text VaryingAxis+parseVaryingAxis "control" = Right VaryControl+parseVaryingAxis "cohort" = Right VaryCohort+parseVaryingAxis value | Just name <- Text.stripPrefix "dim:" value, not (Text.null name) = Right (VaryDimension name)+parseVaryingAxis value | Just name <- Text.stripPrefix "knob:" value, not (Text.null name) = Right (VaryKnob name)+parseVaryingAxis value = Left ("invalid varying axis " <> value)++compatibilityInputs :: Value -> Maybe Value+compatibilityInputs (Object root) = do+  Object compatibility <- KeyMap.lookup "compatibility" root+  KeyMap.lookup "inputs" compatibility+compatibilityInputs _ = Nothing++environmentFingerprint :: Value -> Maybe Value+environmentFingerprint (Object root) = do+  Object fingerprint <- KeyMap.lookup "fingerprint" root+  pure (Object (KeyMap.filterWithKey (\key _ -> Key.toText key `elem` ["host", "runtime", "postgres", "placement", "machineProfile"]) fingerprint))+environmentFingerprint _ = Nothing++compatibleExcept :: NonEmpty VaryingAxis -> Value -> Value -> Either Text ()+compatibleExcept axes left right =+  let normalized value = foldr removeAxis value (NonEmpty.toList axes)+   in if normalized left == normalized right+        then Right ()+        else Left "run compatibility inputs differ outside the declared varying axes"+  where+    removeAxis axis value = case value of+      Object objectValue -> Object case axis of+        VaryControl -> objectValue+        VaryCohort -> KeyMap.delete "cohortPlanHash" objectValue+        VaryDimension name -> updateNested "dimensions" name objectValue+        VaryKnob name -> updateNested "knobs" name objectValue+      other -> other+    deleteNested name (Object objectValue) = Object (KeyMap.delete (Key.fromText name) objectValue)+    deleteNested _ other = other+    updateNested parent name objectValue = case KeyMap.lookup parent objectValue of+      Nothing -> objectValue+      Just nested -> KeyMap.insert parent (deleteNested name nested) objectValue++varyingValue :: VaryingAxis -> Value -> Maybe Value+varyingValue axis (Object inputs) = case axis of+  VaryControl -> Just (Object inputs)+  VaryCohort -> KeyMap.lookup "cohortPlanHash" inputs+  VaryDimension name -> nested "dimensions" name+  VaryKnob name -> nested "knobs" name+  where+    nested parent name = do Object values <- KeyMap.lookup parent inputs; KeyMap.lookup (Key.fromText name) values+varyingValue _ _ = Nothing
+ src/Kenshou/Measure/Compare/Ordering.hs view
@@ -0,0 +1,44 @@+module Kenshou.Measure.Compare.Ordering+  ( Arm (..),+    PairedOrdering (..),+    TrialSlot (..),+    pairedSchedule,+    validateInterleaving,+  )+where++import Data.List (sortOn)+import Data.Text (Text)+import Data.Time (UTCTime)+import Data.Word (Word64)+import System.Random.SplitMix (mkSMGen, nextWord64)++data Arm = Baseline | Candidate deriving stock (Eq, Ord, Show)++data PairedOrdering = ABBA | BAAB deriving stock (Eq, Show)++data TrialSlot = TrialSlot {position :: Int, pairIndex :: Int, arm :: Arm, pairSeed :: Word64}+  deriving stock (Eq, Show)++pairedSchedule :: PairedOrdering -> Int -> Word64 -> [TrialSlot]+pairedSchedule ordering pairs seed = concat (zipWith slots [0 .. max 0 pairs - 1] seeds)+  where+    seeds = randomWords (mkSMGen seed)+    randomWords generator = let (value, next) = nextWord64 generator in value : randomWords next+    slots pair pairSeed =+      let firstBaseline = case ordering of ABBA -> even pair; BAAB -> odd pair+          arms = if firstBaseline then [Baseline, Candidate] else [Candidate, Baseline]+          start = pair * 2+       in zipWith (\offset arm -> TrialSlot (start + offset) pair arm pairSeed) [0, 1] arms++validateInterleaving :: [(Arm, Int, UTCTime)] -> Either Text ()+validateInterleaving trials =+  let ordered = sortOn (\(_, _, started) -> started) trials+      pairs = foldr collect [] ordered+      collect (arm, pair, _) accumulated = case lookup pair accumulated of+        Nothing -> (pair, [arm]) : accumulated+        Just _ -> fmap (\entry@(key, arms) -> if key == pair then (key, arm : arms) else entry) accumulated+      validPair (_, arms) = length arms == 2 && Baseline `elem` arms && Candidate `elem` arms+      firstArms = [arm | (arm, _, _) <- ordered]+      balanced = abs (length (filter (== Baseline) (take (length firstArms `div` 2) firstArms)) - length firstArms `div` 4) <= 1+   in if all validPair pairs && balanced then Right () else Left "runs are not paired and interleaved"
+ src/Kenshou/Measure/Compare/Policy.hs view
@@ -0,0 +1,81 @@+module Kenshou.Measure.Compare.Policy+  ( Direction (..),+    MetricRule (..),+    Policy (..),+    decodePolicy,+    matchesRule,+  )+where++import Data.Aeson+import Data.ByteString (ByteString)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)++data Direction = HigherIsBetter | LowerIsBetter deriving stock (Eq, Show)++data MetricRule = MetricRule {match :: Text, direction :: Direction, relativeLimit :: Double, absoluteFloor :: Double}+  deriving stock (Eq, Show)++data Policy = Policy+  { name :: Text,+    minimumPairs :: Int,+    confidenceLevel :: Double,+    bootstrapIterations :: Int,+    resamplingSeed :: Word64,+    requireInterleaving :: Bool,+    requireGrade :: Text,+    maxCiRelativeWidth :: Double,+    maxCheckpointAsymmetry :: Double,+    metricRules :: [MetricRule]+  }+  deriving stock (Eq, Show)++instance FromJSON Direction where+  parseJSON = withText "Direction" \value -> case value of+    "higher-is-better" -> pure HigherIsBetter+    "lower-is-better" -> pure LowerIsBetter+    _ -> fail "direction must be higher-is-better or lower-is-better"++instance ToJSON Direction where+  toJSON HigherIsBetter = String "higher-is-better"+  toJSON LowerIsBetter = String "lower-is-better"++instance FromJSON MetricRule where+  parseJSON = withObject "MetricRule" \value -> MetricRule <$> value .: "match" <*> value .: "direction" <*> value .: "relativeLimit" <*> value .: "absoluteFloor"++instance ToJSON MetricRule where+  toJSON rule = object ["match" .= rule.match, "direction" .= rule.direction, "relativeLimit" .= rule.relativeLimit, "absoluteFloor" .= rule.absoluteFloor]++instance FromJSON Policy where+  parseJSON = withObject "Policy" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.comparison-policy/v1" :: Text) then fail "unsupported comparison policy" else pure ()+    decodedPolicy <- Policy <$> value .: "name" <*> value .: "minimumPairs" <*> value .: "confidenceLevel" <*> value .: "bootstrapIterations" <*> value .: "resamplingSeed" <*> value .: "requireInterleaving" <*> value .: "requireGrade" <*> value .: "maxCiRelativeWidth" <*> value .: "maxCheckpointAsymmetry" <*> value .: "metrics"+    if decodedPolicy.minimumPairs < 3 then fail "minimumPairs must be at least 3" else pure ()+    if decodedPolicy.bootstrapIterations < 1_000 then fail "bootstrapIterations must be at least 1000" else pure decodedPolicy++instance ToJSON Policy where+  toJSON policy =+    object+      [ "schema" .= ("kenshou.comparison-policy/v1" :: Text),+        "name" .= policy.name,+        "minimumPairs" .= policy.minimumPairs,+        "confidenceLevel" .= policy.confidenceLevel,+        "bootstrapIterations" .= policy.bootstrapIterations,+        "resamplingSeed" .= policy.resamplingSeed,+        "requireInterleaving" .= policy.requireInterleaving,+        "requireGrade" .= policy.requireGrade,+        "maxCiRelativeWidth" .= policy.maxCiRelativeWidth,+        "maxCheckpointAsymmetry" .= policy.maxCheckpointAsymmetry,+        "metrics" .= policy.metricRules+      ]++decodePolicy :: ByteString -> Either Text Policy+decodePolicy = either (Left . Text.pack) Right . eitherDecodeStrict'++matchesRule :: Text -> Text -> Bool+matchesRule patternValue metricName = case Text.breakOn "*" patternValue of+  (_, suffixWithStar) | Text.null suffixWithStar -> patternValue == metricName+  (prefix, suffixWithStar) -> prefix `Text.isPrefixOf` metricName && Text.drop 1 suffixWithStar `Text.isSuffixOf` metricName
+ src/Kenshou/Measure/Health.hs view
@@ -0,0 +1,302 @@+module Kenshou.Measure.Health+  ( Severity (..),+    HealthObservation (..),+    HealthConfig (..),+    defaultHealthConfig,+    captureHealthNotices,+    evaluateHealth,+    healthOutcome,+  )+where++import Control.Applicative ((<|>))+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List (isSuffixOf)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe, mapMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Encoding+import Data.Text.IO qualified as Text+import Data.Time (UTCTime)+import Data.Time.Clock.POSIX (posixSecondsToUTCTime)+import Data.Word (Word64)+import Kenshou.Core.Outcome (Outcome (..))+import System.Directory (copyFile, doesFileExist, listDirectory)+import System.Environment (lookupEnv)+import System.FilePath ((</>))+import Text.Read (readMaybe)++data Severity = Info | Soft | Hard deriving stock (Eq, Ord, Show)++data HealthObservation = HealthObservation+  { gate :: Text,+    severity :: Severity,+    fromMonoNs :: Word64,+    toMonoNs :: Word64,+    detail :: Text,+    evidence :: Value+  }+  deriving stock (Eq, Show)++data HealthConfig = HealthConfig+  { cpuSoft :: Double,+    cpuHard :: Double,+    stealSoft :: Double,+    clockStepNs :: Word64,+    pauseNs :: Word64,+    minSteadySamples :: Word64+  }+  deriving stock (Eq, Show)++defaultHealthConfig :: HealthConfig+defaultHealthConfig = HealthConfig 0.70 0.90 0.02 50_000_000 2_000_000_000 1_000++instance ToJSON Severity where+  toJSON Info = String "info"+  toJSON Soft = String "soft"+  toJSON Hard = String "hard"++instance FromJSON Severity where+  parseJSON = withText "Severity" \case+    "info" -> pure Info+    "soft" -> pure Soft+    "hard" -> pure Hard+    _ -> fail "severity must be info, soft, or hard"++instance ToJSON HealthObservation where+  toJSON value =+    object+      [ "gate" .= value.gate,+        "severity" .= value.severity,+        "fromMonoNs" .= value.fromMonoNs,+        "toMonoNs" .= value.toMonoNs,+        "detail" .= value.detail,+        "evidence" .= value.evidence+      ]++instance FromJSON HealthObservation where+  parseJSON = withObject "HealthObservation" \value -> HealthObservation <$> value .: "gate" <*> value .: "severity" <*> value .: "fromMonoNs" <*> value .: "toMonoNs" <*> value .: "detail" <*> value .: "evidence"++data HealthNotice = HealthNotice Text Severity UTCTime Text++instance FromJSON HealthNotice where+  parseJSON = withObject "HealthNotice" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.health-notice/v1" :: Text) then fail "unsupported health notice" else pure ()+    HealthNotice <$> value .: "source" <*> value .: "severity" <*> value .: "at" <*> value .: "detail"++type CsvRow = Map Text Text++captureHealthNotices :: FilePath -> IO (Maybe FilePath)+captureHealthNotices runDir = do+  let captured = runDir </> "health-notices.jsonl"+  source <- lookupEnv "KENSHOU_HEALTH_NOTICES"+  case source of+    Just path | path /= captured -> do+      exists <- doesFileExist path+      if exists then copyFile path captured >> pure (Just captured) else existing captured+    _ -> existing captured+  where+    existing path = do+      exists <- doesFileExist path+      pure (if exists then Just path else Nothing)++evaluateHealth :: HealthConfig -> FilePath -> IO [HealthObservation]+evaluateHealth config runDir = do+  samplerRows <- steadyRows (runDir </> "series" </> "sampler.csv")+  loadRows <- readCsvRows (runDir </> "series" </> "load.csv")+  hostRows <- steadyRows (runDir </> "series" </> "host.csv")+  processRows <- steadyRows (runDir </> "series" </> "proc.csv")+  rtsRows <- steadyRows (runDir </> "series" </> "rts.csv")+  checkpointRows <- steadyRows (runDir </> "series" </> "pg-checkpointer.csv")+  spec <- decodeValue (runDir </> "run-spec.json")+  sampleObservations <- sampleFileObservations config runDir+  noticeObservations <- noticeFileObservations runDir samplerRows+  let bounds = steadyWindow loadRows samplerRows+      intervalNs = round (numberAt ["knobs", "measure.sample-interval-ms"] spec * 1_000_000)+      effectivePauseNs = max config.pauseNs (5 * intervalNs)+      cpu = driverCpu config bounds hostRows processRows rtsRows+      driverHard = any (\item -> item.gate == "driver-cpu-saturation" && item.severity == Hard) cpu+      overload = backlogObservations bounds driverHard spec loadRows+      clocks = samplerObservations config effectivePauseNs bounds samplerRows+      checkpoint = checkpointObservations bounds checkpointRows+      rts = [observation bounds "rts-stats-unavailable" Info "GHC RTS statistics were unavailable" Null | null rtsRows]+  pure (cpu <> overload <> clocks <> checkpoint <> sampleObservations bounds <> rts <> noticeObservations bounds)++healthOutcome :: [HealthObservation] -> Maybe Outcome+healthOutcome observations+  | any ((== Hard) . (.severity)) observations = Just InfrastructureFailure+  | any ((== Soft) . (.severity)) observations = Just Inconclusive+  | otherwise = Nothing++driverCpu :: HealthConfig -> (Word64, Word64) -> [CsvRow] -> [CsvRow] -> [CsvRow] -> [HealthObservation]+driverCpu config bounds hostRows processRows rtsRows = saturation <> steal+  where+    hostDeltas = fmap (\fieldName -> max 0 (delta fieldName hostRows)) ["cpu_user", "cpu_nice", "cpu_system", "cpu_idle", "cpu_iowait", "cpu_irq", "cpu_softirq", "cpu_steal"]+    hostTotal = sum hostDeltas+    hostBusy = if hostTotal <= 0 then Nothing else Just ((hostTotal - hostDeltas !! 3 - hostDeltas !! 4) / hostTotal)+    capabilities = max 1 (fromMaybe 1 (lastNumber "capabilities" rtsRows))+    elapsed = max 1 (delta "t_mono_ns" processRows)+    processBusy = if length processRows < 2 then Nothing else Just (delta "cpu_total_ns" processRows / elapsed / capabilities)+    busy = if length hostRows >= 2 then hostBusy else processBusy+    saturation = case busy of+      Just value | value > config.cpuHard -> [cpuObservation Hard value]+      Just value | value > config.cpuSoft -> [cpuObservation Soft value]+      _ -> []+    stealFraction = if hostTotal <= 0 then 0 else hostDeltas !! 7 / hostTotal+    steal = [observation bounds "cpu-steal" Soft "host CPU steal exceeded the health threshold" (object ["fraction" .= stealFraction, "threshold" .= config.stealSoft]) | stealFraction > config.stealSoft]+    cpuObservation level value = observation bounds "driver-cpu-saturation" level "load driver CPU use exceeded the health threshold" (object ["utilisation" .= value, "softThreshold" .= config.cpuSoft, "hardThreshold" .= config.cpuHard])++backlogObservations :: (Word64, Word64) -> Bool -> Value -> [CsvRow] -> [HealthObservation]+backlogObservations bounds driverHard spec rows =+  case maximumMaybe (mapMaybe (fieldNumber "max_lag_ns") rows) of+    Just maximumLag+      | modelName `elem` [Just "open-constant", Just "open-poisson"] && maximumLag > maximumThreshold ->+          let aborted = maximumLag > abortThreshold+              level = if driverHard || aborted then Hard else Soft+           in [observation bounds "open-loop-backlog" level "open-loop arrivals accumulated sustained lag" (object ["maximumLagNs" .= maximumLag, "thresholdNs" .= maximumThreshold, "abortedEarly" .= aborted])]+    _ -> []+  where+    modelName = textAt ["knobs", "load.model"] spec+    maximumThreshold = numberAt ["knobs", "load.max-lag-ms"] spec * 1_000_000+    abortThreshold = numberAt ["knobs", "load.abort-lag-ms"] spec * 1_000_000++samplerObservations :: HealthConfig -> Word64 -> (Word64, Word64) -> [CsvRow] -> [HealthObservation]+samplerObservations config effectivePauseNs bounds rows = clock <> late <> overrun+  where+    maximumClockStep = maximumOrZero (fmap abs (mapMaybe (fieldNumber "wall_minus_mono_ns") rows))+    maximumLate = maximumOrZero (mapMaybe (fieldNumber "late_ns") rows)+    maximumCost = maximumOrZero (mapMaybe (fieldNumber "cost_ns") rows)+    interval = if length rows < 2 then 0 else delta "scheduled_mono_ns" rows / fromIntegral (length rows - 1)+    clock = [observation bounds "clock-anomaly" Soft "wall and monotonic clocks changed by more than 50 ms" (object ["maximumStepNs" .= maximumClockStep, "thresholdNs" .= config.clockStepNs]) | maximumClockStep > fromIntegral config.clockStepNs]+    late = [observation bounds "clock-anomaly" Hard "the sampler observed a process or virtual-machine pause" (object ["maximumLateNs" .= maximumLate, "thresholdNs" .= effectivePauseNs]) | maximumLate > fromIntegral effectivePauseNs]+    overrun = [observation bounds "sampler-overrun" Soft "sampling cost exceeded the configured interval" (object ["maximumCostNs" .= maximumCost, "intervalNs" .= interval]) | interval > 0 && maximumCost > interval]++checkpointObservations :: (Word64, Word64) -> [CsvRow] -> [HealthObservation]+checkpointObservations _ [] = []+checkpointObservations bounds rows =+  [ observation+      bounds+      "checkpoint-in-window"+      Info+      "PostgreSQL checkpoint activity in the steady window"+      (object ["started" .= started, "completed" .= completed, "overlapFraction" .= overlapFraction])+  ]+  where+    started = max 0 (delta "num_timed" rows) + max 0 (delta "num_requested" rows)+    lsnChanges = length [() | (left, right) <- adjacent rows, Map.lookup "checkpoint_lsn" left /= Map.lookup "checkpoint_lsn" right]+    completed = case lastNumber "num_done" rows of Just _ -> max 0 (delta "num_done" rows); Nothing -> fromIntegral lsnChanges+    intervals = adjacent rows+    active (left, right) = Map.lookup "checkpoint_lsn" left /= Map.lookup "checkpoint_lsn" right || fromMaybe 0 (fieldNumber "buffers_written" right) > fromMaybe 0 (fieldNumber "buffers_written" left)+    overlapFraction :: Double+    overlapFraction = if null intervals then 0 else fromIntegral (length (filter active intervals)) / fromIntegral (length intervals)++sampleFileObservations :: HealthConfig -> FilePath -> IO ((Word64, Word64) -> [HealthObservation])+sampleFileObservations config runDir = do+  let directory = runDir </> "samples"+  files <- listDirectory directory+  decoded <- traverse (decodeValue . (directory </>)) [file | file <- files, ".meta.json" `isSuffixOf` file]+  pure \bounds -> concatMap (fromMeta bounds) decoded+  where+    fromMeta bounds value =+      let operation = fromMaybe "unknown" (textAt ["operation"] value)+          successes = numberAt ["phaseCounts", "steady", "successes"] value+          failures = numberAt ["phaseCounts", "steady", "failures"] value+          samples = successes + failures+          backpressure = numberAt ["backpressureCount"] value+       in [observation bounds "insufficient-samples" Soft ("operation " <> operation <> " has too few steady samples") (object ["operation" .= operation, "samples" .= samples, "minimum" .= config.minSteadySamples]) | samples < fromIntegral config.minSteadySamples]+            <> [observation bounds "recorder-backpressure" Soft ("operation " <> operation <> " experienced recorder back-pressure") (object ["operation" .= operation, "count" .= backpressure]) | backpressure > 0]++noticeFileObservations :: FilePath -> [CsvRow] -> IO ((Word64, Word64) -> [HealthObservation])+noticeFileObservations runDir samplerRows = do+  let path = runDir </> "health-notices.jsonl"+  exists <- doesFileExist path+  if not exists+    then pure (const [])+    else do+      notices <- fmap (mapMaybe decodeNotice . Text.lines) (Text.readFile path)+      let wallBounds = wallTimeBounds samplerRows+      pure \bounds -> [noticeObservation bounds notice | notice@(HealthNotice _ _ noticeAt _) <- notices, within wallBounds noticeAt]+  where+    decodeNotice :: Text -> Maybe HealthNotice+    decodeNotice = decodeStrict' . Encoding.encodeUtf8+    noticeObservation bounds (HealthNotice noticeSource noticeSeverity noticeAt noticeDetail) = observation bounds "host-notice" noticeSeverity noticeDetail (object ["source" .= noticeSource, "at" .= noticeAt])+    within Nothing _ = True+    within (Just (start, end)) instant = instant >= start && instant <= end++observation :: (Word64, Word64) -> Text -> Severity -> Text -> Value -> HealthObservation+observation (start, end) gate severity detail evidence = HealthObservation gate severity start end detail evidence++steadyWindow :: [CsvRow] -> [CsvRow] -> (Word64, Word64)+steadyWindow loadRows samplerRows =+  let steadyStart = listToMaybeNumber "t_mono_ns" [row | row <- loadRows, Map.lookup "phase" row == Just "steady"]+      steadyEnd = listToMaybeNumber "t_mono_ns" [row | row <- loadRows, Map.lookup "phase" row `elem` [Just "drain", Just "done"]]+      fallback = boundsOf samplerRows+   in (floor (fromMaybe (fromIntegral (fst fallback)) steadyStart), floor (fromMaybe (fromIntegral (snd fallback)) steadyEnd))++wallTimeBounds :: [CsvRow] -> Maybe (UTCTime, UTCTime)+wallTimeBounds rows = do+  start <- listToMaybeNumber "t_wall_ms" rows+  end <- lastNumber "t_wall_ms" rows+  pure (toUtc start, toUtc end)+  where+    toUtc value = posixSecondsToUTCTime (realToFrac (value / 1_000))++readCsvRows :: FilePath -> IO [CsvRow]+readCsvRows path = do+  exists <- doesFileExist path+  if not exists+    then pure []+    else do+      rows <- Text.lines <$> Text.readFile path+      pure case rows of+        [] -> []+        header : body -> fmap (Map.fromList . zip (Text.splitOn "," header) . Text.splitOn ",") body++steadyRows :: FilePath -> IO [CsvRow]+steadyRows path = filter ((== Just "steady") . Map.lookup "phase") <$> readCsvRows path++decodeValue :: FilePath -> IO Value+decodeValue path = do+  decoded <- eitherDecodeFileStrict' path+  pure (either (const Null) id decoded)++textAt :: [Text] -> Value -> Maybe Text+textAt [] (String value) = Just value+textAt (name : rest) (Object value) = KeyMap.lookup (Key.fromText name) value >>= textAt rest+textAt _ _ = Nothing++numberAt :: [Text] -> Value -> Double+numberAt [] (Number value) = realToFrac value+numberAt (name : rest) (Object value) = maybe 0 (numberAt rest) (KeyMap.lookup (Key.fromText name) value)+numberAt _ _ = 0++fieldNumber :: Text -> CsvRow -> Maybe Double+fieldNumber name row = Map.lookup name row >>= readMaybe . Text.unpack++delta :: Text -> [CsvRow] -> Double+delta name rows = fromMaybe 0 ((-) <$> (lastNumber name rows) <*> (listToMaybeNumber name rows))++listToMaybeNumber :: Text -> [CsvRow] -> Maybe Double+listToMaybeNumber _ [] = Nothing+listToMaybeNumber name (row : rest) = fieldNumber name row <|> listToMaybeNumber name rest++lastNumber :: Text -> [CsvRow] -> Maybe Double+lastNumber name = listToMaybeNumber name . reverse++boundsOf :: [CsvRow] -> (Word64, Word64)+boundsOf rows = (floor (fromMaybe 0 (listToMaybeNumber "t_mono_ns" rows)), floor (fromMaybe 0 (lastNumber "t_mono_ns" rows)))++adjacent :: [value] -> [(value, value)]+adjacent values = zip values (drop 1 values)++maximumMaybe :: (Ord value) => [value] -> Maybe value+maximumMaybe [] = Nothing+maximumMaybe values = Just (maximum values)++maximumOrZero :: (Ord value, Num value) => [value] -> value+maximumOrZero = fromMaybe 0 . maximumMaybe
+ src/Kenshou/Measure/Histogram.hs view
@@ -0,0 +1,271 @@+module Kenshou.Measure.Histogram+  ( HistogramConfig (..),+    defaultHistogramConfig,+    MutableHistogram,+    Histogram,+    newHistogram,+    recordValue,+    freeze,+    merge,+    valueAtQuantile,+    totalCount,+    minValue,+    maxValue,+    overflowCount,+    meanValue,+    equivalentRange,+    histogramConfig,+    histogramCounts,+    histogramSum,+    histogramArrayLength,+    histogramFromCounts,+  )+where++import Control.Monad (forM_)+import Control.Monad.ST (runST)+import Data.Bits+import Data.IORef+import Data.Primitive.PrimArray+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import GHC.Exts (RealWorld)++data HistogramConfig = HistogramConfig+  { lowestDiscernible :: !Word64,+    highestTrackable :: !Word64,+    significantDigits :: !Int+  }+  deriving stock (Eq, Show)++defaultHistogramConfig :: HistogramConfig+defaultHistogramConfig = HistogramConfig 1 3_600_000_000_000 3++data Layout = Layout+  { unitMagnitude :: !Int,+    subBucketCountMagnitude :: !Int,+    subBucketCount :: !Int,+    subBucketHalfCount :: !Int,+    subBucketHalfCountMagnitude :: !Int,+    subBucketMask :: !Word64,+    arrayLength :: !Int+  }++data MutableHistogram = MutableHistogram+  { config :: !HistogramConfig,+    layout :: !Layout,+    counts :: !(MutablePrimArray RealWorld Word64),+    count :: !(IORef Word64),+    minimum :: !(IORef Word64),+    maximum :: !(IORef Word64),+    total :: !(IORef Word64),+    overflow :: !(IORef Word64)+  }++data Histogram = Histogram+  { config :: !HistogramConfig,+    layout :: !Layout,+    counts :: !(PrimArray Word64),+    count :: !Word64,+    minimum :: !Word64,+    maximum :: !Word64,+    total :: !Word64,+    overflow :: !Word64+  }++instance Eq Histogram where+  left == right =+    left.config == right.config+      && left.count == right.count+      && left.minimum == right.minimum+      && left.maximum == right.maximum+      && left.total == right.total+      && left.overflow == right.overflow+      && histogramCounts left == histogramCounts right++instance Show Histogram where+  show value =+    "Histogram {config="+      <> show value.config+      <> ", count="+      <> show value.count+      <> ", min="+      <> show value.minimum+      <> ", max="+      <> show value.maximum+      <> ", overflow="+      <> show value.overflow+      <> "}"++newHistogram :: HistogramConfig -> IO MutableHistogram+newHistogram config = case makeLayout config of+  Left message -> ioError (userError (Text.unpack message))+  Right layout -> do+    counts <- newPrimArray layout.arrayLength+    setPrimArray counts 0 layout.arrayLength 0+    MutableHistogram config layout counts+      <$> newIORef 0+      <*> newIORef maxBound+      <*> newIORef 0+      <*> newIORef 0+      <*> newIORef 0++recordValue :: MutableHistogram -> Word64 -> IO ()+recordValue histogram value = do+  modifyIORef' histogram.count (+ 1)+  modifyIORef' histogram.minimum (min value)+  modifyIORef' histogram.maximum (max value)+  modifyIORef' histogram.total (+ value)+  if value > histogram.config.highestTrackable+    then modifyIORef' histogram.overflow (+ 1)+    else do+      let index = countsIndex histogram.layout value+      previous <- readPrimArray histogram.counts index+      writePrimArray histogram.counts index (previous + 1)++freeze :: MutableHistogram -> IO Histogram+freeze mutable = do+  immutable <- freezePrimArray mutable.counts 0 mutable.layout.arrayLength+  count <- readIORef mutable.count+  observedMinimum <- readIORef mutable.minimum+  observedMaximum <- readIORef mutable.maximum+  total <- readIORef mutable.total+  overflow <- readIORef mutable.overflow+  pure $ Histogram mutable.config mutable.layout immutable count (if count == 0 then 0 else observedMinimum) observedMaximum total overflow++merge :: Histogram -> Histogram -> Either Text Histogram+merge left right+  | left.config /= right.config = Left "histogram configurations differ"+  | otherwise =+      Right+        Histogram+          { config = left.config,+            layout = left.layout,+            counts = runST do+              mutable <- newPrimArray left.layout.arrayLength+              forM_ [0 .. left.layout.arrayLength - 1] \index ->+                writePrimArray mutable index (indexPrimArray left.counts index + indexPrimArray right.counts index)+              unsafeFreezePrimArray mutable,+            count = left.count + right.count,+            minimum = case (left.count, right.count) of+              (0, _) -> right.minimum+              (_, 0) -> left.minimum+              _ -> min left.minimum right.minimum,+            maximum = max left.maximum right.maximum,+            total = left.total + right.total,+            overflow = left.overflow + right.overflow+          }++valueAtQuantile :: Histogram -> Double -> Word64+valueAtQuantile histogram quantile+  | histogram.count == 0 = 0+  | quantile >= 1 = histogram.maximum+  | target > histogram.count - histogram.overflow = histogram.maximum+  | otherwise = findIndex 0 0+  where+    bounded = max 0 (min 1 quantile)+    target = max 1 (ceiling (bounded * fromIntegral histogram.count))+    findIndex index cumulative+      | index >= histogram.layout.arrayLength = histogram.maximum+      | next >= target =+          let (_, highest) = equivalentRangeAtIndex histogram.layout index+           in max histogram.minimum (min histogram.maximum highest)+      | otherwise = findIndex (index + 1) next+      where+        next = cumulative + indexPrimArray histogram.counts index++totalCount :: Histogram -> Word64+totalCount = (.count)++minValue :: Histogram -> Word64+minValue = (.minimum)++maxValue :: Histogram -> Word64+maxValue = (.maximum)++overflowCount :: Histogram -> Word64+overflowCount = (.overflow)++meanValue :: Histogram -> Double+meanValue histogram+  | histogram.count == 0 = 0+  | otherwise = fromIntegral histogram.total / fromIntegral histogram.count++equivalentRange :: HistogramConfig -> Word64 -> Either Text (Word64, Word64)+equivalentRange config value = do+  layout <- makeLayout config+  pure (equivalentRangeAtIndex layout (countsIndex layout (min value config.highestTrackable)))++histogramConfig :: Histogram -> HistogramConfig+histogramConfig = (.config)++histogramCounts :: Histogram -> [Word64]+histogramCounts histogram = [indexPrimArray histogram.counts index | index <- [0 .. histogram.layout.arrayLength - 1]]++histogramSum :: Histogram -> Word64+histogramSum = (.total)++histogramArrayLength :: HistogramConfig -> Either Text Int+histogramArrayLength config = (.arrayLength) <$> makeLayout config++histogramFromCounts :: HistogramConfig -> [Word64] -> Word64 -> Word64 -> Word64 -> Word64 -> Word64 -> Either Text Histogram+histogramFromCounts config values count observedMinimum observedMaximum total overflow = do+  layout <- makeLayout config+  if length values /= layout.arrayLength+    then Left "histogram count array has the wrong length"+    else+      pure+        Histogram+          { config,+            layout,+            counts = primArrayFromList values,+            count,+            minimum = if count == 0 then 0 else observedMinimum,+            maximum = observedMaximum,+            total,+            overflow+          }++makeLayout :: HistogramConfig -> Either Text Layout+makeLayout config+  | config.lowestDiscernible == 0 = Left "lowestDiscernible must be positive"+  | config.highestTrackable < config.lowestDiscernible = Left "highestTrackable must not be below lowestDiscernible"+  | config.significantDigits < 1 || config.significantDigits > 5 = Left "significantDigits must be between 1 and 5"+  | otherwise = Right Layout {unitMagnitude, subBucketCountMagnitude, subBucketCount, subBucketHalfCount, subBucketHalfCountMagnitude, subBucketMask, arrayLength}+  where+    unitMagnitude = integerLog2 config.lowestDiscernible+    required = 2 * (10 ^ config.significantDigits)+    subBucketCountMagnitude = ceilingLog2 required+    subBucketCount = 1 `shiftL` subBucketCountMagnitude+    subBucketHalfCount = subBucketCount `div` 2+    subBucketHalfCountMagnitude = subBucketCountMagnitude - 1+    subBucketMask = fromIntegral (subBucketCount - 1) `shiftL` unitMagnitude+    initial = fromIntegral subBucketCount `shiftL` unitMagnitude :: Word64+    bucketCount = length (takeWhile (<= config.highestTrackable) (iterate (`shiftL` 1) initial)) + 1+    arrayLength = (bucketCount + 1) * subBucketHalfCount++countsIndex :: Layout -> Word64 -> Int+countsIndex layout value =+  ((bucketIndex + 1) `shiftL` layout.subBucketHalfCountMagnitude)+    + (subBucketIndex - layout.subBucketHalfCount)+  where+    bucketIndex = (64 - layout.unitMagnitude - layout.subBucketCountMagnitude) - countLeadingZeros (value .|. layout.subBucketMask)+    subBucketIndex = fromIntegral (value `shiftR` (bucketIndex + layout.unitMagnitude))++equivalentRangeAtIndex :: Layout -> Int -> (Word64, Word64)+equivalentRangeAtIndex layout index = (lowest, lowest + rangeSize - 1)+  where+    initialBucket = (index `shiftR` layout.subBucketHalfCountMagnitude) - 1+    initialSub = (index .&. (layout.subBucketHalfCount - 1)) + layout.subBucketHalfCount+    (bucket, subBucket) = if initialBucket < 0 then (0, initialSub - layout.subBucketHalfCount) else (initialBucket, initialSub)+    lowest = fromIntegral subBucket `shiftL` (bucket + layout.unitMagnitude)+    rangeSize = 1 `shiftL` (bucket + layout.unitMagnitude)++integerLog2 :: Word64 -> Int+integerLog2 value = finiteBitSize value - 1 - countLeadingZeros value++ceilingLog2 :: Int -> Int+ceilingLog2 value = go 0 1+  where+    go magnitude power | power >= value = magnitude | otherwise = go (magnitude + 1) (power * 2)
+ src/Kenshou/Measure/Histogram/Codec.hs view
@@ -0,0 +1,129 @@+module Kenshou.Measure.Histogram.Codec+  ( encodeHistogram,+    decodeHistogram,+  )+where++import Data.Bits+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.ByteString.Builder+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.Text (Text)+import Data.Word (Word16, Word32, Word64, Word8)+import Kenshou.Measure.Histogram++encodeHistogram :: Histogram -> ByteString+encodeHistogram histogram =+  LazyByteString.toStrict . toLazyByteString $+    byteString "KHST"+      <> word16LE 1+      <> word16LE 0+      <> word64LE config.lowestDiscernible+      <> word64LE config.highestTrackable+      <> word8 (fromIntegral config.significantDigits)+      <> word64LE (totalCount histogram)+      <> word64LE (minValue histogram)+      <> word64LE (maxValue histogram)+      <> word64LE (histogramSum histogram)+      <> word64LE (overflowCount histogram)+      <> word32LE (fromIntegral (length tokens))+      <> foldMap (putVarWord . zigZag) tokens+  where+    config = histogramConfig histogram+    tokens = runLengthEncode (histogramCounts histogram)++decodeHistogram :: ByteString -> Either Text Histogram+decodeHistogram bytes = do+  ensure (ByteString.take 4 bytes == "KHST") "invalid histogram magic"+  version <- word16At bytes 4+  ensure (version == 1) "unsupported histogram version"+  flags <- word16At bytes 6+  ensure (flags == 0) "unsupported histogram flags"+  lowest <- word64At bytes 8+  highest <- word64At bytes 16+  digits <- word8At bytes 24+  count <- word64At bytes 25+  observedMinimum <- word64At bytes 33+  observedMaximum <- word64At bytes 41+  total <- word64At bytes 49+  overflow <- word64At bytes 57+  tokenCount <- word32At bytes 65+  (tokens, _) <- readTokens bytes 69 (fromIntegral tokenCount)+  let config = HistogramConfig lowest highest (fromIntegral digits)+  expectedLength <- histogramLength config+  values <- expandTokens expectedLength tokens+  histogramFromCounts config values count observedMinimum observedMaximum total overflow++histogramLength :: HistogramConfig -> Either Text Int+histogramLength = histogramArrayLength++runLengthEncode :: [Word64] -> [Int64]+runLengthEncode [] = []+runLengthEncode values = go values+  where+    go [] = []+    go (0 : rest) = let (zeros, remaining) = span (== 0) rest in negate (fromIntegral (1 + length zeros)) : go remaining+    go (value : rest) = fromIntegral value : go rest++expandTokens :: Int -> [Int64] -> Either Text [Word64]+expandTokens expected tokens = do+  values <- fmap concat $ traverse expand tokens+  ensure (length values <= expected) "histogram token stream exceeds the configured layout"+  pure (values <> replicate (expected - length values) 0)+  where+    expand value | value < 0 = Right (replicate (fromIntegral (negate value)) 0)+    expand value = Right [fromIntegral value]++readTokens :: ByteString -> Int -> Int -> Either Text ([Int64], Int)+readTokens bytes = go []+  where+    go acc offset 0 = Right (reverse acc, offset)+    go acc offset remaining = do+      (encoded, next) <- readVarWord bytes offset+      go (unZigZag encoded : acc) next (remaining - 1)++putVarWord :: Word64 -> Builder+putVarWord value+  | value < 0x80 = word8 (fromIntegral value)+  | otherwise = word8 (fromIntegral (value .&. 0x7f) .|. 0x80) <> putVarWord (value `shiftR` 7)++readVarWord :: ByteString -> Int -> Either Text (Word64, Int)+readVarWord bytes = go 0 0+  where+    go bitOffset accumulator offset+      | bitOffset >= 64 = Left "varint is too long"+      | offset >= ByteString.length bytes = Left "truncated varint"+      | otherwise =+          let byte = ByteString.index bytes offset+              value = accumulator .|. (fromIntegral (byte .&. 0x7f) `shiftL` bitOffset)+           in if byte .&. 0x80 == 0 then Right (value, offset + 1) else go (bitOffset + 7) value (offset + 1)++zigZag :: Int64 -> Word64+zigZag value = fromIntegral ((value `shiftL` 1) `xor` (value `shiftR` 63))++unZigZag :: Word64 -> Int64+unZigZag value = fromIntegral (value `shiftR` 1) `xor` negate (fromIntegral (value .&. 1))++word8At :: ByteString -> Int -> Either Text Word8+word8At bytes offset+  | offset < ByteString.length bytes = Right (ByteString.index bytes offset)+  | otherwise = Left "truncated histogram header"++word16At :: ByteString -> Int -> Either Text Word16+word16At bytes offset = fromIntegral <$> littleEndianAt bytes offset 2++word32At :: ByteString -> Int -> Either Text Word32+word32At bytes offset = fromIntegral <$> littleEndianAt bytes offset 4++word64At :: ByteString -> Int -> Either Text Word64+word64At bytes offset = littleEndianAt bytes offset 8++littleEndianAt :: ByteString -> Int -> Int -> Either Text Word64+littleEndianAt bytes offset width+  | offset + width > ByteString.length bytes = Left "truncated histogram header"+  | otherwise = Right $ foldr (.|.) 0 [fromIntegral (ByteString.index bytes (offset + index)) `shiftL` (8 * index) | index <- [0 .. width - 1]]++ensure :: Bool -> Text -> Either Text ()+ensure condition message = if condition then Right () else Left message
+ src/Kenshou/Measure/Knobs.hs view
@@ -0,0 +1,83 @@+module Kenshou.Measure.Knobs+  ( LoadDefaults (..),+    defaultLoadDefaults,+    loadKnobs,+    measureKnobs,+    loadModelFromKnobs,+  )+where++import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Word (Word64)+import Kenshou.Core.Id (Kind (..))+import Kenshou.Core.Knob+import Kenshou.Measure.Load.Types++data LoadDefaults = LoadDefaults+  { model :: Text,+    workers :: Int64,+    thinkTimeUs :: Int64,+    ratePerSecond :: Double,+    executors :: Int64,+    maxLagMs :: Int64,+    abortLagMs :: Int64+  }+  deriving stock (Eq, Show)++defaultLoadDefaults :: LoadDefaults+defaultLoadDefaults = LoadDefaults "closed" 4 0 100 256 1_000 30_000++loadKnobs :: LoadDefaults -> [KnobSpec]+loadKnobs defaults =+  [ textKnob "load.model" "Load generation model" defaults.model ["closed", "open-constant", "open-poisson"],+    intKnob "load.workers" "Closed-loop workers" defaults.workers 1 65_535,+    intKnob "load.think-time-us" "Delay between closed-loop operations" defaults.thinkTimeUs 0 60_000_000,+    doubleKnob "load.rate-per-second" "Open-loop arrival rate" defaults.ratePerSecond 0.001 10_000_000,+    intKnob "load.executors" "Maximum open-loop operations in flight" defaults.executors 1 65_535,+    intKnob "load.max-lag-ms" "Lag that marks sustained overload" defaults.maxLagMs 1 3_600_000,+    intKnob "load.abort-lag-ms" "Lag that aborts an invalid open-loop run" defaults.abortLagMs 1 3_600_000+  ]++measureKnobs :: Kind -> [KnobSpec]+measureKnobs kind =+  [ intKnob "measure.sample-interval-ms" "Time-series sampling interval" 1_000 100 60_000,+    textKnob "measure.raw-samples" "Raw sample retention" (if kind == Benchmark then "full" else "off") ["full", "sampled", "off"],+    intKnob "measure.raw-sample-one-in" "Blocks retained under sampled policy" 100 1 1_000_000,+    intKnob "measure.histogram-digits" "Histogram significant decimal digits" 3 1 5,+    intKnob "measure.interval-histogram-seconds" "Interval histogram frame duration; soaks default to one frame per day to bound retained histograms" (if kind == Soak then 86_400 else 10) 1 86_400,+    textKnob "measure.pg-statements" "PostgreSQL statement sampling" "snapshots" ["off", "snapshots", "periodic"]+  ]++loadModelFromKnobs :: ResolvedKnobs -> Either Text LoadModel+loadModelFromKnobs knobs = case knobText knobs (name "load.model") of+  "closed" -> Right (ClosedLoop (ClosedConfig workers thinkTime (if workers <= 1 then 0 else 1_000_000_000 `div` fromIntegral workers)))+  "open-constant" -> Right (OpenLoop (openConfig (ConstantRate rate)))+  "open-poisson" -> Right (OpenLoop (openConfig (PoissonRate rate)))+  value -> Left ("unknown load.model " <> value)+  where+    workers = fromIntegral (knobInt knobs (name "load.workers"))+    thinkTime = micros (knobInt knobs (name "load.think-time-us"))+    rate = knobDouble knobs (name "load.rate-per-second")+    executors = fromIntegral (knobInt knobs (name "load.executors"))+    maxLag = millis (knobInt knobs (name "load.max-lag-ms"))+    abortLag = millis (knobInt knobs (name "load.abort-lag-ms"))+    openConfig arrival = OpenConfig arrival executors 1 (OverloadConfig maxLag 3 abortLag)++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt def) (IntRange low high) []++doubleKnob :: Text -> Text -> Double -> Double -> Double -> KnobSpec+doubleKnob knobName summary def low high = KnobSpec (name knobName) summary KnobDouble (VDouble def) (DoubleRange low high) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) (fmap VText (first : rest))+textKnob knobName _ _ [] = error ("text knob has no allowed values: " <> show knobName)++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++micros, millis :: Int64 -> Word64+micros value = fromIntegral value * 1_000+millis value = fromIntegral value * 1_000_000
+ src/Kenshou/Measure/Load.hs view
@@ -0,0 +1,15 @@+module Kenshou.Measure.Load+  ( module Kenshou.Measure.Load.Types,+    runLoad,+  )+where++import Kenshou.Measure.Load.Closed (runClosed)+import Kenshou.Measure.Load.Open (runOpen)+import Kenshou.Measure.Load.Types+import Kenshou.Measure.Session (Measurement)++runLoad :: Measurement -> LoadModel -> Operation -> IO LoadReport+runLoad measurement model operation = case model of+  ClosedLoop config -> runClosed measurement config operation+  OpenLoop config -> runOpen measurement config operation
+ src/Kenshou/Measure/Load/Arrival.hs view
@@ -0,0 +1,29 @@+module Kenshou.Measure.Load.Arrival+  ( constantSchedule,+    poissonSchedule,+    nextGapNs,+  )+where++import Data.Word (Word64)+import Kenshou.Measure.Load.Types (Arrival (..))+import System.Random.SplitMix (SMGen, nextWord64)++constantSchedule :: Double -> Word64 -> Int -> [Word64]+constantSchedule rate start count = take count [start + fromIntegral index * gap | index <- [0 :: Int ..]]+  where+    gap = max 1 (round (1_000_000_000 / rate))++poissonSchedule :: Double -> SMGen -> Word64 -> Int -> [Word64]+poissonSchedule rate generator start count = take count (drop 1 (scanl (+) start gaps))+  where+    gaps = unfold generator+    unfold current = let (gap, next) = nextGapNs (PoissonRate rate) current in gap : unfold next++nextGapNs :: Arrival -> SMGen -> (Word64, SMGen)+nextGapNs (ConstantRate rate) generator = (max 1 (round (1_000_000_000 / rate)), generator)+nextGapNs (PoissonRate rate) generator =+  (max 1 (round ((-log uniform / rate) * 1_000_000_000)), next)+  where+    (word, next) = nextWord64 generator+    uniform = (fromIntegral word + 0.5) / 18_446_744_073_709_551_616.0
+ src/Kenshou/Measure/Load/Closed.hs view
@@ -0,0 +1,93 @@+{-# LANGUAGE BangPatterns #-}++module Kenshou.Measure.Load.Closed (runClosed) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async+import Control.Exception (SomeAsyncException, SomeException, fromException, throwIO, try)+import Control.Monad (forM, when)+import Data.IORef+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Load.Series+import Kenshou.Measure.Load.Types+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Timeout (timeout)++runClosed :: Measurement -> ClosedConfig -> Operation -> IO LoadReport+runClosed measurement config operation = do+  opHandle <- registerOp (measurementRecorder measurement) operation.name+  offered <- newIORef 0+  completed <- newIORef 0+  failed <- newIORef 0+  steadyCompleted <- newIORef 0+  maxLag <- newIORef 0+  loadSeries <- openLoadSeries measurement+  let phaseClock = measurementPhaseClock measurement+      plan = (measurementConfig measurement).defaultPhases+  enterPhase phaseClock WarmUp+  sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+  workers <- forM [0 .. config.workers - 1] \workerId -> do+    workerRecorder <- newWorkerRecorder opHandle workerId+    async do+      when (config.staggerNs > 0) (sleepNanos (fromIntegral workerId * config.staggerNs `div` fromIntegral (max 1 config.workers)))+      workerLoop phaseClock workerRecorder workerId 0 offered completed failed steadyCompleted+  sleepNanos (unNanos plan.warmUp)+  enterPhase phaseClock Steady+  sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+  case plan.steady of+    SteadyFor duration -> sleepNanos (unNanos duration)+    SteadyCount target -> waitForCount steadyCompleted target+  enterPhase phaseClock Drain+  sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+  let drainMicros = fromIntegral (min (unNanos plan.drain `div` 1_000) (fromIntegral (maxBound :: Int)))+  drained <- timeout drainMicros (mapM_ wait workers)+  case drained of+    Just () -> pure ()+    Nothing -> mapM_ cancel workers+  enterPhase phaseClock Done+  sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+  closeLoadSeries loadSeries+  offeredValue <- readIORef offered+  completedValue <- readIORef completed+  failedValue <- readIORef failed+  let report = (emptyLoadReport (ClosedLoop config)) {offered = offeredValue, started = offeredValue, completed = completedValue, failed = failedValue}+  appendLoadReport measurement report+  pure report+  where+    workerLoop phaseClock workerRecorder workerId sequenceNumber offered completed failed steadyCompleted = do+      phase <- currentPhase phaseClock+      case phase of+        Drain -> pure ()+        Done -> pure ()+        _ -> do+          intended <- nowNs+          increment offered+          result <- try (operation.run workerId sequenceNumber)+          ended <- nowNs+          opResult <- case result of+            Left err | Just (_ :: SomeAsyncException) <- fromException err -> throwIO err+            Left (_ :: SomeException) -> pure (OpFailed (ErrorCause "exception"))+            Right value -> pure value+          recordOp workerRecorder intended intended ended opResult+          increment completed+          case opResult of OpFailed _ -> increment failed; OpOk _ -> pure ()+          when (phase == Steady) (increment steadyCompleted)+          when (config.thinkTimeNs > 0) (sleepNanos config.thinkTimeNs)+          workerLoop phaseClock workerRecorder workerId (sequenceNumber + 1) offered completed failed steadyCompleted++increment :: IORef Word64 -> IO ()+increment counter = atomicModifyIORef' counter (\value -> let !next = value + 1 in (next, ()))++waitForCount :: IORef Word64 -> Word64 -> IO ()+waitForCount counter target = do+  current <- readIORef counter+  if current >= target then pure () else threadDelay 1_000 >> waitForCount counter target++sleepNanos :: Word64 -> IO ()+sleepNanos duration = nowNs >>= sleepUntilNs . (+ duration)++unNanos :: Nanos -> Word64+unNanos (Nanos value) = value
+ src/Kenshou/Measure/Load/Open.hs view
@@ -0,0 +1,107 @@+{-# LANGUAGE BangPatterns #-}++module Kenshou.Measure.Load.Open (runOpen) where++import Control.Concurrent.Async+import Control.Exception (SomeAsyncException, SomeException, fromException, throwIO, try)+import Control.Monad (forM, when)+import Data.IORef+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Load.Arrival+import Kenshou.Measure.Load.Series+import Kenshou.Measure.Load.Types+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Random.SplitMix (SMGen, mkSMGen)+import System.Timeout (timeout)++data TicketState = TicketState !Word64 !Word64 !SMGen++runOpen :: Measurement -> OpenConfig -> Operation -> IO LoadReport+runOpen measurement config operation = do+  opHandle <- registerOp (measurementRecorder measurement) operation.name+  offered <- newIORef 0+  started <- newIORef 0+  completed <- newIORef 0+  failed <- newIORef 0+  maxLag <- newIORef 0+  aborted <- newIORef False+  loadSeries <- openLoadSeries measurement+  start <- nowNs+  tickets <- newIORef (TicketState 0 start (mkSMGen (measurementEnv measurement).seed))+  let phaseClock = measurementPhaseClock measurement+      plan = (measurementConfig measurement).defaultPhases+  enterPhase phaseClock WarmUp+  sampleLoadSeries loadSeries measurement offered started completed failed maxLag+  executors <- forM [0 .. config.executors - 1] \executorId -> do+    workerRecorder <- newWorkerRecorder opHandle executorId+    async (executorLoop phaseClock workerRecorder executorId tickets offered started completed failed maxLag aborted)+  sleepNanos (unNanos plan.warmUp)+  enterPhase phaseClock Steady+  sampleLoadSeries loadSeries measurement offered started completed failed maxLag+  case plan.steady of+    SteadyFor duration -> sleepNanos (unNanos duration)+    SteadyCount target -> waitForCount completed target+  enterPhase phaseClock Drain+  sampleLoadSeries loadSeries measurement offered started completed failed maxLag+  let drainMicros = fromIntegral (min (unNanos plan.drain `div` 1_000) (fromIntegral (maxBound :: Int)))+  drained <- timeout drainMicros (mapM_ wait executors)+  case drained of Just () -> pure (); Nothing -> mapM_ cancel executors+  enterPhase phaseClock Done+  sampleLoadSeries loadSeries measurement offered started completed failed maxLag+  closeLoadSeries loadSeries+  offeredValue <- readIORef offered+  startedValue <- readIORef started+  completedValue <- readIORef completed+  failedValue <- readIORef failed+  maxLagValue <- readIORef maxLag+  abortedValue <- readIORef aborted+  let overload = if maxLagValue > config.overload.maxLagNs then Just (OverloadEvidence config.overload.maxLagNs maxLagValue) else Nothing+      report = LoadReport (OpenLoop config) offeredValue startedValue completedValue failedValue maxLagValue overload abortedValue+  appendLoadReport measurement report+  pure report+  where+    executorLoop phaseClock workerRecorder executorId tickets offered started completed failed maxLag aborted = do+      phase <- currentPhase phaseClock+      abortNow <- readIORef aborted+      case phase of+        Drain -> pure ()+        Done -> pure ()+        _ | abortNow -> pure ()+        _ -> do+          (sequenceNumber, intended) <- atomicModifyIORef' tickets \(TicketState index next generator) ->+            let (gap, nextGenerator) = nextGapNs config.arrival generator+             in (TicketState (index + 1) (next + gap) nextGenerator, (index, next))+          increment offered+          sleepUntilNs intended+          actual <- nowNs+          let lag = actual - min actual intended+          atomicModifyIORef' maxLag (\value -> let !next = max value lag in (next, ()))+          when (lag > config.overload.abortLagNs) (writeIORef aborted True)+          increment started+          result <- try (operation.run executorId sequenceNumber)+          ended <- nowNs+          opResult <- case result of+            Left err | Just (_ :: SomeAsyncException) <- fromException err -> throwIO err+            Left (_ :: SomeException) -> pure (OpFailed (ErrorCause "exception"))+            Right value -> pure value+          recordOp workerRecorder intended actual ended opResult+          increment completed+          case opResult of OpFailed _ -> increment failed; OpOk _ -> pure ()+          executorLoop phaseClock workerRecorder executorId tickets offered started completed failed maxLag aborted++increment :: IORef Word64 -> IO ()+increment counter = atomicModifyIORef' counter (\value -> let !next = value + 1 in (next, ()))++waitForCount :: IORef Word64 -> Word64 -> IO ()+waitForCount counter target = do+  current <- readIORef counter+  if current >= target then pure () else sleepNanos 1_000_000 >> waitForCount counter target++sleepNanos :: Word64 -> IO ()+sleepNanos duration = nowNs >>= sleepUntilNs . (+ duration)++unNanos :: Nanos -> Word64+unNanos (Nanos value) = value
+ src/Kenshou/Measure/Load/Series.hs view
@@ -0,0 +1,32 @@+module Kenshou.Measure.Load.Series+  ( LoadSeries,+    openLoadSeries,+    sampleLoadSeries,+    closeLoadSeries,+  )+where++import Data.IORef+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Sampler.Csv+import Kenshou.Measure.Session+import System.FilePath ((</>))++newtype LoadSeries = LoadSeries CsvWriter++openLoadSeries :: Measurement -> IO LoadSeries+openLoadSeries measurement = do+  let path = (measurementEnv measurement).runDir </> "series" </> "load.csv"+  writer <- openCsv path ["t_mono_ns", "t_wall_ms", "phase", "offered", "started", "completed", "failed", "max_lag_ns"]+  (measurementEnv measurement).declareArtifact path "text/csv"+  pure (LoadSeries writer)++sampleLoadSeries :: LoadSeries -> Measurement -> IORef Word64 -> IORef Word64 -> IORef Word64 -> IORef Word64 -> IORef Word64 -> IO ()+sampleLoadSeries (LoadSeries writer) measurement offered started completed failed maxLag = do+  (_, prefix) <- timestampColumns (measurementEnv measurement).origin (measurementPhaseClock measurement)+  values <- traverse readIORef [offered, started, completed, failed, maxLag]+  appendCsv writer (prefix <> fmap (Text.pack . show) values)++closeLoadSeries :: LoadSeries -> IO ()+closeLoadSeries (LoadSeries writer) = closeCsv writer
+ src/Kenshou/Measure/Load/Types.hs view
@@ -0,0 +1,89 @@+module Kenshou.Measure.Load.Types+  ( Operation (..),+    Arrival (..),+    ClosedConfig (..),+    OverloadConfig (..),+    OpenConfig (..),+    LoadModel (..),+    OverloadEvidence (..),+    LoadReport (..),+    emptyLoadReport,+  )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.Word (Word64)+import Kenshou.Measure.Recorder (OpName, OpResult)++data Operation = Operation+  { name :: OpName,+    run :: Int -> Word64 -> IO OpResult+  }++data Arrival = ConstantRate !Double | PoissonRate !Double+  deriving stock (Eq, Show)++data ClosedConfig = ClosedConfig+  { workers :: !Int,+    thinkTimeNs :: !Word64,+    staggerNs :: !Word64+  }+  deriving stock (Eq, Show)++data OverloadConfig = OverloadConfig+  { maxLagNs :: !Word64,+    sustainedIntervals :: !Int,+    abortLagNs :: !Word64+  }+  deriving stock (Eq, Show)++data OpenConfig = OpenConfig+  { arrival :: !Arrival,+    executors :: !Int,+    rampFrom :: !Double,+    overload :: !OverloadConfig+  }+  deriving stock (Eq, Show)++data LoadModel = ClosedLoop !ClosedConfig | OpenLoop !OpenConfig+  deriving stock (Eq, Show)++data OverloadEvidence = OverloadEvidence+  { thresholdNs :: !Word64,+    observedLagNs :: !Word64+  }+  deriving stock (Eq, Show)++data LoadReport = LoadReport+  { model :: !LoadModel,+    offered :: !Word64,+    started :: !Word64,+    completed :: !Word64,+    failed :: !Word64,+    maxLagNs :: !Word64,+    overloaded :: !(Maybe OverloadEvidence),+    abortedEarly :: !Bool+  }+  deriving stock (Eq, Show)++emptyLoadReport :: LoadModel -> LoadReport+emptyLoadReport model = LoadReport model 0 0 0 0 0 Nothing False++instance ToJSON OverloadEvidence where+  toJSON value = object ["thresholdNs" .= value.thresholdNs, "observedLagNs" .= value.observedLagNs]++instance ToJSON LoadReport where+  toJSON report =+    object+      [ "model" .= modelName report.model,+        "offered" .= report.offered,+        "started" .= report.started,+        "completed" .= report.completed,+        "failed" .= report.failed,+        "maxLagNs" .= report.maxLagNs,+        "overloaded" .= report.overloaded,+        "abortedEarly" .= report.abortedEarly+      ]+    where+      modelName (ClosedLoop _) = "closed" :: String+      modelName (OpenLoop config) = case config.arrival of ConstantRate _ -> "open-constant"; PoissonRate _ -> "open-poisson"
+ src/Kenshou/Measure/Methodology.hs view
@@ -0,0 +1,19 @@+module Kenshou.Measure.Methodology+  ( PoolMethodology (..),+    poolMethodologyFor,+    minimumTrials,+    defaultTrials,+  )+where++import Kenshou.Core.Bundle (LayerBundle (..), Registry, lookupBundle)+import Kenshou.Core.Contribution (PoolMethodology (..))+import Kenshou.Core.Id (LayerId)++-- | The connection-pool methodology a layer's registered bundle contributes.+poolMethodologyFor :: Registry -> LayerId -> Maybe PoolMethodology+poolMethodologyFor registry layer = lookupBundle registry layer >>= (.methodology)++minimumTrials, defaultTrials :: Int+minimumTrials = 3+defaultTrials = 5
+ src/Kenshou/Measure/Metrics.hs view
@@ -0,0 +1,16 @@+module Kenshou.Measure.Metrics+  ( Metric (..),+  )+where++import Data.Aeson+import Data.Text (Text)++data Metric = Metric {value :: Double, unit :: Text}+  deriving stock (Eq, Show)++instance ToJSON Metric where+  toJSON metric = object ["value" .= metric.value, "unit" .= metric.unit]++instance FromJSON Metric where+  parseJSON = withObject "Metric" \value -> Metric <$> value .: "value" <*> value .: "unit"
+ src/Kenshou/Measure/Perf.hs view
@@ -0,0 +1,145 @@+module Kenshou.Measure.Perf+  ( AdmissionProblem (..),+    requiredMetricProblems,+    runtimeResultProblems,+    validateProfilePolicy,+    verifyPerformanceRun,+  )+where++import Control.Exception (IOException, displayException, try)+import Control.Monad (unless, when)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseMaybe)+import Data.ByteString qualified as ByteString+import Data.Char (isAlphaNum)+import Data.IORef+import Data.List (nub)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Compare.Policy qualified as Policy+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (decodeHistogram)+import Kenshou.Measure.Metrics (Metric (..))+import Kenshou.Measure.Samples+import Kenshou.Measure.Summary+import Kenshou.Perf.Artifacts (verifyPerfManifest)+import Kenshou.Perf.Plan (PerfProfile (..), RequiredMetric (..))+import System.FilePath ((</>))++data AdmissionProblem+  = MissingRequiredMetric Text+  | InsufficientSteadySamples Text Int Word64+  | InvalidPerformanceEvidence Text+  | RuntimeWorkFailure Text+  deriving stock (Eq, Show)++-- A known defect may deliberately have a nonblocking exit code. Performance+-- acceptance still retains the failed runtime observation independently.+runtimeResultProblems :: Value -> [AdmissionProblem]+runtimeResultProblems document = case parseMaybe (withObject "runtime result" (.: "outcome")) document :: Maybe Text of+  Just "passed" -> []+  Just "failed" -> [RuntimeWorkFailure "runtime assertions failed (including nonblocking known defects)"]+  Just "errored" -> [RuntimeWorkFailure "runtime execution errored", InvalidPerformanceEvidence "runtime execution lacks authoritative measurements"]+  Just "inconclusive" -> [InvalidPerformanceEvidence "runtime result is inconclusive"]+  Just "infrastructure-failure" -> [InvalidPerformanceEvidence "runtime infrastructure failed"]+  _ -> [InvalidPerformanceEvidence "runtime outcome is missing or unsupported"]++requiredMetricProblems :: [RequiredMetric] -> MeasurementSummary -> [AdmissionProblem]+requiredMetricProblems required summary = concatMap check required+  where+    check metric = case Map.lookup metric.name summary.metrics of+      Nothing -> [MissingRequiredMetric metric.name]+      Just value+        | value.unit /= metric.unit -> [InvalidPerformanceEvidence ("metric unit mismatch: " <> metric.name)]+        | isNaN value.value || isInfinite value.value -> [InvalidPerformanceEvidence ("non-finite metric: " <> metric.name)]+        | otherwise -> case operationFor metric.name >>= (`Map.lookup` summary.ops) >>= count of+            Nothing -> [InvalidPerformanceEvidence ("required metric has no steady sample count: " <> metric.name)]+            Just samples | samples < fromIntegral metric.minimumSteadySamples -> [InsufficientSteadySamples metric.name metric.minimumSteadySamples samples]+            _ -> []+    count = parseMaybe (withObject "operation summary" (.: "count"))++validateProfilePolicy :: PerfProfile -> Policy.Policy -> Either Text ()+validateProfilePolicy profile policy = do+  unless (profile.minimumPairs >= policy.minimumPairs) (Left "profile pair count is below the comparison policy minimum")+  unless (policy.requireInterleaving && policy.requireGrade == "benchmark") (Left "profile policy must require interleaved benchmark-grade evidence")+  unless (policy.confidenceLevel > 0 && policy.confidenceLevel < 1) (Left "invalid policy confidence level")+  unless (all (\rule -> rule.relativeLimit >= 0 && rule.absoluteFloor >= 0) policy.metricRules) (Left "invalid comparison policy effect limits")+  mapM_ required profile.requiredMetrics+  where+    required metric = unless (any (\rule -> Policy.matchesRule rule.match metric.name) policy.metricRules) (Left ("required metric has no policy rule: " <> metric.name))++-- Recompute from sealed raw samples and compare the retained histogram and+-- summary. The ordinary comparison engine trusts a stored summary if present;+-- performance orchestration must call this admission check before comparing.+verifyPerformanceRun :: PerfProfile -> FilePath -> IO (Either [AdmissionProblem] MeasurementSummary)+verifyPerformanceRun profile directory = do+  checked <- try @IOException verify+  pure case checked of+    Left problem -> Left [InvalidPerformanceEvidence (Text.pack (displayException problem))]+    Right result -> result+  where+    verify = do+      manifest <- verifyPerfManifest directory+      case manifest of+        Left problems -> pure (Left [InvalidPerformanceEvidence (Text.pack (show problems))])+        Right () -> do+          summarized <- summarizeRunDir directory+          case summarized of+            Left problem -> pure (Left [InvalidPerformanceEvidence (Text.pack (show problem))])+            Right summary -> do+              rawProblems <- concat <$> traverse (verifyOperation directory summary.window profile.steadyCount) operations+              result <- eitherDecodeFileStrict' (directory </> "run-result.json") :: IO (Either String Value)+              let retained = result >>= \document -> maybe (Left "missing retained measurement summary") Right (jsonAt ["summaries", "measurements", "measurements"] document)+                  summaryProblems = case retained of+                    Left problem -> [InvalidPerformanceEvidence (Text.pack problem)]+                    Right value | value /= toJSON summary -> [InvalidPerformanceEvidence "retained measurement summary differs from raw evidence"]+                    _ -> []+                  outcomeProblems = either (\problem -> [InvalidPerformanceEvidence (Text.pack problem)]) runtimeResultProblems result+                  problems = requiredMetricProblems profile.requiredMetrics summary <> rawProblems <> summaryProblems <> outcomeProblems+              pure if null problems then Right summary else Left problems+    operations = nub [operation | metric <- profile.requiredMetrics, Just operation <- [operationFor metric.name]]++verifyOperation :: FilePath -> SummaryWindow -> Int -> Text -> IO [AdmissionProblem]+verifyOperation directory window plannedCount operation+  | Text.null operation || not (Text.all (\c -> isAlphaNum c || c `elem` ['-', '_']) operation) = pure [InvalidPerformanceEvidence "unsafe required operation name"]+  | otherwise = do+      let base = directory </> "samples" </> Text.unpack operation+      latency <- decodeHistogram <$> ByteString.readFile (base <> ".hist")+      service <- decodeHistogram <$> ByteString.readFile (base <> ".service.hist")+      case (latency, service) of+        (Right expectedLatency, Right expectedService) -> do+          rebuiltLatency <- Histogram.newHistogram (Histogram.histogramConfig expectedLatency)+          rebuiltService <- Histogram.newHistogram (Histogram.histogramConfig expectedService)+          counts <- newIORef (0 :: Word64, 0 :: Word64, False)+          report <- readSamples (base <> ".raw") \sample -> do+            let steady = sample.intendedStartNs >= window.steadyStartMonoNs && sample.intendedStartNs < window.steadyEndMonoNs+                invalid = sample.actualStartNs > sample.endNs || sample.intendedStartNs > sample.endNs || sample.outcomeCode > 1+            modifyIORef' counts (\(count, failures, bad) -> (count + if steady then 1 else 0, failures + if steady && sample.outcomeCode /= 0 then 1 else 0, bad || invalid || not steady))+            when (steady && not invalid) do+              Histogram.recordValue rebuiltLatency (sample.endNs - sample.intendedStartNs)+              Histogram.recordValue rebuiltService (sample.endNs - sample.actualStartNs)+          actualLatency <- Histogram.freeze rebuiltLatency+          actualService <- Histogram.freeze rebuiltService+          (count, failures, malformed) <- readIORef counts+          pure+            ( [InvalidPerformanceEvidence (operation <> ": raw sample clock/schema/phase mismatch") | malformed || report.ignoredBytes /= 0 || report.header.clockKind /= MonotonicClock || report.header.operation /= operation]+                <> [InvalidPerformanceEvidence (operation <> ": histogram differs from raw samples") | expectedLatency /= actualLatency || expectedService /= actualService]+                <> [RuntimeWorkFailure (operation <> ": completed steady work differs from declared count") | count /= fromIntegral plannedCount]+                <> [RuntimeWorkFailure (operation <> ": failed steady operations") | failures > 0]+            )+        _ -> pure [InvalidPerformanceEvidence (operation <> ": invalid histogram schema")]++operationFor :: Text -> Maybe Text+operationFor name = case Text.splitOn "." name of+  "op" : operation : _ -> Just operation+  _ -> Nothing++jsonAt :: [Text] -> Value -> Maybe Value+jsonAt [] value = Just value+jsonAt (key : rest) (Object value) = KeyMap.lookup (Key.fromText key) value >>= jsonAt rest+jsonAt _ _ = Nothing
+ src/Kenshou/Measure/Phase.hs view
@@ -0,0 +1,75 @@+module Kenshou.Measure.Phase+  ( Phase (..),+    SteadyBound (..),+    PhasePlan (..),+    PhaseClock,+    newPhaseClock,+    enterPhase,+    currentPhase,+    phaseOf,+    addBoundaryListener,+    renderPhase,+  )+where++import Data.IORef+import Data.Text (Text)+import Data.Word (Word64)+import Kenshou.Measure.Clock++data Phase = Setup | WarmUp | Steady | Drain | Done+  deriving stock (Eq, Ord, Show, Enum, Bounded)++data SteadyBound = SteadyFor !Nanos | SteadyCount !Word64+  deriving stock (Eq, Show)++data PhasePlan = PhasePlan+  { warmUp :: !Nanos,+    steady :: !SteadyBound,+    drain :: !Nanos+  }+  deriving stock (Eq, Show)++data PhaseClock = PhaseClock+  { current :: !(IORef Phase),+    boundaries :: !(IORef [(Word64, Phase)]),+    boundaryListeners :: !(IORef [IO ()]),+    onPhase :: !(Phase -> Origin -> IO ())+  }++newPhaseClock :: (Phase -> Origin -> IO ()) -> PhasePlan -> IO PhaseClock+newPhaseClock onPhase _plan = do+  origin <- captureOrigin+  current <- newIORef Setup+  boundaries <- newIORef [(origin.monoNs, Setup)]+  boundaryListeners <- newIORef []+  onPhase Setup origin+  pure PhaseClock {current, boundaries, boundaryListeners, onPhase}++enterPhase :: PhaseClock -> Phase -> IO ()+enterPhase clock phase = do+  origin <- captureOrigin+  writeIORef clock.current phase+  modifyIORef' clock.boundaries (<> [(origin.monoNs, phase)])+  clock.onPhase phase origin+  readIORef clock.boundaryListeners >>= sequence_++currentPhase :: PhaseClock -> IO Phase+currentPhase = readIORef . (.current)++phaseOf :: PhaseClock -> Word64 -> IO Phase+phaseOf clock intendedStart = do+  boundaries <- readIORef clock.boundaries+  pure (foldl choose Setup boundaries)+  where+    choose selected (at, phase) = if at <= intendedStart then phase else selected++addBoundaryListener :: PhaseClock -> IO () -> IO ()+addBoundaryListener clock listener = modifyIORef' clock.boundaryListeners (<> [listener])++renderPhase :: Phase -> Text+renderPhase Setup = "setup"+renderPhase WarmUp = "warm-up"+renderPhase Steady = "steady"+renderPhase Drain = "drain"+renderPhase Done = "done"
+ src/Kenshou/Measure/Recorder.hs view
@@ -0,0 +1,301 @@+{-# LANGUAGE BangPatterns #-}++module Kenshou.Measure.Recorder+  ( OpName (..),+    ErrorCause (..),+    OpResult (..),+    RawSamplePolicy (..),+    RecorderConfig (..),+    Recorder,+    OpHandle,+    WorkerRecorder,+    OpReport (..),+    RecorderReport (..),+    newRecorder,+    registerOp,+    newWorkerRecorder,+    recordOp,+    recordDuration,+    timeOp,+    finishRecorder,+  )+where++import Control.Monad (forM, forM_, when)+import Data.Aeson (ToJSON (..), encode, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.ByteString qualified as ByteString+import Data.ByteString.Builder (byteString, toLazyByteString, word16LE, word32LE, word64LE, word8)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word+import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram (Histogram, HistogramConfig)+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (encodeHistogram)+import Kenshou.Measure.Phase+import Kenshou.Measure.Samples+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))++newtype OpName = OpName Text+  deriving stock (Eq, Ord, Show)++newtype ErrorCause = ErrorCause Text+  deriving stock (Eq, Ord, Show)++data OpResult = OpOk !Int | OpFailed !ErrorCause+  deriving stock (Eq, Show)++data RawSamplePolicy = RawFull | RawSampled !Word64 | RawOff+  deriving stock (Eq, Show)++instance ToJSON RawSamplePolicy where+  toJSON RawFull = toJSON ("full" :: Text)+  toJSON (RawSampled oneIn) = object ["mode" .= ("sampled" :: Text), "oneIn" .= oneIn]+  toJSON RawOff = toJSON ("off" :: Text)++data RecorderConfig = RecorderConfig+  { runDir :: FilePath,+    origin :: Origin,+    processLabel :: Maybe Text,+    histogram :: HistogramConfig,+    rawSamples :: RawSamplePolicy,+    intervalHistogramSeconds :: Word64,+    declareArtifact :: FilePath -> Text -> IO ()+  }++data Recorder = Recorder+  { config :: RecorderConfig,+    phaseClock :: PhaseClock,+    operations :: IORef [OpHandle]+  }++data OpHandle = OpHandle+  { recorder :: Recorder,+    name :: OpName,+    writer :: Maybe SampleWriter,+    workers :: IORef [WorkerRecorder]+  }++data WorkerRecorder = WorkerRecorder+  { operation :: OpHandle,+    workerId :: Word16,+    latency :: Histogram.MutableHistogram,+    service :: Histogram.MutableHistogram,+    phaseCounts :: IORef (Map Phase (Word64, Word64, Word64)),+    errors :: IORef (Map ErrorCause Word64),+    intervals :: IORef (Map Word64 Histogram.MutableHistogram),+    pending :: IORef [SampleRecord],+    pendingCount :: IORef Int,+    seen :: IORef Word64+  }++data OpReport = OpReport+  { name :: OpName,+    latency :: Histogram,+    service :: Histogram,+    phaseCounts :: Map Phase (Word64, Word64, Word64),+    errors :: Map ErrorCause Word64,+    backpressureCount :: Word64,+    histogramFile :: FilePath,+    serviceHistogramFile :: FilePath,+    rawFile :: Maybe FilePath,+    intervalHistogramFile :: Maybe FilePath,+    metadataFile :: FilePath+  }++data RecorderReport = RecorderReport+  { operations :: [OpReport],+    backpressureCount :: Word64+  }++newRecorder :: RecorderConfig -> PhaseClock -> IO Recorder+newRecorder config phaseClock = do+  createDirectoryIfMissing True (config.runDir </> "samples")+  Recorder config phaseClock <$> newIORef []++registerOp :: Recorder -> OpName -> IO OpHandle+registerOp recorder name@(OpName nameText) = do+  let rawPath = samplePath recorder.config nameText "raw"+      label = maybe "harness" id recorder.config.processLabel+  writer <- case recorder.config.rawSamples of+    RawOff -> pure Nothing+    _ -> Just <$> openSampleWriter rawPath (SampleHeader recorder.config.origin MonotonicClock nameText label)+  handle <- OpHandle recorder name writer <$> newIORef []+  modifyIORef' recorder.operations (<> [handle])+  pure handle++newWorkerRecorder :: OpHandle -> Int -> IO WorkerRecorder+newWorkerRecorder operation workerNumber = do+  latency <- Histogram.newHistogram operation.recorder.config.histogram+  service <- Histogram.newHistogram operation.recorder.config.histogram+  worker <-+    WorkerRecorder operation (fromIntegral workerNumber) latency service+      <$> newIORef Map.empty+      <*> newIORef Map.empty+      <*> newIORef Map.empty+      <*> newIORef []+      <*> newIORef 0+      <*> newIORef 0+  modifyIORef' operation.workers (<> [worker])+  pure worker++recordOp :: WorkerRecorder -> Word64 -> Word64 -> Word64 -> OpResult -> IO ()+recordOp worker intended actual end result = do+  phase <- phaseOf worker.operation.recorder.phaseClock intended+  let serviceNs = end - min end actual+      latencyNs = end - min end intended+      (successes, failures, units) = case result of+        OpOk completed -> (1, 0, fromIntegral (max 0 completed))+        OpFailed _ -> (0, 1, 0)+  when (phase == Steady) do+    Histogram.recordValue worker.latency latencyNs+    Histogram.recordValue worker.service serviceNs+    when (worker.operation.recorder.config.rawSamples /= RawFull) (recordInterval worker intended latencyNs)+  modifyIORef' worker.phaseCounts (Map.insertWith addCounts phase (successes, failures, units))+  case result of+    OpFailed cause -> modifyIORef' worker.errors (Map.insertWith (+) cause 1)+    OpOk _ -> pure ()+  sequenceNumber <- atomicModifyIORef' worker.seen (\value -> (value + 1, value))+  when (retain worker.operation.recorder.config.rawSamples sequenceNumber) do+    let outcomeCode = case result of OpOk _ -> 0; OpFailed _ -> 1+    modifyIORef' worker.pending (SampleRecord intended actual end outcomeCode units :)+    count <- atomicModifyIORef' worker.pendingCount (\value -> let next = value + 1 in (next, next))+    when (count >= 512) (flushWorker worker)++recordDuration :: WorkerRecorder -> Word64 -> Word64 -> OpResult -> IO ()+recordDuration worker end duration = recordOp worker (end - min end duration) (end - min end duration) end++timeOp :: WorkerRecorder -> IO OpResult -> IO OpResult+timeOp worker action = do+  start <- nowNs+  result <- action+  end <- nowNs+  recordOp worker start start end result+  pure result++finishRecorder :: Recorder -> IO RecorderReport+finishRecorder recorder = do+  operations <- readIORef recorder.operations >>= mapM finishOperation+  pure RecorderReport {operations, backpressureCount = sum (fmap (.backpressureCount) operations)}++finishOperation :: OpHandle -> IO OpReport+finishOperation operation = do+  workers <- readIORef operation.workers+  mapM_ flushWorker workers+  forM_ operation.writer closeSampleWriter+  latency <- mergeWorkerHistograms operation.recorder.config.histogram workers (.latency)+  service <- mergeWorkerHistograms operation.recorder.config.histogram workers (.service)+  phaseCounts <- fmap (Map.unionsWith addCounts) (mapM (readIORef . (.phaseCounts)) workers)+  errors <- fmap (Map.unionsWith (+)) (mapM (readIORef . (.errors)) workers)+  backpressureCount <- maybe (pure 0) sampleWriterBackpressure operation.writer+  intervalHistograms <- finishIntervals operation.recorder.config workers+  let OpName nameText = operation.name+      histogramFile = samplePath operation.recorder.config nameText "hist"+      serviceHistogramFile = samplePath operation.recorder.config nameText "service.hist"+      metadataFile = samplePath operation.recorder.config nameText "meta.json"+      rawFile = case operation.writer of Nothing -> Nothing; Just _ -> Just (samplePath operation.recorder.config nameText "raw")+      intervalHistogramFile = if null intervalHistograms then Nothing else Just (samplePath operation.recorder.config nameText "ihist")+  writeBytes histogramFile (encodeHistogram latency)+  writeBytes serviceHistogramFile (encodeHistogram service)+  forM_ intervalHistogramFile \path -> writeIntervalHistograms operation.recorder.config path intervalHistograms+  LazyByteString.writeFile metadataFile $+    encode+      ( object+          [ "schema" .= ("kenshou.sample-meta/v1" :: Text),+            "operation" .= nameText,+            "latencyBasis" .= ("intended-start" :: Text),+            "clock" .= ("monotonic" :: Text),+            "rawSamples" .= operation.recorder.config.rawSamples,+            "phaseCounts" .= object [Key.fromText (renderPhase phase) .= object ["successes" .= successes, "failures" .= failures, "units" .= units] | (phase, (successes, failures, units)) <- Map.toAscList phaseCounts],+            "errorCauses" .= object [Key.fromText cause .= count | (ErrorCause cause, count) <- Map.toAscList errors],+            "files" .= object ["histogram" .= histogramFile, "serviceHistogram" .= serviceHistogramFile, "raw" .= rawFile, "intervalHistogram" .= intervalHistogramFile],+            "backpressureCount" .= backpressureCount+          ]+      )+  let artifacts = [(histogramFile, "application/vnd.kenshou.histogram"), (serviceHistogramFile, "application/vnd.kenshou.histogram"), (metadataFile, "application/json")] <> maybe [] (\path -> [(path, "application/vnd.kenshou.samples")]) rawFile <> maybe [] (\path -> [(path, "application/vnd.kenshou.interval-histograms")]) intervalHistogramFile+  mapM_ (uncurry operation.recorder.config.declareArtifact) artifacts+  pure OpReport {name = operation.name, latency, service, phaseCounts, errors, backpressureCount, histogramFile, serviceHistogramFile, rawFile, intervalHistogramFile, metadataFile}++flushWorker :: WorkerRecorder -> IO ()+flushWorker worker = case worker.operation.writer of+  Nothing -> writeIORef worker.pending [] >> writeIORef worker.pendingCount 0+  Just writer -> do+    records <- atomicModifyIORef' worker.pending (\pending -> ([], reverse pending))+    writeIORef worker.pendingCount 0+    writeSampleBlock writer worker.workerId records++mergeWorkerHistograms :: HistogramConfig -> [WorkerRecorder] -> (WorkerRecorder -> Histogram.MutableHistogram) -> IO Histogram+mergeWorkerHistograms config workers project = do+  histograms <- mapM (Histogram.freeze . project) workers+  case histograms of+    [] -> Histogram.newHistogram config >>= Histogram.freeze+    first : rest -> case foldl' combine (Right first) rest of+      Left message -> ioError (userError (Text.unpack message))+      Right value -> pure value+  where+    combine accumulated next = accumulated >>= (`Histogram.merge` next)++samplePath :: RecorderConfig -> Text -> FilePath -> FilePath+samplePath config operation extension = config.runDir </> "samples" </> Text.unpack operation <> maybe "" (("." <>) . Text.unpack) config.processLabel <> "." <> extension++writeBytes :: FilePath -> ByteString.ByteString -> IO ()+writeBytes = ByteString.writeFile++retain :: RawSamplePolicy -> Word64 -> Bool+retain RawFull _ = True+retain (RawSampled oneIn) sequenceNumber = oneIn > 0 && sequenceNumber `mod` oneIn == 0+retain RawOff _ = False++recordInterval :: WorkerRecorder -> Word64 -> Word64 -> IO ()+recordInterval worker intended latencyNs = do+  let seconds = max 1 worker.operation.recorder.config.intervalHistogramSeconds+      intervalNs = seconds * 1_000_000_000+      epoch = intended `div` intervalNs+  existing <- Map.lookup epoch <$> readIORef worker.intervals+  histogram <- case existing of+    Just value -> pure value+    Nothing -> do+      value <- Histogram.newHistogram worker.operation.recorder.config.histogram+      modifyIORef' worker.intervals (Map.insert epoch value)+      pure value+  Histogram.recordValue histogram latencyNs++finishIntervals :: RecorderConfig -> [WorkerRecorder] -> IO [(Word64, Histogram)]+finishIntervals config workers = do+  workerMaps <- mapM (readIORef . (.intervals)) workers+  let epochs = Set.toAscList (Set.unions (fmap Map.keysSet workerMaps))+  forM epochs \epoch -> do+    histograms <- forM workerMaps \intervals -> traverse Histogram.freeze (Map.lookup epoch intervals)+    empty <- Histogram.newHistogram config.histogram >>= Histogram.freeze+    let mergeOne accumulated Nothing = accumulated+        mergeOne (Left message) _ = Left message+        mergeOne (Right accumulated) (Just next) = Histogram.merge accumulated next+    case foldl mergeOne (Right empty) histograms of+      Left message -> ioError (userError (Text.unpack message))+      Right histogram -> pure (epoch, histogram)++writeIntervalHistograms :: RecorderConfig -> FilePath -> [(Word64, Histogram)] -> IO ()+writeIntervalHistograms config path histograms =+  ByteString.writeFile path . LazyByteString.toStrict . toLazyByteString $+    byteString "KIHS" <> word16LE 1 <> word16LE 0 <> foldMap frame histograms+  where+    intervalNs = max 1 config.intervalHistogramSeconds * 1_000_000_000+    frame (epoch, histogram) =+      let encoded = encodeHistogram histogram+          absoluteStart = epoch * intervalNs+          start = absoluteStart - min absoluteStart config.origin.monoNs+       in word64LE start <> word64LE (start + intervalNs) <> word8 2 <> word32LE (fromIntegral (ByteString.length encoded)) <> byteString encoded++addCounts :: (Word64, Word64, Word64) -> (Word64, Word64, Word64) -> (Word64, Word64, Word64)+addCounts (a, b, c) (x, y, z) =+  let !successes = a + x+      !failures = b + y+      !units = c + z+   in (successes, failures, units)
+ src/Kenshou/Measure/Sampler.hs view
@@ -0,0 +1,174 @@+module Kenshou.Measure.Sampler+  ( Sampler (..),+    SamplerConfig (..),+    SamplerReport (..),+    SamplingHandle,+    startSampling,+    stopSampling,+  )+where++import Control.Concurrent.Async+import Control.Concurrent.STM+import Control.Exception (finally)+import Control.Monad (forM_)+import Data.IORef+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Phase+import Kenshou.Measure.Sampler.Csv+import Kenshou.Measure.Sampler.Host+import Kenshou.Measure.Sampler.Postgres+import Kenshou.Measure.Sampler.Process+import Kenshou.Measure.Sampler.Rts+import System.FilePath ((</>))++data Sampler = Sampler+  { name :: Text,+    sample :: [Text] -> IO ()+  }++data SamplerConfig = SamplerConfig+  { runDir :: FilePath,+    origin :: Origin,+    intervalMs :: Int,+    postgres :: Maybe PgSamplerConfig,+    extraSamplers :: [Sampler],+    phaseClock :: PhaseClock,+    declareArtifact :: FilePath -> Text -> IO (),+    logLine :: Text -> IO ()+  }++data SamplerReport = SamplerReport+  { rtsAvailable :: Bool,+    processAvailable :: Bool,+    hostAvailable :: Bool,+    postgresAvailable :: Bool,+    pgStatementsAvailable :: Bool,+    ticks :: Word64+  }+  deriving stock (Eq, Show)++data Event = Boundary | Scheduled | Stop++data SamplingHandle = SamplingHandle+  { threads :: [Async ()],+    events :: TChan Event,+    closeResources :: IO (),+    report :: IO SamplerReport+  }++startSampling :: SamplerConfig -> IO SamplingHandle+startSampling config = do+  let seriesDir = config.runDir </> "series"+      path name = seriesDir </> name+      declare name = config.declareArtifact (path name) "text/csv"+  events <- newBroadcastTChanIO+  phaseEvents <- atomically (dupTChan events)+  postgresEvents <- atomically (dupTChan events)+  addBoundaryListener config.phaseClock (atomically (writeTChan events Boundary))+  samplerWriter <- openCsv (path "sampler.csv") ["t_mono_ns", "t_wall_ms", "phase", "scheduled_mono_ns", "late_ns", "cost_ns", "wall_minus_mono_ns"]+  declare "sampler.csv"+  rts <- openRtsSampler (path "rts.csv")+  forM_ rts (const (declare "rts.csv"))+  process <- openProcessSampler (path "proc.csv")+  declare "proc.csv"+  host <- openHostSampler (path "host.csv")+  forM_ host (const (declare "host.csv"))+  previousClockDelta <- newIORef Nothing+  tickCount <- newIORef 0+  mainThread <- async (mainLoop config phaseEvents samplerWriter rts process host previousClockDelta tickCount 1)+  postgresResult <- case config.postgres of+    Nothing -> pure Nothing+    Just pgConfig -> do+      opened <- openPostgresSampler seriesDir pgConfig config.logLine+      case opened of+        Left err -> config.logLine ("PostgreSQL sampler unavailable: " <> err) >> pure Nothing+        Right sampler -> do+          mapM_ declare (postgresArtifactNames (postgresHasStatements sampler))+          thread <- async (postgresLoop config postgresEvents sampler 1 `finally` closePostgresSampler sampler)+          pure (Just (thread, postgresHasStatements sampler))+  let closeResources = do+        closeCsv samplerWriter+        forM_ rts closeRtsSampler+        closeProcessSampler process+        forM_ host closeHostSampler+      readReport = do+        count <- readIORef tickCount+        pure+          SamplerReport+            { rtsAvailable = maybe False (const True) rts,+              processAvailable = True,+              hostAvailable = maybe False (const True) host,+              postgresAvailable = maybe False (const True) postgresResult,+              pgStatementsAvailable = maybe False snd postgresResult,+              ticks = count+            }+  pure+    SamplingHandle+      { threads = mainThread : maybe [] (pure . fst) postgresResult,+        events,+        closeResources,+        report = readReport+      }++stopSampling :: SamplingHandle -> IO SamplerReport+stopSampling handle = do+  atomically (writeTChan handle.events Stop)+  mapM_ wait handle.threads `finally` handle.closeResources+  handle.report++mainLoop :: SamplerConfig -> TChan Event -> CsvWriter -> Maybe RtsSampler -> ProcessSampler -> Maybe HostSampler -> IORef (Maybe Integer) -> IORef Word64 -> Word64 -> IO ()+mainLoop config events samplerWriter rts process host previousClockDelta tickCount tickNumber = do+  let intervalNs = fromIntegral config.intervalMs * 1_000_000+      scheduled = config.origin.monoNs + tickNumber * intervalNs+  event <- waitForEvent events scheduled+  case event of+    Stop -> pure ()+    Boundary -> nowNs >>= sampleAll >> mainLoop config events samplerWriter rts process host previousClockDelta tickCount tickNumber+    Scheduled -> sampleAll scheduled >> mainLoop config events samplerWriter rts process host previousClockDelta tickCount (tickNumber + 1)+  where+    sampleAll scheduled = do+      started <- nowNs+      (_, prefix) <- timestampColumns config.origin config.phaseClock+      forM_ rts (`sampleRts` prefix)+      sampleProcess process prefix+      forM_ host (`sampleHost` prefix)+      forM_ config.extraSamplers (\sampler -> sampler.sample prefix)+      ended <- nowNs+      wallOrigin <- captureOrigin+      let delta = fromIntegral wallOrigin.wallUnixNs - fromIntegral wallOrigin.monoNs+      previous <- readIORef previousClockDelta+      writeIORef previousClockDelta (Just delta)+      let clockChange = maybe 0 (delta -) previous+          value = Text.pack . show+      appendCsv samplerWriter (prefix <> fmap value [scheduled - min scheduled config.origin.monoNs, started - min started scheduled, ended - started] <> [Text.pack (show clockChange)])+      atomicModifyIORef' tickCount (\count -> (count + 1, ()))++postgresLoop :: SamplerConfig -> TChan Event -> PostgresSampler -> Word64 -> IO ()+postgresLoop config events sampler tickNumber = do+  let intervalNs = fromIntegral config.intervalMs * 1_000_000+      scheduled = config.origin.monoNs + tickNumber * intervalNs+  event <- waitForEvent events scheduled+  case event of+    Stop -> pure ()+    Boundary -> sampleNow True >> postgresLoop config events sampler tickNumber+    Scheduled -> sampleNow (postgresPeriodic sampler && tickNumber `mod` 10 == 0) >> postgresLoop config events sampler (tickNumber + 1)+  where+    sampleNow includeStatements = do+      (_, prefix) <- timestampColumns config.origin config.phaseClock+      samplePostgres sampler prefix includeStatements++waitForEvent :: TChan Event -> Word64 -> IO Event+waitForEvent events deadline = do+  current <- nowNs+  let remainingNs = deadline - min deadline current+      remainingMicros = fromIntegral (min (fromIntegral (maxBound :: Int)) ((remainingNs + 999) `div` 1_000))+  elapsed <- registerDelay remainingMicros+  atomically $+    readTChan events+      `orElse` do+        readTVar elapsed >>= check+        pure Scheduled
+ src/Kenshou/Measure/Sampler/Csv.hs view
@@ -0,0 +1,53 @@+module Kenshou.Measure.Sampler.Csv+  ( CsvWriter,+    openCsv,+    appendCsv,+    closeCsv,+    timestampColumns,+  )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Data.Time.Clock.POSIX (getPOSIXTime)+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Phase+import System.Directory (createDirectoryIfMissing)+import System.FilePath (takeDirectory)+import System.IO++newtype CsvWriter = CsvWriter Handle++openCsv :: FilePath -> [Text] -> IO CsvWriter+openCsv path columns = do+  createDirectoryIfMissing True (takeDirectory path)+  handle <- openFile path WriteMode+  hSetBuffering handle LineBuffering+  let writer = CsvWriter handle+  appendCsv writer columns+  pure writer++appendCsv :: CsvWriter -> [Text] -> IO ()+appendCsv (CsvWriter handle) fields = Text.hPutStrLn handle (Text.intercalate "," (fmap quote fields))+  where+    quote value+      | Text.any (`elem` [',', '"', '\r', '\n']) value = "\"" <> Text.replace "\"" "\"\"" value <> "\""+      | otherwise = value++closeCsv :: CsvWriter -> IO ()+closeCsv (CsvWriter handle) = hClose handle++timestampColumns :: Origin -> PhaseClock -> IO (Word64, [Text])+timestampColumns origin phaseClock = do+  mono <- nowNs+  wall <- getPOSIXTime+  phase <- currentPhase phaseClock+  pure+    ( mono,+      [ Text.pack (show (mono - min mono origin.monoNs)),+        Text.pack (show (floor (wall * 1_000) :: Integer)),+        renderPhase phase+      ]+    )
+ src/Kenshou/Measure/Sampler/Host.hs view
@@ -0,0 +1,95 @@+{-# LANGUAGE CPP #-}++module Kenshou.Measure.Sampler.Host+  ( HostSample (..),+    parseHostSample,+    HostSampler,+    openHostSampler,+    sampleHost,+    closeHostSampler,+  )+where++import Data.List (find)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Sampler.Csv+import Text.Read (readMaybe)++#ifdef linux_HOST_OS+import Data.Text.IO qualified as Text+#endif++data HostSample = HostSample+  { cpuUser :: Maybe Word64,+    cpuNice :: Maybe Word64,+    cpuSystem :: Maybe Word64,+    cpuIdle :: Maybe Word64,+    cpuIowait :: Maybe Word64,+    cpuIrq :: Maybe Word64,+    cpuSoftirq :: Maybe Word64,+    cpuSteal :: Maybe Word64,+    loadAverage1 :: Maybe Double,+    memAvailableBytes :: Maybe Word64+  }+  deriving stock (Eq, Show)++newtype HostSampler = HostSampler CsvWriter++openHostSampler :: FilePath -> IO (Maybe HostSampler)+#ifdef linux_HOST_OS+openHostSampler path = Just . HostSampler <$> openCsv path+  [ "t_mono_ns", "t_wall_ms", "phase", "cpu_user", "cpu_nice", "cpu_system", "cpu_idle", "cpu_iowait",+    "cpu_irq", "cpu_softirq", "cpu_steal", "loadavg1", "mem_available_bytes"+  ]+#else+openHostSampler _ = pure Nothing+#endif++sampleHost :: HostSampler -> [Text] -> IO ()+#ifdef linux_HOST_OS+sampleHost (HostSampler writer) prefix = do+  stat <- Text.readFile "/proc/stat"+  loadavg <- Text.readFile "/proc/loadavg"+  meminfo <- Text.readFile "/proc/meminfo"+  let sample = parseHostSample stat loadavg meminfo+      integral = maybe "" (Text.pack . show)+      floating = maybe "" (Text.pack . show)+  appendCsv writer (prefix <> fmap integral+    [ sample.cpuUser, sample.cpuNice, sample.cpuSystem, sample.cpuIdle, sample.cpuIowait,+      sample.cpuIrq, sample.cpuSoftirq, sample.cpuSteal+    ] <> [floating sample.loadAverage1, integral sample.memAvailableBytes])+#else+sampleHost _ _ = pure ()+#endif++closeHostSampler :: HostSampler -> IO ()+closeHostSampler (HostSampler writer) = closeCsv writer++parseHostSample :: Text -> Text -> Text -> HostSample+parseHostSample stat loadavg meminfo =+  HostSample+    { cpuUser = cpu 0,+      cpuNice = cpu 1,+      cpuSystem = cpu 2,+      cpuIdle = cpu 3,+      cpuIowait = cpu 4,+      cpuIrq = cpu 5,+      cpuSoftirq = cpu 6,+      cpuSteal = cpu 7,+      loadAverage1 = case Text.words loadavg of value : _ -> readDouble value; [] -> Nothing,+      memAvailableBytes = (* 1_024) <$> memAvailable+    }+  where+    cpuFields = maybe [] (drop 1 . Text.words) (find (Text.isPrefixOf "cpu ") (Text.lines stat))+    cpu index = case drop index cpuFields of value : _ -> readWord value; [] -> Nothing+    memAvailable = do+      line <- find (Text.isPrefixOf "MemAvailable:") (Text.lines meminfo)+      case Text.words line of _ : value : _ -> readWord value; _ -> Nothing++readWord :: Text -> Maybe Word64+readWord = readMaybe . Text.unpack++readDouble :: Text -> Maybe Double+readDouble = readMaybe . Text.unpack
+ src/Kenshou/Measure/Sampler/Postgres.hs view
@@ -0,0 +1,137 @@+module Kenshou.Measure.Sampler.Postgres+  ( PgStatementsMode (..),+    PgSamplerConfig (..),+    PostgresSampler,+    openPostgresSampler,+    samplePostgres,+    closePostgresSampler,+    postgresHasStatements,+    postgresPeriodic,+    postgresArtifactNames,+  )+where++import Control.Exception (finally)+import Control.Monad (forM_, when)+import Data.Text (Text)+import Data.Text qualified as Text+import Hasql.Connection qualified as Connection+import Hasql.Connection.Settings qualified as Settings+import Hasql.Decoders qualified as Decoders+import Hasql.Encoders qualified as Encoders+import Hasql.Session qualified as Session+import Hasql.Statement qualified as Statement+import Kenshou.Measure.Sampler.Csv++data PgStatementsMode = PgStatementsOff | PgStatementsSnapshots | PgStatementsPeriodic+  deriving stock (Eq, Show)++data PgSamplerConfig = PgSamplerConfig+  { connectionString :: Text,+    relations :: [Text],+    statements :: PgStatementsMode,+    serverVersionNum :: Int+  }+  deriving stock (Eq, Show)++data PostgresSampler = PostgresSampler+  { connection :: Connection.Connection,+    activity :: CsvWriter,+    checkpointer :: CsvWriter,+    wal :: CsvWriter,+    database :: CsvWriter,+    relations :: CsvWriter,+    statements :: Maybe CsvWriter,+    watchedRelations :: [Text],+    versionNumber :: Int,+    statementsMode :: PgStatementsMode,+    logLine :: Text -> IO ()+  }++postgresArtifactNames :: Bool -> [FilePath]+postgresArtifactNames hasStatements =+  ["pg-activity.csv", "pg-checkpointer.csv", "pg-wal.csv", "pg-database.csv", "pg-relations.csv"]+    <> ["pg-statements.csv" | hasStatements]++openPostgresSampler :: FilePath -> PgSamplerConfig -> (Text -> IO ()) -> IO (Either Text PostgresSampler)+openPostgresSampler seriesDir config logger = do+  acquired <- Connection.acquire (Settings.connectionString config.connectionString <> Settings.applicationName "kenshou-sampler")+  case acquired of+    Left err -> pure (Left (Text.pack (show err)))+    Right connection -> do+      activity <- openCsv (seriesDir <> "/pg-activity.csv") (base <> ["application_name", "state", "wait_event_type", "wait_event", "connections"])+      checkpointer <- openCsv (seriesDir <> "/pg-checkpointer.csv") (base <> ["num_timed", "num_requested", "write_time_ms", "sync_time_ms", "buffers_written", "checkpoint_lsn", "checkpoint_time_unix", "num_done"])+      wal <- openCsv (seriesDir <> "/pg-wal.csv") (base <> ["current_wal_bytes", "wal_records", "wal_fpi", "wal_bytes", "wal_buffers_full"])+      database <- openCsv (seriesDir <> "/pg-database.csv") (base <> ["xact_commit", "xact_rollback", "blks_read", "blks_hit", "tup_inserted", "tup_updated", "tup_deleted", "deadlocks", "temp_bytes"])+      relations <- openCsv (seriesDir <> "/pg-relations.csv") (base <> ["relation", "relation_bytes", "indexes_bytes", "total_bytes", "n_live_tup", "n_dead_tup", "n_tup_ins", "n_tup_upd", "n_tup_del", "autovacuum_count"])+      statements <-+        if config.statements == PgStatementsOff+          then pure Nothing+          else do+            created <- Connection.use connection (Session.statement () createStatements)+            probed <- case created of Left err -> pure (Left err); Right () -> Connection.use connection (Session.statement () probeStatements)+            case probed of+              Left err -> logger ("pg_stat_statements unavailable: " <> Text.pack (show err)) >> pure Nothing+              Right () -> Just <$> openCsv (seriesDir <> "/pg-statements.csv") (base <> ["queryid", "calls", "total_exec_time_ms", "rows", "shared_blks_hit", "shared_blks_read", "wal_bytes"])+      pure (Right PostgresSampler {connection, activity, checkpointer, wal, database, relations, statements, watchedRelations = config.relations, versionNumber = config.serverVersionNum, statementsMode = config.statements, logLine = logger})+  where+    base = ["t_mono_ns", "t_wall_ms", "phase"]++samplePostgres :: PostgresSampler -> [Text] -> Bool -> IO ()+samplePostgres sampler prefix includeStatements = do+  writeRows sampler sampler.activity prefix activityQuery+  writeRows sampler sampler.checkpointer prefix (if sampler.versionNumber >= 180000 then checkpointer18Query else checkpointer17Query)+  writeRows sampler sampler.wal prefix walQuery+  writeRows sampler sampler.database prefix databaseQuery+  forM_ sampler.watchedRelations \relation -> writeParamRows sampler sampler.relations prefix relation relationQuery+  when includeStatements $ forM_ sampler.statements \writer -> writeRows sampler writer prefix statementsQuery++closePostgresSampler :: PostgresSampler -> IO ()+closePostgresSampler sampler =+  let writers = [sampler.activity, sampler.checkpointer, sampler.wal, sampler.database, sampler.relations] <> maybe [] pure sampler.statements+   in mapM_ closeCsv writers `finally` Connection.release sampler.connection++postgresHasStatements :: PostgresSampler -> Bool+postgresHasStatements = maybe False (const True) . (.statements)++postgresPeriodic :: PostgresSampler -> Bool+postgresPeriodic sampler = sampler.statementsMode == PgStatementsPeriodic++writeRows :: PostgresSampler -> CsvWriter -> [Text] -> Statement.Statement () [Text] -> IO ()+writeRows sampler writer prefix statement = do+  result <- Connection.use sampler.connection (Session.statement () statement)+  case result of+    Left err -> sampler.logLine ("PostgreSQL sampler query failed: " <> Text.pack (show err))+    Right rows -> forM_ rows (appendCsv writer . (prefix <>) . Text.splitOn separator)++writeParamRows :: PostgresSampler -> CsvWriter -> [Text] -> Text -> Statement.Statement Text [Text] -> IO ()+writeParamRows sampler writer prefix parameter statement = do+  result <- Connection.use sampler.connection (Session.statement parameter statement)+  case result of+    Left err -> sampler.logLine ("PostgreSQL relation sampler query failed: " <> Text.pack (show err))+    Right rows -> forM_ rows (appendCsv writer . (prefix <>) . Text.splitOn separator)++separator :: Text+separator = Text.singleton '\x1f'++textRows :: Text -> Statement.Statement () [Text]+textRows sql = Statement.unpreparable sql Encoders.noParams (Decoders.rowList (Decoders.column (Decoders.nonNullable Decoders.text)))++createStatements, probeStatements :: Statement.Statement () ()+createStatements = Statement.unpreparable "CREATE EXTENSION IF NOT EXISTS pg_stat_statements" Encoders.noParams Decoders.noResult+probeStatements = Statement.unpreparable "SELECT 1 FROM pg_stat_statements(false) LIMIT 0" Encoders.noParams Decoders.noResult++activityQuery, checkpointer17Query, checkpointer18Query, walQuery, databaseQuery, statementsQuery :: Statement.Statement () [Text]+activityQuery = textRows "SELECT concat_ws(chr(31), coalesce(application_name,''), coalesce(state,''), coalesce(wait_event_type,''), coalesce(wait_event,''), count(*)::text) FROM pg_stat_activity WHERE backend_type='client backend' AND application_name <> 'kenshou-sampler' GROUP BY application_name,state,wait_event_type,wait_event ORDER BY application_name,state,wait_event_type,wait_event"+checkpointer17Query = textRows "SELECT concat_ws(chr(31), c.num_timed::text, c.num_requested::text, c.write_time::text, c.sync_time::text, c.buffers_written::text, (pg_control_checkpoint()).checkpoint_lsn::text, extract(epoch FROM (pg_control_checkpoint()).checkpoint_time)::bigint::text, '') FROM pg_stat_checkpointer c"+checkpointer18Query = textRows "SELECT concat_ws(chr(31), c.num_timed::text, c.num_requested::text, c.write_time::text, c.sync_time::text, c.buffers_written::text, (pg_control_checkpoint()).checkpoint_lsn::text, extract(epoch FROM (pg_control_checkpoint()).checkpoint_time)::bigint::text, c.num_done::text) FROM pg_stat_checkpointer c"+walQuery = textRows "SELECT concat_ws(chr(31), pg_wal_lsn_diff(pg_current_wal_lsn(),'0/0')::bigint::text, w.wal_records::text, w.wal_fpi::text, w.wal_bytes::bigint::text, w.wal_buffers_full::text) FROM pg_stat_wal w"+databaseQuery = textRows "SELECT concat_ws(chr(31), xact_commit::text, xact_rollback::text, blks_read::text, blks_hit::text, tup_inserted::text, tup_updated::text, tup_deleted::text, deadlocks::text, temp_bytes::text) FROM pg_stat_database WHERE datname=current_database()"+statementsQuery = textRows "SELECT concat_ws(chr(31), queryid::text, calls::text, total_exec_time::text, rows::text, shared_blks_hit::text, shared_blks_read::text, wal_bytes::bigint::text) FROM pg_stat_statements(false) WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database()) ORDER BY queryid"++relationQuery :: Statement.Statement Text [Text]+relationQuery =+  Statement.unpreparable+    "SELECT concat_ws(chr(31), $1::text, pg_relation_size($1::regclass)::text, pg_indexes_size($1::regclass)::text, pg_total_relation_size($1::regclass)::text, s.n_live_tup::text, s.n_dead_tup::text, s.n_tup_ins::text, s.n_tup_upd::text, s.n_tup_del::text, s.autovacuum_count::text) FROM pg_stat_all_tables s WHERE s.relid=$1::regclass"+    (Encoders.param (Encoders.nonNullable Encoders.text))+    (Decoders.rowList (Decoders.column (Decoders.nonNullable Decoders.text)))
+ src/Kenshou/Measure/Sampler/Process.hs view
@@ -0,0 +1,144 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE ForeignFunctionInterface #-}++module Kenshou.Measure.Sampler.Process+  ( ProcessSample (..),+    parseProcSample,+    readProcessSample,+    ProcessSampler,+    openProcessSampler,+    sampleProcess,+    closeProcessSampler,+  )+where++import Data.Char (isSpace)+import Data.List (find)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Sampler.Csv+import System.CPUTime (getCPUTime)+import System.Directory (listDirectory)+import Text.Read (readMaybe)++#ifdef linux_HOST_OS+import Data.Text.IO qualified as Text+#endif++#ifdef darwin_HOST_OS+import Foreign (Ptr, alloca, peek)+import Foreign.C.Types++foreign import ccall unsafe "kenshou_proc_taskinfo" c_procTaskInfo :: Ptr Word64 -> Ptr CInt -> IO CInt+#endif++data ProcessSample = ProcessSample+  { rssBytes :: Maybe Word64,+    rssMaxBytes :: Maybe Word64,+    osThreads :: Maybe Word64,+    openFds :: Maybe Word64,+    cpuUserNs :: Maybe Word64,+    cpuSystemNs :: Maybe Word64,+    cpuTotalNs :: Maybe Word64,+    voluntaryContextSwitches :: Maybe Word64,+    nonvoluntaryContextSwitches :: Maybe Word64+  }+  deriving stock (Eq, Show)++newtype ProcessSampler = ProcessSampler CsvWriter++openProcessSampler :: FilePath -> IO ProcessSampler+openProcessSampler path =+  ProcessSampler+    <$> openCsv+      path+      [ "t_mono_ns",+        "t_wall_ms",+        "phase",+        "rss_bytes",+        "rss_max_bytes",+        "os_threads",+        "open_fds",+        "cpu_user_ns",+        "cpu_system_ns",+        "cpu_total_ns",+        "voluntary_ctxt_switches",+        "nonvoluntary_ctxt_switches"+      ]++sampleProcess :: ProcessSampler -> [Text] -> IO ()+sampleProcess (ProcessSampler writer) prefix = do+  sample <- readProcessSample+  let field = maybe "" (Text.pack . show)+  appendCsv+    writer+    ( prefix+        <> fmap+          field+          [ sample.rssBytes,+            sample.rssMaxBytes,+            sample.osThreads,+            sample.openFds,+            sample.cpuUserNs,+            sample.cpuSystemNs,+            sample.cpuTotalNs,+            sample.voluntaryContextSwitches,+            sample.nonvoluntaryContextSwitches+          ]+    )++closeProcessSampler :: ProcessSampler -> IO ()+closeProcessSampler (ProcessSampler writer) = closeCsv writer++readProcessSample :: IO ProcessSample+#ifdef linux_HOST_OS+readProcessSample = do+  stat <- Text.readFile "/proc/self/stat"+  status <- Text.readFile "/proc/self/status"+  fds <- listDirectory "/proc/self/fd"+  pure (parseProcSample stat status (fromIntegral (length fds)))+#elif darwin_HOST_OS+readProcessSample = do+  total <- (`div` 1_000) <$> getCPUTime+  fds <- listDirectory "/dev/fd"+  alloca \rssPointer -> alloca \threadsPointer -> do+    ok <- c_procTaskInfo rssPointer threadsPointer+    rss <- if ok == 0 then pure Nothing else Just <$> peek rssPointer+    threads <- if ok == 0 then pure Nothing else Just . fromIntegral <$> peek threadsPointer+    pure (ProcessSample rss Nothing threads (Just (fromIntegral (length fds))) Nothing Nothing (Just (fromIntegral total)) Nothing Nothing)+#else+readProcessSample = do+  total <- (`div` 1_000) <$> getCPUTime+  pure (ProcessSample Nothing Nothing Nothing Nothing Nothing Nothing (Just (fromIntegral total)) Nothing Nothing)+#endif++parseProcSample :: Text -> Text -> Word64 -> ProcessSample+parseProcSample stat status fdCount =+  ProcessSample+    { rssBytes = kilobytes "VmRSS:",+      rssMaxBytes = kilobytes "VmHWM:",+      osThreads = statusNumber "Threads:",+      openFds = Just fdCount,+      cpuUserNs = ticksToNs <$> statNumber 11,+      cpuSystemNs = ticksToNs <$> statNumber 12,+      cpuTotalNs = (+) <$> (ticksToNs <$> statNumber 11) <*> (ticksToNs <$> statNumber 12),+      voluntaryContextSwitches = statusNumber "voluntary_ctxt_switches:",+      nonvoluntaryContextSwitches = statusNumber "nonvoluntary_ctxt_switches:"+    }+  where+    -- Linux kernels expose USER_HZ as 100 on supported benchmark hosts.+    ticksToNs value = value * 10_000_000+    afterCommand = snd (Text.breakOnEnd ") " stat)+    statFields = Text.words afterCommand+    statNumber index = atMay statFields index >>= readWord+    statusLines = Text.lines status+    statusValue key = Text.strip . Text.drop (Text.length key) <$> find (Text.isPrefixOf key) statusLines+    statusNumber key = statusValue key >>= readWord . Text.takeWhile (not . isSpace)+    kilobytes key = (* 1_024) <$> statusNumber key++atMay :: [value] -> Int -> Maybe value+atMay values index = case drop index values of value : _ -> Just value; [] -> Nothing++readWord :: Text -> Maybe Word64+readWord = readMaybe . Text.unpack
+ src/Kenshou/Measure/Sampler/Rts.hs view
@@ -0,0 +1,101 @@+module Kenshou.Measure.Sampler.Rts+  ( RtsSampler,+    openRtsSampler,+    sampleRts,+    closeRtsSampler,+  )+where++import Data.IORef+import Data.Text qualified as Text+import Data.Word (Word32, Word64)+import GHC.Conc (getNumCapabilities, listThreads)+import GHC.Stats+import Kenshou.Measure.Sampler.Csv++data RtsSampler = RtsSampler CsvWriter (IORef (Maybe (Word32, Word64)))++openRtsSampler :: FilePath -> IO (Maybe RtsSampler)+openRtsSampler path = do+  enabled <- getRTSStatsEnabled+  if not enabled+    then pure Nothing+    else do+      writer <- openCsv path (baseColumns <> columns)+      previous <- newIORef Nothing+      pure (Just (RtsSampler writer previous))+  where+    baseColumns = ["t_mono_ns", "t_wall_ms", "phase"]+    columns =+      [ "gcs",+        "major_gcs",+        "allocated_bytes",+        "max_live_bytes",+        "cumulative_live_bytes",+        "live_bytes_major_mean",+        "live_bytes_last_gc",+        "last_gc_gen",+        "mem_in_use_bytes",+        "max_mem_in_use_bytes",+        "large_objects_bytes",+        "compact_bytes",+        "slop_bytes",+        "block_fragmentation_bytes",+        "copied_bytes",+        "gc_cpu_ns",+        "gc_elapsed_ns",+        "mutator_cpu_ns",+        "mutator_elapsed_ns",+        "cpu_ns",+        "elapsed_ns",+        "haskell_threads",+        "capabilities"+      ]++sampleRts :: RtsSampler -> [Text.Text] -> IO ()+sampleRts (RtsSampler writer previousRef) prefix = do+  stats <- getRTSStats+  previous <- readIORef previousRef+  threads <- length <$> listThreads+  capabilities <- getNumCapabilities+  let details = stats.gc+      majorMean = case previous of+        Just (oldMajor, oldCumulative) | stats.major_gcs > oldMajor -> Just ((stats.cumulative_live_bytes - oldCumulative) `div` fromIntegral (stats.major_gcs - oldMajor))+        _ -> Nothing+      value = Text.pack . show+      maybeValue = maybe "" (value . toInteger)+  appendCsv writer $+    prefix+      <> fmap+        value+        [ toInteger stats.gcs,+          toInteger stats.major_gcs,+          toInteger stats.allocated_bytes,+          toInteger stats.max_live_bytes,+          toInteger stats.cumulative_live_bytes+        ]+      <> [maybeValue majorMean]+      <> fmap+        value+        [ toInteger details.gcdetails_live_bytes,+          toInteger details.gcdetails_gen,+          toInteger details.gcdetails_mem_in_use_bytes,+          toInteger stats.max_mem_in_use_bytes,+          toInteger details.gcdetails_large_objects_bytes,+          toInteger details.gcdetails_compact_bytes,+          toInteger details.gcdetails_slop_bytes,+          toInteger details.gcdetails_block_fragmentation_bytes,+          toInteger stats.copied_bytes,+          toInteger stats.gc_cpu_ns,+          toInteger stats.gc_elapsed_ns,+          toInteger stats.mutator_cpu_ns,+          toInteger stats.mutator_elapsed_ns,+          toInteger stats.cpu_ns,+          toInteger stats.elapsed_ns,+          toInteger threads,+          toInteger capabilities+        ]+  writeIORef previousRef (Just (stats.major_gcs, stats.cumulative_live_bytes))++closeRtsSampler :: RtsSampler -> IO ()+closeRtsSampler (RtsSampler writer _) = closeCsv writer
+ src/Kenshou/Measure/Samples.hs view
@@ -0,0 +1,276 @@+module Kenshou.Measure.Samples+  ( ClockKind (..),+    SampleHeader (..),+    SampleRecord (..),+    SampleWriter,+    SampleFileReport (..),+    openSampleWriter,+    writeSampleBlock,+    closeSampleWriter,+    sampleWriterBackpressure,+    readSamples,+  )+where++import Control.Concurrent.Async (Async, async, wait)+import Control.Concurrent.MVar+import Control.Concurrent.STM+import Data.Bits+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.ByteString.Builder+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Data.Word+import Kenshou.Measure.Clock (Origin (..))+import System.IO++data ClockKind = MonotonicClock | WallClock+  deriving stock (Eq, Show)++data SampleHeader = SampleHeader+  { origin :: !Origin,+    clockKind :: !ClockKind,+    operation :: !Text,+    processLabel :: !Text+  }+  deriving stock (Eq, Show)++data SampleRecord = SampleRecord+  { intendedStartNs :: !Word64,+    actualStartNs :: !Word64,+    endNs :: !Word64,+    outcomeCode :: !Word8,+    units :: !Word64+  }+  deriving stock (Eq, Show)++data SampleCommand = WriteBlock !Word16 ![SampleRecord] | Stop !(MVar ())++data SampleWriter = SampleWriter+  { queue :: !(TBQueue SampleCommand),+    worker :: !(Async ()),+    backpressure :: !(IORef Word64)+  }++data SampleFileReport = SampleFileReport+  { header :: !SampleHeader,+    recordsRead :: !Word64,+    ignoredBytes :: !Word64+  }+  deriving stock (Eq, Show)++openSampleWriter :: FilePath -> SampleHeader -> IO SampleWriter+openSampleWriter path header = do+  handle <- openBinaryFile path WriteMode+  ByteString.hPut handle (builderBytes (headerBuilder header))+  queue <- newTBQueueIO 256+  backpressure <- newIORef 0+  worker <- async (writerLoop handle queue)+  pure SampleWriter {queue, worker, backpressure}++writeSampleBlock :: SampleWriter -> Word16 -> [SampleRecord] -> IO ()+writeSampleBlock _ _ [] = pure ()+writeSampleBlock writer workerId records = do+  wasFull <- atomically do+    full <- isFullTBQueue writer.queue+    writeTBQueue writer.queue (WriteBlock workerId records)+    pure full+  whenFull wasFull do+    modifyIORef' writer.backpressure (+ 1)+  where+    whenFull True action = action+    whenFull False _ = pure ()++closeSampleWriter :: SampleWriter -> IO ()+closeSampleWriter writer = do+  stopped <- newEmptyMVar+  atomically (writeTBQueue writer.queue (Stop stopped))+  takeMVar stopped+  wait writer.worker++sampleWriterBackpressure :: SampleWriter -> IO Word64+sampleWriterBackpressure = readIORef . (.backpressure)++writerLoop :: Handle -> TBQueue SampleCommand -> IO ()+writerLoop handle queue = do+  command <- atomically (readTBQueue queue)+  case command of+    WriteBlock workerId records -> ByteString.hPut handle (encodeBlock workerId records) >> writerLoop handle queue+    Stop stopped -> hFlush handle >> hClose handle >> putMVar stopped ()++encodeBlock :: Word16 -> [SampleRecord] -> ByteString+encodeBlock _ [] = ByteString.empty+encodeBlock workerId records@(firstRecord : _) =+  builderBytes $+    word16LE workerId+      <> word32LE (fromIntegral (length records))+      <> word32LE (fromIntegral (ByteString.length payload))+      <> word64LE first+      <> byteString payload+  where+    first = firstRecord.intendedStartNs+    payload = builderBytes (recordsBuilder first records)++recordsBuilder :: Word64 -> [SampleRecord] -> Builder+recordsBuilder _ [] = mempty+recordsBuilder previous (record : rest) =+  putVarWord (zigZag (wordDelta record.intendedStartNs previous))+    <> putVarWord (record.actualStartNs - min record.actualStartNs record.intendedStartNs)+    <> putVarWord (record.endNs - min record.endNs record.actualStartNs)+    <> word8 record.outcomeCode+    <> putVarWord record.units+    <> recordsBuilder record.intendedStartNs rest++readSamples :: FilePath -> (SampleRecord -> IO ()) -> IO SampleFileReport+readSamples path consume = do+  bytes <- ByteString.readFile path+  case parseHeader bytes of+    Left message -> ioError (userError (Text.unpack message))+    Right (header, bodyOffset) -> do+      (count, ignored) <- readBlocks bytes bodyOffset consume+      pure (SampleFileReport header count ignored)++readBlocks :: ByteString -> Int -> (SampleRecord -> IO ()) -> IO (Word64, Word64)+readBlocks bytes = go 0+  where+    totalLength = ByteString.length bytes+    go count offset consume+      | offset == totalLength = pure (count, 0)+      | offset + 18 > totalLength = pure (count, fromIntegral (totalLength - offset))+      | otherwise = case parseBlockHeader bytes offset of+          Left _ -> pure (count, fromIntegral (totalLength - offset))+          Right (recordCount, payloadBytes, first, payloadOffset)+            | payloadOffset + payloadBytes > totalLength -> pure (count, fromIntegral (totalLength - offset))+            | otherwise -> case parseRecords bytes payloadOffset payloadBytes recordCount first of+                Left _ -> pure (count, fromIntegral (totalLength - offset))+                Right records -> mapM_ consume records >> go (count + fromIntegral (length records)) (payloadOffset + payloadBytes) consume++parseBlockHeader :: ByteString -> Int -> Either Text (Int, Int, Word64, Int)+parseBlockHeader bytes offset = do+  _workerId <- word16At bytes offset+  count <- word32At bytes (offset + 2)+  payloadBytes <- word32At bytes (offset + 6)+  first <- word64At bytes (offset + 10)+  pure (fromIntegral count, fromIntegral payloadBytes, first, offset + 18)++parseRecords :: ByteString -> Int -> Int -> Int -> Word64 -> Either Text [SampleRecord]+parseRecords bytes offset payloadLength count first = do+  (records, finalOffset) <- go [] offset first count+  ensure (finalOffset == offset + payloadLength) "sample block payload length mismatch"+  pure (reverse records)+  where+    limit = offset + payloadLength+    go records current _ 0 = Right (records, current)+    go records current previous remaining = do+      ensure (current < limit) "truncated sample block"+      (delta, afterDelta) <- readVarWord bytes current limit+      (lag, afterLag) <- readVarWord bytes afterDelta limit+      (service, afterService) <- readVarWord bytes afterLag limit+      outcome <- word8AtLimit bytes afterService limit+      (units, afterUnits) <- readVarWord bytes (afterService + 1) limit+      let intended = addSigned previous (unZigZag delta)+          actual = intended + lag+          record = SampleRecord intended actual (actual + service) outcome units+      go (record : records) afterUnits intended (remaining - 1)++headerBuilder :: SampleHeader -> Builder+headerBuilder header =+  byteString "KSMP"+    <> word16LE 1+    <> word16LE 0+    <> word64LE header.origin.monoNs+    <> int64LE header.origin.wallUnixNs+    <> word8 (case header.clockKind of MonotonicClock -> 0; WallClock -> 1)+    <> textBuilder header.operation+    <> textBuilder header.processLabel++parseHeader :: ByteString -> Either Text (SampleHeader, Int)+parseHeader bytes = do+  ensure (ByteString.take 4 bytes == "KSMP") "invalid sample magic"+  version <- word16At bytes 4+  ensure (version == 1) "unsupported sample version"+  flags <- word16At bytes 6+  ensure (flags == 0) "unsupported sample flags"+  mono <- word64At bytes 8+  wall <- fromIntegral <$> word64At bytes 16+  clockByte <- word8At bytes 24+  clock <- case clockByte of 0 -> Right MonotonicClock; 1 -> Right WallClock; _ -> Left "unknown sample clock"+  (operation, afterOperation) <- textAt bytes 25+  (processLabel, bodyOffset) <- textAt bytes afterOperation+  pure (SampleHeader (Origin mono wall) clock operation processLabel, bodyOffset)++textBuilder :: Text -> Builder+textBuilder value = word16LE (fromIntegral (ByteString.length encoded)) <> byteString encoded+  where+    encoded = Text.encodeUtf8 value++textAt :: ByteString -> Int -> Either Text (Text, Int)+textAt bytes offset = do+  lengthValue <- word16At bytes offset+  let start = offset + 2+      end = start + fromIntegral lengthValue+  ensure (end <= ByteString.length bytes) "truncated sample text"+  case Text.decodeUtf8' (ByteString.take (fromIntegral lengthValue) (ByteString.drop start bytes)) of+    Left _ -> Left "invalid UTF-8 in sample header"+    Right value -> Right (value, end)++builderBytes :: Builder -> ByteString+builderBytes = LazyByteString.toStrict . toLazyByteString++putVarWord :: Word64 -> Builder+putVarWord value+  | value < 0x80 = word8 (fromIntegral value)+  | otherwise = word8 (fromIntegral (value .&. 0x7f) .|. 0x80) <> putVarWord (value `shiftR` 7)++readVarWord :: ByteString -> Int -> Int -> Either Text (Word64, Int)+readVarWord bytes = go 0 0+  where+    go bitOffset accumulator offset limit+      | bitOffset >= 64 = Left "varint is too long"+      | offset >= limit = Left "truncated varint"+      | otherwise =+          let byte = ByteString.index bytes offset+              value = accumulator .|. (fromIntegral (byte .&. 0x7f) `shiftL` bitOffset)+           in if byte .&. 0x80 == 0 then Right (value, offset + 1) else go (bitOffset + 7) value (offset + 1) limit++wordDelta :: Word64 -> Word64 -> Int64+wordDelta current previous = fromIntegral current - fromIntegral previous++addSigned :: Word64 -> Int64 -> Word64+addSigned value delta = fromIntegral (fromIntegral value + delta :: Int64)++zigZag :: Int64 -> Word64+zigZag value = fromIntegral ((value `shiftL` 1) `xor` (value `shiftR` 63))++unZigZag :: Word64 -> Int64+unZigZag value = fromIntegral (value `shiftR` 1) `xor` negate (fromIntegral (value .&. 1))++word8At :: ByteString -> Int -> Either Text Word8+word8At bytes offset = word8AtLimit bytes offset (ByteString.length bytes)++word8AtLimit :: ByteString -> Int -> Int -> Either Text Word8+word8AtLimit bytes offset limit+  | offset < limit = Right (ByteString.index bytes offset)+  | otherwise = Left "truncated sample file"++word16At :: ByteString -> Int -> Either Text Word16+word16At bytes offset = fromIntegral <$> littleEndianAt bytes offset 2++word32At :: ByteString -> Int -> Either Text Word32+word32At bytes offset = fromIntegral <$> littleEndianAt bytes offset 4++word64At :: ByteString -> Int -> Either Text Word64+word64At bytes offset = littleEndianAt bytes offset 8++littleEndianAt :: ByteString -> Int -> Int -> Either Text Word64+littleEndianAt bytes offset width+  | offset + width > ByteString.length bytes = Left "truncated sample file"+  | otherwise = Right $ foldr (.|.) 0 [fromIntegral (ByteString.index bytes (offset + index)) `shiftL` (8 * index) | index <- [0 .. width - 1]]++ensure :: Bool -> Text -> Either Text ()+ensure condition message = if condition then Right () else Left message
+ src/Kenshou/Measure/Selftest.hs view
@@ -0,0 +1,10 @@+module Kenshou.Measure.Selftest (bundle) where++import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Id (selftestLayer)+import Kenshou.Measure.Selftest.PgInsert (pgInsertScenario)+import Kenshou.Measure.Selftest.RegressionInjected (regressionInjectedScenario)+import Kenshou.Measure.Selftest.SleepService (sleepServiceScenario)++bundle :: LayerBundle+bundle = mkBundle selftestLayer [sleepServiceScenario, pgInsertScenario, regressionInjectedScenario] []
+ src/Kenshou/Measure/Selftest/PgInsert.hs view
@@ -0,0 +1,151 @@+module Kenshou.Measure.Selftest.PgInsert (pgInsertScenario) where++import Control.Exception (bracket)+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Data.Word (Word64)+import Hasql.Connection qualified as Connection+import Hasql.Connection.Settings qualified as Settings+import Hasql.Decoders qualified as Decoders+import Hasql.Encoders qualified as Encoders+import Hasql.Session qualified as Session+import Hasql.Statement qualified as Statement+import Kenshou.Core.Context (RunContext (..), requirePostgres)+import Kenshou.Core.Dimension+import Kenshou.Core.Env+import Kenshou.Core.Env.Postgres (PostgresEnv (..))+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Recorder+import Kenshou.Measure.Sampler.Postgres+import Kenshou.Measure.Session+import System.Directory (doesFileExist)+import System.FilePath ((</>))++pgInsertScenario :: Scenario+pgInsertScenario =+  Scenario+    { id = either (error . show) id (parseScenarioId "selftest/measure/benchmark/pg-insert"),+      revision = 2,+      summary = "Verifies runtime, process, host, and PostgreSQL measurement series.",+      tier = TierSmoke,+      placement = PlaceEither,+      knobs = loadKnobs (defaultLoadDefaults {model = "closed", workers = 8}) <> measureKnobs Benchmark <> [payloadKnob],+      dimensions = postgresDimensions (PgDurable :| []) (Pg17 :| [Pg18]) telemetryOff,+      phases = Core.PhasePlan 3 15 2,+      requires = pgRequirements,+      knownDefect = Nothing,+      run = runPgInsert+    }++runPgInsert :: RunContext -> IO ScenarioReport+runPgInsert context = case (loadModelFromKnobs context.knobs, measureConfigFromKnobs context (phasePlanFromCore context.phases)) of+  (Left message, _) -> pure (failedWith ["invalid-load-config"] message)+  (_, Left message) -> pure (failedWith ["invalid-measure-config"] message)+  (Right loadModel, Right baseConfig) -> do+    let environment = requirePostgres context+        workers = case loadModel of ClosedLoop closed -> closed.workers; OpenLoop open -> open.executors+        payloadBytes = knobInt context.knobs (name "pg-insert.payload-bytes")+        settings = Settings.connectionString environment.connectionString <> Settings.applicationName "kenshou-selftest-writer"+        config = watchInserts baseConfig+    acquired <- acquireMany workers settings+    case acquired of+      Left message -> pure (infrastructureFailureBecause message)+      Right [] -> pure (infrastructureFailureBecause "load.workers produced no PostgreSQL connections")+      Right connections@(firstConnection : _) -> bracket (pure connections) (mapM_ Connection.release) \openConnections -> do+        setup <- Connection.use firstConnection setupSession+        case setup of+          Left err -> pure (infrastructureFailureBecause (Text.pack (show err)))+          Right () -> do+            let operation = Operation (OpName "insert") (insertOne openConnections payloadBytes)+            (_, report) <- withMeasurement context config (\measurement -> runLoad measurement loadModel operation)+            counted <- Connection.use firstConnection (Session.statement () countRows)+            assess context report counted++insertOne :: [Connection.Connection] -> Int64 -> Int -> Word64 -> IO OpResult+insertOne connections payloadBytes worker sequenceNumber = do+  let connection = connections !! (worker `mod` length connections)+      parameter = Text.intercalate "," [Text.pack (show worker), Text.pack (show sequenceNumber), Text.pack (show payloadBytes)]+  result <- Connection.use connection (Session.statement parameter insertRow)+  pure (either (const (OpFailed (ErrorCause "postgres"))) (const (OpOk 1)) result)++assess :: RunContext -> MeasurementReport -> Either error Int64 -> IO ScenarioReport+assess context report counted = do+  seriesOk <- and <$> traverse hasRows expectedSeries+  let successful = case report.recorder.operations of+        operation : _ -> sum [successes | (successes, _, _) <- Map.elems operation.phaseCounts]+        [] -> 0+      rowCount = either (const Nothing) (Just . fromIntegral) counted+      failures = ["row-count" | rowCount /= Just successful] <> ["series-rows" | not seriesOk]+      scenarioReport = if null failures then passed else failedWith failures ("row count=" <> showText rowCount <> ", recorded successes=" <> showText successful)+  pure (scenarioReport {outcome = measuredOutcome report scenarioReport.outcome})+  where+    expectedSeries = ["sampler.csv", "rts.csv", "proc.csv", "load.csv", "pg-activity.csv", "pg-checkpointer.csv", "pg-wal.csv", "pg-database.csv", "pg-relations.csv", "pg-statements.csv"]+    hasRows file = do+      let path = context.outDir </> "series" </> file+      exists <- doesFileExist path+      if not exists then pure False else (> 2) . length . Text.lines <$> Text.readFile path++acquireMany :: Int -> Settings.Settings -> IO (Either Text [Connection.Connection])+acquireMany count settings = go count []+  where+    go 0 acquired = pure (Right (reverse acquired))+    go remaining acquired = do+      result <- Connection.acquire settings+      case result of+        Left err -> mapM_ Connection.release acquired >> pure (Left (Text.pack (show err)))+        Right connection -> go (remaining - 1) (connection : acquired)++setupSession :: Session.Session ()+setupSession = do+  Session.statement () (unitStatement "CREATE SCHEMA IF NOT EXISTS kenshou_selftest")+  Session.statement () (unitStatement "CREATE TABLE IF NOT EXISTS kenshou_selftest.inserts (id bigserial PRIMARY KEY, worker int NOT NULL, seq bigint NOT NULL, payload bytea NOT NULL)")+  Session.statement () (unitStatement "TRUNCATE kenshou_selftest.inserts")++unitStatement :: Text -> Statement.Statement () ()+unitStatement sql = Statement.unpreparable sql Encoders.noParams Decoders.noResult++insertRow :: Statement.Statement Text ()+insertRow =+  Statement.unpreparable+    "INSERT INTO kenshou_selftest.inserts(worker,seq,payload) VALUES (split_part($1,',',1)::int, split_part($1,',',2)::bigint, decode(repeat('00',split_part($1,',',3)::int),'hex'))"+    (Encoders.param (Encoders.nonNullable Encoders.text))+    Decoders.noResult++countRows :: Statement.Statement () Int64+countRows = Statement.unpreparable "SELECT count(*)::bigint FROM kenshou_selftest.inserts" Encoders.noParams (Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.int8)))++payloadKnob :: KnobSpec+payloadKnob = KnobSpec (name "pg-insert.payload-bytes") "Inserted payload size" KnobInt (VInt 256) (IntRange 1 65_536) []++telemetryOff :: DimensionSupport+telemetryOff =+  DimensionSupport+    (Supported (Support (TracingOff :| []) TracingOff))+    (Supported (Support (MetricsOff :| []) MetricsOff))+    NotApplicable+    NotApplicable++pgRequirements :: EnvRequirements+pgRequirements = EnvRequirements (Just (PostgresRequirement [] [("shared_preload_libraries", "'pg_stat_statements'")] False)) [] []++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++showText :: (Show value) => value -> Text+showText = Text.pack . show++watchInserts :: MeasureConfig -> MeasureConfig+watchInserts (MeasureConfig phases histogram rawSamples sampleInterval intervalSeconds postgres extraSamplers healthConfig) =+  MeasureConfig phases histogram rawSamples sampleInterval intervalSeconds (fmap addRelation postgres) extraSamplers healthConfig+  where+    addRelation :: PgSamplerConfig -> PgSamplerConfig+    addRelation pg = pg {relations = ["kenshou_selftest.inserts"]}
+ src/Kenshou/Measure/Selftest/RegressionInjected.hs view
@@ -0,0 +1,86 @@+module Kenshou.Measure.Selftest.RegressionInjected (regressionInjectedScenario) where++import Control.Concurrent (threadDelay)+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Dimension+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Id qualified as Id+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Random.SplitMix (mkSMGen, nextWord64)++regressionInjectedScenario :: Scenario+regressionInjectedScenario =+  Scenario+    { id = either (error . show) id (parseScenarioId "selftest/measure/benchmark/regression-injected"),+      revision = 2,+      summary = "Produces controlled slowdown and noise for comparator verification.",+      tier = TierSmoke,+      placement = PlaceEither,+      knobs = loadKnobs (defaultLoadDefaults {model = "closed", workers = 4}) <> measureKnobs Benchmark <> serviceKnobs,+      dimensions = telemetryOff,+      phases = Core.PhasePlan 2 10 1,+      requires = noEnvironment,+      knownDefect = Nothing,+      run = runRegressionInjected+    }++runRegressionInjected :: RunContext -> IO ScenarioReport+runRegressionInjected context = case (loadModelFromKnobs context.knobs, measureConfigFromKnobs context (phasePlanFromCore context.phases)) of+  (Left message, _) -> pure (failedWith ["invalid-load-config"] message)+  (_, Left message) -> pure (failedWith ["invalid-measure-config"] message)+  (Right model, Right config) -> do+    let base = knobInt context.knobs (name "service.base-latency-us")+        extra = knobInt context.knobs (name "service.extra-latency-us")+        noise = knobText context.knobs (name "service.noise")+        factor = if noise == "high" then noiseFactor (Id.unSeed context.seed) else 1+        delay = max 1 (floor (fromIntegral (base + extra) * factor))+        operation = Operation (OpName "work") (\_ _ -> threadDelay delay >> pure (OpOk 1))+    (_, report) <- withMeasurement context config (\measurement -> runLoad measurement model operation)+    let count = case report.recorder.operations of operationReport : _ -> sum [successes | (successes, _, _) <- Map.elems operationReport.phaseCounts]; [] -> 0+        scenarioReport = if count >= 1_000 then passed else failedWith ["insufficient-samples"] ("steady samples=" <> Text.pack (show count))+    pure (scenarioReport {outcome = measuredOutcome report scenarioReport.outcome})++noiseFactor :: Word64 -> Double+noiseFactor seed =+  let (word, _) = nextWord64 (mkSMGen seed)+      unit = fromIntegral word / fromIntegral (maxBound :: Word64)+   in exp (log 0.6 + unit * (log 1.6 - log 0.6))++serviceKnobs :: [KnobSpec]+serviceKnobs =+  [ intKnob "service.base-latency-us" "Base operation latency" 2_000 1 60_000_000,+    intKnob "service.extra-latency-us" "Injected operation latency" 0 0 60_000_000,+    textKnob "service.noise" "Run-level latency noise" "none" ["none", "high"],+    textKnob "service.label" "Behaviour-neutral comparison label" "a" ["a", "b"]+  ]++telemetryOff :: DimensionSupport+telemetryOff =+  DimensionSupport+    (Supported (Support (TracingOff :| []) TracingOff))+    (Supported (Support (MetricsOff :| []) MetricsOff))+    NotApplicable+    NotApplicable++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt def) (IntRange low high) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []+textKnob knobName _ _ [] = error ("no values for " <> Text.unpack knobName)++name :: Text -> KnobName+name = either (error . show) id . mkKnobName
+ src/Kenshou/Measure/Selftest/SleepService.hs view
@@ -0,0 +1,142 @@+module Kenshou.Measure.Selftest.SleepService (sleepServiceScenario) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async)+import Control.Monad (void)+import Data.Aeson (encode, object, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Data.Word (Word64)+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Dimension+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Environment (lookupEnv)+import System.Posix.Process (getProcessID)+import System.Posix.Signals (sigSTOP, signalProcess)+import System.Process (createProcess, proc, waitForProcess)++sleepServiceScenario :: Scenario+sleepServiceScenario =+  Scenario+    { id = either (error . show) id (parseScenarioId "selftest/measure/benchmark/sleep-service"),+      revision = 2,+      summary = "Demonstrates intended-start latency and coordinated-omission correction.",+      tier = TierSmoke,+      placement = PlaceEither,+      knobs = loadKnobs defaults <> measureKnobs Benchmark <> serviceKnobs,+      dimensions = telemetryOff,+      phases = Core.PhasePlan 5 30 5,+      requires = noEnvironment,+      knownDefect = Nothing,+      run = runSleepService+    }+  where+    defaults = defaultLoadDefaults {model = "open-constant", workers = 1, ratePerSecond = 100, executors = 1, maxLagMs = 5_000}++runSleepService :: RunContext -> IO ScenarioReport+runSleepService context = case (loadModelFromKnobs context.knobs, measureConfigFromKnobs context (phasePlanFromCore context.phases)) of+  (Left message, _) -> pure (failedWith ["invalid-load-config"] message)+  (_, Left message) -> pure (failedWith ["invalid-measure-config"] message)+  (Right configuredModel, Right config) -> do+    serviceStart <- nowNs+    let injection = knobText context.knobs (name "selftest.inject")+        model = injectedModel injection configuredModel+        baseLatencyUs = knobInt context.knobs (name "service.base-latency-us")+        stallMs = knobInt context.knobs (name "service.stall-ms")+        stallEveryMs = knobInt context.knobs (name "service.stall-every-ms")+        operation = Operation (OpName "request") (\_ _ -> serviceCall serviceStart (fromIntegral baseLatencyUs) (fromIntegral stallMs) (fromIntegral stallEveryMs))+    (_, report) <- withMeasurement context config \measurement -> do+      startInjection injection+      runLoad measurement model operation+    let scenarioReport = if injection == "none" then assess model (fromIntegral baseLatencyUs * 1_000) report else passed+    pure (scenarioReport {outcome = measuredOutcome report scenarioReport.outcome})++serviceCall :: Word64 -> Word64 -> Word64 -> Word64 -> IO OpResult+serviceCall serviceStart baseLatencyUs stallMs stallEveryMs = do+  current <- nowNs+  let intervalNs = stallEveryMs * 1_000_000+      stallNs = stallMs * 1_000_000+      elapsed = current - min current serviceStart+      offset = elapsed `mod` intervalNs+  if offset < stallNs then sleepUntilNs (current + stallNs - offset) else pure ()+  threadDelay (fromIntegral baseLatencyUs)+  pure (OpOk 1)++assess :: LoadModel -> Word64 -> MeasurementReport -> ScenarioReport+assess model baseLatencyNs report = case find ((== OpName "request") . (.name)) report.recorder.operations of+  Nothing -> failedWith ["missing-request-report"] "request operation was not recorded"+  Just operation ->+    let latencyP99 = Histogram.valueAtQuantile operation.latency 0.99+        serviceP99 = Histogram.valueAtQuantile operation.service 0.99+        maximumLatency = Histogram.maxValue operation.latency+        failures = case model of+          OpenLoop _ -> ["intended-p99" | latencyP99 < 800_000_000 || latencyP99 > 1_000_000_000] <> ["service-p99" | serviceP99 >= 5 * baseLatencyNs]+          ClosedLoop _ -> ["closed-p99" | latencyP99 >= 5 * baseLatencyNs] <> ["closed-max" | maximumLatency <= 900_000_000]+     in if null failures then passed else failedWith failures ("unexpected latency distribution: p99=" <> showText latencyP99 <> " service-p99=" <> showText serviceP99 <> " max=" <> showText maximumLatency)++serviceKnobs :: [KnobSpec]+serviceKnobs =+  [ intKnob "service.base-latency-us" "Base service latency" 1_000 1 60_000_000,+    intKnob "service.stall-ms" "Periodic stall duration" 1_000 1 60_000,+    intKnob "service.stall-every-ms" "Period between stall starts" 10_000 2 600_000,+    textKnob "selftest.inject" "Inject a measurement health condition" "none" ["none", "backlog", "process-pause", "maintenance-notice"]+  ]++injectedModel :: Text -> LoadModel -> LoadModel+injectedModel "backlog" (OpenLoop config) = OpenLoop (config {arrival = ConstantRate 2_000, executors = 1})+injectedModel "backlog" _ = OpenLoop (OpenConfig (ConstantRate 2_000) 1 1 (OverloadConfig 5_000_000_000 3 30_000_000_000))+injectedModel _ model = model++startInjection :: Text -> IO ()+startInjection "process-pause" = void . async $ do+  threadDelay 10_000_000+  processId <- getProcessID+  (_, _, _, processHandle) <- createProcess (proc "sh" ["-c", "sleep 8; kill -CONT " <> show processId])+  signalProcess sigSTOP processId+  void (waitForProcess processHandle)+startInjection "maintenance-notice" = void . async $ do+  threadDelay 8_000_000+  path <- lookupEnv "KENSHOU_HEALTH_NOTICES"+  whenJust path \noticePath -> do+    now <- getCurrentTime+    LazyByteString.appendFile noticePath (encode (object ["schema" .= ("kenshou.health-notice/v1" :: Text), "source" .= ("selftest-maintenance" :: Text), "severity" .= ("hard" :: Text), "at" .= now, "detail" .= ("injected host maintenance notice" :: Text)]) <> "\n")+startInjection _ = pure ()++telemetryOff :: DimensionSupport+telemetryOff =+  DimensionSupport+    (Supported (Support (TracingOff :| []) TracingOff))+    (Supported (Support (MetricsOff :| []) MetricsOff))+    NotApplicable+    NotApplicable++intKnob :: Text -> Text -> Int -> Int -> Int -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt (fromIntegral def)) (IntRange (fromIntegral low) (fromIntegral high)) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []+textKnob knobName _ _ [] = error ("no values for " <> Text.unpack knobName)++whenJust :: Maybe value -> (value -> IO ()) -> IO ()+whenJust Nothing _ = pure ()+whenJust (Just value) action = action value++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++showText :: (Show value) => value -> Text+showText = Text.pack . show
+ src/Kenshou/Measure/Session.hs view
@@ -0,0 +1,218 @@+module Kenshou.Measure.Session+  ( MeasureEnv (..),+    MeasureConfig (..),+    Measurement,+    MeasurementReport (..),+    measureEnvFromRunContext,+    phasePlanFromCore,+    measureConfigFromKnobs,+    withMeasurement,+    measurementPhaseClock,+    measurementRecorder,+    measurementConfig,+    measurementEnv,+    appendLoadReport,+    measuredOutcome,+  )+where++import Control.Exception (SomeException, mask, throwIO, try)+import Control.Monad (forM_, when)+import Data.Aeson (Value, toJSON)+import Data.IORef+import Data.Text (Text)+import Data.Word (Word64)+import Kenshou.Core.Context qualified as Core+import Kenshou.Core.Dimension (Dimensions (..), renderDurability)+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot (..), PostgresEnv (..))+import Kenshou.Core.Id (Kind, ScenarioId (..), unSeed)+import Kenshou.Core.Knob (ResolvedKnobs)+import Kenshou.Core.Knob qualified as Knob+import Kenshou.Core.Log qualified as Log+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.Phase qualified as Core+import Kenshou.Measure.Clock+import Kenshou.Measure.Health+import Kenshou.Measure.Histogram (HistogramConfig (..), defaultHistogramConfig)+import Kenshou.Measure.Load.Types (LoadReport)+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Sampler+import Kenshou.Measure.Sampler.Postgres+import Kenshou.Measure.Summary+import System.FilePath (makeRelative)++data MeasureEnv = MeasureEnv+  { runDir :: FilePath,+    seed :: Word64,+    origin :: Origin,+    processLabel :: Maybe Text,+    scenarioKind :: Kind,+    pgDurability :: Maybe Text,+    specPhases :: Maybe PhasePlan,+    knobs :: ResolvedKnobs,+    dimensions :: Dimensions,+    postgres :: Maybe PostgresEnv,+    onPhase :: Phase -> Origin -> IO (),+    registerSection :: Text -> Value -> IO (),+    declareArtifact :: FilePath -> Text -> IO (),+    logLine :: Text -> IO ()+  }++data MeasureConfig = MeasureConfig+  { defaultPhases :: PhasePlan,+    histogram :: HistogramConfig,+    rawSamples :: RawSamplePolicy,+    sampleIntervalMs :: Int,+    intervalHistogramSeconds :: Word64,+    postgres :: Maybe PgSamplerConfig,+    extraSamplers :: [Sampler],+    healthConfig :: HealthConfig+  }++data Measurement = Measurement MeasureEnv PhaseClock Recorder MeasureConfig (IORef [LoadReport])++data MeasurementReport = MeasurementReport+  { recorder :: RecorderReport,+    loads :: [LoadReport],+    samplers :: SamplerReport,+    summary :: MeasurementSummary,+    health :: [HealthObservation]+  }++measureEnvFromRunContext :: Core.RunContext -> IO MeasureEnv+measureEnvFromRunContext context = do+  origin <- captureOrigin+  let ScenarioId _ _ scenarioKind _ = context.scenario+      Dimensions _ _ durability _ = context.dimensions+  pure+    MeasureEnv+      { runDir = context.outDir,+        seed = unSeed context.seed,+        origin,+        processLabel = Nothing,+        scenarioKind,+        pgDurability = renderDurability <$> durability,+        specPhases = Just (phasePlanFromCore context.phases),+        knobs = context.knobs,+        dimensions = context.dimensions,+        postgres = context.env.postgres,+        onPhase = \phase _ -> Log.logAt context.logger Log.Debug ("measurement phase " <> renderPhase phase) [],+        registerSection = Core.putSummary context Core.Measurements,+        declareArtifact = \path mediaType -> Core.declareMediaType context (makeRelative context.outDir path) mediaType,+        logLine = \message -> Log.logAt context.logger Log.Info message []+      }++phasePlanFromCore :: Core.PhasePlan -> PhasePlan+phasePlanFromCore plan =+  PhasePlan+    { warmUp = seconds plan.warmUpSeconds,+      steady = SteadyFor (seconds plan.steadySeconds),+      drain = seconds plan.drainSeconds+    }+  where+    seconds value = Nanos (floor (value * 1_000_000_000))++measureConfigFromKnobs :: Core.RunContext -> PhasePlan -> Either Text MeasureConfig+measureConfigFromKnobs context defaultPhases = do+  rawSamples <- case Knob.knobText context.knobs (knobName "measure.raw-samples") of+    "full" -> Right RawFull+    "sampled" -> Right (RawSampled (fromIntegral (Knob.knobInt context.knobs (knobName "measure.raw-sample-one-in"))))+    "off" -> Right RawOff+    value -> Left ("unknown measure.raw-samples " <> value)+  pure+    MeasureConfig+      { defaultPhases,+        histogram = defaultHistogramConfig {significantDigits = fromIntegral (Knob.knobInt context.knobs (knobName "measure.histogram-digits"))},+        rawSamples,+        sampleIntervalMs = fromIntegral (Knob.knobInt context.knobs (knobName "measure.sample-interval-ms")),+        intervalHistogramSeconds = fromIntegral (Knob.knobInt context.knobs (knobName "measure.interval-histogram-seconds")),+        postgres = postgresConfig,+        extraSamplers = [],+        healthConfig = defaultHealthConfig+      }+  where+    postgresConfig =+      fmap+        ( \environment ->+            PgSamplerConfig+              { connectionString = environment.connectionString,+                relations = [],+                statements = case Knob.knobText context.knobs (knobName "measure.pg-statements") of+                  "off" -> PgStatementsOff+                  "periodic" -> PgStatementsPeriodic+                  _ -> PgStatementsSnapshots,+                serverVersionNum = environment.snapshot.serverVersionNum+              }+        )+        context.env.postgres++withMeasurement :: Core.RunContext -> MeasureConfig -> (Measurement -> IO value) -> IO (value, MeasurementReport)+withMeasurement context config action = mask \restore -> do+  env <- measureEnvFromRunContext context+  phaseClock <- newPhaseClock env.onPhase config.defaultPhases+  recorder <-+    newRecorder+      RecorderConfig+        { runDir = env.runDir,+          origin = env.origin,+          processLabel = env.processLabel,+          histogram = config.histogram,+          rawSamples = config.rawSamples,+          intervalHistogramSeconds = config.intervalHistogramSeconds,+          declareArtifact = env.declareArtifact+        }+      phaseClock+  loadReports <- newIORef []+  let measurement = Measurement env phaseClock recorder config loadReports+  sampling <-+    startSampling+      SamplerConfig+        { runDir = env.runDir,+          origin = env.origin,+          intervalMs = config.sampleIntervalMs,+          postgres = config.postgres,+          extraSamplers = config.extraSamplers,+          phaseClock,+          declareArtifact = env.declareArtifact,+          logLine = env.logLine+        }+  result <- try (restore (action measurement))+  phase <- currentPhase phaseClock+  when (phase /= Done) (enterPhase phaseClock Done)+  samplerReport <- stopSampling sampling+  recorderReport <- finishRecorder recorder+  loads <- readIORef loadReports+  case result of+    Left exception -> throwIO (exception :: SomeException)+    Right value -> do+      capturedNotices <- captureHealthNotices env.runDir+      forM_ capturedNotices (\path -> env.declareArtifact path "application/x-ndjson")+      summarized <- summarizeRunDirWithHealth config.healthConfig env.runDir+      summary <- either (ioError . userError . show) pure summarized+      env.registerSection "measurements" (toJSON summary)+      pure (value, MeasurementReport recorderReport loads samplerReport summary summary.health)++measurementPhaseClock :: Measurement -> PhaseClock+measurementPhaseClock (Measurement _ phaseClock _ _ _) = phaseClock++measurementRecorder :: Measurement -> Recorder+measurementRecorder (Measurement _ _ recorder _ _) = recorder++measurementConfig :: Measurement -> MeasureConfig+measurementConfig (Measurement _ _ _ config _) = config++measurementEnv :: Measurement -> MeasureEnv+measurementEnv (Measurement env _ _ _ _) = env++appendLoadReport :: Measurement -> LoadReport -> IO ()+appendLoadReport (Measurement _ _ _ _ loadReports) report = modifyIORef' loadReports (<> [report])++measuredOutcome :: MeasurementReport -> Outcome -> Outcome+measuredOutcome report scenarioOutcome = case healthOutcome report.health of+  Just InfrastructureFailure -> InfrastructureFailure+  Just Inconclusive | scenarioOutcome `elem` [Passed, Failed] -> Inconclusive+  _ -> scenarioOutcome++knobName :: Text -> Knob.KnobName+knobName = either (error . show) id . Knob.mkKnobName
+ src/Kenshou/Measure/Stats.hs view
@@ -0,0 +1,85 @@+module Kenshou.Measure.Stats+  ( Interval (..),+    exactQuantile,+    arithmeticMean,+    geometricMean,+    coefficientOfVariation,+    bootstrapInterval,+    studentTInterval,+    intervalEnvelope,+  )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.List (sort)+import Data.Word (Word64)+import System.Random.SplitMix (mkSMGen, nextWord64)++data Interval = Interval {low :: Double, estimate :: Double, high :: Double}+  deriving stock (Eq, Show)++instance ToJSON Interval where+  toJSON value = object ["low" .= value.low, "estimate" .= value.estimate, "high" .= value.high]++exactQuantile :: Double -> [Double] -> Double+exactQuantile _ [] = 0+exactQuantile quantile values =+  let ordered = sort values+      position = max 0 (min 1 quantile) * fromIntegral (length ordered - 1)+      lower = floor position+      upper = ceiling position+      fraction = position - fromIntegral lower+   in ordered !! lower * (1 - fraction) + ordered !! upper * fraction++arithmeticMean :: [Double] -> Double+arithmeticMean [] = 0+arithmeticMean values = sum values / fromIntegral (length values)++geometricMean :: [Double] -> Double+geometricMean [] = 0+geometricMean values = exp (arithmeticMean (fmap log values))++coefficientOfVariation :: [Double] -> Double+coefficientOfVariation values =+  let mean = arithmeticMean values+      variance = if length values < 2 then 0 else sum [square (value - mean) | value <- values] / fromIntegral (length values - 1)+   in if mean == 0 then 0 else sqrt variance / abs mean+  where+    square value = value * value++bootstrapInterval :: Word64 -> Int -> Double -> ([Double] -> Double) -> [Double] -> Interval+bootstrapInterval _ _ _ _ [] = Interval 0 0 0+bootstrapInterval seed iterations confidence statistic values =+  let estimates = take (max 1 iterations) (resamples (mkSMGen seed))+      alpha = (1 - confidence) / 2+   in Interval (exactQuantile alpha estimates) (statistic values) (exactQuantile (1 - alpha) estimates)+  where+    count = length values+    resamples generator =+      let (sample, next) = draw count generator []+       in statistic sample : resamples next+    draw 0 generator accumulated = (accumulated, generator)+    draw remaining generator accumulated =+      let (word, next) = nextWord64 generator+          value = values !! fromIntegral (word `mod` fromIntegral count)+       in draw (remaining - 1) next (value : accumulated)++studentTInterval :: Double -> [Double] -> Interval+studentTInterval _ [] = Interval 0 0 0+studentTInterval _ values =+  let mean = arithmeticMean values+      count = length values+      variance = if count < 2 then 0 else sum [(value - mean) ^ (2 :: Int) | value <- values] / fromIntegral (count - 1)+      margin = critical (count - 1) * sqrt variance / sqrt (fromIntegral count)+   in Interval (mean - margin) mean (mean + margin)++intervalEnvelope :: Interval -> Interval -> Interval+intervalEnvelope left right = Interval (min left.low right.low) left.estimate (max left.high right.high)++critical :: Int -> Double+critical degrees+  | degrees <= 0 = 0+  | degrees <= length table = table !! (degrees - 1)+  | otherwise = 1.96+  where+    table = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045]
+ src/Kenshou/Measure/Summary.hs view
@@ -0,0 +1,287 @@+module Kenshou.Measure.Summary+  ( SummaryError (..),+    SummaryWindow (..),+    MeasurementSummary (..),+    summarizeRunDir,+    summarizeRunDirWithHealth,+  )+where++import Control.Exception (SomeException, try)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.List (isSuffixOf)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, mapMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Health+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (decodeHistogram)+import Kenshou.Measure.Metrics+import Kenshou.Measure.Samples+import Kenshou.Measure.Stats (arithmeticMean)+import System.Directory (doesDirectoryExist, doesFileExist, listDirectory)+import System.FilePath ((</>))+import Text.Read (readMaybe)++newtype SummaryError = SummaryError Text deriving stock (Eq, Show)++data SummaryWindow = SummaryWindow+  { steadyStartMonoNs :: Word64,+    steadyEndMonoNs :: Word64,+    steadySeconds :: Double+  }+  deriving stock (Eq, Show)++data MeasurementSummary = MeasurementSummary+  { grade :: Text,+    gradeReasons :: [Text],+    window :: SummaryWindow,+    ops :: Map Text Value,+    metrics :: Map Text Metric,+    health :: [HealthObservation]+  }+  deriving stock (Eq, Show)++instance ToJSON SummaryWindow where+  toJSON value = object ["steadyStartMonoNs" .= value.steadyStartMonoNs, "steadyEndMonoNs" .= value.steadyEndMonoNs, "steadySeconds" .= value.steadySeconds]++instance FromJSON SummaryWindow where+  parseJSON = withObject "SummaryWindow" \value -> SummaryWindow <$> value .: "steadyStartMonoNs" <*> value .: "steadyEndMonoNs" <*> value .: "steadySeconds"++instance ToJSON MeasurementSummary where+  toJSON summary =+    object+      [ "schema" .= ("kenshou.measurements/v1" :: Text),+        "algorithm" .= object ["name" .= ("kenshou-summary" :: Text), "version" .= (1 :: Int)],+        "grade" .= summary.grade,+        "gradeReasons" .= summary.gradeReasons,+        "window" .= summary.window,+        "ops" .= summary.ops,+        "metrics" .= summary.metrics,+        "health" .= summary.health+      ]++instance FromJSON MeasurementSummary where+  parseJSON = withObject "MeasurementSummary" \value -> do+    schema <- value .: "schema"+    if schema /= ("kenshou.measurements/v1" :: Text) then fail "unsupported measurement summary" else pure ()+    MeasurementSummary <$> value .: "grade" <*> value .:? "gradeReasons" .!= [] <*> value .: "window" <*> value .: "ops" <*> value .: "metrics" <*> value .:? "health" .!= []++data Counts = Counts {successes :: Word64, failures :: Word64, units :: Word64}++data SampleMeta = SampleMeta {operation :: Text, latencyBasis :: Text, rawFull :: Bool, steadyCounts :: Counts, errorCauses :: Map Text Word64}++instance FromJSON Counts where+  parseJSON = withObject "Counts" \value -> Counts <$> value .:? "successes" .!= 0 <*> value .:? "failures" .!= 0 <*> value .:? "units" .!= 0++instance FromJSON SampleMeta where+  parseJSON = withObject "SampleMeta" \value -> do+    phaseCounts <- value .: "phaseCounts"+    steady <- withObject "phaseCounts" (\counts -> counts .:? "steady" .!= Counts 0 0 0) phaseCounts+    raw <- value .: "rawSamples"+    SampleMeta <$> value .: "operation" <*> value .:? "latencyBasis" .!= "intended-start" <*> pure (raw == String "full") <*> pure steady <*> value .:? "errorCauses" .!= Map.empty++summarizeRunDir :: FilePath -> IO (Either SummaryError MeasurementSummary)+summarizeRunDir = summarizeRunDirWithHealth defaultHealthConfig++summarizeRunDirWithHealth :: HealthConfig -> FilePath -> IO (Either SummaryError MeasurementSummary)+summarizeRunDirWithHealth healthConfig runDir = do+  exists <- doesDirectoryExist runDir+  if not exists+    then pure (Left (SummaryError "run directory does not exist"))+    else do+      result <- try (summarize healthConfig runDir)+      pure (either (Left . SummaryError . Text.pack . show) id (result :: Either SomeException (Either SummaryError MeasurementSummary)))++summarize :: HealthConfig -> FilePath -> IO (Either SummaryError MeasurementSummary)+summarize healthConfig runDir = do+  windowResult <- readWindow (runDir </> "series" </> "load.csv")+  case windowResult of+    Left err -> pure (Left err)+    Right window -> do+      files <- listDirectory (runDir </> "samples")+      operationResults <- traverse (summarizeOperation runDir window) [file | file <- files, ".meta.json" `isSuffixOf` file]+      case sequence operationResults of+        Left err -> pure (Left err)+        Right operations -> do+          let operationMap = Map.fromList [(name, value) | (name, value, _, _) <- operations]+              operationMetrics = Map.unions [values | (_, _, values, _) <- operations]+              totalOps = sum [metric.value | (name, metric) <- Map.toList operationMetrics, ".throughput" `Text.isSuffixOf` name] * window.steadySeconds+          runtimeMetrics <- summarizeSeries runDir window totalOps+          reasons <- gradeReasonsFromRun runDir (all (\(_, _, _, full) -> full) operations)+          health <- evaluateHealth healthConfig runDir+          let healthReasons = ["health:" <> item.gate | item <- health, item.severity /= Info]+              allReasons = reasons <> healthReasons+          pure (Right (MeasurementSummary (if null allReasons then "benchmark" else "exploratory") allReasons window operationMap (operationMetrics <> runtimeMetrics) health))++summarizeOperation :: FilePath -> SummaryWindow -> FilePath -> IO (Either SummaryError (Text, Value, Map Text Metric, Bool))+summarizeOperation runDir window metaFile = do+  decoded <- eitherDecodeFileStrict' (runDir </> "samples" </> metaFile) :: IO (Either String SampleMeta)+  case decoded of+    Left message -> pure (Left (SummaryError (Text.pack message)))+    Right meta -> do+      let base = Text.unpack meta.operation+          histogramPath = runDir </> "samples" </> base <> ".hist"+          servicePath = runDir </> "samples" </> base <> ".service.hist"+          rawPath = runDir </> "samples" </> base <> ".raw"+      latency <- decodeHistogram <$> ByteString.readFile histogramPath+      service <- decodeHistogram <$> ByteString.readFile servicePath+      rawOk <-+        if meta.rawFull+          then do+            rawExists <- doesFileExist rawPath+            if not rawExists then pure False else (== 0) . (.ignoredBytes) <$> readSamples rawPath (const (pure ()))+          else pure True+      case (latency, service, rawOk) of+        (_, _, False) -> pure (Left (SummaryError ("raw sample file is missing or truncated for " <> meta.operation)))+        (Left message, _, _) -> pure (Left (SummaryError message))+        (_, Left message, _) -> pure (Left (SummaryError message))+        (Right latencyHistogram, Right serviceHistogram, _) -> do+          let count = Histogram.totalCount latencyHistogram+              failures = meta.steadyCounts.failures+              throughput = fromIntegral count / window.steadySeconds+              unitsPerSecond = fromIntegral meta.steadyCounts.units / window.steadySeconds+              errorRate = if count + failures == 0 then 0 else fromIntegral failures / fromIntegral (count + failures)+              quantile histogram q = fromIntegral (Histogram.valueAtQuantile histogram q) :: Double+              latencyValues = metricSet ("op." <> meta.operation <> ".latency") latencyHistogram+              serviceP99 = quantile serviceHistogram 0.99+              metrics =+                Map.fromList+                  [ ("op." <> meta.operation <> ".throughput", Metric throughput "ops/s"),+                    ("op." <> meta.operation <> ".units-throughput", Metric unitsPerSecond "units/s"),+                    ("op." <> meta.operation <> ".service.p99", Metric serviceP99 "ns"),+                    ("op." <> meta.operation <> ".error-rate", Metric errorRate "ratio")+                  ]+                  <> latencyValues+              value =+                object+                  [ "latencyBasis" .= meta.latencyBasis,+                    "coordinatedOmissionRisk" .= (meta.latencyBasis == "actual-start"),+                    "count" .= count,+                    "reportedSuccesses" .= meta.steadyCounts.successes,+                    "failures" .= failures,+                    "errors" .= meta.errorCauses,+                    "throughput" .= throughput,+                    "unitsPerSecond" .= unitsPerSecond,+                    "latencyNs" .= distribution latencyHistogram,+                    "serviceNs" .= distribution serviceHistogram+                  ]+          pure (Right (meta.operation, value, metrics, meta.rawFull))+  where+    distribution histogram =+      object+        [ "p50" .= Histogram.valueAtQuantile histogram 0.50,+          "p90" .= Histogram.valueAtQuantile histogram 0.90,+          "p99" .= Histogram.valueAtQuantile histogram 0.99,+          "p999" .= Histogram.valueAtQuantile histogram 0.999,+          "max" .= Histogram.maxValue histogram,+          "mean" .= Histogram.meanValue histogram+        ]+    metricSet prefix histogram =+      Map.fromList+        [ (prefix <> ".p50", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.50)) "ns"),+          (prefix <> ".p90", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.90)) "ns"),+          (prefix <> ".p95", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.95)) "ns"),+          (prefix <> ".p99", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.99)) "ns"),+          (prefix <> ".p999", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.999)) "ns"),+          (prefix <> ".max", Metric (fromIntegral (Histogram.maxValue histogram)) "ns"),+          (prefix <> ".mean", Metric (Histogram.meanValue histogram) "ns")+        ]++readWindow :: FilePath -> IO (Either SummaryError SummaryWindow)+readWindow path = do+  rows <- readCsv path+  let steady = [readWord (row !! 0) | row <- rows, length row > 2, row !! 2 == "steady"]+      drain = [readWord (row !! 0) | row <- rows, length row > 2, row !! 2 == "drain"]+  pure case (catMaybes steady, catMaybes drain) of+    (start : _, end : _) | end > start -> Right (SummaryWindow start end (fromIntegral (end - start) / 1e9))+    _ -> Left (SummaryError "load.csv has no complete steady window")++summarizeSeries :: FilePath -> SummaryWindow -> Double -> IO (Map Text Metric)+summarizeSeries runDir window totalOps = do+  rts <- numericRows (runDir </> "series" </> "rts.csv") "phase" "steady"+  proc <- numericRows (runDir </> "series" </> "proc.csv") "phase" "steady"+  wal <- numericRows (runDir </> "series" </> "pg-wal.csv") "phase" "steady"+  checkpointer <- numericRows (runDir </> "series" </> "pg-checkpointer.csv") "phase" "steady"+  let delta column rows = (-) <$> (Map.lookup column =<< lastMay rows) <*> (Map.lookup column =<< headMay rows)+      maximumOf column rows = maximumMaybe (mapMaybe (Map.lookup column) rows)+      meanOf column rows = case mapMaybe (Map.lookup column) rows of [] -> Nothing; values -> Just (arithmeticMean values)+      elapsed = fromIntegral (window.steadyEndMonoNs - window.steadyStartMonoNs)+      capabilities = max 1 (maybe 1 id (Map.lookup "capabilities" =<< lastMay rts))+      metrics =+        [ metric "rts.alloc-rate" "bytes/s" ((/ window.steadySeconds) <$> delta "allocated_bytes" rts),+          metric "rts.alloc-bytes-per-op" "bytes/op" ((/ max 1 totalOps) <$> delta "allocated_bytes" rts),+          metric "rts.gc-productivity" "ratio" ((\gc total -> 1 - gc / max 1 total) <$> delta "gc_elapsed_ns" rts <*> delta "elapsed_ns" rts),+          metric "rts.max-live-bytes" "bytes" (maximumOf "max_live_bytes" rts),+          metric "rts.live-bytes-major-mean" "bytes" (meanOf "live_bytes_major_mean" rts),+          metric "proc.cpu-utilisation" "ratio" ((/ (max 1 elapsed * capabilities)) <$> delta "cpu_total_ns" proc),+          metric "proc.rss-max" "bytes" (maximumOf "rss_bytes" proc),+          metric "pg.wal-bytes-per-op" "bytes/op" ((/ max 1 totalOps) <$> delta "wal_bytes" wal),+          metric "pg.checkpoints-in-window" "count" ((+) <$> delta "num_timed" checkpointer <*> delta "num_requested" checkpointer)+        ]+  pure (Map.fromList (catMaybes metrics))+  where+    metric name unit value = (name,) . (`Metric` unit) <$> value++gradeReasonsFromRun :: FilePath -> Bool -> IO [Text]+gradeReasonsFromRun runDir rawFull = do+  decoded <- eitherDecodeFileStrict' (runDir </> "run-spec.json") :: IO (Either String Value)+  let durability = case decoded of+        Right (Object root) -> do+          Object dimensions <- KeyMap.lookup "dimensions" root+          String value <- KeyMap.lookup "pg.durability" dimensions+          pure value+        _ -> Nothing+  pure (["raw samples are not retained in full" | not rawFull] <> ["pg.durability=fsync-off" | durability == Just "fsync-off"])++readCsv :: FilePath -> IO [[Text]]+readCsv path = do+  exists <- doesFileExist path+  if not exists then pure [] else fmap (Text.splitOn ",") . drop 1 . Text.lines <$> Text.readFile path++numericRows :: FilePath -> Text -> Text -> IO [Map Text Double]+numericRows path filterColumn filterValue = do+  exists <- doesFileExist path+  if not exists+    then pure []+    else do+      linesValue <- Text.lines <$> Text.readFile path+      case linesValue of+        [] -> pure []+        header : rows -> do+          let columns = Text.splitOn "," header+              decoded = fmap (Map.fromList . zip columns . fmap readDouble . Text.splitOn ",") rows+          pure [Map.mapMaybe id row | (raw, row) <- zip rows decoded, field columns filterColumn raw == Just filterValue]++field :: [Text] -> Text -> Text -> Maybe Text+field columns wanted row = do+  index <- lookupIndex wanted columns+  atMay (Text.splitOn "," row) index++lookupIndex :: (Eq value) => value -> [value] -> Maybe Int+lookupIndex wanted = go 0+  where+    go _ [] = Nothing; go index (value : rest) = if value == wanted then Just index else go (index + 1) rest++readDouble :: Text -> Maybe Double+readDouble value | Text.null value = Nothing+readDouble value = readMaybe (Text.unpack value)++readWord :: Text -> Maybe Word64+readWord = readMaybe . Text.unpack++headMay, lastMay :: [value] -> Maybe value+headMay [] = Nothing; headMay (value : _) = Just value+lastMay [] = Nothing; lastMay values = Just (last values)++maximumMaybe :: (Ord value) => [value] -> Maybe value+maximumMaybe [] = Nothing; maximumMaybe values = Just (maximum values)++atMay :: [value] -> Int -> Maybe value+atMay values index = case drop index values of value : _ -> Just value; [] -> Nothing
+ test/Kenshou/Measure/ClockSpec.hs view
@@ -0,0 +1,18 @@+module Kenshou.Measure.ClockSpec (spec) where++import Kenshou.Measure.Clock+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Clock" do+  it "sleepUntilNs never returns before the deadline" do+    start <- nowNs+    let deadline = start + 2_000_000+    sleepUntilNs deadline+    finished <- nowNs+    finished `shouldSatisfy` (>= deadline)++  it "captures paired monotonic and wall-clock origins" do+    origin <- captureOrigin+    origin.monoNs `shouldSatisfy` (> 0)+    origin.wallUnixNs `shouldSatisfy` (> 0)
+ test/Kenshou/Measure/CompareSpec.hs view
@@ -0,0 +1,141 @@+module Kenshou.Measure.CompareSpec (spec) where++import Data.Aeson (Value (Null), encode, object, (.=))+import Data.ByteString.Char8 qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility+import Kenshou.Measure.Compare.Ordering+import Kenshou.Measure.Compare.Policy+import Kenshou.Measure.Health+import Kenshou.Measure.Stats+import Kenshou.Measure.Summary (MeasurementSummary (..), SummaryWindow (..))+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = do+  describe "comparison source references" do+    it "retains the baseline and candidate run IDs in pair order" do+      withSystemTempDirectory "kenshou-comparison-arms" $ \root -> do+        let baselineIds = ["baseline-1", "baseline-2", "baseline-3"] :: [Text]+            candidateIds = ["candidate-1", "candidate-2", "candidate-3"] :: [Text]+            baselineDirs = [root </> show index </> "baseline" | index <- [1 :: Int .. 3]]+            candidateDirs = [root </> show index </> "candidate" | index <- [1 :: Int .. 3]]+        sequence_ [writeRun dir runId "released" | (dir, runId) <- zip baselineDirs baselineIds]+        sequence_ [writeRun dir runId "head" | (dir, runId) <- zip candidateDirs candidateIds]+        compared <- compareRuns testPolicy (VaryCohort :| []) baselineDirs candidateDirs+        case compared of+          Left err -> expectationFailure (show err)+          Right result -> do+            result.baselineRuns `shouldBe` baselineIds+            result.candidateRuns `shouldBe` candidateIds+            result.startedAt `shouldSatisfy` (<= result.finishedAt)+            result.design `shouldBe` "sequential"++  describe "pairedSchedule" do+    it "assigns one shared seed to each pair in ABBA order" do+      let schedule = pairedSchedule ABBA 3 17+      fmap (.arm) schedule `shouldBe` [Baseline, Candidate, Candidate, Baseline, Baseline, Candidate]+      fmap (.pairIndex) schedule `shouldBe` [0, 0, 1, 1, 2, 2]+      fmap (.pairSeed) schedule `shouldSatisfy` pairSeedsMatch++  describe "statistics" do+    it "interpolates exact quantiles" do+      exactQuantile 0.25 [10, 0] `shouldBe` 2.5++    it "repeats bootstrap intervals exactly for a fixed seed" do+      let interval = bootstrapInterval 41 1_000 0.95 arithmeticMean [1, 2, 3, 4]+      bootstrapInterval 41 1_000 0.95 arithmeticMean [1, 2, 3, 4] `shouldBe` interval++  describe "comparison policy" do+    it "parses the explicit A/A control axis" do+      parseVaryingAxis "control" `shouldBe` Right VaryControl++    it "rejects fewer than three pairs" do+      decodePolicy (policyJson 2 1_000) `shouldSatisfy` isLeft++    it "rejects fewer than one thousand bootstrap iterations" do+      decodePolicy (policyJson 3 999) `shouldSatisfy` isLeft++    it "accepts the minimum supported policy" do+      decodePolicy (policyJson 3 1_000) `shouldSatisfy` isRight++  describe "paired metric classification" do+    it "keeps a clear regression when another metric is inconclusive" do+      decideVerdict False [] [MetricRegression, MetricInconclusive] `shouldBe` VerdictRegression++    it "reports a clear regression only after both gates are exceeded" do+      let result = compareMetricPairs testPolicy "latency" "ns" latencyRule [(100, 130), (100, 130), (100, 130)]+      result.status `shouldBe` MetricRegression++    it "passes stable identical measurements" do+      let result = compareMetricPairs testPolicy "latency" "ns" latencyRule [(100, 100), (101, 101), (99, 99)]+      result.status `shouldBe` MetricPass++    it "reports noisy mixed measurements as inconclusive" do+      let result = compareMetricPairs testPolicy "latency" "ns" latencyRule [(100, 140), (100, 60), (100, 100)]+      result.status `shouldBe` MetricInconclusive++    it "lets a hard health observation override a statistical regression" do+      decideVerdict True [] [MetricRegression] `shouldBe` VerdictInfrastructureFailure++    it "lets checkpoint asymmetry make a statistical regression inconclusive" do+      decideVerdict False ["checkpoint overlap differs"] [MetricRegression] `shouldBe` VerdictInconclusive++  describe "health outcomes" do+    it "maps soft observations to inconclusive and hard observations to infrastructure failure" do+      healthOutcome [health Soft] `shouldBe` Just Inconclusive+      healthOutcome [health Hard] `shouldBe` Just InfrastructureFailure++pairSeedsMatch :: (Eq a) => [a] -> Bool+pairSeedsMatch [firstA, firstB, secondA, secondB, thirdA, thirdB] = firstA == firstB && secondA == secondB && thirdA == thirdB+pairSeedsMatch _ = False++testPolicy :: Policy+testPolicy = Policy "test" 3 0.95 1_000 42 False "benchmark" 0.5 1 []++latencyRule :: MetricRule+latencyRule = MetricRule "latency" LowerIsBetter 0.1 5++health :: Severity -> HealthObservation+health severity = HealthObservation "test" severity 0 1 "test observation" Null++policyJson :: Int -> Int -> ByteString.ByteString+policyJson pairs iterations =+  ByteString.pack+    ( concat+        [ "{\"schema\":\"kenshou.comparison-policy/v1\",\"name\":\"test\",\"minimumPairs\":",+          show pairs,+          ",\"confidenceLevel\":0.95,\"bootstrapIterations\":",+          show iterations,+          ",\"resamplingSeed\":1,\"requireInterleaving\":false,\"requireGrade\":\"benchmark\",\"maxCiRelativeWidth\":0.5,\"maxCheckpointAsymmetry\":1,\"metrics\":[]}"+        ]+    )++writeRun :: FilePath -> Text -> Text -> IO ()+writeRun directory runId cohort = do+  createDirectoryIfMissing True directory+  let summary = MeasurementSummary "benchmark" [] (SummaryWindow 0 1 1) Map.empty Map.empty []+      document =+        object+          [ "runId" .= runId,+            "outcome" .= ("passed" :: Text),+            "compatibility" .= object ["inputs" .= object ["cohortPlanHash" .= cohort]],+            "fingerprint" .= object ["host" .= object ["os" .= ("darwin" :: Text)]],+            "summaries" .= object ["measurements" .= object ["measurements" .= summary]]+          ]+  LazyByteString.writeFile (directory </> "run-result.json") (encode document)++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft (Right _) = False++isRight :: Either left right -> Bool+isRight = not . isLeft
+ test/Kenshou/Measure/HistogramSpec.hs view
@@ -0,0 +1,74 @@+module Kenshou.Measure.HistogramSpec (spec) where++import Control.Monad (replicateM_)+import Data.ByteString qualified as ByteString+import Data.Char (digitToInt, isHexDigit)+import Data.Either (isLeft)+import Data.Time.Clock (diffUTCTime, getCurrentTime)+import Data.Word (Word64)+import Hedgehog+import Hedgehog.Gen qualified as Gen+import Hedgehog.Range qualified as Range+import Kenshou.Measure.Histogram+import Kenshou.Measure.Histogram.Codec+import Test.Hspec+import Test.Hspec.Hedgehog (hedgehog)++spec :: Spec+spec = describe "Kenshou.Measure.Histogram" do+  it "keeps exact aggregates and bounded quantiles" do+    histogram <- build [1_000, 2_048, 2_000_000, 3_600_000_000_000]+    totalCount histogram `shouldBe` 4+    minValue histogram `shouldBe` 1_000+    maxValue histogram `shouldBe` 3_600_000_000_000+    valueAtQuantile histogram 0.5 `shouldSatisfy` \value -> value >= 2_048 && value <= 2_050+    valueAtQuantile histogram 1 `shouldBe` 3_600_000_000_000++  it "stays within the configured 0.1 percent equivalent range" $ hedgehog do+    value <- forAll (Gen.word64 (Range.linear 1 3_600_000_000_000))+    (low, high) <- evalEither (equivalentRange defaultHistogramConfig value)+    assert (low <= value && high >= value)+    assert (high - low <= max 1 (value `div` 1_000))++  it "merges without changing the recorded distribution" do+    left <- build [1, 2, 3]+    right <- build [4, 5, 6]+    together <- build [1, 2, 3, 4, 5, 6]+    merge left right `shouldBe` Right together+    merge right left `shouldBe` Right together++  it "round-trips the KHST version 1 codec" do+    histogram <- build ([0 .. 10_000] <> [2_000_000, 3_600_000_000_001])+    decodeHistogram (encodeHistogram histogram) `shouldBe` Right histogram++  it "keeps the KHST version 1 golden bytes stable" do+    histogram <- build [1, 1_000, 2_000_000]+    golden <- decodeHex . filter isHexDigit <$> readFile "test/golden/histogram-v1.hex"+    encodeHistogram histogram `shouldBe` golden++  it "records ten million samples inside the cost budget" do+    mutable <- newHistogram defaultHistogramConfig+    started <- getCurrentTime+    replicateM_ 10_000_000 (recordValue mutable 1_000)+    finished <- getCurrentTime+    total <- totalCount <$> freeze mutable+    total `shouldBe` 10_000_000+    realToFrac (diffUTCTime finished started) `shouldSatisfy` (< (2.5 :: Double))++  it "rejects a mismatched merge configuration" do+    left <- build [1]+    mutable <- newHistogram (HistogramConfig 1 10_000 2)+    recordValue mutable 1+    right <- freeze mutable+    merge left right `shouldSatisfy` isLeft++build :: [Word64] -> IO Histogram+build values = do+  mutable <- newHistogram defaultHistogramConfig+  mapM_ (recordValue mutable) values+  freeze mutable++decodeHex :: String -> ByteString.ByteString+decodeHex [] = ByteString.empty+decodeHex (high : low : rest) = ByteString.cons (fromIntegral (digitToInt high * 16 + digitToInt low)) (decodeHex rest)+decodeHex _ = error "golden hex has an odd number of digits"
+ test/Kenshou/Measure/LoadSpec.hs view
@@ -0,0 +1,21 @@+module Kenshou.Measure.LoadSpec (spec) where++import Kenshou.Measure.Load.Arrival+import System.Random.SplitMix (mkSMGen)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Load.Arrival" do+  it "uses exact constant-rate spacing" do+    constantSchedule 100 1_000 5 `shouldBe` [1_000, 10_001_000, 20_001_000, 30_001_000, 40_001_000]++  it "replays Poisson schedules from the same seed" do+    let first = poissonSchedule 100 (mkSMGen 42) 0 1_000+        second = poissonSchedule 100 (mkSMGen 42) 0 1_000+    first `shouldBe` second++  it "keeps the Poisson mean near the requested rate" do+    let arrivals = poissonSchedule 100 (mkSMGen 42) 0 20_000+        gaps = zipWith (-) arrivals (0 : arrivals)+        meanGap = fromIntegral (sum gaps) / fromIntegral (length gaps) :: Double+    meanGap `shouldSatisfy` \value -> value > 9_700_000 && value < 10_300_000
+ test/Kenshou/Measure/MethodologySpec.hs view
@@ -0,0 +1,15 @@+module Kenshou.Measure.MethodologySpec (spec) where++import Kenshou.Core.Bundle (LayerBundle (..), mkBundle, mkRegistry)+import Kenshou.Core.Id (mkLayerId, selftestLayer)+import Kenshou.Measure.Methodology (PoolMethodology (..), poolMethodologyFor)+import Test.Hspec++spec :: Spec+spec = describe "pool methodology" do+  it "reads the methodology a layer's bundle contributes" do+    let methodology = PoolMethodology 12 (10, 13)+        other = either (error . show) id (mkLayerId "acme")+    registry <- either (fail . show) pure (mkRegistry [(mkBundle selftestLayer [] []) {methodology = Just methodology}, mkBundle other [] []])+    poolMethodologyFor registry selftestLayer `shouldBe` Just methodology+    poolMethodologyFor registry other `shouldBe` Nothing
+ test/Kenshou/Measure/PerfSpec.hs view
@@ -0,0 +1,168 @@+module Kenshou.Measure.PerfSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Core.Id (newRunId)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Measure.Clock (Origin (..))+import Kenshou.Measure.Compare.Policy qualified as Policy+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (encodeHistogram)+import Kenshou.Measure.Metrics (Metric (..))+import Kenshou.Measure.Perf+import Kenshou.Measure.Samples+import Kenshou.Measure.Summary+import Kenshou.Perf.Plan (PerfProfile (..), ProfileCatalog (..), RequiredMetric (..))+import System.Directory (createDirectoryIfMissing, doesFileExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance metric admission" do+  it "cannot accept missing p99 while throughput remains present" do+    requiredMetricProblems required summary {metrics = Map.delete p99 summary.metrics}+      `shouldBe` [MissingRequiredMetric p99]+  it "classifies a changed metric unit as invalid evidence" do+    requiredMetricProblems required summary {metrics = Map.insert p99 (Metric 10 "us") summary.metrics}+      `shouldBe` [InvalidPerformanceEvidence ("metric unit mismatch: " <> p99)]+  it "keeps an undersampled required metric inconclusive" do+    let under = summary {ops = Map.singleton "publish-to-finalize" (object ["count" .= (999 :: Int)])}+    requiredMetricProblems required under `shouldBe` [InsufficientSteadySamples p99 1000 999]+  it "requires a policy rule for every required metric" do+    profile <- loadProfile+    policy <- loadPolicy+    validateProfilePolicy profile policy {Policy.metricRules = filter (not . (\rule -> Policy.matchesRule rule.match p99)) policy.metricRules}+      `shouldBe` Left ("required metric has no policy rule: " <> p99)+  it "accepts a sealed run only after raw histogram and summary reconstruction" do+    withSystemTempDirectory "kenshou-perf-admission" \directory -> do+      profile <- loadProfile+      writeRun directory+      admitted <- verifyPerformanceRun profile directory+      admitted `shouldSatisfy` isRight+  it "retains a known runtime failure even when it is nonblocking and exits zero" do+    withSystemTempDirectory "kenshou-perf-known-defect" \directory -> do+      profile <- loadProfile+      writeRun directory+      document <- eitherDecodeFileStrict' (directory </> "run-result.json") >>= either fail pure+      case document of+        Object fields -> encodeFile (directory </> "run-result.json") (Object (KeyMap.insert "exitCode" (Number 0) (KeyMap.insert "blocking" (Bool False) (KeyMap.insert "outcome" (String "failed") fields))))+        _ -> fail "fixture is not an object"+      seal directory+      admitted <- verifyPerformanceRun profile directory+      admitted `shouldBe` Left [RuntimeWorkFailure "runtime assertions failed (including nonblocking known defects)"]+  it "does not admit absent, unknown or infrastructure runtime outcomes" do+    runtimeResultProblems (object []) `shouldBe` [InvalidPerformanceEvidence "runtime outcome is missing or unsupported"]+    runtimeResultProblems (object ["outcome" .= ("infrastructure-failure" :: Text)]) `shouldBe` [InvalidPerformanceEvidence "runtime infrastructure failed"]+  it "detects storage tampering before using a retained summary" do+    withSystemTempDirectory "kenshou-perf-tamper" \directory -> do+      profile <- loadProfile+      writeRun directory+      ByteString.appendFile (directory </> "samples/publish-to-finalize.raw") "tampered"+      admitted <- verifyPerformanceRun profile directory+      admitted `shouldSatisfy` isInvalid+  it "rejects a corrupt histogram even if someone reseals its digest" do+    withSystemTempDirectory "kenshou-perf-histogram" \directory -> do+      profile <- loadProfile+      writeRun directory+      changed <- Histogram.newHistogram Histogram.defaultHistogramConfig+      mapM_ (const (Histogram.recordValue changed 20)) [1 :: Int .. 1000]+      histogram <- Histogram.freeze changed+      ByteString.writeFile (directory </> "samples/publish-to-finalize.hist") (encodeHistogram histogram)+      seal directory+      admitted <- verifyPerformanceRun profile directory+      admitted `shouldSatisfy` isInvalid+  it "rejects a missing retained required metric even with a resealed summary" do+    withSystemTempDirectory "kenshou-perf-summary" \directory -> do+      profile <- loadProfile+      writeRun directory+      recomputed <- summarizeRunDir directory >>= either (fail . show) pure+      let incomplete = recomputed {metrics = Map.delete p99 recomputed.metrics}+      writeResult directory incomplete+      seal directory+      admitted <- verifyPerformanceRun profile directory+      admitted `shouldSatisfy` isInvalid+  where+    isRight (Right _) = True+    isRight _ = False+    isInvalid (Left problems) = any invalid problems+    isInvalid _ = False+    invalid (InvalidPerformanceEvidence _) = True+    invalid _ = False++p99 :: Text+p99 = "op.publish-to-finalize.latency.p99"++required :: [RequiredMetric]+required = [RequiredMetric p99 "ns" 1000]++summary :: MeasurementSummary+summary =+  MeasurementSummary+    "benchmark"+    []+    (SummaryWindow 1000 4000 0.000003)+    (Map.singleton "publish-to-finalize" (object ["count" .= (1000 :: Int)]))+    (Map.fromList [(p99, Metric 10 "ns"), ("op.publish-to-finalize.throughput", Metric 200 "ops/s")])+    []++writeRun :: FilePath -> IO ()+writeRun directory = do+  createDirectoryIfMissing True (directory </> "samples")+  createDirectoryIfMissing True (directory </> "series")+  ByteString.writeFile (directory </> "series/load.csv") "mono_ns,wall,phase\n1000,0,steady\n4000,0,drain\n"+  encodeFile (directory </> "run-spec.json") (object ["dimensions" .= object []])+  mutable <- Histogram.newHistogram Histogram.defaultHistogramConfig+  mapM_ (const (Histogram.recordValue mutable 10)) [1 :: Int .. 1000]+  histogram <- Histogram.freeze mutable+  let base = directory </> "samples/publish-to-finalize"+  ByteString.writeFile (base <> ".hist") (encodeHistogram histogram)+  ByteString.writeFile (base <> ".service.hist") (encodeHistogram histogram)+  writer <- openSampleWriter (base <> ".raw") (SampleHeader (Origin 0 0) MonotonicClock "publish-to-finalize" "fixture")+  writeSampleBlock writer 0 [SampleRecord (1000 + index) (1000 + index) (1010 + index) 0 1 | index <- [0 .. 999]]+  closeSampleWriter writer+  encodeFile+    (base <> ".meta.json")+    ( object+        [ "operation" .= ("publish-to-finalize" :: Text),+          "latencyBasis" .= ("intended-start" :: Text),+          "rawSamples" .= ("full" :: Text),+          "phaseCounts" .= object ["steady" .= object ["successes" .= (1000 :: Int), "failures" .= (0 :: Int), "units" .= (1000 :: Int)]]+        ]+    )+  recomputed <- summarizeRunDir directory >>= either (fail . show) pure+  writeResult directory recomputed+  seal directory++writeResult :: FilePath -> MeasurementSummary -> IO ()+writeResult directory measurements =+  encodeFile+    (directory </> "run-result.json")+    (object ["outcome" .= ("passed" :: Text), "summaries" .= object ["measurements" .= object ["measurements" .= measurements]]])++seal :: FilePath -> IO ()+seal directory = do+  runId <- newRunId+  manifest <- writeManifest directory runId Map.empty+  encodeFile (directory </> "manifest.json") manifest++loadProfile :: IO PerfProfile+loadProfile = do+  path <- repositoryFile "fixtures/consumer-baseline/policies/performance/catalog.v1.json"+  catalog <- eitherDecodeFileStrict' path >>= either fail pure+  case (catalog :: ProfileCatalog).profiles of+    profile : _ -> pure profile+    [] -> fail "empty profile catalog"++loadPolicy :: IO Policy.Policy+loadPolicy = do+  path <- repositoryFile "policies/default.json"+  ByteString.readFile path >>= either (fail . show) pure . Policy.decodePolicy++repositoryFile :: FilePath -> IO FilePath+repositoryFile path = do+  exists <- doesFileExist path+  pure (if exists then path else "../" <> path)
+ test/Kenshou/Measure/RecorderSpec.hs view
@@ -0,0 +1,88 @@+module Kenshou.Measure.RecorderSpec (spec) where++import Data.ByteString qualified as ByteString+import Data.IORef+import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (decodeHistogram)+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Samples (SampleFileReport (..), SampleRecord (..), readSamples)+import System.Directory (doesFileExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Recorder" do+  it "retains warm-up raw samples while excluding them from steady histograms" $ withSystemTempDirectory "kenshou-recorder" \directory -> do+    origin <- captureOrigin+    clock <- newPhaseClock (\_ _ -> pure ()) (PhasePlan (Nanos 1) (SteadyFor (Nanos 1)) (Nanos 1))+    recorder <-+      newRecorder+        RecorderConfig+          { runDir = directory,+            origin,+            processLabel = Nothing,+            histogram = Histogram.defaultHistogramConfig,+            rawSamples = RawFull,+            intervalHistogramSeconds = 10,+            declareArtifact = \_ _ -> pure ()+          }+        clock+    operation <- registerOp recorder (OpName "request")+    worker <- newWorkerRecorder operation 1+    enterPhase clock WarmUp+    warm <- nowNs+    recordOp worker warm warm (warm + 10) (OpOk 1)+    enterPhase clock Steady+    steady <- nowNs+    recordOp worker steady steady (steady + 20) (OpOk 1)+    recordOp worker (steady + 30) (steady + 35) (steady + 60) (OpFailed (ErrorCause "store"))+    report <- finishRecorder recorder+    case report.operations of+      [operationReport] -> do+        Histogram.totalCount operationReport.latency `shouldBe` 2+        Histogram.valueAtQuantile operationReport.latency 1 `shouldBe` 30+        doesFileExist (directory </> "samples/request.hist") `shouldReturn` True+        encoded <- ByteString.readFile (directory </> "samples/request.hist")+        fmap Histogram.totalCount (decodeHistogram encoded) `shouldBe` Right 2+        rawCount <- newIORef (0 :: Int)+        recomputedMutable <- Histogram.newHistogram Histogram.defaultHistogramConfig+        rawReport <- readSamples (directory </> "samples/request.raw") \(SampleRecord intended _ end _ _) -> do+          modifyIORef' rawCount (+ 1)+          if intended >= steady then Histogram.recordValue recomputedMutable (end - intended) else pure ()+        readIORef rawCount `shouldReturn` 3+        recomputed <- Histogram.freeze recomputedMutable+        recomputed `shouldBe` operationReport.latency+        let SampleFileReport _ _ ignored = rawReport+        ignored `shouldBe` 0+      unexpected -> expectationFailure ("expected one operation report, got " <> show (length unexpected))++  it "writes interval histograms when full raw retention is disabled" $ withSystemTempDirectory "kenshou-intervals" \directory -> do+    origin <- captureOrigin+    clock <- newPhaseClock (\_ _ -> pure ()) (PhasePlan (Nanos 1) (SteadyFor (Nanos 1)) (Nanos 1))+    recorder <-+      newRecorder+        RecorderConfig+          { runDir = directory,+            origin,+            processLabel = Nothing,+            histogram = Histogram.defaultHistogramConfig,+            rawSamples = RawOff,+            intervalHistogramSeconds = 10,+            declareArtifact = \_ _ -> pure ()+          }+        clock+    operation <- registerOp recorder (OpName "request")+    worker <- newWorkerRecorder operation 1+    enterPhase clock Steady+    started <- nowNs+    recordOp worker started started (started + 50) (OpOk 1)+    report <- finishRecorder recorder+    case report.operations of+      [operationReport] -> do+        operationReport.rawFile `shouldBe` Nothing+        operationReport.intervalHistogramFile `shouldBe` Just (directory </> "samples/request.ihist")+        doesFileExist (directory </> "samples/request.ihist") `shouldReturn` True+      unexpected -> expectationFailure ("expected one operation report, got " <> show (length unexpected))
+ test/Kenshou/Measure/SamplerSpec.hs view
@@ -0,0 +1,31 @@+module Kenshou.Measure.SamplerSpec (spec) where++import Data.Text.IO qualified as Text+import Kenshou.Measure.Sampler.Host+import Kenshou.Measure.Sampler.Process+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Sampler parsers" do+  it "parses Linux process statistics after a command name containing spaces and parentheses" do+    stat <- Text.readFile "test/fixtures/proc/stat.txt"+    status <- Text.readFile "test/fixtures/proc/status.txt"+    let sample = parseProcSample stat status 17+    sample.rssBytes `shouldBe` Just (1_024 * 1_024)+    sample.rssMaxBytes `shouldBe` Just (2_048 * 1_024)+    sample.osThreads `shouldBe` Just 9+    sample.openFds `shouldBe` Just 17+    sample.cpuUserNs `shouldBe` Just 1_100_000_000+    sample.cpuSystemNs `shouldBe` Just 220_000_000+    sample.voluntaryContextSwitches `shouldBe` Just 31+    sample.nonvoluntaryContextSwitches `shouldBe` Just 7++  it "parses aggregate Linux host counters" do+    stat <- Text.readFile "test/fixtures/proc/host-stat.txt"+    loadavg <- Text.readFile "test/fixtures/proc/loadavg.txt"+    meminfo <- Text.readFile "test/fixtures/proc/meminfo.txt"+    let sample = parseHostSample stat loadavg meminfo+    sample.cpuUser `shouldBe` Just 100+    sample.cpuSteal `shouldBe` Just 8+    sample.loadAverage1 `shouldBe` Just 1.25+    sample.memAvailableBytes `shouldBe` Just (4_096 * 1_024)
+ test/Kenshou/Measure/SamplesSpec.hs view
@@ -0,0 +1,27 @@+module Kenshou.Measure.SamplesSpec (spec) where++import Data.IORef+import Kenshou.Measure.Clock+import Kenshou.Measure.Samples+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Samples" do+  it "round-trips worker blocks through KSMP version 1" $ withSystemTempDirectory "kenshou-samples" \directory -> do+    origin <- captureOrigin+    let path = directory </> "request.raw"+        records =+          [ SampleRecord (origin.monoNs + 10) (origin.monoNs + 12) (origin.monoNs + 22) 0 1,+            SampleRecord (origin.monoNs + 30) (origin.monoNs + 35) (origin.monoNs + 50) 1 0+          ]+    writer <- openSampleWriter path (SampleHeader origin MonotonicClock "request" "harness")+    writeSampleBlock writer 3 records+    closeSampleWriter writer+    observed <- newIORef []+    report <- readSamples path (\record -> modifyIORef' observed (<> [record]))+    readIORef observed `shouldReturn` records+    let SampleFileReport _ recordsRead ignoredBytes = report+    recordsRead `shouldBe` 2+    ignoredBytes `shouldBe` 0
+ test/Main.hs view
@@ -0,0 +1,24 @@+module Main (main) where++import Kenshou.Measure.ClockSpec qualified+import Kenshou.Measure.CompareSpec qualified+import Kenshou.Measure.HistogramSpec qualified+import Kenshou.Measure.LoadSpec qualified+import Kenshou.Measure.MethodologySpec qualified+import Kenshou.Measure.PerfSpec qualified+import Kenshou.Measure.RecorderSpec qualified+import Kenshou.Measure.SamplerSpec qualified+import Kenshou.Measure.SamplesSpec qualified+import Test.Hspec (hspec)++main :: IO ()+main = hspec do+  Kenshou.Measure.ClockSpec.spec+  Kenshou.Measure.CompareSpec.spec+  Kenshou.Measure.PerfSpec.spec+  Kenshou.Measure.HistogramSpec.spec+  Kenshou.Measure.LoadSpec.spec+  Kenshou.Measure.MethodologySpec.spec+  Kenshou.Measure.SamplesSpec.spec+  Kenshou.Measure.RecorderSpec.spec+  Kenshou.Measure.SamplerSpec.spec
+ test/fixtures/health-notice.json view
@@ -0,0 +1,7 @@+{+  "schema": "kenshou.health-notice/v1",+  "source": "selftest-maintenance",+  "severity": "hard",+  "at": "2026-09-21T16:00:00Z",+  "detail": "fixture host-maintenance notice"+}
+ test/fixtures/proc/host-stat.txt view
@@ -0,0 +1,2 @@+cpu  100 2 30 400 5 6 7 8 0 0+cpu0 50 1 15 200 2 3 4 4 0 0
+ test/fixtures/proc/loadavg.txt view
@@ -0,0 +1,1 @@+1.25 0.75 0.50 2/100 123
+ test/fixtures/proc/meminfo.txt view
@@ -0,0 +1,2 @@+MemTotal:       8192 kB+MemAvailable:   4096 kB
+ test/fixtures/proc/stat.txt view
@@ -0,0 +1,1 @@+123 (kenshou worker) S 1 2 3 4 5 6 7 8 9 10 110 22 0 0 20 0 9 0 1000 2000 300 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+ test/fixtures/proc/status.txt view
@@ -0,0 +1,6 @@+Name:	kenshou worker+VmHWM:	2048 kB+VmRSS:	1024 kB+Threads:	9+voluntary_ctxt_switches:	31+nonvoluntary_ctxt_switches:	7
+ test/golden/histogram-v1.hex view
@@ -0,0 +1,1 @@+4b48535401000000010000000000000000a0b83046030000030300000000000000010000000000000080841e000000000069881e00000000000000000000000000070000000102cb0f02efae0102bbd102