kenshou-measure (empty) → 0.1.0.0
raw patch · 57 files changed
+5199/−0 lines, 57 filesdep +aesondep +asyncdep +base
Dependencies added: aeson, async, base, bytestring, containers, directory, filepath, hasql, hedgehog, hspec, hspec-hedgehog, kenshou-core, kenshou-measure, optparse-applicative, primitive, process, splitmix, stm, temporary, text, time, unix
Files
- CHANGELOG.md +12/−0
- LICENSE +28/−0
- cbits/kenshou_proc_darwin.c +12/−0
- kenshou-measure.cabal +143/−0
- src/Kenshou/Measure.hs +19/−0
- src/Kenshou/Measure/Cli.hs +129/−0
- src/Kenshou/Measure/Clock.hs +42/−0
- src/Kenshou/Measure/Compare.hs +323/−0
- src/Kenshou/Measure/Compare/Compatibility.hs +74/−0
- src/Kenshou/Measure/Compare/Ordering.hs +44/−0
- src/Kenshou/Measure/Compare/Policy.hs +81/−0
- src/Kenshou/Measure/Health.hs +302/−0
- src/Kenshou/Measure/Histogram.hs +271/−0
- src/Kenshou/Measure/Histogram/Codec.hs +129/−0
- src/Kenshou/Measure/Knobs.hs +83/−0
- src/Kenshou/Measure/Load.hs +15/−0
- src/Kenshou/Measure/Load/Arrival.hs +29/−0
- src/Kenshou/Measure/Load/Closed.hs +93/−0
- src/Kenshou/Measure/Load/Open.hs +107/−0
- src/Kenshou/Measure/Load/Series.hs +32/−0
- src/Kenshou/Measure/Load/Types.hs +89/−0
- src/Kenshou/Measure/Methodology.hs +19/−0
- src/Kenshou/Measure/Metrics.hs +16/−0
- src/Kenshou/Measure/Perf.hs +145/−0
- src/Kenshou/Measure/Phase.hs +75/−0
- src/Kenshou/Measure/Recorder.hs +301/−0
- src/Kenshou/Measure/Sampler.hs +174/−0
- src/Kenshou/Measure/Sampler/Csv.hs +53/−0
- src/Kenshou/Measure/Sampler/Host.hs +95/−0
- src/Kenshou/Measure/Sampler/Postgres.hs +137/−0
- src/Kenshou/Measure/Sampler/Process.hs +144/−0
- src/Kenshou/Measure/Sampler/Rts.hs +101/−0
- src/Kenshou/Measure/Samples.hs +276/−0
- src/Kenshou/Measure/Selftest.hs +10/−0
- src/Kenshou/Measure/Selftest/PgInsert.hs +151/−0
- src/Kenshou/Measure/Selftest/RegressionInjected.hs +86/−0
- src/Kenshou/Measure/Selftest/SleepService.hs +142/−0
- src/Kenshou/Measure/Session.hs +218/−0
- src/Kenshou/Measure/Stats.hs +85/−0
- src/Kenshou/Measure/Summary.hs +287/−0
- test/Kenshou/Measure/ClockSpec.hs +18/−0
- test/Kenshou/Measure/CompareSpec.hs +141/−0
- test/Kenshou/Measure/HistogramSpec.hs +74/−0
- test/Kenshou/Measure/LoadSpec.hs +21/−0
- test/Kenshou/Measure/MethodologySpec.hs +15/−0
- test/Kenshou/Measure/PerfSpec.hs +168/−0
- test/Kenshou/Measure/RecorderSpec.hs +88/−0
- test/Kenshou/Measure/SamplerSpec.hs +31/−0
- test/Kenshou/Measure/SamplesSpec.hs +27/−0
- test/Main.hs +24/−0
- test/fixtures/health-notice.json +7/−0
- test/fixtures/proc/host-stat.txt +2/−0
- test/fixtures/proc/loadavg.txt +1/−0
- test/fixtures/proc/meminfo.txt +2/−0
- test/fixtures/proc/stat.txt +1/−0
- test/fixtures/proc/status.txt +6/−0
- test/golden/histogram-v1.hex +1/−0
+ CHANGELOG.md view
@@ -0,0 +1,12 @@+# Changelog++## 0.1.0.0 - 2026-10-28++### New Features++- First public release of the measurement toolkit: load models, latency recording, samplers, statistics, health gates, and paired comparison.+- `poolMethodologyFor` reads a layer's recommended pool size from its bundle instead of a built-in constant.++### Other Changes++- Extracted from `mori://shinzui/keiro-runtime-kenshou` at commit `a1648293443ad88ff1695ed800ff44c99993574e`, with its package and module names unchanged.
+ LICENSE view
@@ -0,0 +1,28 @@+BSD 3-Clause License++Copyright (c) 2026, Nadeem Bitar++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its contributors+ may be used to endorse or promote products derived from this software without+ specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ cbits/kenshou_proc_darwin.c view
@@ -0,0 +1,12 @@+#include <libproc.h>+#include <stdint.h>+#include <unistd.h>++int kenshou_proc_taskinfo(uint64_t *resident_size, int *thread_count) {+ struct proc_taskinfo info;+ int bytes = proc_pidinfo(getpid(), PROC_PIDTASKINFO, 0, &info, sizeof(info));+ if (bytes != sizeof(info)) return 0;+ *resident_size = info.pti_resident_size;+ *thread_count = info.pti_threadnum;+ return 1;+}
+ kenshou-measure.cabal view
@@ -0,0 +1,143 @@+cabal-version: 3.0+name: kenshou-measure+version: 0.1.0.0+synopsis: Measurement and comparison toolkit for Kenshou scenarios+description: Load generation, latency recording, sampling, summaries, health gates, and paired comparisons.+license: BSD-3-Clause+license-file: LICENSE+author: Nadeem Bitar+maintainer: nadeem@gmail.com+homepage: https://github.com/shinzui/kenshou+bug-reports: https://github.com/shinzui/kenshou/issues+category: Testing+build-type: Simple+extra-doc-files: CHANGELOG.md+tested-with: ghc >=9.12 && <9.13+extra-source-files:+ test/fixtures/health-notice.json+ test/fixtures/proc/host-stat.txt+ test/fixtures/proc/loadavg.txt+ test/fixtures/proc/meminfo.txt+ test/fixtures/proc/stat.txt+ test/fixtures/proc/status.txt+ test/golden/histogram-v1.hex++common warnings+ ghc-options:+ -Wall+ -Wcompat+ -Widentities+ -Wincomplete-record-updates+ -Wincomplete-uni-patterns+ -Wpartial-fields+ -Wredundant-constraints++ default-language: GHC2024+ default-extensions:+ BlockArguments+ DuplicateRecordFields+ ImportQualifiedPost+ NoFieldSelectors+ OverloadedLabels+ OverloadedRecordDot+ OverloadedStrings++library+ import: warnings+ hs-source-dirs: src+ exposed-modules:+ Kenshou.Measure+ Kenshou.Measure.Cli+ Kenshou.Measure.Clock+ Kenshou.Measure.Compare+ Kenshou.Measure.Compare.Compatibility+ Kenshou.Measure.Compare.Ordering+ Kenshou.Measure.Compare.Policy+ Kenshou.Measure.Health+ Kenshou.Measure.Histogram+ Kenshou.Measure.Histogram.Codec+ Kenshou.Measure.Knobs+ Kenshou.Measure.Load+ Kenshou.Measure.Load.Arrival+ Kenshou.Measure.Load.Closed+ Kenshou.Measure.Load.Open+ Kenshou.Measure.Load.Series+ Kenshou.Measure.Load.Types+ Kenshou.Measure.Methodology+ Kenshou.Measure.Metrics+ Kenshou.Measure.Perf+ Kenshou.Measure.Phase+ Kenshou.Measure.Recorder+ Kenshou.Measure.Sampler+ Kenshou.Measure.Sampler.Csv+ Kenshou.Measure.Sampler.Host+ Kenshou.Measure.Sampler.Postgres+ Kenshou.Measure.Sampler.Process+ Kenshou.Measure.Sampler.Rts+ Kenshou.Measure.Samples+ Kenshou.Measure.Selftest+ Kenshou.Measure.Selftest.PgInsert+ Kenshou.Measure.Selftest.RegressionInjected+ Kenshou.Measure.Selftest.SleepService+ Kenshou.Measure.Session+ Kenshou.Measure.Stats+ Kenshou.Measure.Summary++ build-depends:+ aeson >=2.2 && <2.3,+ async >=2.2 && <2.3,+ base >=4.21 && <4.22,+ bytestring >=0.12 && <0.13,+ containers >=0.7 && <0.8,+ directory >=1.3 && <1.4,+ filepath >=1.5 && <1.6,+ hasql >=1.10 && <1.11,+ kenshou-core ^>=0.1.0.0,+ optparse-applicative >=0.19 && <0.20,+ primitive >=0.9 && <0.10,+ process >=1.6 && <1.7,+ splitmix >=0.1 && <0.2,+ stm >=2.5 && <2.6,+ text >=2.1 && <2.2,+ time >=1.14 && <1.15,+ unix >=2.8 && <2.9,++ if os(osx)+ c-sources: cbits/kenshou_proc_darwin.c++test-suite kenshou-measure-test+ import: warnings+ type: exitcode-stdio-1.0+ hs-source-dirs: test+ main-is: Main.hs+ other-modules:+ Kenshou.Measure.ClockSpec+ Kenshou.Measure.CompareSpec+ Kenshou.Measure.HistogramSpec+ Kenshou.Measure.LoadSpec+ Kenshou.Measure.MethodologySpec+ Kenshou.Measure.PerfSpec+ Kenshou.Measure.RecorderSpec+ Kenshou.Measure.SamplerSpec+ Kenshou.Measure.SamplesSpec++ build-depends:+ aeson,+ base >=4.21 && <4.22,+ bytestring,+ containers,+ directory >=1.3 && <1.4,+ filepath,+ hedgehog >=1.4 && <1.8,+ hspec >=2.11 && <2.12,+ hspec-hedgehog >=0.0 && <0.4,+ kenshou-core ^>=0.1.0.0,+ kenshou-measure,+ splitmix >=0.1 && <0.2,+ temporary,+ text,+ time >=1.14 && <1.15,++source-repository head+ type: git+ location: https://github.com/shinzui/kenshou.git
+ src/Kenshou/Measure.hs view
@@ -0,0 +1,19 @@+-- | Measurement, load generation, sampling, and comparison support for Kenshou scenarios.+module Kenshou.Measure+ ( module Kenshou.Measure.Clock,+ module Kenshou.Measure.Histogram,+ module Kenshou.Measure.Knobs,+ module Kenshou.Measure.Load,+ module Kenshou.Measure.Phase,+ module Kenshou.Measure.Recorder,+ module Kenshou.Measure.Session,+ )+where++import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session
+ src/Kenshou/Measure/Cli.hs view
@@ -0,0 +1,129 @@+module Kenshou.Measure.Cli+ ( compareCommand,+ summarizeCommand,+ )+where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Kenshou.Core.Cli+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility+import Kenshou.Measure.Compare.Policy+import Kenshou.Measure.Metrics+import Kenshou.Measure.Stats+import Kenshou.Measure.Summary+import Options.Applicative+import System.Exit (ExitCode (..))+import System.IO (stderr)++data SummarizeOptions = SummarizeOptions {runDir :: FilePath, json :: Bool, verify :: Bool}++data CompareOptions = CompareOptions+ { baselines :: [FilePath],+ candidates :: [FilePath],+ policySource :: InputSource,+ vary :: [Text],+ output :: Maybe FilePath,+ json :: Bool+ }++summarizeCommand :: CliCommand+summarizeCommand = CliCommand "summarize" "Recompute and verify run measurements" Analysis False (runSummarize <$> summarizeParser)++compareCommand :: CliCommand+compareCommand = CliCommand "compare" "Compare paired baseline and candidate runs" Analysis False (runCompare <$> compareParser)++summarizeParser :: Parser SummarizeOptions+summarizeParser =+ SummarizeOptions+ <$> strArgument (metavar "RUN_DIR" <> help "Completed run directory")+ <*> switch (long "json" <> help "Write JSON to standard output")+ <*> switch (long "verify" <> help "Compare recomputed figures with run-result.json")++compareParser :: Parser CompareOptions+compareParser =+ CompareOptions+ <$> some (strOption (long "baseline" <> metavar "DIR" <> help "Baseline run directory; repeat once per pair"))+ <*> some (strOption (long "candidate" <> metavar "DIR" <> help "Candidate run directory; repeat once per pair"))+ <*> (parseInputSource <$> strOption (long "policy" <> metavar "FILE" <> help "Comparison policy file, or - for stdin"))+ <*> many (Text.pack <$> strOption (long "vary" <> metavar "AXIS" <> help "Allowed varying axis: cohort, dim:NAME, or knob:NAME"))+ <*> optional (strOption (long "out" <> metavar "FILE" <> help "Write the comparison JSON document"))+ <*> switch (long "json" <> help "Write JSON to standard output")++runSummarize :: SummarizeOptions -> CliEnv -> IO ExitCode+runSummarize options _ = do+ result <- summarizeRunDir options.runDir+ case result of+ Left (SummaryError message) -> Text.hPutStrLn stderr ("kenshou summarize: " <> message) >> pure (ExitFailure 4)+ Right summary -> do+ let document = toJSON summary+ if options.json then LazyByteString.putStr (encode document <> "\n") else renderSummary summary+ if not options.verify then pure ExitSuccess else verifyStored options.runDir document++verifyStored :: FilePath -> Value -> IO ExitCode+verifyStored runDir recomputed = do+ decoded <- eitherDecodeFileStrict' (runDir <> "/run-result.json") :: IO (Either String Value)+ case decoded of+ Left message -> Text.hPutStrLn stderr ("kenshou summarize: " <> Text.pack message) >> pure (ExitFailure 4)+ Right result -> case storedMeasurements result of+ Nothing -> Text.hPutStrLn stderr "kenshou summarize: run-result.json has no stored measurements" >> pure (ExitFailure 1)+ Just stored | stored == recomputed -> pure ExitSuccess+ Just _ -> Text.hPutStrLn stderr "kenshou summarize: stored measurements differ from recomputed measurements" >> pure (ExitFailure 1)++storedMeasurements :: Value -> Maybe Value+storedMeasurements (Object root) = do+ Object summaries <- KeyMap.lookup "summaries" root+ Object measurements <- KeyMap.lookup "measurements" summaries+ KeyMap.lookup "measurements" measurements+storedMeasurements _ = Nothing++runCompare :: CompareOptions -> CliEnv -> IO ExitCode+runCompare options _ = do+ policyBytes <- readInputSource options.policySource+ case decodePolicy policyBytes of+ Left message -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)+ Right policy -> case traverse parseVaryingAxis (if null options.vary then ["cohort"] else options.vary) of+ Left message -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)+ Right [] -> pure (ExitFailure 2)+ Right (axis : axes) -> do+ result <- compareRuns policy (axis :| axes) options.baselines options.candidates+ case result of+ Left (CompareError message) -> Text.hPutStrLn stderr ("kenshou compare: " <> message) >> pure (ExitFailure 2)+ Right comparison -> do+ let bytes = encode comparison+ maybe (pure ()) (\path -> LazyByteString.writeFile path (bytes <> "\n")) options.output+ if options.json then LazyByteString.putStr (bytes <> "\n") else renderComparison comparison+ pure case verdictExitCode comparison.verdict of 0 -> ExitSuccess; code -> ExitFailure code++renderSummary :: MeasurementSummary -> IO ()+renderSummary summary = do+ Text.putStrLn ("grade: " <> summary.grade)+ mapM_ (\(name, metric) -> Text.putStrLn (name <> ": " <> Text.pack (show metric.value) <> " " <> metric.unit)) (toList summary.metrics)++renderComparison :: Comparison -> IO ()+renderComparison comparison = do+ Text.putStrLn ("comparison " <> comparison.comparisonId <> " verdict: " <> verdictName comparison.verdict)+ mapM_ (\(_, metric) -> Text.putStrLn (metric.metric <> " " <> statusName metric.status <> " ratio=" <> Text.pack (show metric.ratio.estimate))) (toList comparison.metrics)+ mapM_ (Text.putStrLn . ("reason: " <>)) comparison.reasons++verdictName :: Verdict -> Text+verdictName VerdictPass = "pass"+verdictName VerdictRegression = "regression"+verdictName VerdictInconclusive = "inconclusive"+verdictName VerdictInfrastructureFailure = "infrastructure-failure"++statusName :: MetricStatus -> Text+statusName MetricPass = "pass"+statusName MetricRegression = "regression"+statusName MetricInconclusive = "inconclusive"++toList :: Map key value -> [(key, value)]+toList = Map.toList
+ src/Kenshou/Measure/Clock.hs view
@@ -0,0 +1,42 @@+module Kenshou.Measure.Clock+ ( Nanos (..),+ Origin (..),+ nowNs,+ captureOrigin,+ sleepUntilNs,+ )+where++import Control.Concurrent (threadDelay)+import Data.Int (Int64)+import Data.Time.Clock.POSIX (getPOSIXTime)+import Data.Word (Word64)+import GHC.Clock (getMonotonicTimeNSec)++newtype Nanos = Nanos Word64+ deriving stock (Eq, Ord, Show)++data Origin = Origin+ { monoNs :: !Word64,+ wallUnixNs :: !Int64+ }+ deriving stock (Eq, Show)++nowNs :: IO Word64+nowNs = getMonotonicTimeNSec++captureOrigin :: IO Origin+captureOrigin = do+ monotonic <- nowNs+ wall <- getPOSIXTime+ pure (Origin monotonic (floor (wall * 1_000_000_000)))++sleepUntilNs :: Word64 -> IO ()+sleepUntilNs deadline = do+ current <- nowNs+ if current >= deadline+ then pure ()+ else do+ let remainingMicros = max 1 ((deadline - current) `div` 1_000)+ threadDelay (fromIntegral (min remainingMicros 1_000_000))+ sleepUntilNs deadline
+ src/Kenshou/Measure/Compare.hs view
@@ -0,0 +1,323 @@+module Kenshou.Measure.Compare+ ( Verdict (..),+ MetricStatus (..),+ MetricComparison (..),+ Comparison (..),+ CompareError (..),+ compareRuns,+ compareMetricPairs,+ decideVerdict,+ verdictExitCode,+ )+where++import Data.Aeson+import Data.Aeson.Key (fromText)+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List (find, nub)+import Data.List.NonEmpty (NonEmpty)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, fromMaybe, mapMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (UTCTime, getCurrentTime)+import Data.Time.Format (defaultTimeLocale, parseTimeM)+import Kenshou.Core.Fingerprint (HostFingerprint (..), collectHostFingerprint)+import Kenshou.Core.Id (newRunId, renderRunId)+import Kenshou.Measure.Compare.Compatibility+import Kenshou.Measure.Compare.Ordering+import Kenshou.Measure.Compare.Policy+import Kenshou.Measure.Health+import Kenshou.Measure.Metrics+import Kenshou.Measure.Stats+import Kenshou.Measure.Summary++data Verdict = VerdictPass | VerdictRegression | VerdictInconclusive | VerdictInfrastructureFailure deriving stock (Eq, Ord, Show)++data MetricStatus = MetricPass | MetricRegression | MetricInconclusive deriving stock (Eq, Ord, Show)++data MetricComparison = MetricComparison+ { metric :: Text,+ unit :: Text,+ pairs :: [(Double, Double)],+ ratio :: Interval,+ delta :: Interval,+ relativeLimit :: Double,+ absoluteFloor :: Double,+ baselineCv :: Double,+ status :: MetricStatus,+ message :: Text+ }+ deriving stock (Eq, Show)++data Comparison = Comparison+ { comparisonId :: Text,+ startedAt :: UTCTime,+ finishedAt :: UTCTime,+ harnessRevision :: Maybe Text,+ harnessDirty :: Maybe Bool,+ design :: Text,+ baselineRuns :: [Text],+ candidateRuns :: [Text],+ policy :: Policy,+ variedFactors :: [VaryingAxis],+ pairCount :: Int,+ metrics :: Map Text MetricComparison,+ reasons :: [Text],+ verdict :: Verdict+ }+ deriving stock (Eq, Show)++newtype CompareError = CompareError Text deriving stock (Eq, Show)++instance ToJSON Verdict where toJSON = String . verdictText++instance ToJSON MetricStatus where toJSON = String . statusText++instance ToJSON MetricComparison where+ toJSON value =+ object+ [ "metric" .= value.metric,+ "unit" .= value.unit,+ "pairs" .= [object ["baseline" .= baseline, "candidate" .= candidate] | (baseline, candidate) <- value.pairs],+ "ratio" .= value.ratio,+ "delta" .= value.delta,+ "relativeLimit" .= value.relativeLimit,+ "absoluteFloor" .= value.absoluteFloor,+ "baselineCv" .= value.baselineCv,+ "status" .= value.status,+ "message" .= value.message+ ]++instance ToJSON Comparison where+ toJSON value =+ object+ [ "schema" .= ("kenshou.comparison/v1" :: Text),+ "comparisonId" .= value.comparisonId,+ "startedAt" .= value.startedAt,+ "finishedAt" .= value.finishedAt,+ "harnessRevision" .= value.harnessRevision,+ "harnessDirty" .= value.harnessDirty,+ "design" .= value.design,+ "baselineRuns" .= value.baselineRuns,+ "candidateRuns" .= value.candidateRuns,+ "algorithm" .= object ["name" .= ("paired-bootstrap-t-envelope" :: Text), "version" .= (1 :: Int), "generator" .= ("splitmix" :: Text), "iterations" .= value.policy.bootstrapIterations, "seed" .= value.policy.resamplingSeed, "confidenceLevel" .= value.policy.confidenceLevel],+ "policy" .= value.policy,+ "variedFactors" .= value.variedFactors,+ "pairCount" .= value.pairCount,+ "metrics" .= value.metrics,+ "reasons" .= value.reasons,+ "verdict" .= value.verdict,+ "exitCode" .= verdictExitCode value.verdict+ ]++data RunData = RunData+ { runId :: Text,+ result :: Value,+ inputs :: Value,+ fingerprint :: Value,+ summary :: MeasurementSummary,+ outcome :: Text,+ startedAt :: Maybe UTCTime+ }++compareRuns :: Policy -> NonEmpty VaryingAxis -> [FilePath] -> [FilePath] -> IO (Either CompareError Comparison)+compareRuns policy axes baselineDirs candidateDirs+ | null baselineDirs || length baselineDirs /= length candidateDirs = pure (Left (CompareError "baseline and candidate runs must form the same non-zero number of pairs"))+ | otherwise = do+ baselineResults <- traverse loadRun baselineDirs+ candidateResults <- traverse loadRun candidateDirs+ case (sequence baselineResults, sequence candidateResults) of+ (Left err, _) -> pure (Left err)+ (_, Left err) -> pure (Left err)+ (Right baselines, Right candidates) -> compareLoaded baselines candidates+ where+ compareLoaded baselines candidates = case compatibilityProblem axes (baselines <> candidates) of+ Just message -> pure (Left (CompareError message))+ Nothing -> case varyingProblem axes baselines candidates of+ Just message -> pure (Left (CompareError message))+ Nothing -> do+ startedAt <- getCurrentTime+ identifier <- renderRunId <$> newRunId+ fingerprint <- collectHostFingerprint+ let referenceFingerprint = maybe Null (.fingerprint) (headMay baselines)+ environmentChanged = any ((/= referenceFingerprint) . (.fingerprint)) (tailSafe baselines <> candidates)+ badOutcomes = any ((`elem` ["errored", "infrastructure-failure"]) . (.outcome)) (baselines <> candidates)+ hardHealth = any (hasSeverity Hard . (.summary)) (baselines <> candidates)+ softHealth = any (hasSeverity Soft . (.summary)) (baselines <> candidates)+ checkpointAsymmetry = any (uncurry (checkpointDiffers policy.maxCheckpointAsymmetry)) (zip baselines candidates)+ metricComparisons = compareMetrics policy baselines candidates+ interleaving = validateInterleaving (catMaybes (concat (zipWith trialRows [0 ..] (zip baselines candidates))))+ lowGrade = any ((/= policy.requireGrade) . (.grade) . (.summary)) (baselines <> candidates)+ reasons =+ ["machine profiles differ" | environmentChanged]+ <> ["an input run has an infrastructure outcome" | badOutcomes]+ <> ["an input run has a hard health observation" | hardHealth]+ <> ["fewer than the required number of pairs" | length baselines < policy.minimumPairs]+ <> ["runs are not interleaved" | policy.requireInterleaving && either (const True) (const False) interleaving]+ <> ["an input run is below the required evidence grade" | lowGrade]+ <> ["an input run has a soft health observation" | softHealth]+ <> ["checkpoint overlap differs too much within a pair" | checkpointAsymmetry]+ statuses = fmap (.status) (Map.elems metricComparisons)+ verdict = decideVerdict (environmentChanged || badOutcomes || hardHealth) reasons statuses+ finishedAt <- getCurrentTime+ pure (Right (Comparison identifier startedAt finishedAt fingerprint.revision fingerprint.dirty (comparisonDesign baselines candidates) (map (.runId) baselines) (map (.runId) candidates) policy (toList axes) (length baselines) metricComparisons reasons verdict))++comparisonDesign :: [RunData] -> [RunData] -> Text+comparisonDesign baselines candidates = case traverse firstArm (zip baselines candidates) of+ Just arms+ | arms == take (length arms) (cycle [Baseline, Candidate]) -> "abba"+ | arms == take (length arms) (cycle [Candidate, Baseline]) -> "baab"+ _ -> "sequential"+ where+ firstArm (baseline, candidate) = do+ baselineStart <- baseline.startedAt+ candidateStart <- candidate.startedAt+ if baselineStart < candidateStart then Just Baseline else if candidateStart < baselineStart then Just Candidate else Nothing++loadRun :: FilePath -> IO (Either CompareError RunData)+loadRun directory = do+ decoded <- eitherDecodeFileStrict' (directory <> "/run-result.json") :: IO (Either String Value)+ case decoded of+ Left message -> pure (Left (CompareError (Text.pack message)))+ Right result -> do+ summarized <- case jsonAt ["summaries", "measurements", "measurements"] result of+ Just value -> pure case fromJSON value of Error message -> Left (SummaryError (Text.pack message)); Success summary -> Right summary+ Nothing -> summarizeRunDir directory+ pure do+ summary <- either (Left . CompareError . (\(SummaryError message) -> message)) Right summarized+ inputs <- maybe (Left (CompareError "run result has no compatibility inputs")) Right (compatibilityInputs result)+ fingerprint <- maybe (Left (CompareError "run result has no environment fingerprint")) Right (environmentFingerprint result)+ runId <- maybe (Left (CompareError "run result has no run ID")) Right (textAt ["runId"] result)+ let outcome = fromMaybe "unknown" (textAt ["outcome"] result)+ started = textAt ["timings", "startedAt"] result >>= parseUtc+ Right RunData {runId, result, inputs, fingerprint, summary, outcome, startedAt = started}++compatibilityProblem :: NonEmpty VaryingAxis -> [RunData] -> Maybe Text+compatibilityProblem _ [] = Just "no runs"+compatibilityProblem axes (first : rest) = case [message | run <- rest, Left message <- [compatibleExcept axes first.inputs run.inputs]] of+ message : _ -> Just message+ [] -> Nothing++varyingProblem :: NonEmpty VaryingAxis -> [RunData] -> [RunData] -> Maybe Text+varyingProblem axes baselines candidates = findProblem (toList axes)+ where+ findProblem [] = Nothing+ findProblem (VaryControl : rest) =+ let baselineValues = nub (fmap (.inputs) baselines)+ candidateValues = nub (fmap (.inputs) candidates)+ in if length baselineValues /= 1 || baselineValues /= candidateValues+ then Just "control arms must have identical compatibility inputs"+ else findProblem rest+ findProblem (axis : rest) =+ let baselineValues = nub (fmap (varyingValue axis . (.inputs)) baselines)+ candidateValues = nub (fmap (varyingValue axis . (.inputs)) candidates)+ in if length baselineValues /= 1 || length candidateValues /= 1 || baselineValues == candidateValues+ then Just "each varying axis must have one distinct value per arm"+ else findProblem rest++compareMetrics :: Policy -> [RunData] -> [RunData] -> Map Text MetricComparison+compareMetrics policy baselines candidates = Map.fromList (mapMaybe compareOne commonNames)+ where+ allMetricMaps = fmap ((.metrics) . (.summary)) (baselines <> candidates)+ commonNames = case allMetricMaps of [] -> []; first : rest -> filter (\name -> all (Map.member name) rest) (Map.keys first)+ compareOne name = do+ rule <- find (\candidate -> matchesRule candidate.match name) policy.metricRules+ pairedValues <- traverse (pairMetric name) (zip baselines candidates)+ let unit = fromMaybe "" do+ baseline <- headMay baselines+ (.unit) <$> Map.lookup name baseline.summary.metrics+ pure (name, compareMetricPairs policy name unit rule pairedValues)+ pairMetric name (baseline, candidate) = do+ baselineMetric <- Map.lookup name baseline.summary.metrics+ candidateMetric <- Map.lookup name candidate.summary.metrics+ pure (baselineMetric.value, candidateMetric.value)++compareMetricPairs :: Policy -> Text -> Text -> MetricRule -> [(Double, Double)] -> MetricComparison+compareMetricPairs policy name unit rule pairedValues =+ let baselineValues = fmap fst pairedValues+ adverseRatios = fmap (adverseRatio rule.direction) pairedValues+ adverseDeltas = fmap (adverseDelta rule.direction) pairedValues+ ratioBootstrap = bootstrapInterval policy.resamplingSeed policy.bootstrapIterations policy.confidenceLevel geometricMean adverseRatios+ logT = studentTInterval policy.confidenceLevel (fmap log adverseRatios)+ ratioT = Interval (exp logT.low) (exp logT.estimate) (exp logT.high)+ ratioInterval = intervalEnvelope ratioBootstrap ratioT+ deltaInterval = intervalEnvelope (bootstrapInterval (policy.resamplingSeed + 1) policy.bootstrapIterations policy.confidenceLevel arithmeticMean adverseDeltas) (studentTInterval policy.confidenceLevel adverseDeltas)+ tooWide = ratioInterval.low <= 0 || ratioInterval.high / ratioInterval.low > 1 + policy.maxCiRelativeWidth+ comparisonStatus+ | tooWide = MetricInconclusive+ | ratioInterval.low > 1 + rule.relativeLimit && deltaInterval.low > rule.absoluteFloor = MetricRegression+ | ratioInterval.high <= 1 + rule.relativeLimit || deltaInterval.high <= rule.absoluteFloor = MetricPass+ | otherwise = MetricInconclusive+ comparisonMessage = case comparisonStatus of MetricPass -> "within policy limits"; MetricRegression -> "both relative and absolute regression limits exceeded"; MetricInconclusive -> "confidence interval crosses a policy limit or is too wide"+ in MetricComparison name unit pairedValues ratioInterval deltaInterval rule.relativeLimit rule.absoluteFloor (coefficientOfVariation baselineValues) comparisonStatus comparisonMessage+ where+ adverseRatio LowerIsBetter (baseline, candidate) = candidate / max 1e-12 baseline+ adverseRatio HigherIsBetter (baseline, candidate) = baseline / max 1e-12 candidate+ adverseDelta LowerIsBetter (baseline, candidate) = candidate - baseline+ adverseDelta HigherIsBetter (baseline, candidate) = baseline - candidate++hasSeverity :: Severity -> MeasurementSummary -> Bool+hasSeverity wanted summary = any ((== wanted) . (.severity)) summary.health++checkpointDiffers :: Double -> RunData -> RunData -> Bool+checkpointDiffers limit baseline candidate = abs (checkpointOverlap baseline.summary - checkpointOverlap candidate.summary) > limit++checkpointOverlap :: MeasurementSummary -> Double+checkpointOverlap summary = fromMaybe 0 do+ health <- find ((== "checkpoint-in-window") . (.gate)) summary.health+ case health.evidence of+ Object value -> case KeyMap.lookup "overlapFraction" value of Just (Number number) -> Just (realToFrac number); _ -> Nothing+ _ -> Nothing++decideVerdict :: Bool -> [Text] -> [MetricStatus] -> Verdict+decideVerdict infrastructureIssue reasons statuses+ | infrastructureIssue = VerdictInfrastructureFailure+ | not (null reasons) || null statuses = VerdictInconclusive+ | MetricRegression `elem` statuses = VerdictRegression+ | MetricInconclusive `elem` statuses = VerdictInconclusive+ | otherwise = VerdictPass++trialRows :: Int -> (RunData, RunData) -> [Maybe (Arm, Int, UTCTime)]+trialRows pair (baseline, candidate) = [fmap (Baseline,pair,) baseline.startedAt, fmap (Candidate,pair,) candidate.startedAt]++tailSafe :: [value] -> [value]+tailSafe [] = []; tailSafe (_ : rest) = rest++textAt :: [Text] -> Value -> Maybe Text+textAt [] (String value) = Just value+textAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= textAt rest+textAt _ _ = Nothing++jsonAt :: [Text] -> Value -> Maybe Value+jsonAt [] value = Just value+jsonAt (key : rest) (Object objectValue) = KeyMap.lookup (fromText key) objectValue >>= jsonAt rest+jsonAt _ _ = Nothing++headMay :: [value] -> Maybe value+headMay [] = Nothing+headMay (value : _) = Just value++parseUtc :: Text -> Maybe UTCTime+parseUtc = parseTimeM True defaultTimeLocale "%Y-%m-%dT%H:%M:%S%QZ" . Text.unpack++verdictExitCode :: Verdict -> Int+verdictExitCode VerdictPass = 0+verdictExitCode VerdictRegression = 1+verdictExitCode VerdictInconclusive = 3+verdictExitCode VerdictInfrastructureFailure = 4++verdictText :: Verdict -> Text+verdictText VerdictPass = "pass"+verdictText VerdictRegression = "regression"+verdictText VerdictInconclusive = "inconclusive"+verdictText VerdictInfrastructureFailure = "infrastructure-failure"++statusText :: MetricStatus -> Text+statusText MetricPass = "pass"+statusText MetricRegression = "regression"+statusText MetricInconclusive = "inconclusive"++toList :: NonEmpty value -> [value]+toList = foldr (:) []
+ src/Kenshou/Measure/Compare/Compatibility.hs view
@@ -0,0 +1,74 @@+module Kenshou.Measure.Compare.Compatibility+ ( VaryingAxis (..),+ parseVaryingAxis,+ compatibilityInputs,+ environmentFingerprint,+ compatibleExcept,+ varyingValue,+ )+where++import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List.NonEmpty (NonEmpty)+import Data.List.NonEmpty qualified as NonEmpty+import Data.Text (Text)+import Data.Text qualified as Text++data VaryingAxis = VaryControl | VaryCohort | VaryDimension Text | VaryKnob Text deriving stock (Eq, Ord, Show)++instance ToJSON VaryingAxis where+ toJSON VaryControl = String "control"+ toJSON VaryCohort = String "cohort"+ toJSON (VaryDimension name) = String ("dim:" <> name)+ toJSON (VaryKnob name) = String ("knob:" <> name)++parseVaryingAxis :: Text -> Either Text VaryingAxis+parseVaryingAxis "control" = Right VaryControl+parseVaryingAxis "cohort" = Right VaryCohort+parseVaryingAxis value | Just name <- Text.stripPrefix "dim:" value, not (Text.null name) = Right (VaryDimension name)+parseVaryingAxis value | Just name <- Text.stripPrefix "knob:" value, not (Text.null name) = Right (VaryKnob name)+parseVaryingAxis value = Left ("invalid varying axis " <> value)++compatibilityInputs :: Value -> Maybe Value+compatibilityInputs (Object root) = do+ Object compatibility <- KeyMap.lookup "compatibility" root+ KeyMap.lookup "inputs" compatibility+compatibilityInputs _ = Nothing++environmentFingerprint :: Value -> Maybe Value+environmentFingerprint (Object root) = do+ Object fingerprint <- KeyMap.lookup "fingerprint" root+ pure (Object (KeyMap.filterWithKey (\key _ -> Key.toText key `elem` ["host", "runtime", "postgres", "placement", "machineProfile"]) fingerprint))+environmentFingerprint _ = Nothing++compatibleExcept :: NonEmpty VaryingAxis -> Value -> Value -> Either Text ()+compatibleExcept axes left right =+ let normalized value = foldr removeAxis value (NonEmpty.toList axes)+ in if normalized left == normalized right+ then Right ()+ else Left "run compatibility inputs differ outside the declared varying axes"+ where+ removeAxis axis value = case value of+ Object objectValue -> Object case axis of+ VaryControl -> objectValue+ VaryCohort -> KeyMap.delete "cohortPlanHash" objectValue+ VaryDimension name -> updateNested "dimensions" name objectValue+ VaryKnob name -> updateNested "knobs" name objectValue+ other -> other+ deleteNested name (Object objectValue) = Object (KeyMap.delete (Key.fromText name) objectValue)+ deleteNested _ other = other+ updateNested parent name objectValue = case KeyMap.lookup parent objectValue of+ Nothing -> objectValue+ Just nested -> KeyMap.insert parent (deleteNested name nested) objectValue++varyingValue :: VaryingAxis -> Value -> Maybe Value+varyingValue axis (Object inputs) = case axis of+ VaryControl -> Just (Object inputs)+ VaryCohort -> KeyMap.lookup "cohortPlanHash" inputs+ VaryDimension name -> nested "dimensions" name+ VaryKnob name -> nested "knobs" name+ where+ nested parent name = do Object values <- KeyMap.lookup parent inputs; KeyMap.lookup (Key.fromText name) values+varyingValue _ _ = Nothing
+ src/Kenshou/Measure/Compare/Ordering.hs view
@@ -0,0 +1,44 @@+module Kenshou.Measure.Compare.Ordering+ ( Arm (..),+ PairedOrdering (..),+ TrialSlot (..),+ pairedSchedule,+ validateInterleaving,+ )+where++import Data.List (sortOn)+import Data.Text (Text)+import Data.Time (UTCTime)+import Data.Word (Word64)+import System.Random.SplitMix (mkSMGen, nextWord64)++data Arm = Baseline | Candidate deriving stock (Eq, Ord, Show)++data PairedOrdering = ABBA | BAAB deriving stock (Eq, Show)++data TrialSlot = TrialSlot {position :: Int, pairIndex :: Int, arm :: Arm, pairSeed :: Word64}+ deriving stock (Eq, Show)++pairedSchedule :: PairedOrdering -> Int -> Word64 -> [TrialSlot]+pairedSchedule ordering pairs seed = concat (zipWith slots [0 .. max 0 pairs - 1] seeds)+ where+ seeds = randomWords (mkSMGen seed)+ randomWords generator = let (value, next) = nextWord64 generator in value : randomWords next+ slots pair pairSeed =+ let firstBaseline = case ordering of ABBA -> even pair; BAAB -> odd pair+ arms = if firstBaseline then [Baseline, Candidate] else [Candidate, Baseline]+ start = pair * 2+ in zipWith (\offset arm -> TrialSlot (start + offset) pair arm pairSeed) [0, 1] arms++validateInterleaving :: [(Arm, Int, UTCTime)] -> Either Text ()+validateInterleaving trials =+ let ordered = sortOn (\(_, _, started) -> started) trials+ pairs = foldr collect [] ordered+ collect (arm, pair, _) accumulated = case lookup pair accumulated of+ Nothing -> (pair, [arm]) : accumulated+ Just _ -> fmap (\entry@(key, arms) -> if key == pair then (key, arm : arms) else entry) accumulated+ validPair (_, arms) = length arms == 2 && Baseline `elem` arms && Candidate `elem` arms+ firstArms = [arm | (arm, _, _) <- ordered]+ balanced = abs (length (filter (== Baseline) (take (length firstArms `div` 2) firstArms)) - length firstArms `div` 4) <= 1+ in if all validPair pairs && balanced then Right () else Left "runs are not paired and interleaved"
+ src/Kenshou/Measure/Compare/Policy.hs view
@@ -0,0 +1,81 @@+module Kenshou.Measure.Compare.Policy+ ( Direction (..),+ MetricRule (..),+ Policy (..),+ decodePolicy,+ matchesRule,+ )+where++import Data.Aeson+import Data.ByteString (ByteString)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)++data Direction = HigherIsBetter | LowerIsBetter deriving stock (Eq, Show)++data MetricRule = MetricRule {match :: Text, direction :: Direction, relativeLimit :: Double, absoluteFloor :: Double}+ deriving stock (Eq, Show)++data Policy = Policy+ { name :: Text,+ minimumPairs :: Int,+ confidenceLevel :: Double,+ bootstrapIterations :: Int,+ resamplingSeed :: Word64,+ requireInterleaving :: Bool,+ requireGrade :: Text,+ maxCiRelativeWidth :: Double,+ maxCheckpointAsymmetry :: Double,+ metricRules :: [MetricRule]+ }+ deriving stock (Eq, Show)++instance FromJSON Direction where+ parseJSON = withText "Direction" \value -> case value of+ "higher-is-better" -> pure HigherIsBetter+ "lower-is-better" -> pure LowerIsBetter+ _ -> fail "direction must be higher-is-better or lower-is-better"++instance ToJSON Direction where+ toJSON HigherIsBetter = String "higher-is-better"+ toJSON LowerIsBetter = String "lower-is-better"++instance FromJSON MetricRule where+ parseJSON = withObject "MetricRule" \value -> MetricRule <$> value .: "match" <*> value .: "direction" <*> value .: "relativeLimit" <*> value .: "absoluteFloor"++instance ToJSON MetricRule where+ toJSON rule = object ["match" .= rule.match, "direction" .= rule.direction, "relativeLimit" .= rule.relativeLimit, "absoluteFloor" .= rule.absoluteFloor]++instance FromJSON Policy where+ parseJSON = withObject "Policy" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.comparison-policy/v1" :: Text) then fail "unsupported comparison policy" else pure ()+ decodedPolicy <- Policy <$> value .: "name" <*> value .: "minimumPairs" <*> value .: "confidenceLevel" <*> value .: "bootstrapIterations" <*> value .: "resamplingSeed" <*> value .: "requireInterleaving" <*> value .: "requireGrade" <*> value .: "maxCiRelativeWidth" <*> value .: "maxCheckpointAsymmetry" <*> value .: "metrics"+ if decodedPolicy.minimumPairs < 3 then fail "minimumPairs must be at least 3" else pure ()+ if decodedPolicy.bootstrapIterations < 1_000 then fail "bootstrapIterations must be at least 1000" else pure decodedPolicy++instance ToJSON Policy where+ toJSON policy =+ object+ [ "schema" .= ("kenshou.comparison-policy/v1" :: Text),+ "name" .= policy.name,+ "minimumPairs" .= policy.minimumPairs,+ "confidenceLevel" .= policy.confidenceLevel,+ "bootstrapIterations" .= policy.bootstrapIterations,+ "resamplingSeed" .= policy.resamplingSeed,+ "requireInterleaving" .= policy.requireInterleaving,+ "requireGrade" .= policy.requireGrade,+ "maxCiRelativeWidth" .= policy.maxCiRelativeWidth,+ "maxCheckpointAsymmetry" .= policy.maxCheckpointAsymmetry,+ "metrics" .= policy.metricRules+ ]++decodePolicy :: ByteString -> Either Text Policy+decodePolicy = either (Left . Text.pack) Right . eitherDecodeStrict'++matchesRule :: Text -> Text -> Bool+matchesRule patternValue metricName = case Text.breakOn "*" patternValue of+ (_, suffixWithStar) | Text.null suffixWithStar -> patternValue == metricName+ (prefix, suffixWithStar) -> prefix `Text.isPrefixOf` metricName && Text.drop 1 suffixWithStar `Text.isSuffixOf` metricName
+ src/Kenshou/Measure/Health.hs view
@@ -0,0 +1,302 @@+module Kenshou.Measure.Health+ ( Severity (..),+ HealthObservation (..),+ HealthConfig (..),+ defaultHealthConfig,+ captureHealthNotices,+ evaluateHealth,+ healthOutcome,+ )+where++import Control.Applicative ((<|>))+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.List (isSuffixOf)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (fromMaybe, mapMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Encoding+import Data.Text.IO qualified as Text+import Data.Time (UTCTime)+import Data.Time.Clock.POSIX (posixSecondsToUTCTime)+import Data.Word (Word64)+import Kenshou.Core.Outcome (Outcome (..))+import System.Directory (copyFile, doesFileExist, listDirectory)+import System.Environment (lookupEnv)+import System.FilePath ((</>))+import Text.Read (readMaybe)++data Severity = Info | Soft | Hard deriving stock (Eq, Ord, Show)++data HealthObservation = HealthObservation+ { gate :: Text,+ severity :: Severity,+ fromMonoNs :: Word64,+ toMonoNs :: Word64,+ detail :: Text,+ evidence :: Value+ }+ deriving stock (Eq, Show)++data HealthConfig = HealthConfig+ { cpuSoft :: Double,+ cpuHard :: Double,+ stealSoft :: Double,+ clockStepNs :: Word64,+ pauseNs :: Word64,+ minSteadySamples :: Word64+ }+ deriving stock (Eq, Show)++defaultHealthConfig :: HealthConfig+defaultHealthConfig = HealthConfig 0.70 0.90 0.02 50_000_000 2_000_000_000 1_000++instance ToJSON Severity where+ toJSON Info = String "info"+ toJSON Soft = String "soft"+ toJSON Hard = String "hard"++instance FromJSON Severity where+ parseJSON = withText "Severity" \case+ "info" -> pure Info+ "soft" -> pure Soft+ "hard" -> pure Hard+ _ -> fail "severity must be info, soft, or hard"++instance ToJSON HealthObservation where+ toJSON value =+ object+ [ "gate" .= value.gate,+ "severity" .= value.severity,+ "fromMonoNs" .= value.fromMonoNs,+ "toMonoNs" .= value.toMonoNs,+ "detail" .= value.detail,+ "evidence" .= value.evidence+ ]++instance FromJSON HealthObservation where+ parseJSON = withObject "HealthObservation" \value -> HealthObservation <$> value .: "gate" <*> value .: "severity" <*> value .: "fromMonoNs" <*> value .: "toMonoNs" <*> value .: "detail" <*> value .: "evidence"++data HealthNotice = HealthNotice Text Severity UTCTime Text++instance FromJSON HealthNotice where+ parseJSON = withObject "HealthNotice" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.health-notice/v1" :: Text) then fail "unsupported health notice" else pure ()+ HealthNotice <$> value .: "source" <*> value .: "severity" <*> value .: "at" <*> value .: "detail"++type CsvRow = Map Text Text++captureHealthNotices :: FilePath -> IO (Maybe FilePath)+captureHealthNotices runDir = do+ let captured = runDir </> "health-notices.jsonl"+ source <- lookupEnv "KENSHOU_HEALTH_NOTICES"+ case source of+ Just path | path /= captured -> do+ exists <- doesFileExist path+ if exists then copyFile path captured >> pure (Just captured) else existing captured+ _ -> existing captured+ where+ existing path = do+ exists <- doesFileExist path+ pure (if exists then Just path else Nothing)++evaluateHealth :: HealthConfig -> FilePath -> IO [HealthObservation]+evaluateHealth config runDir = do+ samplerRows <- steadyRows (runDir </> "series" </> "sampler.csv")+ loadRows <- readCsvRows (runDir </> "series" </> "load.csv")+ hostRows <- steadyRows (runDir </> "series" </> "host.csv")+ processRows <- steadyRows (runDir </> "series" </> "proc.csv")+ rtsRows <- steadyRows (runDir </> "series" </> "rts.csv")+ checkpointRows <- steadyRows (runDir </> "series" </> "pg-checkpointer.csv")+ spec <- decodeValue (runDir </> "run-spec.json")+ sampleObservations <- sampleFileObservations config runDir+ noticeObservations <- noticeFileObservations runDir samplerRows+ let bounds = steadyWindow loadRows samplerRows+ intervalNs = round (numberAt ["knobs", "measure.sample-interval-ms"] spec * 1_000_000)+ effectivePauseNs = max config.pauseNs (5 * intervalNs)+ cpu = driverCpu config bounds hostRows processRows rtsRows+ driverHard = any (\item -> item.gate == "driver-cpu-saturation" && item.severity == Hard) cpu+ overload = backlogObservations bounds driverHard spec loadRows+ clocks = samplerObservations config effectivePauseNs bounds samplerRows+ checkpoint = checkpointObservations bounds checkpointRows+ rts = [observation bounds "rts-stats-unavailable" Info "GHC RTS statistics were unavailable" Null | null rtsRows]+ pure (cpu <> overload <> clocks <> checkpoint <> sampleObservations bounds <> rts <> noticeObservations bounds)++healthOutcome :: [HealthObservation] -> Maybe Outcome+healthOutcome observations+ | any ((== Hard) . (.severity)) observations = Just InfrastructureFailure+ | any ((== Soft) . (.severity)) observations = Just Inconclusive+ | otherwise = Nothing++driverCpu :: HealthConfig -> (Word64, Word64) -> [CsvRow] -> [CsvRow] -> [CsvRow] -> [HealthObservation]+driverCpu config bounds hostRows processRows rtsRows = saturation <> steal+ where+ hostDeltas = fmap (\fieldName -> max 0 (delta fieldName hostRows)) ["cpu_user", "cpu_nice", "cpu_system", "cpu_idle", "cpu_iowait", "cpu_irq", "cpu_softirq", "cpu_steal"]+ hostTotal = sum hostDeltas+ hostBusy = if hostTotal <= 0 then Nothing else Just ((hostTotal - hostDeltas !! 3 - hostDeltas !! 4) / hostTotal)+ capabilities = max 1 (fromMaybe 1 (lastNumber "capabilities" rtsRows))+ elapsed = max 1 (delta "t_mono_ns" processRows)+ processBusy = if length processRows < 2 then Nothing else Just (delta "cpu_total_ns" processRows / elapsed / capabilities)+ busy = if length hostRows >= 2 then hostBusy else processBusy+ saturation = case busy of+ Just value | value > config.cpuHard -> [cpuObservation Hard value]+ Just value | value > config.cpuSoft -> [cpuObservation Soft value]+ _ -> []+ stealFraction = if hostTotal <= 0 then 0 else hostDeltas !! 7 / hostTotal+ steal = [observation bounds "cpu-steal" Soft "host CPU steal exceeded the health threshold" (object ["fraction" .= stealFraction, "threshold" .= config.stealSoft]) | stealFraction > config.stealSoft]+ cpuObservation level value = observation bounds "driver-cpu-saturation" level "load driver CPU use exceeded the health threshold" (object ["utilisation" .= value, "softThreshold" .= config.cpuSoft, "hardThreshold" .= config.cpuHard])++backlogObservations :: (Word64, Word64) -> Bool -> Value -> [CsvRow] -> [HealthObservation]+backlogObservations bounds driverHard spec rows =+ case maximumMaybe (mapMaybe (fieldNumber "max_lag_ns") rows) of+ Just maximumLag+ | modelName `elem` [Just "open-constant", Just "open-poisson"] && maximumLag > maximumThreshold ->+ let aborted = maximumLag > abortThreshold+ level = if driverHard || aborted then Hard else Soft+ in [observation bounds "open-loop-backlog" level "open-loop arrivals accumulated sustained lag" (object ["maximumLagNs" .= maximumLag, "thresholdNs" .= maximumThreshold, "abortedEarly" .= aborted])]+ _ -> []+ where+ modelName = textAt ["knobs", "load.model"] spec+ maximumThreshold = numberAt ["knobs", "load.max-lag-ms"] spec * 1_000_000+ abortThreshold = numberAt ["knobs", "load.abort-lag-ms"] spec * 1_000_000++samplerObservations :: HealthConfig -> Word64 -> (Word64, Word64) -> [CsvRow] -> [HealthObservation]+samplerObservations config effectivePauseNs bounds rows = clock <> late <> overrun+ where+ maximumClockStep = maximumOrZero (fmap abs (mapMaybe (fieldNumber "wall_minus_mono_ns") rows))+ maximumLate = maximumOrZero (mapMaybe (fieldNumber "late_ns") rows)+ maximumCost = maximumOrZero (mapMaybe (fieldNumber "cost_ns") rows)+ interval = if length rows < 2 then 0 else delta "scheduled_mono_ns" rows / fromIntegral (length rows - 1)+ clock = [observation bounds "clock-anomaly" Soft "wall and monotonic clocks changed by more than 50 ms" (object ["maximumStepNs" .= maximumClockStep, "thresholdNs" .= config.clockStepNs]) | maximumClockStep > fromIntegral config.clockStepNs]+ late = [observation bounds "clock-anomaly" Hard "the sampler observed a process or virtual-machine pause" (object ["maximumLateNs" .= maximumLate, "thresholdNs" .= effectivePauseNs]) | maximumLate > fromIntegral effectivePauseNs]+ overrun = [observation bounds "sampler-overrun" Soft "sampling cost exceeded the configured interval" (object ["maximumCostNs" .= maximumCost, "intervalNs" .= interval]) | interval > 0 && maximumCost > interval]++checkpointObservations :: (Word64, Word64) -> [CsvRow] -> [HealthObservation]+checkpointObservations _ [] = []+checkpointObservations bounds rows =+ [ observation+ bounds+ "checkpoint-in-window"+ Info+ "PostgreSQL checkpoint activity in the steady window"+ (object ["started" .= started, "completed" .= completed, "overlapFraction" .= overlapFraction])+ ]+ where+ started = max 0 (delta "num_timed" rows) + max 0 (delta "num_requested" rows)+ lsnChanges = length [() | (left, right) <- adjacent rows, Map.lookup "checkpoint_lsn" left /= Map.lookup "checkpoint_lsn" right]+ completed = case lastNumber "num_done" rows of Just _ -> max 0 (delta "num_done" rows); Nothing -> fromIntegral lsnChanges+ intervals = adjacent rows+ active (left, right) = Map.lookup "checkpoint_lsn" left /= Map.lookup "checkpoint_lsn" right || fromMaybe 0 (fieldNumber "buffers_written" right) > fromMaybe 0 (fieldNumber "buffers_written" left)+ overlapFraction :: Double+ overlapFraction = if null intervals then 0 else fromIntegral (length (filter active intervals)) / fromIntegral (length intervals)++sampleFileObservations :: HealthConfig -> FilePath -> IO ((Word64, Word64) -> [HealthObservation])+sampleFileObservations config runDir = do+ let directory = runDir </> "samples"+ files <- listDirectory directory+ decoded <- traverse (decodeValue . (directory </>)) [file | file <- files, ".meta.json" `isSuffixOf` file]+ pure \bounds -> concatMap (fromMeta bounds) decoded+ where+ fromMeta bounds value =+ let operation = fromMaybe "unknown" (textAt ["operation"] value)+ successes = numberAt ["phaseCounts", "steady", "successes"] value+ failures = numberAt ["phaseCounts", "steady", "failures"] value+ samples = successes + failures+ backpressure = numberAt ["backpressureCount"] value+ in [observation bounds "insufficient-samples" Soft ("operation " <> operation <> " has too few steady samples") (object ["operation" .= operation, "samples" .= samples, "minimum" .= config.minSteadySamples]) | samples < fromIntegral config.minSteadySamples]+ <> [observation bounds "recorder-backpressure" Soft ("operation " <> operation <> " experienced recorder back-pressure") (object ["operation" .= operation, "count" .= backpressure]) | backpressure > 0]++noticeFileObservations :: FilePath -> [CsvRow] -> IO ((Word64, Word64) -> [HealthObservation])+noticeFileObservations runDir samplerRows = do+ let path = runDir </> "health-notices.jsonl"+ exists <- doesFileExist path+ if not exists+ then pure (const [])+ else do+ notices <- fmap (mapMaybe decodeNotice . Text.lines) (Text.readFile path)+ let wallBounds = wallTimeBounds samplerRows+ pure \bounds -> [noticeObservation bounds notice | notice@(HealthNotice _ _ noticeAt _) <- notices, within wallBounds noticeAt]+ where+ decodeNotice :: Text -> Maybe HealthNotice+ decodeNotice = decodeStrict' . Encoding.encodeUtf8+ noticeObservation bounds (HealthNotice noticeSource noticeSeverity noticeAt noticeDetail) = observation bounds "host-notice" noticeSeverity noticeDetail (object ["source" .= noticeSource, "at" .= noticeAt])+ within Nothing _ = True+ within (Just (start, end)) instant = instant >= start && instant <= end++observation :: (Word64, Word64) -> Text -> Severity -> Text -> Value -> HealthObservation+observation (start, end) gate severity detail evidence = HealthObservation gate severity start end detail evidence++steadyWindow :: [CsvRow] -> [CsvRow] -> (Word64, Word64)+steadyWindow loadRows samplerRows =+ let steadyStart = listToMaybeNumber "t_mono_ns" [row | row <- loadRows, Map.lookup "phase" row == Just "steady"]+ steadyEnd = listToMaybeNumber "t_mono_ns" [row | row <- loadRows, Map.lookup "phase" row `elem` [Just "drain", Just "done"]]+ fallback = boundsOf samplerRows+ in (floor (fromMaybe (fromIntegral (fst fallback)) steadyStart), floor (fromMaybe (fromIntegral (snd fallback)) steadyEnd))++wallTimeBounds :: [CsvRow] -> Maybe (UTCTime, UTCTime)+wallTimeBounds rows = do+ start <- listToMaybeNumber "t_wall_ms" rows+ end <- lastNumber "t_wall_ms" rows+ pure (toUtc start, toUtc end)+ where+ toUtc value = posixSecondsToUTCTime (realToFrac (value / 1_000))++readCsvRows :: FilePath -> IO [CsvRow]+readCsvRows path = do+ exists <- doesFileExist path+ if not exists+ then pure []+ else do+ rows <- Text.lines <$> Text.readFile path+ pure case rows of+ [] -> []+ header : body -> fmap (Map.fromList . zip (Text.splitOn "," header) . Text.splitOn ",") body++steadyRows :: FilePath -> IO [CsvRow]+steadyRows path = filter ((== Just "steady") . Map.lookup "phase") <$> readCsvRows path++decodeValue :: FilePath -> IO Value+decodeValue path = do+ decoded <- eitherDecodeFileStrict' path+ pure (either (const Null) id decoded)++textAt :: [Text] -> Value -> Maybe Text+textAt [] (String value) = Just value+textAt (name : rest) (Object value) = KeyMap.lookup (Key.fromText name) value >>= textAt rest+textAt _ _ = Nothing++numberAt :: [Text] -> Value -> Double+numberAt [] (Number value) = realToFrac value+numberAt (name : rest) (Object value) = maybe 0 (numberAt rest) (KeyMap.lookup (Key.fromText name) value)+numberAt _ _ = 0++fieldNumber :: Text -> CsvRow -> Maybe Double+fieldNumber name row = Map.lookup name row >>= readMaybe . Text.unpack++delta :: Text -> [CsvRow] -> Double+delta name rows = fromMaybe 0 ((-) <$> (lastNumber name rows) <*> (listToMaybeNumber name rows))++listToMaybeNumber :: Text -> [CsvRow] -> Maybe Double+listToMaybeNumber _ [] = Nothing+listToMaybeNumber name (row : rest) = fieldNumber name row <|> listToMaybeNumber name rest++lastNumber :: Text -> [CsvRow] -> Maybe Double+lastNumber name = listToMaybeNumber name . reverse++boundsOf :: [CsvRow] -> (Word64, Word64)+boundsOf rows = (floor (fromMaybe 0 (listToMaybeNumber "t_mono_ns" rows)), floor (fromMaybe 0 (lastNumber "t_mono_ns" rows)))++adjacent :: [value] -> [(value, value)]+adjacent values = zip values (drop 1 values)++maximumMaybe :: (Ord value) => [value] -> Maybe value+maximumMaybe [] = Nothing+maximumMaybe values = Just (maximum values)++maximumOrZero :: (Ord value, Num value) => [value] -> value+maximumOrZero = fromMaybe 0 . maximumMaybe
+ src/Kenshou/Measure/Histogram.hs view
@@ -0,0 +1,271 @@+module Kenshou.Measure.Histogram+ ( HistogramConfig (..),+ defaultHistogramConfig,+ MutableHistogram,+ Histogram,+ newHistogram,+ recordValue,+ freeze,+ merge,+ valueAtQuantile,+ totalCount,+ minValue,+ maxValue,+ overflowCount,+ meanValue,+ equivalentRange,+ histogramConfig,+ histogramCounts,+ histogramSum,+ histogramArrayLength,+ histogramFromCounts,+ )+where++import Control.Monad (forM_)+import Control.Monad.ST (runST)+import Data.Bits+import Data.IORef+import Data.Primitive.PrimArray+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import GHC.Exts (RealWorld)++data HistogramConfig = HistogramConfig+ { lowestDiscernible :: !Word64,+ highestTrackable :: !Word64,+ significantDigits :: !Int+ }+ deriving stock (Eq, Show)++defaultHistogramConfig :: HistogramConfig+defaultHistogramConfig = HistogramConfig 1 3_600_000_000_000 3++data Layout = Layout+ { unitMagnitude :: !Int,+ subBucketCountMagnitude :: !Int,+ subBucketCount :: !Int,+ subBucketHalfCount :: !Int,+ subBucketHalfCountMagnitude :: !Int,+ subBucketMask :: !Word64,+ arrayLength :: !Int+ }++data MutableHistogram = MutableHistogram+ { config :: !HistogramConfig,+ layout :: !Layout,+ counts :: !(MutablePrimArray RealWorld Word64),+ count :: !(IORef Word64),+ minimum :: !(IORef Word64),+ maximum :: !(IORef Word64),+ total :: !(IORef Word64),+ overflow :: !(IORef Word64)+ }++data Histogram = Histogram+ { config :: !HistogramConfig,+ layout :: !Layout,+ counts :: !(PrimArray Word64),+ count :: !Word64,+ minimum :: !Word64,+ maximum :: !Word64,+ total :: !Word64,+ overflow :: !Word64+ }++instance Eq Histogram where+ left == right =+ left.config == right.config+ && left.count == right.count+ && left.minimum == right.minimum+ && left.maximum == right.maximum+ && left.total == right.total+ && left.overflow == right.overflow+ && histogramCounts left == histogramCounts right++instance Show Histogram where+ show value =+ "Histogram {config="+ <> show value.config+ <> ", count="+ <> show value.count+ <> ", min="+ <> show value.minimum+ <> ", max="+ <> show value.maximum+ <> ", overflow="+ <> show value.overflow+ <> "}"++newHistogram :: HistogramConfig -> IO MutableHistogram+newHistogram config = case makeLayout config of+ Left message -> ioError (userError (Text.unpack message))+ Right layout -> do+ counts <- newPrimArray layout.arrayLength+ setPrimArray counts 0 layout.arrayLength 0+ MutableHistogram config layout counts+ <$> newIORef 0+ <*> newIORef maxBound+ <*> newIORef 0+ <*> newIORef 0+ <*> newIORef 0++recordValue :: MutableHistogram -> Word64 -> IO ()+recordValue histogram value = do+ modifyIORef' histogram.count (+ 1)+ modifyIORef' histogram.minimum (min value)+ modifyIORef' histogram.maximum (max value)+ modifyIORef' histogram.total (+ value)+ if value > histogram.config.highestTrackable+ then modifyIORef' histogram.overflow (+ 1)+ else do+ let index = countsIndex histogram.layout value+ previous <- readPrimArray histogram.counts index+ writePrimArray histogram.counts index (previous + 1)++freeze :: MutableHistogram -> IO Histogram+freeze mutable = do+ immutable <- freezePrimArray mutable.counts 0 mutable.layout.arrayLength+ count <- readIORef mutable.count+ observedMinimum <- readIORef mutable.minimum+ observedMaximum <- readIORef mutable.maximum+ total <- readIORef mutable.total+ overflow <- readIORef mutable.overflow+ pure $ Histogram mutable.config mutable.layout immutable count (if count == 0 then 0 else observedMinimum) observedMaximum total overflow++merge :: Histogram -> Histogram -> Either Text Histogram+merge left right+ | left.config /= right.config = Left "histogram configurations differ"+ | otherwise =+ Right+ Histogram+ { config = left.config,+ layout = left.layout,+ counts = runST do+ mutable <- newPrimArray left.layout.arrayLength+ forM_ [0 .. left.layout.arrayLength - 1] \index ->+ writePrimArray mutable index (indexPrimArray left.counts index + indexPrimArray right.counts index)+ unsafeFreezePrimArray mutable,+ count = left.count + right.count,+ minimum = case (left.count, right.count) of+ (0, _) -> right.minimum+ (_, 0) -> left.minimum+ _ -> min left.minimum right.minimum,+ maximum = max left.maximum right.maximum,+ total = left.total + right.total,+ overflow = left.overflow + right.overflow+ }++valueAtQuantile :: Histogram -> Double -> Word64+valueAtQuantile histogram quantile+ | histogram.count == 0 = 0+ | quantile >= 1 = histogram.maximum+ | target > histogram.count - histogram.overflow = histogram.maximum+ | otherwise = findIndex 0 0+ where+ bounded = max 0 (min 1 quantile)+ target = max 1 (ceiling (bounded * fromIntegral histogram.count))+ findIndex index cumulative+ | index >= histogram.layout.arrayLength = histogram.maximum+ | next >= target =+ let (_, highest) = equivalentRangeAtIndex histogram.layout index+ in max histogram.minimum (min histogram.maximum highest)+ | otherwise = findIndex (index + 1) next+ where+ next = cumulative + indexPrimArray histogram.counts index++totalCount :: Histogram -> Word64+totalCount = (.count)++minValue :: Histogram -> Word64+minValue = (.minimum)++maxValue :: Histogram -> Word64+maxValue = (.maximum)++overflowCount :: Histogram -> Word64+overflowCount = (.overflow)++meanValue :: Histogram -> Double+meanValue histogram+ | histogram.count == 0 = 0+ | otherwise = fromIntegral histogram.total / fromIntegral histogram.count++equivalentRange :: HistogramConfig -> Word64 -> Either Text (Word64, Word64)+equivalentRange config value = do+ layout <- makeLayout config+ pure (equivalentRangeAtIndex layout (countsIndex layout (min value config.highestTrackable)))++histogramConfig :: Histogram -> HistogramConfig+histogramConfig = (.config)++histogramCounts :: Histogram -> [Word64]+histogramCounts histogram = [indexPrimArray histogram.counts index | index <- [0 .. histogram.layout.arrayLength - 1]]++histogramSum :: Histogram -> Word64+histogramSum = (.total)++histogramArrayLength :: HistogramConfig -> Either Text Int+histogramArrayLength config = (.arrayLength) <$> makeLayout config++histogramFromCounts :: HistogramConfig -> [Word64] -> Word64 -> Word64 -> Word64 -> Word64 -> Word64 -> Either Text Histogram+histogramFromCounts config values count observedMinimum observedMaximum total overflow = do+ layout <- makeLayout config+ if length values /= layout.arrayLength+ then Left "histogram count array has the wrong length"+ else+ pure+ Histogram+ { config,+ layout,+ counts = primArrayFromList values,+ count,+ minimum = if count == 0 then 0 else observedMinimum,+ maximum = observedMaximum,+ total,+ overflow+ }++makeLayout :: HistogramConfig -> Either Text Layout+makeLayout config+ | config.lowestDiscernible == 0 = Left "lowestDiscernible must be positive"+ | config.highestTrackable < config.lowestDiscernible = Left "highestTrackable must not be below lowestDiscernible"+ | config.significantDigits < 1 || config.significantDigits > 5 = Left "significantDigits must be between 1 and 5"+ | otherwise = Right Layout {unitMagnitude, subBucketCountMagnitude, subBucketCount, subBucketHalfCount, subBucketHalfCountMagnitude, subBucketMask, arrayLength}+ where+ unitMagnitude = integerLog2 config.lowestDiscernible+ required = 2 * (10 ^ config.significantDigits)+ subBucketCountMagnitude = ceilingLog2 required+ subBucketCount = 1 `shiftL` subBucketCountMagnitude+ subBucketHalfCount = subBucketCount `div` 2+ subBucketHalfCountMagnitude = subBucketCountMagnitude - 1+ subBucketMask = fromIntegral (subBucketCount - 1) `shiftL` unitMagnitude+ initial = fromIntegral subBucketCount `shiftL` unitMagnitude :: Word64+ bucketCount = length (takeWhile (<= config.highestTrackable) (iterate (`shiftL` 1) initial)) + 1+ arrayLength = (bucketCount + 1) * subBucketHalfCount++countsIndex :: Layout -> Word64 -> Int+countsIndex layout value =+ ((bucketIndex + 1) `shiftL` layout.subBucketHalfCountMagnitude)+ + (subBucketIndex - layout.subBucketHalfCount)+ where+ bucketIndex = (64 - layout.unitMagnitude - layout.subBucketCountMagnitude) - countLeadingZeros (value .|. layout.subBucketMask)+ subBucketIndex = fromIntegral (value `shiftR` (bucketIndex + layout.unitMagnitude))++equivalentRangeAtIndex :: Layout -> Int -> (Word64, Word64)+equivalentRangeAtIndex layout index = (lowest, lowest + rangeSize - 1)+ where+ initialBucket = (index `shiftR` layout.subBucketHalfCountMagnitude) - 1+ initialSub = (index .&. (layout.subBucketHalfCount - 1)) + layout.subBucketHalfCount+ (bucket, subBucket) = if initialBucket < 0 then (0, initialSub - layout.subBucketHalfCount) else (initialBucket, initialSub)+ lowest = fromIntegral subBucket `shiftL` (bucket + layout.unitMagnitude)+ rangeSize = 1 `shiftL` (bucket + layout.unitMagnitude)++integerLog2 :: Word64 -> Int+integerLog2 value = finiteBitSize value - 1 - countLeadingZeros value++ceilingLog2 :: Int -> Int+ceilingLog2 value = go 0 1+ where+ go magnitude power | power >= value = magnitude | otherwise = go (magnitude + 1) (power * 2)
+ src/Kenshou/Measure/Histogram/Codec.hs view
@@ -0,0 +1,129 @@+module Kenshou.Measure.Histogram.Codec+ ( encodeHistogram,+ decodeHistogram,+ )+where++import Data.Bits+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.ByteString.Builder+import Data.ByteString.Lazy qualified as LazyByteString+import Data.Int (Int64)+import Data.Text (Text)+import Data.Word (Word16, Word32, Word64, Word8)+import Kenshou.Measure.Histogram++encodeHistogram :: Histogram -> ByteString+encodeHistogram histogram =+ LazyByteString.toStrict . toLazyByteString $+ byteString "KHST"+ <> word16LE 1+ <> word16LE 0+ <> word64LE config.lowestDiscernible+ <> word64LE config.highestTrackable+ <> word8 (fromIntegral config.significantDigits)+ <> word64LE (totalCount histogram)+ <> word64LE (minValue histogram)+ <> word64LE (maxValue histogram)+ <> word64LE (histogramSum histogram)+ <> word64LE (overflowCount histogram)+ <> word32LE (fromIntegral (length tokens))+ <> foldMap (putVarWord . zigZag) tokens+ where+ config = histogramConfig histogram+ tokens = runLengthEncode (histogramCounts histogram)++decodeHistogram :: ByteString -> Either Text Histogram+decodeHistogram bytes = do+ ensure (ByteString.take 4 bytes == "KHST") "invalid histogram magic"+ version <- word16At bytes 4+ ensure (version == 1) "unsupported histogram version"+ flags <- word16At bytes 6+ ensure (flags == 0) "unsupported histogram flags"+ lowest <- word64At bytes 8+ highest <- word64At bytes 16+ digits <- word8At bytes 24+ count <- word64At bytes 25+ observedMinimum <- word64At bytes 33+ observedMaximum <- word64At bytes 41+ total <- word64At bytes 49+ overflow <- word64At bytes 57+ tokenCount <- word32At bytes 65+ (tokens, _) <- readTokens bytes 69 (fromIntegral tokenCount)+ let config = HistogramConfig lowest highest (fromIntegral digits)+ expectedLength <- histogramLength config+ values <- expandTokens expectedLength tokens+ histogramFromCounts config values count observedMinimum observedMaximum total overflow++histogramLength :: HistogramConfig -> Either Text Int+histogramLength = histogramArrayLength++runLengthEncode :: [Word64] -> [Int64]+runLengthEncode [] = []+runLengthEncode values = go values+ where+ go [] = []+ go (0 : rest) = let (zeros, remaining) = span (== 0) rest in negate (fromIntegral (1 + length zeros)) : go remaining+ go (value : rest) = fromIntegral value : go rest++expandTokens :: Int -> [Int64] -> Either Text [Word64]+expandTokens expected tokens = do+ values <- fmap concat $ traverse expand tokens+ ensure (length values <= expected) "histogram token stream exceeds the configured layout"+ pure (values <> replicate (expected - length values) 0)+ where+ expand value | value < 0 = Right (replicate (fromIntegral (negate value)) 0)+ expand value = Right [fromIntegral value]++readTokens :: ByteString -> Int -> Int -> Either Text ([Int64], Int)+readTokens bytes = go []+ where+ go acc offset 0 = Right (reverse acc, offset)+ go acc offset remaining = do+ (encoded, next) <- readVarWord bytes offset+ go (unZigZag encoded : acc) next (remaining - 1)++putVarWord :: Word64 -> Builder+putVarWord value+ | value < 0x80 = word8 (fromIntegral value)+ | otherwise = word8 (fromIntegral (value .&. 0x7f) .|. 0x80) <> putVarWord (value `shiftR` 7)++readVarWord :: ByteString -> Int -> Either Text (Word64, Int)+readVarWord bytes = go 0 0+ where+ go bitOffset accumulator offset+ | bitOffset >= 64 = Left "varint is too long"+ | offset >= ByteString.length bytes = Left "truncated varint"+ | otherwise =+ let byte = ByteString.index bytes offset+ value = accumulator .|. (fromIntegral (byte .&. 0x7f) `shiftL` bitOffset)+ in if byte .&. 0x80 == 0 then Right (value, offset + 1) else go (bitOffset + 7) value (offset + 1)++zigZag :: Int64 -> Word64+zigZag value = fromIntegral ((value `shiftL` 1) `xor` (value `shiftR` 63))++unZigZag :: Word64 -> Int64+unZigZag value = fromIntegral (value `shiftR` 1) `xor` negate (fromIntegral (value .&. 1))++word8At :: ByteString -> Int -> Either Text Word8+word8At bytes offset+ | offset < ByteString.length bytes = Right (ByteString.index bytes offset)+ | otherwise = Left "truncated histogram header"++word16At :: ByteString -> Int -> Either Text Word16+word16At bytes offset = fromIntegral <$> littleEndianAt bytes offset 2++word32At :: ByteString -> Int -> Either Text Word32+word32At bytes offset = fromIntegral <$> littleEndianAt bytes offset 4++word64At :: ByteString -> Int -> Either Text Word64+word64At bytes offset = littleEndianAt bytes offset 8++littleEndianAt :: ByteString -> Int -> Int -> Either Text Word64+littleEndianAt bytes offset width+ | offset + width > ByteString.length bytes = Left "truncated histogram header"+ | otherwise = Right $ foldr (.|.) 0 [fromIntegral (ByteString.index bytes (offset + index)) `shiftL` (8 * index) | index <- [0 .. width - 1]]++ensure :: Bool -> Text -> Either Text ()+ensure condition message = if condition then Right () else Left message
+ src/Kenshou/Measure/Knobs.hs view
@@ -0,0 +1,83 @@+module Kenshou.Measure.Knobs+ ( LoadDefaults (..),+ defaultLoadDefaults,+ loadKnobs,+ measureKnobs,+ loadModelFromKnobs,+ )+where++import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Word (Word64)+import Kenshou.Core.Id (Kind (..))+import Kenshou.Core.Knob+import Kenshou.Measure.Load.Types++data LoadDefaults = LoadDefaults+ { model :: Text,+ workers :: Int64,+ thinkTimeUs :: Int64,+ ratePerSecond :: Double,+ executors :: Int64,+ maxLagMs :: Int64,+ abortLagMs :: Int64+ }+ deriving stock (Eq, Show)++defaultLoadDefaults :: LoadDefaults+defaultLoadDefaults = LoadDefaults "closed" 4 0 100 256 1_000 30_000++loadKnobs :: LoadDefaults -> [KnobSpec]+loadKnobs defaults =+ [ textKnob "load.model" "Load generation model" defaults.model ["closed", "open-constant", "open-poisson"],+ intKnob "load.workers" "Closed-loop workers" defaults.workers 1 65_535,+ intKnob "load.think-time-us" "Delay between closed-loop operations" defaults.thinkTimeUs 0 60_000_000,+ doubleKnob "load.rate-per-second" "Open-loop arrival rate" defaults.ratePerSecond 0.001 10_000_000,+ intKnob "load.executors" "Maximum open-loop operations in flight" defaults.executors 1 65_535,+ intKnob "load.max-lag-ms" "Lag that marks sustained overload" defaults.maxLagMs 1 3_600_000,+ intKnob "load.abort-lag-ms" "Lag that aborts an invalid open-loop run" defaults.abortLagMs 1 3_600_000+ ]++measureKnobs :: Kind -> [KnobSpec]+measureKnobs kind =+ [ intKnob "measure.sample-interval-ms" "Time-series sampling interval" 1_000 100 60_000,+ textKnob "measure.raw-samples" "Raw sample retention" (if kind == Benchmark then "full" else "off") ["full", "sampled", "off"],+ intKnob "measure.raw-sample-one-in" "Blocks retained under sampled policy" 100 1 1_000_000,+ intKnob "measure.histogram-digits" "Histogram significant decimal digits" 3 1 5,+ intKnob "measure.interval-histogram-seconds" "Interval histogram frame duration; soaks default to one frame per day to bound retained histograms" (if kind == Soak then 86_400 else 10) 1 86_400,+ textKnob "measure.pg-statements" "PostgreSQL statement sampling" "snapshots" ["off", "snapshots", "periodic"]+ ]++loadModelFromKnobs :: ResolvedKnobs -> Either Text LoadModel+loadModelFromKnobs knobs = case knobText knobs (name "load.model") of+ "closed" -> Right (ClosedLoop (ClosedConfig workers thinkTime (if workers <= 1 then 0 else 1_000_000_000 `div` fromIntegral workers)))+ "open-constant" -> Right (OpenLoop (openConfig (ConstantRate rate)))+ "open-poisson" -> Right (OpenLoop (openConfig (PoissonRate rate)))+ value -> Left ("unknown load.model " <> value)+ where+ workers = fromIntegral (knobInt knobs (name "load.workers"))+ thinkTime = micros (knobInt knobs (name "load.think-time-us"))+ rate = knobDouble knobs (name "load.rate-per-second")+ executors = fromIntegral (knobInt knobs (name "load.executors"))+ maxLag = millis (knobInt knobs (name "load.max-lag-ms"))+ abortLag = millis (knobInt knobs (name "load.abort-lag-ms"))+ openConfig arrival = OpenConfig arrival executors 1 (OverloadConfig maxLag 3 abortLag)++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt def) (IntRange low high) []++doubleKnob :: Text -> Text -> Double -> Double -> Double -> KnobSpec+doubleKnob knobName summary def low high = KnobSpec (name knobName) summary KnobDouble (VDouble def) (DoubleRange low high) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) (fmap VText (first : rest))+textKnob knobName _ _ [] = error ("text knob has no allowed values: " <> show knobName)++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++micros, millis :: Int64 -> Word64+micros value = fromIntegral value * 1_000+millis value = fromIntegral value * 1_000_000
+ src/Kenshou/Measure/Load.hs view
@@ -0,0 +1,15 @@+module Kenshou.Measure.Load+ ( module Kenshou.Measure.Load.Types,+ runLoad,+ )+where++import Kenshou.Measure.Load.Closed (runClosed)+import Kenshou.Measure.Load.Open (runOpen)+import Kenshou.Measure.Load.Types+import Kenshou.Measure.Session (Measurement)++runLoad :: Measurement -> LoadModel -> Operation -> IO LoadReport+runLoad measurement model operation = case model of+ ClosedLoop config -> runClosed measurement config operation+ OpenLoop config -> runOpen measurement config operation
+ src/Kenshou/Measure/Load/Arrival.hs view
@@ -0,0 +1,29 @@+module Kenshou.Measure.Load.Arrival+ ( constantSchedule,+ poissonSchedule,+ nextGapNs,+ )+where++import Data.Word (Word64)+import Kenshou.Measure.Load.Types (Arrival (..))+import System.Random.SplitMix (SMGen, nextWord64)++constantSchedule :: Double -> Word64 -> Int -> [Word64]+constantSchedule rate start count = take count [start + fromIntegral index * gap | index <- [0 :: Int ..]]+ where+ gap = max 1 (round (1_000_000_000 / rate))++poissonSchedule :: Double -> SMGen -> Word64 -> Int -> [Word64]+poissonSchedule rate generator start count = take count (drop 1 (scanl (+) start gaps))+ where+ gaps = unfold generator+ unfold current = let (gap, next) = nextGapNs (PoissonRate rate) current in gap : unfold next++nextGapNs :: Arrival -> SMGen -> (Word64, SMGen)+nextGapNs (ConstantRate rate) generator = (max 1 (round (1_000_000_000 / rate)), generator)+nextGapNs (PoissonRate rate) generator =+ (max 1 (round ((-log uniform / rate) * 1_000_000_000)), next)+ where+ (word, next) = nextWord64 generator+ uniform = (fromIntegral word + 0.5) / 18_446_744_073_709_551_616.0
+ src/Kenshou/Measure/Load/Closed.hs view
@@ -0,0 +1,93 @@+{-# LANGUAGE BangPatterns #-}++module Kenshou.Measure.Load.Closed (runClosed) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async+import Control.Exception (SomeAsyncException, SomeException, fromException, throwIO, try)+import Control.Monad (forM, when)+import Data.IORef+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Load.Series+import Kenshou.Measure.Load.Types+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Timeout (timeout)++runClosed :: Measurement -> ClosedConfig -> Operation -> IO LoadReport+runClosed measurement config operation = do+ opHandle <- registerOp (measurementRecorder measurement) operation.name+ offered <- newIORef 0+ completed <- newIORef 0+ failed <- newIORef 0+ steadyCompleted <- newIORef 0+ maxLag <- newIORef 0+ loadSeries <- openLoadSeries measurement+ let phaseClock = measurementPhaseClock measurement+ plan = (measurementConfig measurement).defaultPhases+ enterPhase phaseClock WarmUp+ sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+ workers <- forM [0 .. config.workers - 1] \workerId -> do+ workerRecorder <- newWorkerRecorder opHandle workerId+ async do+ when (config.staggerNs > 0) (sleepNanos (fromIntegral workerId * config.staggerNs `div` fromIntegral (max 1 config.workers)))+ workerLoop phaseClock workerRecorder workerId 0 offered completed failed steadyCompleted+ sleepNanos (unNanos plan.warmUp)+ enterPhase phaseClock Steady+ sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+ case plan.steady of+ SteadyFor duration -> sleepNanos (unNanos duration)+ SteadyCount target -> waitForCount steadyCompleted target+ enterPhase phaseClock Drain+ sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+ let drainMicros = fromIntegral (min (unNanos plan.drain `div` 1_000) (fromIntegral (maxBound :: Int)))+ drained <- timeout drainMicros (mapM_ wait workers)+ case drained of+ Just () -> pure ()+ Nothing -> mapM_ cancel workers+ enterPhase phaseClock Done+ sampleLoadSeries loadSeries measurement offered offered completed failed maxLag+ closeLoadSeries loadSeries+ offeredValue <- readIORef offered+ completedValue <- readIORef completed+ failedValue <- readIORef failed+ let report = (emptyLoadReport (ClosedLoop config)) {offered = offeredValue, started = offeredValue, completed = completedValue, failed = failedValue}+ appendLoadReport measurement report+ pure report+ where+ workerLoop phaseClock workerRecorder workerId sequenceNumber offered completed failed steadyCompleted = do+ phase <- currentPhase phaseClock+ case phase of+ Drain -> pure ()+ Done -> pure ()+ _ -> do+ intended <- nowNs+ increment offered+ result <- try (operation.run workerId sequenceNumber)+ ended <- nowNs+ opResult <- case result of+ Left err | Just (_ :: SomeAsyncException) <- fromException err -> throwIO err+ Left (_ :: SomeException) -> pure (OpFailed (ErrorCause "exception"))+ Right value -> pure value+ recordOp workerRecorder intended intended ended opResult+ increment completed+ case opResult of OpFailed _ -> increment failed; OpOk _ -> pure ()+ when (phase == Steady) (increment steadyCompleted)+ when (config.thinkTimeNs > 0) (sleepNanos config.thinkTimeNs)+ workerLoop phaseClock workerRecorder workerId (sequenceNumber + 1) offered completed failed steadyCompleted++increment :: IORef Word64 -> IO ()+increment counter = atomicModifyIORef' counter (\value -> let !next = value + 1 in (next, ()))++waitForCount :: IORef Word64 -> Word64 -> IO ()+waitForCount counter target = do+ current <- readIORef counter+ if current >= target then pure () else threadDelay 1_000 >> waitForCount counter target++sleepNanos :: Word64 -> IO ()+sleepNanos duration = nowNs >>= sleepUntilNs . (+ duration)++unNanos :: Nanos -> Word64+unNanos (Nanos value) = value
+ src/Kenshou/Measure/Load/Open.hs view
@@ -0,0 +1,107 @@+{-# LANGUAGE BangPatterns #-}++module Kenshou.Measure.Load.Open (runOpen) where++import Control.Concurrent.Async+import Control.Exception (SomeAsyncException, SomeException, fromException, throwIO, try)+import Control.Monad (forM, when)+import Data.IORef+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Load.Arrival+import Kenshou.Measure.Load.Series+import Kenshou.Measure.Load.Types+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Random.SplitMix (SMGen, mkSMGen)+import System.Timeout (timeout)++data TicketState = TicketState !Word64 !Word64 !SMGen++runOpen :: Measurement -> OpenConfig -> Operation -> IO LoadReport+runOpen measurement config operation = do+ opHandle <- registerOp (measurementRecorder measurement) operation.name+ offered <- newIORef 0+ started <- newIORef 0+ completed <- newIORef 0+ failed <- newIORef 0+ maxLag <- newIORef 0+ aborted <- newIORef False+ loadSeries <- openLoadSeries measurement+ start <- nowNs+ tickets <- newIORef (TicketState 0 start (mkSMGen (measurementEnv measurement).seed))+ let phaseClock = measurementPhaseClock measurement+ plan = (measurementConfig measurement).defaultPhases+ enterPhase phaseClock WarmUp+ sampleLoadSeries loadSeries measurement offered started completed failed maxLag+ executors <- forM [0 .. config.executors - 1] \executorId -> do+ workerRecorder <- newWorkerRecorder opHandle executorId+ async (executorLoop phaseClock workerRecorder executorId tickets offered started completed failed maxLag aborted)+ sleepNanos (unNanos plan.warmUp)+ enterPhase phaseClock Steady+ sampleLoadSeries loadSeries measurement offered started completed failed maxLag+ case plan.steady of+ SteadyFor duration -> sleepNanos (unNanos duration)+ SteadyCount target -> waitForCount completed target+ enterPhase phaseClock Drain+ sampleLoadSeries loadSeries measurement offered started completed failed maxLag+ let drainMicros = fromIntegral (min (unNanos plan.drain `div` 1_000) (fromIntegral (maxBound :: Int)))+ drained <- timeout drainMicros (mapM_ wait executors)+ case drained of Just () -> pure (); Nothing -> mapM_ cancel executors+ enterPhase phaseClock Done+ sampleLoadSeries loadSeries measurement offered started completed failed maxLag+ closeLoadSeries loadSeries+ offeredValue <- readIORef offered+ startedValue <- readIORef started+ completedValue <- readIORef completed+ failedValue <- readIORef failed+ maxLagValue <- readIORef maxLag+ abortedValue <- readIORef aborted+ let overload = if maxLagValue > config.overload.maxLagNs then Just (OverloadEvidence config.overload.maxLagNs maxLagValue) else Nothing+ report = LoadReport (OpenLoop config) offeredValue startedValue completedValue failedValue maxLagValue overload abortedValue+ appendLoadReport measurement report+ pure report+ where+ executorLoop phaseClock workerRecorder executorId tickets offered started completed failed maxLag aborted = do+ phase <- currentPhase phaseClock+ abortNow <- readIORef aborted+ case phase of+ Drain -> pure ()+ Done -> pure ()+ _ | abortNow -> pure ()+ _ -> do+ (sequenceNumber, intended) <- atomicModifyIORef' tickets \(TicketState index next generator) ->+ let (gap, nextGenerator) = nextGapNs config.arrival generator+ in (TicketState (index + 1) (next + gap) nextGenerator, (index, next))+ increment offered+ sleepUntilNs intended+ actual <- nowNs+ let lag = actual - min actual intended+ atomicModifyIORef' maxLag (\value -> let !next = max value lag in (next, ()))+ when (lag > config.overload.abortLagNs) (writeIORef aborted True)+ increment started+ result <- try (operation.run executorId sequenceNumber)+ ended <- nowNs+ opResult <- case result of+ Left err | Just (_ :: SomeAsyncException) <- fromException err -> throwIO err+ Left (_ :: SomeException) -> pure (OpFailed (ErrorCause "exception"))+ Right value -> pure value+ recordOp workerRecorder intended actual ended opResult+ increment completed+ case opResult of OpFailed _ -> increment failed; OpOk _ -> pure ()+ executorLoop phaseClock workerRecorder executorId tickets offered started completed failed maxLag aborted++increment :: IORef Word64 -> IO ()+increment counter = atomicModifyIORef' counter (\value -> let !next = value + 1 in (next, ()))++waitForCount :: IORef Word64 -> Word64 -> IO ()+waitForCount counter target = do+ current <- readIORef counter+ if current >= target then pure () else sleepNanos 1_000_000 >> waitForCount counter target++sleepNanos :: Word64 -> IO ()+sleepNanos duration = nowNs >>= sleepUntilNs . (+ duration)++unNanos :: Nanos -> Word64+unNanos (Nanos value) = value
+ src/Kenshou/Measure/Load/Series.hs view
@@ -0,0 +1,32 @@+module Kenshou.Measure.Load.Series+ ( LoadSeries,+ openLoadSeries,+ sampleLoadSeries,+ closeLoadSeries,+ )+where++import Data.IORef+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Sampler.Csv+import Kenshou.Measure.Session+import System.FilePath ((</>))++newtype LoadSeries = LoadSeries CsvWriter++openLoadSeries :: Measurement -> IO LoadSeries+openLoadSeries measurement = do+ let path = (measurementEnv measurement).runDir </> "series" </> "load.csv"+ writer <- openCsv path ["t_mono_ns", "t_wall_ms", "phase", "offered", "started", "completed", "failed", "max_lag_ns"]+ (measurementEnv measurement).declareArtifact path "text/csv"+ pure (LoadSeries writer)++sampleLoadSeries :: LoadSeries -> Measurement -> IORef Word64 -> IORef Word64 -> IORef Word64 -> IORef Word64 -> IORef Word64 -> IO ()+sampleLoadSeries (LoadSeries writer) measurement offered started completed failed maxLag = do+ (_, prefix) <- timestampColumns (measurementEnv measurement).origin (measurementPhaseClock measurement)+ values <- traverse readIORef [offered, started, completed, failed, maxLag]+ appendCsv writer (prefix <> fmap (Text.pack . show) values)++closeLoadSeries :: LoadSeries -> IO ()+closeLoadSeries (LoadSeries writer) = closeCsv writer
+ src/Kenshou/Measure/Load/Types.hs view
@@ -0,0 +1,89 @@+module Kenshou.Measure.Load.Types+ ( Operation (..),+ Arrival (..),+ ClosedConfig (..),+ OverloadConfig (..),+ OpenConfig (..),+ LoadModel (..),+ OverloadEvidence (..),+ LoadReport (..),+ emptyLoadReport,+ )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.Word (Word64)+import Kenshou.Measure.Recorder (OpName, OpResult)++data Operation = Operation+ { name :: OpName,+ run :: Int -> Word64 -> IO OpResult+ }++data Arrival = ConstantRate !Double | PoissonRate !Double+ deriving stock (Eq, Show)++data ClosedConfig = ClosedConfig+ { workers :: !Int,+ thinkTimeNs :: !Word64,+ staggerNs :: !Word64+ }+ deriving stock (Eq, Show)++data OverloadConfig = OverloadConfig+ { maxLagNs :: !Word64,+ sustainedIntervals :: !Int,+ abortLagNs :: !Word64+ }+ deriving stock (Eq, Show)++data OpenConfig = OpenConfig+ { arrival :: !Arrival,+ executors :: !Int,+ rampFrom :: !Double,+ overload :: !OverloadConfig+ }+ deriving stock (Eq, Show)++data LoadModel = ClosedLoop !ClosedConfig | OpenLoop !OpenConfig+ deriving stock (Eq, Show)++data OverloadEvidence = OverloadEvidence+ { thresholdNs :: !Word64,+ observedLagNs :: !Word64+ }+ deriving stock (Eq, Show)++data LoadReport = LoadReport+ { model :: !LoadModel,+ offered :: !Word64,+ started :: !Word64,+ completed :: !Word64,+ failed :: !Word64,+ maxLagNs :: !Word64,+ overloaded :: !(Maybe OverloadEvidence),+ abortedEarly :: !Bool+ }+ deriving stock (Eq, Show)++emptyLoadReport :: LoadModel -> LoadReport+emptyLoadReport model = LoadReport model 0 0 0 0 0 Nothing False++instance ToJSON OverloadEvidence where+ toJSON value = object ["thresholdNs" .= value.thresholdNs, "observedLagNs" .= value.observedLagNs]++instance ToJSON LoadReport where+ toJSON report =+ object+ [ "model" .= modelName report.model,+ "offered" .= report.offered,+ "started" .= report.started,+ "completed" .= report.completed,+ "failed" .= report.failed,+ "maxLagNs" .= report.maxLagNs,+ "overloaded" .= report.overloaded,+ "abortedEarly" .= report.abortedEarly+ ]+ where+ modelName (ClosedLoop _) = "closed" :: String+ modelName (OpenLoop config) = case config.arrival of ConstantRate _ -> "open-constant"; PoissonRate _ -> "open-poisson"
+ src/Kenshou/Measure/Methodology.hs view
@@ -0,0 +1,19 @@+module Kenshou.Measure.Methodology+ ( PoolMethodology (..),+ poolMethodologyFor,+ minimumTrials,+ defaultTrials,+ )+where++import Kenshou.Core.Bundle (LayerBundle (..), Registry, lookupBundle)+import Kenshou.Core.Contribution (PoolMethodology (..))+import Kenshou.Core.Id (LayerId)++-- | The connection-pool methodology a layer's registered bundle contributes.+poolMethodologyFor :: Registry -> LayerId -> Maybe PoolMethodology+poolMethodologyFor registry layer = lookupBundle registry layer >>= (.methodology)++minimumTrials, defaultTrials :: Int+minimumTrials = 3+defaultTrials = 5
+ src/Kenshou/Measure/Metrics.hs view
@@ -0,0 +1,16 @@+module Kenshou.Measure.Metrics+ ( Metric (..),+ )+where++import Data.Aeson+import Data.Text (Text)++data Metric = Metric {value :: Double, unit :: Text}+ deriving stock (Eq, Show)++instance ToJSON Metric where+ toJSON metric = object ["value" .= metric.value, "unit" .= metric.unit]++instance FromJSON Metric where+ parseJSON = withObject "Metric" \value -> Metric <$> value .: "value" <*> value .: "unit"
+ src/Kenshou/Measure/Perf.hs view
@@ -0,0 +1,145 @@+module Kenshou.Measure.Perf+ ( AdmissionProblem (..),+ requiredMetricProblems,+ runtimeResultProblems,+ validateProfilePolicy,+ verifyPerformanceRun,+ )+where++import Control.Exception (IOException, displayException, try)+import Control.Monad (unless, when)+import Data.Aeson+import Data.Aeson.Key qualified as Key+import Data.Aeson.KeyMap qualified as KeyMap+import Data.Aeson.Types (parseMaybe)+import Data.ByteString qualified as ByteString+import Data.Char (isAlphaNum)+import Data.IORef+import Data.List (nub)+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Compare.Policy qualified as Policy+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (decodeHistogram)+import Kenshou.Measure.Metrics (Metric (..))+import Kenshou.Measure.Samples+import Kenshou.Measure.Summary+import Kenshou.Perf.Artifacts (verifyPerfManifest)+import Kenshou.Perf.Plan (PerfProfile (..), RequiredMetric (..))+import System.FilePath ((</>))++data AdmissionProblem+ = MissingRequiredMetric Text+ | InsufficientSteadySamples Text Int Word64+ | InvalidPerformanceEvidence Text+ | RuntimeWorkFailure Text+ deriving stock (Eq, Show)++-- A known defect may deliberately have a nonblocking exit code. Performance+-- acceptance still retains the failed runtime observation independently.+runtimeResultProblems :: Value -> [AdmissionProblem]+runtimeResultProblems document = case parseMaybe (withObject "runtime result" (.: "outcome")) document :: Maybe Text of+ Just "passed" -> []+ Just "failed" -> [RuntimeWorkFailure "runtime assertions failed (including nonblocking known defects)"]+ Just "errored" -> [RuntimeWorkFailure "runtime execution errored", InvalidPerformanceEvidence "runtime execution lacks authoritative measurements"]+ Just "inconclusive" -> [InvalidPerformanceEvidence "runtime result is inconclusive"]+ Just "infrastructure-failure" -> [InvalidPerformanceEvidence "runtime infrastructure failed"]+ _ -> [InvalidPerformanceEvidence "runtime outcome is missing or unsupported"]++requiredMetricProblems :: [RequiredMetric] -> MeasurementSummary -> [AdmissionProblem]+requiredMetricProblems required summary = concatMap check required+ where+ check metric = case Map.lookup metric.name summary.metrics of+ Nothing -> [MissingRequiredMetric metric.name]+ Just value+ | value.unit /= metric.unit -> [InvalidPerformanceEvidence ("metric unit mismatch: " <> metric.name)]+ | isNaN value.value || isInfinite value.value -> [InvalidPerformanceEvidence ("non-finite metric: " <> metric.name)]+ | otherwise -> case operationFor metric.name >>= (`Map.lookup` summary.ops) >>= count of+ Nothing -> [InvalidPerformanceEvidence ("required metric has no steady sample count: " <> metric.name)]+ Just samples | samples < fromIntegral metric.minimumSteadySamples -> [InsufficientSteadySamples metric.name metric.minimumSteadySamples samples]+ _ -> []+ count = parseMaybe (withObject "operation summary" (.: "count"))++validateProfilePolicy :: PerfProfile -> Policy.Policy -> Either Text ()+validateProfilePolicy profile policy = do+ unless (profile.minimumPairs >= policy.minimumPairs) (Left "profile pair count is below the comparison policy minimum")+ unless (policy.requireInterleaving && policy.requireGrade == "benchmark") (Left "profile policy must require interleaved benchmark-grade evidence")+ unless (policy.confidenceLevel > 0 && policy.confidenceLevel < 1) (Left "invalid policy confidence level")+ unless (all (\rule -> rule.relativeLimit >= 0 && rule.absoluteFloor >= 0) policy.metricRules) (Left "invalid comparison policy effect limits")+ mapM_ required profile.requiredMetrics+ where+ required metric = unless (any (\rule -> Policy.matchesRule rule.match metric.name) policy.metricRules) (Left ("required metric has no policy rule: " <> metric.name))++-- Recompute from sealed raw samples and compare the retained histogram and+-- summary. The ordinary comparison engine trusts a stored summary if present;+-- performance orchestration must call this admission check before comparing.+verifyPerformanceRun :: PerfProfile -> FilePath -> IO (Either [AdmissionProblem] MeasurementSummary)+verifyPerformanceRun profile directory = do+ checked <- try @IOException verify+ pure case checked of+ Left problem -> Left [InvalidPerformanceEvidence (Text.pack (displayException problem))]+ Right result -> result+ where+ verify = do+ manifest <- verifyPerfManifest directory+ case manifest of+ Left problems -> pure (Left [InvalidPerformanceEvidence (Text.pack (show problems))])+ Right () -> do+ summarized <- summarizeRunDir directory+ case summarized of+ Left problem -> pure (Left [InvalidPerformanceEvidence (Text.pack (show problem))])+ Right summary -> do+ rawProblems <- concat <$> traverse (verifyOperation directory summary.window profile.steadyCount) operations+ result <- eitherDecodeFileStrict' (directory </> "run-result.json") :: IO (Either String Value)+ let retained = result >>= \document -> maybe (Left "missing retained measurement summary") Right (jsonAt ["summaries", "measurements", "measurements"] document)+ summaryProblems = case retained of+ Left problem -> [InvalidPerformanceEvidence (Text.pack problem)]+ Right value | value /= toJSON summary -> [InvalidPerformanceEvidence "retained measurement summary differs from raw evidence"]+ _ -> []+ outcomeProblems = either (\problem -> [InvalidPerformanceEvidence (Text.pack problem)]) runtimeResultProblems result+ problems = requiredMetricProblems profile.requiredMetrics summary <> rawProblems <> summaryProblems <> outcomeProblems+ pure if null problems then Right summary else Left problems+ operations = nub [operation | metric <- profile.requiredMetrics, Just operation <- [operationFor metric.name]]++verifyOperation :: FilePath -> SummaryWindow -> Int -> Text -> IO [AdmissionProblem]+verifyOperation directory window plannedCount operation+ | Text.null operation || not (Text.all (\c -> isAlphaNum c || c `elem` ['-', '_']) operation) = pure [InvalidPerformanceEvidence "unsafe required operation name"]+ | otherwise = do+ let base = directory </> "samples" </> Text.unpack operation+ latency <- decodeHistogram <$> ByteString.readFile (base <> ".hist")+ service <- decodeHistogram <$> ByteString.readFile (base <> ".service.hist")+ case (latency, service) of+ (Right expectedLatency, Right expectedService) -> do+ rebuiltLatency <- Histogram.newHistogram (Histogram.histogramConfig expectedLatency)+ rebuiltService <- Histogram.newHistogram (Histogram.histogramConfig expectedService)+ counts <- newIORef (0 :: Word64, 0 :: Word64, False)+ report <- readSamples (base <> ".raw") \sample -> do+ let steady = sample.intendedStartNs >= window.steadyStartMonoNs && sample.intendedStartNs < window.steadyEndMonoNs+ invalid = sample.actualStartNs > sample.endNs || sample.intendedStartNs > sample.endNs || sample.outcomeCode > 1+ modifyIORef' counts (\(count, failures, bad) -> (count + if steady then 1 else 0, failures + if steady && sample.outcomeCode /= 0 then 1 else 0, bad || invalid || not steady))+ when (steady && not invalid) do+ Histogram.recordValue rebuiltLatency (sample.endNs - sample.intendedStartNs)+ Histogram.recordValue rebuiltService (sample.endNs - sample.actualStartNs)+ actualLatency <- Histogram.freeze rebuiltLatency+ actualService <- Histogram.freeze rebuiltService+ (count, failures, malformed) <- readIORef counts+ pure+ ( [InvalidPerformanceEvidence (operation <> ": raw sample clock/schema/phase mismatch") | malformed || report.ignoredBytes /= 0 || report.header.clockKind /= MonotonicClock || report.header.operation /= operation]+ <> [InvalidPerformanceEvidence (operation <> ": histogram differs from raw samples") | expectedLatency /= actualLatency || expectedService /= actualService]+ <> [RuntimeWorkFailure (operation <> ": completed steady work differs from declared count") | count /= fromIntegral plannedCount]+ <> [RuntimeWorkFailure (operation <> ": failed steady operations") | failures > 0]+ )+ _ -> pure [InvalidPerformanceEvidence (operation <> ": invalid histogram schema")]++operationFor :: Text -> Maybe Text+operationFor name = case Text.splitOn "." name of+ "op" : operation : _ -> Just operation+ _ -> Nothing++jsonAt :: [Text] -> Value -> Maybe Value+jsonAt [] value = Just value+jsonAt (key : rest) (Object value) = KeyMap.lookup (Key.fromText key) value >>= jsonAt rest+jsonAt _ _ = Nothing
+ src/Kenshou/Measure/Phase.hs view
@@ -0,0 +1,75 @@+module Kenshou.Measure.Phase+ ( Phase (..),+ SteadyBound (..),+ PhasePlan (..),+ PhaseClock,+ newPhaseClock,+ enterPhase,+ currentPhase,+ phaseOf,+ addBoundaryListener,+ renderPhase,+ )+where++import Data.IORef+import Data.Text (Text)+import Data.Word (Word64)+import Kenshou.Measure.Clock++data Phase = Setup | WarmUp | Steady | Drain | Done+ deriving stock (Eq, Ord, Show, Enum, Bounded)++data SteadyBound = SteadyFor !Nanos | SteadyCount !Word64+ deriving stock (Eq, Show)++data PhasePlan = PhasePlan+ { warmUp :: !Nanos,+ steady :: !SteadyBound,+ drain :: !Nanos+ }+ deriving stock (Eq, Show)++data PhaseClock = PhaseClock+ { current :: !(IORef Phase),+ boundaries :: !(IORef [(Word64, Phase)]),+ boundaryListeners :: !(IORef [IO ()]),+ onPhase :: !(Phase -> Origin -> IO ())+ }++newPhaseClock :: (Phase -> Origin -> IO ()) -> PhasePlan -> IO PhaseClock+newPhaseClock onPhase _plan = do+ origin <- captureOrigin+ current <- newIORef Setup+ boundaries <- newIORef [(origin.monoNs, Setup)]+ boundaryListeners <- newIORef []+ onPhase Setup origin+ pure PhaseClock {current, boundaries, boundaryListeners, onPhase}++enterPhase :: PhaseClock -> Phase -> IO ()+enterPhase clock phase = do+ origin <- captureOrigin+ writeIORef clock.current phase+ modifyIORef' clock.boundaries (<> [(origin.monoNs, phase)])+ clock.onPhase phase origin+ readIORef clock.boundaryListeners >>= sequence_++currentPhase :: PhaseClock -> IO Phase+currentPhase = readIORef . (.current)++phaseOf :: PhaseClock -> Word64 -> IO Phase+phaseOf clock intendedStart = do+ boundaries <- readIORef clock.boundaries+ pure (foldl choose Setup boundaries)+ where+ choose selected (at, phase) = if at <= intendedStart then phase else selected++addBoundaryListener :: PhaseClock -> IO () -> IO ()+addBoundaryListener clock listener = modifyIORef' clock.boundaryListeners (<> [listener])++renderPhase :: Phase -> Text+renderPhase Setup = "setup"+renderPhase WarmUp = "warm-up"+renderPhase Steady = "steady"+renderPhase Drain = "drain"+renderPhase Done = "done"
+ src/Kenshou/Measure/Recorder.hs view
@@ -0,0 +1,301 @@+{-# LANGUAGE BangPatterns #-}++module Kenshou.Measure.Recorder+ ( OpName (..),+ ErrorCause (..),+ OpResult (..),+ RawSamplePolicy (..),+ RecorderConfig (..),+ Recorder,+ OpHandle,+ WorkerRecorder,+ OpReport (..),+ RecorderReport (..),+ newRecorder,+ registerOp,+ newWorkerRecorder,+ recordOp,+ recordDuration,+ timeOp,+ finishRecorder,+ )+where++import Control.Monad (forM, forM_, when)+import Data.Aeson (ToJSON (..), encode, object, (.=))+import Data.Aeson.Key qualified as Key+import Data.ByteString qualified as ByteString+import Data.ByteString.Builder (byteString, toLazyByteString, word16LE, word32LE, word64LE, word8)+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Set qualified as Set+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word+import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram (Histogram, HistogramConfig)+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (encodeHistogram)+import Kenshou.Measure.Phase+import Kenshou.Measure.Samples+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))++newtype OpName = OpName Text+ deriving stock (Eq, Ord, Show)++newtype ErrorCause = ErrorCause Text+ deriving stock (Eq, Ord, Show)++data OpResult = OpOk !Int | OpFailed !ErrorCause+ deriving stock (Eq, Show)++data RawSamplePolicy = RawFull | RawSampled !Word64 | RawOff+ deriving stock (Eq, Show)++instance ToJSON RawSamplePolicy where+ toJSON RawFull = toJSON ("full" :: Text)+ toJSON (RawSampled oneIn) = object ["mode" .= ("sampled" :: Text), "oneIn" .= oneIn]+ toJSON RawOff = toJSON ("off" :: Text)++data RecorderConfig = RecorderConfig+ { runDir :: FilePath,+ origin :: Origin,+ processLabel :: Maybe Text,+ histogram :: HistogramConfig,+ rawSamples :: RawSamplePolicy,+ intervalHistogramSeconds :: Word64,+ declareArtifact :: FilePath -> Text -> IO ()+ }++data Recorder = Recorder+ { config :: RecorderConfig,+ phaseClock :: PhaseClock,+ operations :: IORef [OpHandle]+ }++data OpHandle = OpHandle+ { recorder :: Recorder,+ name :: OpName,+ writer :: Maybe SampleWriter,+ workers :: IORef [WorkerRecorder]+ }++data WorkerRecorder = WorkerRecorder+ { operation :: OpHandle,+ workerId :: Word16,+ latency :: Histogram.MutableHistogram,+ service :: Histogram.MutableHistogram,+ phaseCounts :: IORef (Map Phase (Word64, Word64, Word64)),+ errors :: IORef (Map ErrorCause Word64),+ intervals :: IORef (Map Word64 Histogram.MutableHistogram),+ pending :: IORef [SampleRecord],+ pendingCount :: IORef Int,+ seen :: IORef Word64+ }++data OpReport = OpReport+ { name :: OpName,+ latency :: Histogram,+ service :: Histogram,+ phaseCounts :: Map Phase (Word64, Word64, Word64),+ errors :: Map ErrorCause Word64,+ backpressureCount :: Word64,+ histogramFile :: FilePath,+ serviceHistogramFile :: FilePath,+ rawFile :: Maybe FilePath,+ intervalHistogramFile :: Maybe FilePath,+ metadataFile :: FilePath+ }++data RecorderReport = RecorderReport+ { operations :: [OpReport],+ backpressureCount :: Word64+ }++newRecorder :: RecorderConfig -> PhaseClock -> IO Recorder+newRecorder config phaseClock = do+ createDirectoryIfMissing True (config.runDir </> "samples")+ Recorder config phaseClock <$> newIORef []++registerOp :: Recorder -> OpName -> IO OpHandle+registerOp recorder name@(OpName nameText) = do+ let rawPath = samplePath recorder.config nameText "raw"+ label = maybe "harness" id recorder.config.processLabel+ writer <- case recorder.config.rawSamples of+ RawOff -> pure Nothing+ _ -> Just <$> openSampleWriter rawPath (SampleHeader recorder.config.origin MonotonicClock nameText label)+ handle <- OpHandle recorder name writer <$> newIORef []+ modifyIORef' recorder.operations (<> [handle])+ pure handle++newWorkerRecorder :: OpHandle -> Int -> IO WorkerRecorder+newWorkerRecorder operation workerNumber = do+ latency <- Histogram.newHistogram operation.recorder.config.histogram+ service <- Histogram.newHistogram operation.recorder.config.histogram+ worker <-+ WorkerRecorder operation (fromIntegral workerNumber) latency service+ <$> newIORef Map.empty+ <*> newIORef Map.empty+ <*> newIORef Map.empty+ <*> newIORef []+ <*> newIORef 0+ <*> newIORef 0+ modifyIORef' operation.workers (<> [worker])+ pure worker++recordOp :: WorkerRecorder -> Word64 -> Word64 -> Word64 -> OpResult -> IO ()+recordOp worker intended actual end result = do+ phase <- phaseOf worker.operation.recorder.phaseClock intended+ let serviceNs = end - min end actual+ latencyNs = end - min end intended+ (successes, failures, units) = case result of+ OpOk completed -> (1, 0, fromIntegral (max 0 completed))+ OpFailed _ -> (0, 1, 0)+ when (phase == Steady) do+ Histogram.recordValue worker.latency latencyNs+ Histogram.recordValue worker.service serviceNs+ when (worker.operation.recorder.config.rawSamples /= RawFull) (recordInterval worker intended latencyNs)+ modifyIORef' worker.phaseCounts (Map.insertWith addCounts phase (successes, failures, units))+ case result of+ OpFailed cause -> modifyIORef' worker.errors (Map.insertWith (+) cause 1)+ OpOk _ -> pure ()+ sequenceNumber <- atomicModifyIORef' worker.seen (\value -> (value + 1, value))+ when (retain worker.operation.recorder.config.rawSamples sequenceNumber) do+ let outcomeCode = case result of OpOk _ -> 0; OpFailed _ -> 1+ modifyIORef' worker.pending (SampleRecord intended actual end outcomeCode units :)+ count <- atomicModifyIORef' worker.pendingCount (\value -> let next = value + 1 in (next, next))+ when (count >= 512) (flushWorker worker)++recordDuration :: WorkerRecorder -> Word64 -> Word64 -> OpResult -> IO ()+recordDuration worker end duration = recordOp worker (end - min end duration) (end - min end duration) end++timeOp :: WorkerRecorder -> IO OpResult -> IO OpResult+timeOp worker action = do+ start <- nowNs+ result <- action+ end <- nowNs+ recordOp worker start start end result+ pure result++finishRecorder :: Recorder -> IO RecorderReport+finishRecorder recorder = do+ operations <- readIORef recorder.operations >>= mapM finishOperation+ pure RecorderReport {operations, backpressureCount = sum (fmap (.backpressureCount) operations)}++finishOperation :: OpHandle -> IO OpReport+finishOperation operation = do+ workers <- readIORef operation.workers+ mapM_ flushWorker workers+ forM_ operation.writer closeSampleWriter+ latency <- mergeWorkerHistograms operation.recorder.config.histogram workers (.latency)+ service <- mergeWorkerHistograms operation.recorder.config.histogram workers (.service)+ phaseCounts <- fmap (Map.unionsWith addCounts) (mapM (readIORef . (.phaseCounts)) workers)+ errors <- fmap (Map.unionsWith (+)) (mapM (readIORef . (.errors)) workers)+ backpressureCount <- maybe (pure 0) sampleWriterBackpressure operation.writer+ intervalHistograms <- finishIntervals operation.recorder.config workers+ let OpName nameText = operation.name+ histogramFile = samplePath operation.recorder.config nameText "hist"+ serviceHistogramFile = samplePath operation.recorder.config nameText "service.hist"+ metadataFile = samplePath operation.recorder.config nameText "meta.json"+ rawFile = case operation.writer of Nothing -> Nothing; Just _ -> Just (samplePath operation.recorder.config nameText "raw")+ intervalHistogramFile = if null intervalHistograms then Nothing else Just (samplePath operation.recorder.config nameText "ihist")+ writeBytes histogramFile (encodeHistogram latency)+ writeBytes serviceHistogramFile (encodeHistogram service)+ forM_ intervalHistogramFile \path -> writeIntervalHistograms operation.recorder.config path intervalHistograms+ LazyByteString.writeFile metadataFile $+ encode+ ( object+ [ "schema" .= ("kenshou.sample-meta/v1" :: Text),+ "operation" .= nameText,+ "latencyBasis" .= ("intended-start" :: Text),+ "clock" .= ("monotonic" :: Text),+ "rawSamples" .= operation.recorder.config.rawSamples,+ "phaseCounts" .= object [Key.fromText (renderPhase phase) .= object ["successes" .= successes, "failures" .= failures, "units" .= units] | (phase, (successes, failures, units)) <- Map.toAscList phaseCounts],+ "errorCauses" .= object [Key.fromText cause .= count | (ErrorCause cause, count) <- Map.toAscList errors],+ "files" .= object ["histogram" .= histogramFile, "serviceHistogram" .= serviceHistogramFile, "raw" .= rawFile, "intervalHistogram" .= intervalHistogramFile],+ "backpressureCount" .= backpressureCount+ ]+ )+ let artifacts = [(histogramFile, "application/vnd.kenshou.histogram"), (serviceHistogramFile, "application/vnd.kenshou.histogram"), (metadataFile, "application/json")] <> maybe [] (\path -> [(path, "application/vnd.kenshou.samples")]) rawFile <> maybe [] (\path -> [(path, "application/vnd.kenshou.interval-histograms")]) intervalHistogramFile+ mapM_ (uncurry operation.recorder.config.declareArtifact) artifacts+ pure OpReport {name = operation.name, latency, service, phaseCounts, errors, backpressureCount, histogramFile, serviceHistogramFile, rawFile, intervalHistogramFile, metadataFile}++flushWorker :: WorkerRecorder -> IO ()+flushWorker worker = case worker.operation.writer of+ Nothing -> writeIORef worker.pending [] >> writeIORef worker.pendingCount 0+ Just writer -> do+ records <- atomicModifyIORef' worker.pending (\pending -> ([], reverse pending))+ writeIORef worker.pendingCount 0+ writeSampleBlock writer worker.workerId records++mergeWorkerHistograms :: HistogramConfig -> [WorkerRecorder] -> (WorkerRecorder -> Histogram.MutableHistogram) -> IO Histogram+mergeWorkerHistograms config workers project = do+ histograms <- mapM (Histogram.freeze . project) workers+ case histograms of+ [] -> Histogram.newHistogram config >>= Histogram.freeze+ first : rest -> case foldl' combine (Right first) rest of+ Left message -> ioError (userError (Text.unpack message))+ Right value -> pure value+ where+ combine accumulated next = accumulated >>= (`Histogram.merge` next)++samplePath :: RecorderConfig -> Text -> FilePath -> FilePath+samplePath config operation extension = config.runDir </> "samples" </> Text.unpack operation <> maybe "" (("." <>) . Text.unpack) config.processLabel <> "." <> extension++writeBytes :: FilePath -> ByteString.ByteString -> IO ()+writeBytes = ByteString.writeFile++retain :: RawSamplePolicy -> Word64 -> Bool+retain RawFull _ = True+retain (RawSampled oneIn) sequenceNumber = oneIn > 0 && sequenceNumber `mod` oneIn == 0+retain RawOff _ = False++recordInterval :: WorkerRecorder -> Word64 -> Word64 -> IO ()+recordInterval worker intended latencyNs = do+ let seconds = max 1 worker.operation.recorder.config.intervalHistogramSeconds+ intervalNs = seconds * 1_000_000_000+ epoch = intended `div` intervalNs+ existing <- Map.lookup epoch <$> readIORef worker.intervals+ histogram <- case existing of+ Just value -> pure value+ Nothing -> do+ value <- Histogram.newHistogram worker.operation.recorder.config.histogram+ modifyIORef' worker.intervals (Map.insert epoch value)+ pure value+ Histogram.recordValue histogram latencyNs++finishIntervals :: RecorderConfig -> [WorkerRecorder] -> IO [(Word64, Histogram)]+finishIntervals config workers = do+ workerMaps <- mapM (readIORef . (.intervals)) workers+ let epochs = Set.toAscList (Set.unions (fmap Map.keysSet workerMaps))+ forM epochs \epoch -> do+ histograms <- forM workerMaps \intervals -> traverse Histogram.freeze (Map.lookup epoch intervals)+ empty <- Histogram.newHistogram config.histogram >>= Histogram.freeze+ let mergeOne accumulated Nothing = accumulated+ mergeOne (Left message) _ = Left message+ mergeOne (Right accumulated) (Just next) = Histogram.merge accumulated next+ case foldl mergeOne (Right empty) histograms of+ Left message -> ioError (userError (Text.unpack message))+ Right histogram -> pure (epoch, histogram)++writeIntervalHistograms :: RecorderConfig -> FilePath -> [(Word64, Histogram)] -> IO ()+writeIntervalHistograms config path histograms =+ ByteString.writeFile path . LazyByteString.toStrict . toLazyByteString $+ byteString "KIHS" <> word16LE 1 <> word16LE 0 <> foldMap frame histograms+ where+ intervalNs = max 1 config.intervalHistogramSeconds * 1_000_000_000+ frame (epoch, histogram) =+ let encoded = encodeHistogram histogram+ absoluteStart = epoch * intervalNs+ start = absoluteStart - min absoluteStart config.origin.monoNs+ in word64LE start <> word64LE (start + intervalNs) <> word8 2 <> word32LE (fromIntegral (ByteString.length encoded)) <> byteString encoded++addCounts :: (Word64, Word64, Word64) -> (Word64, Word64, Word64) -> (Word64, Word64, Word64)+addCounts (a, b, c) (x, y, z) =+ let !successes = a + x+ !failures = b + y+ !units = c + z+ in (successes, failures, units)
+ src/Kenshou/Measure/Sampler.hs view
@@ -0,0 +1,174 @@+module Kenshou.Measure.Sampler+ ( Sampler (..),+ SamplerConfig (..),+ SamplerReport (..),+ SamplingHandle,+ startSampling,+ stopSampling,+ )+where++import Control.Concurrent.Async+import Control.Concurrent.STM+import Control.Exception (finally)+import Control.Monad (forM_)+import Data.IORef+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Phase+import Kenshou.Measure.Sampler.Csv+import Kenshou.Measure.Sampler.Host+import Kenshou.Measure.Sampler.Postgres+import Kenshou.Measure.Sampler.Process+import Kenshou.Measure.Sampler.Rts+import System.FilePath ((</>))++data Sampler = Sampler+ { name :: Text,+ sample :: [Text] -> IO ()+ }++data SamplerConfig = SamplerConfig+ { runDir :: FilePath,+ origin :: Origin,+ intervalMs :: Int,+ postgres :: Maybe PgSamplerConfig,+ extraSamplers :: [Sampler],+ phaseClock :: PhaseClock,+ declareArtifact :: FilePath -> Text -> IO (),+ logLine :: Text -> IO ()+ }++data SamplerReport = SamplerReport+ { rtsAvailable :: Bool,+ processAvailable :: Bool,+ hostAvailable :: Bool,+ postgresAvailable :: Bool,+ pgStatementsAvailable :: Bool,+ ticks :: Word64+ }+ deriving stock (Eq, Show)++data Event = Boundary | Scheduled | Stop++data SamplingHandle = SamplingHandle+ { threads :: [Async ()],+ events :: TChan Event,+ closeResources :: IO (),+ report :: IO SamplerReport+ }++startSampling :: SamplerConfig -> IO SamplingHandle+startSampling config = do+ let seriesDir = config.runDir </> "series"+ path name = seriesDir </> name+ declare name = config.declareArtifact (path name) "text/csv"+ events <- newBroadcastTChanIO+ phaseEvents <- atomically (dupTChan events)+ postgresEvents <- atomically (dupTChan events)+ addBoundaryListener config.phaseClock (atomically (writeTChan events Boundary))+ samplerWriter <- openCsv (path "sampler.csv") ["t_mono_ns", "t_wall_ms", "phase", "scheduled_mono_ns", "late_ns", "cost_ns", "wall_minus_mono_ns"]+ declare "sampler.csv"+ rts <- openRtsSampler (path "rts.csv")+ forM_ rts (const (declare "rts.csv"))+ process <- openProcessSampler (path "proc.csv")+ declare "proc.csv"+ host <- openHostSampler (path "host.csv")+ forM_ host (const (declare "host.csv"))+ previousClockDelta <- newIORef Nothing+ tickCount <- newIORef 0+ mainThread <- async (mainLoop config phaseEvents samplerWriter rts process host previousClockDelta tickCount 1)+ postgresResult <- case config.postgres of+ Nothing -> pure Nothing+ Just pgConfig -> do+ opened <- openPostgresSampler seriesDir pgConfig config.logLine+ case opened of+ Left err -> config.logLine ("PostgreSQL sampler unavailable: " <> err) >> pure Nothing+ Right sampler -> do+ mapM_ declare (postgresArtifactNames (postgresHasStatements sampler))+ thread <- async (postgresLoop config postgresEvents sampler 1 `finally` closePostgresSampler sampler)+ pure (Just (thread, postgresHasStatements sampler))+ let closeResources = do+ closeCsv samplerWriter+ forM_ rts closeRtsSampler+ closeProcessSampler process+ forM_ host closeHostSampler+ readReport = do+ count <- readIORef tickCount+ pure+ SamplerReport+ { rtsAvailable = maybe False (const True) rts,+ processAvailable = True,+ hostAvailable = maybe False (const True) host,+ postgresAvailable = maybe False (const True) postgresResult,+ pgStatementsAvailable = maybe False snd postgresResult,+ ticks = count+ }+ pure+ SamplingHandle+ { threads = mainThread : maybe [] (pure . fst) postgresResult,+ events,+ closeResources,+ report = readReport+ }++stopSampling :: SamplingHandle -> IO SamplerReport+stopSampling handle = do+ atomically (writeTChan handle.events Stop)+ mapM_ wait handle.threads `finally` handle.closeResources+ handle.report++mainLoop :: SamplerConfig -> TChan Event -> CsvWriter -> Maybe RtsSampler -> ProcessSampler -> Maybe HostSampler -> IORef (Maybe Integer) -> IORef Word64 -> Word64 -> IO ()+mainLoop config events samplerWriter rts process host previousClockDelta tickCount tickNumber = do+ let intervalNs = fromIntegral config.intervalMs * 1_000_000+ scheduled = config.origin.monoNs + tickNumber * intervalNs+ event <- waitForEvent events scheduled+ case event of+ Stop -> pure ()+ Boundary -> nowNs >>= sampleAll >> mainLoop config events samplerWriter rts process host previousClockDelta tickCount tickNumber+ Scheduled -> sampleAll scheduled >> mainLoop config events samplerWriter rts process host previousClockDelta tickCount (tickNumber + 1)+ where+ sampleAll scheduled = do+ started <- nowNs+ (_, prefix) <- timestampColumns config.origin config.phaseClock+ forM_ rts (`sampleRts` prefix)+ sampleProcess process prefix+ forM_ host (`sampleHost` prefix)+ forM_ config.extraSamplers (\sampler -> sampler.sample prefix)+ ended <- nowNs+ wallOrigin <- captureOrigin+ let delta = fromIntegral wallOrigin.wallUnixNs - fromIntegral wallOrigin.monoNs+ previous <- readIORef previousClockDelta+ writeIORef previousClockDelta (Just delta)+ let clockChange = maybe 0 (delta -) previous+ value = Text.pack . show+ appendCsv samplerWriter (prefix <> fmap value [scheduled - min scheduled config.origin.monoNs, started - min started scheduled, ended - started] <> [Text.pack (show clockChange)])+ atomicModifyIORef' tickCount (\count -> (count + 1, ()))++postgresLoop :: SamplerConfig -> TChan Event -> PostgresSampler -> Word64 -> IO ()+postgresLoop config events sampler tickNumber = do+ let intervalNs = fromIntegral config.intervalMs * 1_000_000+ scheduled = config.origin.monoNs + tickNumber * intervalNs+ event <- waitForEvent events scheduled+ case event of+ Stop -> pure ()+ Boundary -> sampleNow True >> postgresLoop config events sampler tickNumber+ Scheduled -> sampleNow (postgresPeriodic sampler && tickNumber `mod` 10 == 0) >> postgresLoop config events sampler (tickNumber + 1)+ where+ sampleNow includeStatements = do+ (_, prefix) <- timestampColumns config.origin config.phaseClock+ samplePostgres sampler prefix includeStatements++waitForEvent :: TChan Event -> Word64 -> IO Event+waitForEvent events deadline = do+ current <- nowNs+ let remainingNs = deadline - min deadline current+ remainingMicros = fromIntegral (min (fromIntegral (maxBound :: Int)) ((remainingNs + 999) `div` 1_000))+ elapsed <- registerDelay remainingMicros+ atomically $+ readTChan events+ `orElse` do+ readTVar elapsed >>= check+ pure Scheduled
+ src/Kenshou/Measure/Sampler/Csv.hs view
@@ -0,0 +1,53 @@+module Kenshou.Measure.Sampler.Csv+ ( CsvWriter,+ openCsv,+ appendCsv,+ closeCsv,+ timestampColumns,+ )+where++import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Data.Time.Clock.POSIX (getPOSIXTime)+import Data.Word (Word64)+import Kenshou.Measure.Clock+import Kenshou.Measure.Phase+import System.Directory (createDirectoryIfMissing)+import System.FilePath (takeDirectory)+import System.IO++newtype CsvWriter = CsvWriter Handle++openCsv :: FilePath -> [Text] -> IO CsvWriter+openCsv path columns = do+ createDirectoryIfMissing True (takeDirectory path)+ handle <- openFile path WriteMode+ hSetBuffering handle LineBuffering+ let writer = CsvWriter handle+ appendCsv writer columns+ pure writer++appendCsv :: CsvWriter -> [Text] -> IO ()+appendCsv (CsvWriter handle) fields = Text.hPutStrLn handle (Text.intercalate "," (fmap quote fields))+ where+ quote value+ | Text.any (`elem` [',', '"', '\r', '\n']) value = "\"" <> Text.replace "\"" "\"\"" value <> "\""+ | otherwise = value++closeCsv :: CsvWriter -> IO ()+closeCsv (CsvWriter handle) = hClose handle++timestampColumns :: Origin -> PhaseClock -> IO (Word64, [Text])+timestampColumns origin phaseClock = do+ mono <- nowNs+ wall <- getPOSIXTime+ phase <- currentPhase phaseClock+ pure+ ( mono,+ [ Text.pack (show (mono - min mono origin.monoNs)),+ Text.pack (show (floor (wall * 1_000) :: Integer)),+ renderPhase phase+ ]+ )
+ src/Kenshou/Measure/Sampler/Host.hs view
@@ -0,0 +1,95 @@+{-# LANGUAGE CPP #-}++module Kenshou.Measure.Sampler.Host+ ( HostSample (..),+ parseHostSample,+ HostSampler,+ openHostSampler,+ sampleHost,+ closeHostSampler,+ )+where++import Data.List (find)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Sampler.Csv+import Text.Read (readMaybe)++#ifdef linux_HOST_OS+import Data.Text.IO qualified as Text+#endif++data HostSample = HostSample+ { cpuUser :: Maybe Word64,+ cpuNice :: Maybe Word64,+ cpuSystem :: Maybe Word64,+ cpuIdle :: Maybe Word64,+ cpuIowait :: Maybe Word64,+ cpuIrq :: Maybe Word64,+ cpuSoftirq :: Maybe Word64,+ cpuSteal :: Maybe Word64,+ loadAverage1 :: Maybe Double,+ memAvailableBytes :: Maybe Word64+ }+ deriving stock (Eq, Show)++newtype HostSampler = HostSampler CsvWriter++openHostSampler :: FilePath -> IO (Maybe HostSampler)+#ifdef linux_HOST_OS+openHostSampler path = Just . HostSampler <$> openCsv path+ [ "t_mono_ns", "t_wall_ms", "phase", "cpu_user", "cpu_nice", "cpu_system", "cpu_idle", "cpu_iowait",+ "cpu_irq", "cpu_softirq", "cpu_steal", "loadavg1", "mem_available_bytes"+ ]+#else+openHostSampler _ = pure Nothing+#endif++sampleHost :: HostSampler -> [Text] -> IO ()+#ifdef linux_HOST_OS+sampleHost (HostSampler writer) prefix = do+ stat <- Text.readFile "/proc/stat"+ loadavg <- Text.readFile "/proc/loadavg"+ meminfo <- Text.readFile "/proc/meminfo"+ let sample = parseHostSample stat loadavg meminfo+ integral = maybe "" (Text.pack . show)+ floating = maybe "" (Text.pack . show)+ appendCsv writer (prefix <> fmap integral+ [ sample.cpuUser, sample.cpuNice, sample.cpuSystem, sample.cpuIdle, sample.cpuIowait,+ sample.cpuIrq, sample.cpuSoftirq, sample.cpuSteal+ ] <> [floating sample.loadAverage1, integral sample.memAvailableBytes])+#else+sampleHost _ _ = pure ()+#endif++closeHostSampler :: HostSampler -> IO ()+closeHostSampler (HostSampler writer) = closeCsv writer++parseHostSample :: Text -> Text -> Text -> HostSample+parseHostSample stat loadavg meminfo =+ HostSample+ { cpuUser = cpu 0,+ cpuNice = cpu 1,+ cpuSystem = cpu 2,+ cpuIdle = cpu 3,+ cpuIowait = cpu 4,+ cpuIrq = cpu 5,+ cpuSoftirq = cpu 6,+ cpuSteal = cpu 7,+ loadAverage1 = case Text.words loadavg of value : _ -> readDouble value; [] -> Nothing,+ memAvailableBytes = (* 1_024) <$> memAvailable+ }+ where+ cpuFields = maybe [] (drop 1 . Text.words) (find (Text.isPrefixOf "cpu ") (Text.lines stat))+ cpu index = case drop index cpuFields of value : _ -> readWord value; [] -> Nothing+ memAvailable = do+ line <- find (Text.isPrefixOf "MemAvailable:") (Text.lines meminfo)+ case Text.words line of _ : value : _ -> readWord value; _ -> Nothing++readWord :: Text -> Maybe Word64+readWord = readMaybe . Text.unpack++readDouble :: Text -> Maybe Double+readDouble = readMaybe . Text.unpack
+ src/Kenshou/Measure/Sampler/Postgres.hs view
@@ -0,0 +1,137 @@+module Kenshou.Measure.Sampler.Postgres+ ( PgStatementsMode (..),+ PgSamplerConfig (..),+ PostgresSampler,+ openPostgresSampler,+ samplePostgres,+ closePostgresSampler,+ postgresHasStatements,+ postgresPeriodic,+ postgresArtifactNames,+ )+where++import Control.Exception (finally)+import Control.Monad (forM_, when)+import Data.Text (Text)+import Data.Text qualified as Text+import Hasql.Connection qualified as Connection+import Hasql.Connection.Settings qualified as Settings+import Hasql.Decoders qualified as Decoders+import Hasql.Encoders qualified as Encoders+import Hasql.Session qualified as Session+import Hasql.Statement qualified as Statement+import Kenshou.Measure.Sampler.Csv++data PgStatementsMode = PgStatementsOff | PgStatementsSnapshots | PgStatementsPeriodic+ deriving stock (Eq, Show)++data PgSamplerConfig = PgSamplerConfig+ { connectionString :: Text,+ relations :: [Text],+ statements :: PgStatementsMode,+ serverVersionNum :: Int+ }+ deriving stock (Eq, Show)++data PostgresSampler = PostgresSampler+ { connection :: Connection.Connection,+ activity :: CsvWriter,+ checkpointer :: CsvWriter,+ wal :: CsvWriter,+ database :: CsvWriter,+ relations :: CsvWriter,+ statements :: Maybe CsvWriter,+ watchedRelations :: [Text],+ versionNumber :: Int,+ statementsMode :: PgStatementsMode,+ logLine :: Text -> IO ()+ }++postgresArtifactNames :: Bool -> [FilePath]+postgresArtifactNames hasStatements =+ ["pg-activity.csv", "pg-checkpointer.csv", "pg-wal.csv", "pg-database.csv", "pg-relations.csv"]+ <> ["pg-statements.csv" | hasStatements]++openPostgresSampler :: FilePath -> PgSamplerConfig -> (Text -> IO ()) -> IO (Either Text PostgresSampler)+openPostgresSampler seriesDir config logger = do+ acquired <- Connection.acquire (Settings.connectionString config.connectionString <> Settings.applicationName "kenshou-sampler")+ case acquired of+ Left err -> pure (Left (Text.pack (show err)))+ Right connection -> do+ activity <- openCsv (seriesDir <> "/pg-activity.csv") (base <> ["application_name", "state", "wait_event_type", "wait_event", "connections"])+ checkpointer <- openCsv (seriesDir <> "/pg-checkpointer.csv") (base <> ["num_timed", "num_requested", "write_time_ms", "sync_time_ms", "buffers_written", "checkpoint_lsn", "checkpoint_time_unix", "num_done"])+ wal <- openCsv (seriesDir <> "/pg-wal.csv") (base <> ["current_wal_bytes", "wal_records", "wal_fpi", "wal_bytes", "wal_buffers_full"])+ database <- openCsv (seriesDir <> "/pg-database.csv") (base <> ["xact_commit", "xact_rollback", "blks_read", "blks_hit", "tup_inserted", "tup_updated", "tup_deleted", "deadlocks", "temp_bytes"])+ relations <- openCsv (seriesDir <> "/pg-relations.csv") (base <> ["relation", "relation_bytes", "indexes_bytes", "total_bytes", "n_live_tup", "n_dead_tup", "n_tup_ins", "n_tup_upd", "n_tup_del", "autovacuum_count"])+ statements <-+ if config.statements == PgStatementsOff+ then pure Nothing+ else do+ created <- Connection.use connection (Session.statement () createStatements)+ probed <- case created of Left err -> pure (Left err); Right () -> Connection.use connection (Session.statement () probeStatements)+ case probed of+ Left err -> logger ("pg_stat_statements unavailable: " <> Text.pack (show err)) >> pure Nothing+ Right () -> Just <$> openCsv (seriesDir <> "/pg-statements.csv") (base <> ["queryid", "calls", "total_exec_time_ms", "rows", "shared_blks_hit", "shared_blks_read", "wal_bytes"])+ pure (Right PostgresSampler {connection, activity, checkpointer, wal, database, relations, statements, watchedRelations = config.relations, versionNumber = config.serverVersionNum, statementsMode = config.statements, logLine = logger})+ where+ base = ["t_mono_ns", "t_wall_ms", "phase"]++samplePostgres :: PostgresSampler -> [Text] -> Bool -> IO ()+samplePostgres sampler prefix includeStatements = do+ writeRows sampler sampler.activity prefix activityQuery+ writeRows sampler sampler.checkpointer prefix (if sampler.versionNumber >= 180000 then checkpointer18Query else checkpointer17Query)+ writeRows sampler sampler.wal prefix walQuery+ writeRows sampler sampler.database prefix databaseQuery+ forM_ sampler.watchedRelations \relation -> writeParamRows sampler sampler.relations prefix relation relationQuery+ when includeStatements $ forM_ sampler.statements \writer -> writeRows sampler writer prefix statementsQuery++closePostgresSampler :: PostgresSampler -> IO ()+closePostgresSampler sampler =+ let writers = [sampler.activity, sampler.checkpointer, sampler.wal, sampler.database, sampler.relations] <> maybe [] pure sampler.statements+ in mapM_ closeCsv writers `finally` Connection.release sampler.connection++postgresHasStatements :: PostgresSampler -> Bool+postgresHasStatements = maybe False (const True) . (.statements)++postgresPeriodic :: PostgresSampler -> Bool+postgresPeriodic sampler = sampler.statementsMode == PgStatementsPeriodic++writeRows :: PostgresSampler -> CsvWriter -> [Text] -> Statement.Statement () [Text] -> IO ()+writeRows sampler writer prefix statement = do+ result <- Connection.use sampler.connection (Session.statement () statement)+ case result of+ Left err -> sampler.logLine ("PostgreSQL sampler query failed: " <> Text.pack (show err))+ Right rows -> forM_ rows (appendCsv writer . (prefix <>) . Text.splitOn separator)++writeParamRows :: PostgresSampler -> CsvWriter -> [Text] -> Text -> Statement.Statement Text [Text] -> IO ()+writeParamRows sampler writer prefix parameter statement = do+ result <- Connection.use sampler.connection (Session.statement parameter statement)+ case result of+ Left err -> sampler.logLine ("PostgreSQL relation sampler query failed: " <> Text.pack (show err))+ Right rows -> forM_ rows (appendCsv writer . (prefix <>) . Text.splitOn separator)++separator :: Text+separator = Text.singleton '\x1f'++textRows :: Text -> Statement.Statement () [Text]+textRows sql = Statement.unpreparable sql Encoders.noParams (Decoders.rowList (Decoders.column (Decoders.nonNullable Decoders.text)))++createStatements, probeStatements :: Statement.Statement () ()+createStatements = Statement.unpreparable "CREATE EXTENSION IF NOT EXISTS pg_stat_statements" Encoders.noParams Decoders.noResult+probeStatements = Statement.unpreparable "SELECT 1 FROM pg_stat_statements(false) LIMIT 0" Encoders.noParams Decoders.noResult++activityQuery, checkpointer17Query, checkpointer18Query, walQuery, databaseQuery, statementsQuery :: Statement.Statement () [Text]+activityQuery = textRows "SELECT concat_ws(chr(31), coalesce(application_name,''), coalesce(state,''), coalesce(wait_event_type,''), coalesce(wait_event,''), count(*)::text) FROM pg_stat_activity WHERE backend_type='client backend' AND application_name <> 'kenshou-sampler' GROUP BY application_name,state,wait_event_type,wait_event ORDER BY application_name,state,wait_event_type,wait_event"+checkpointer17Query = textRows "SELECT concat_ws(chr(31), c.num_timed::text, c.num_requested::text, c.write_time::text, c.sync_time::text, c.buffers_written::text, (pg_control_checkpoint()).checkpoint_lsn::text, extract(epoch FROM (pg_control_checkpoint()).checkpoint_time)::bigint::text, '') FROM pg_stat_checkpointer c"+checkpointer18Query = textRows "SELECT concat_ws(chr(31), c.num_timed::text, c.num_requested::text, c.write_time::text, c.sync_time::text, c.buffers_written::text, (pg_control_checkpoint()).checkpoint_lsn::text, extract(epoch FROM (pg_control_checkpoint()).checkpoint_time)::bigint::text, c.num_done::text) FROM pg_stat_checkpointer c"+walQuery = textRows "SELECT concat_ws(chr(31), pg_wal_lsn_diff(pg_current_wal_lsn(),'0/0')::bigint::text, w.wal_records::text, w.wal_fpi::text, w.wal_bytes::bigint::text, w.wal_buffers_full::text) FROM pg_stat_wal w"+databaseQuery = textRows "SELECT concat_ws(chr(31), xact_commit::text, xact_rollback::text, blks_read::text, blks_hit::text, tup_inserted::text, tup_updated::text, tup_deleted::text, deadlocks::text, temp_bytes::text) FROM pg_stat_database WHERE datname=current_database()"+statementsQuery = textRows "SELECT concat_ws(chr(31), queryid::text, calls::text, total_exec_time::text, rows::text, shared_blks_hit::text, shared_blks_read::text, wal_bytes::bigint::text) FROM pg_stat_statements(false) WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database()) ORDER BY queryid"++relationQuery :: Statement.Statement Text [Text]+relationQuery =+ Statement.unpreparable+ "SELECT concat_ws(chr(31), $1::text, pg_relation_size($1::regclass)::text, pg_indexes_size($1::regclass)::text, pg_total_relation_size($1::regclass)::text, s.n_live_tup::text, s.n_dead_tup::text, s.n_tup_ins::text, s.n_tup_upd::text, s.n_tup_del::text, s.autovacuum_count::text) FROM pg_stat_all_tables s WHERE s.relid=$1::regclass"+ (Encoders.param (Encoders.nonNullable Encoders.text))+ (Decoders.rowList (Decoders.column (Decoders.nonNullable Decoders.text)))
+ src/Kenshou/Measure/Sampler/Process.hs view
@@ -0,0 +1,144 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE ForeignFunctionInterface #-}++module Kenshou.Measure.Sampler.Process+ ( ProcessSample (..),+ parseProcSample,+ readProcessSample,+ ProcessSampler,+ openProcessSampler,+ sampleProcess,+ closeProcessSampler,+ )+where++import Data.Char (isSpace)+import Data.List (find)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Sampler.Csv+import System.CPUTime (getCPUTime)+import System.Directory (listDirectory)+import Text.Read (readMaybe)++#ifdef linux_HOST_OS+import Data.Text.IO qualified as Text+#endif++#ifdef darwin_HOST_OS+import Foreign (Ptr, alloca, peek)+import Foreign.C.Types++foreign import ccall unsafe "kenshou_proc_taskinfo" c_procTaskInfo :: Ptr Word64 -> Ptr CInt -> IO CInt+#endif++data ProcessSample = ProcessSample+ { rssBytes :: Maybe Word64,+ rssMaxBytes :: Maybe Word64,+ osThreads :: Maybe Word64,+ openFds :: Maybe Word64,+ cpuUserNs :: Maybe Word64,+ cpuSystemNs :: Maybe Word64,+ cpuTotalNs :: Maybe Word64,+ voluntaryContextSwitches :: Maybe Word64,+ nonvoluntaryContextSwitches :: Maybe Word64+ }+ deriving stock (Eq, Show)++newtype ProcessSampler = ProcessSampler CsvWriter++openProcessSampler :: FilePath -> IO ProcessSampler+openProcessSampler path =+ ProcessSampler+ <$> openCsv+ path+ [ "t_mono_ns",+ "t_wall_ms",+ "phase",+ "rss_bytes",+ "rss_max_bytes",+ "os_threads",+ "open_fds",+ "cpu_user_ns",+ "cpu_system_ns",+ "cpu_total_ns",+ "voluntary_ctxt_switches",+ "nonvoluntary_ctxt_switches"+ ]++sampleProcess :: ProcessSampler -> [Text] -> IO ()+sampleProcess (ProcessSampler writer) prefix = do+ sample <- readProcessSample+ let field = maybe "" (Text.pack . show)+ appendCsv+ writer+ ( prefix+ <> fmap+ field+ [ sample.rssBytes,+ sample.rssMaxBytes,+ sample.osThreads,+ sample.openFds,+ sample.cpuUserNs,+ sample.cpuSystemNs,+ sample.cpuTotalNs,+ sample.voluntaryContextSwitches,+ sample.nonvoluntaryContextSwitches+ ]+ )++closeProcessSampler :: ProcessSampler -> IO ()+closeProcessSampler (ProcessSampler writer) = closeCsv writer++readProcessSample :: IO ProcessSample+#ifdef linux_HOST_OS+readProcessSample = do+ stat <- Text.readFile "/proc/self/stat"+ status <- Text.readFile "/proc/self/status"+ fds <- listDirectory "/proc/self/fd"+ pure (parseProcSample stat status (fromIntegral (length fds)))+#elif darwin_HOST_OS+readProcessSample = do+ total <- (`div` 1_000) <$> getCPUTime+ fds <- listDirectory "/dev/fd"+ alloca \rssPointer -> alloca \threadsPointer -> do+ ok <- c_procTaskInfo rssPointer threadsPointer+ rss <- if ok == 0 then pure Nothing else Just <$> peek rssPointer+ threads <- if ok == 0 then pure Nothing else Just . fromIntegral <$> peek threadsPointer+ pure (ProcessSample rss Nothing threads (Just (fromIntegral (length fds))) Nothing Nothing (Just (fromIntegral total)) Nothing Nothing)+#else+readProcessSample = do+ total <- (`div` 1_000) <$> getCPUTime+ pure (ProcessSample Nothing Nothing Nothing Nothing Nothing Nothing (Just (fromIntegral total)) Nothing Nothing)+#endif++parseProcSample :: Text -> Text -> Word64 -> ProcessSample+parseProcSample stat status fdCount =+ ProcessSample+ { rssBytes = kilobytes "VmRSS:",+ rssMaxBytes = kilobytes "VmHWM:",+ osThreads = statusNumber "Threads:",+ openFds = Just fdCount,+ cpuUserNs = ticksToNs <$> statNumber 11,+ cpuSystemNs = ticksToNs <$> statNumber 12,+ cpuTotalNs = (+) <$> (ticksToNs <$> statNumber 11) <*> (ticksToNs <$> statNumber 12),+ voluntaryContextSwitches = statusNumber "voluntary_ctxt_switches:",+ nonvoluntaryContextSwitches = statusNumber "nonvoluntary_ctxt_switches:"+ }+ where+ -- Linux kernels expose USER_HZ as 100 on supported benchmark hosts.+ ticksToNs value = value * 10_000_000+ afterCommand = snd (Text.breakOnEnd ") " stat)+ statFields = Text.words afterCommand+ statNumber index = atMay statFields index >>= readWord+ statusLines = Text.lines status+ statusValue key = Text.strip . Text.drop (Text.length key) <$> find (Text.isPrefixOf key) statusLines+ statusNumber key = statusValue key >>= readWord . Text.takeWhile (not . isSpace)+ kilobytes key = (* 1_024) <$> statusNumber key++atMay :: [value] -> Int -> Maybe value+atMay values index = case drop index values of value : _ -> Just value; [] -> Nothing++readWord :: Text -> Maybe Word64+readWord = readMaybe . Text.unpack
+ src/Kenshou/Measure/Sampler/Rts.hs view
@@ -0,0 +1,101 @@+module Kenshou.Measure.Sampler.Rts+ ( RtsSampler,+ openRtsSampler,+ sampleRts,+ closeRtsSampler,+ )+where++import Data.IORef+import Data.Text qualified as Text+import Data.Word (Word32, Word64)+import GHC.Conc (getNumCapabilities, listThreads)+import GHC.Stats+import Kenshou.Measure.Sampler.Csv++data RtsSampler = RtsSampler CsvWriter (IORef (Maybe (Word32, Word64)))++openRtsSampler :: FilePath -> IO (Maybe RtsSampler)+openRtsSampler path = do+ enabled <- getRTSStatsEnabled+ if not enabled+ then pure Nothing+ else do+ writer <- openCsv path (baseColumns <> columns)+ previous <- newIORef Nothing+ pure (Just (RtsSampler writer previous))+ where+ baseColumns = ["t_mono_ns", "t_wall_ms", "phase"]+ columns =+ [ "gcs",+ "major_gcs",+ "allocated_bytes",+ "max_live_bytes",+ "cumulative_live_bytes",+ "live_bytes_major_mean",+ "live_bytes_last_gc",+ "last_gc_gen",+ "mem_in_use_bytes",+ "max_mem_in_use_bytes",+ "large_objects_bytes",+ "compact_bytes",+ "slop_bytes",+ "block_fragmentation_bytes",+ "copied_bytes",+ "gc_cpu_ns",+ "gc_elapsed_ns",+ "mutator_cpu_ns",+ "mutator_elapsed_ns",+ "cpu_ns",+ "elapsed_ns",+ "haskell_threads",+ "capabilities"+ ]++sampleRts :: RtsSampler -> [Text.Text] -> IO ()+sampleRts (RtsSampler writer previousRef) prefix = do+ stats <- getRTSStats+ previous <- readIORef previousRef+ threads <- length <$> listThreads+ capabilities <- getNumCapabilities+ let details = stats.gc+ majorMean = case previous of+ Just (oldMajor, oldCumulative) | stats.major_gcs > oldMajor -> Just ((stats.cumulative_live_bytes - oldCumulative) `div` fromIntegral (stats.major_gcs - oldMajor))+ _ -> Nothing+ value = Text.pack . show+ maybeValue = maybe "" (value . toInteger)+ appendCsv writer $+ prefix+ <> fmap+ value+ [ toInteger stats.gcs,+ toInteger stats.major_gcs,+ toInteger stats.allocated_bytes,+ toInteger stats.max_live_bytes,+ toInteger stats.cumulative_live_bytes+ ]+ <> [maybeValue majorMean]+ <> fmap+ value+ [ toInteger details.gcdetails_live_bytes,+ toInteger details.gcdetails_gen,+ toInteger details.gcdetails_mem_in_use_bytes,+ toInteger stats.max_mem_in_use_bytes,+ toInteger details.gcdetails_large_objects_bytes,+ toInteger details.gcdetails_compact_bytes,+ toInteger details.gcdetails_slop_bytes,+ toInteger details.gcdetails_block_fragmentation_bytes,+ toInteger stats.copied_bytes,+ toInteger stats.gc_cpu_ns,+ toInteger stats.gc_elapsed_ns,+ toInteger stats.mutator_cpu_ns,+ toInteger stats.mutator_elapsed_ns,+ toInteger stats.cpu_ns,+ toInteger stats.elapsed_ns,+ toInteger threads,+ toInteger capabilities+ ]+ writeIORef previousRef (Just (stats.major_gcs, stats.cumulative_live_bytes))++closeRtsSampler :: RtsSampler -> IO ()+closeRtsSampler (RtsSampler writer _) = closeCsv writer
+ src/Kenshou/Measure/Samples.hs view
@@ -0,0 +1,276 @@+module Kenshou.Measure.Samples+ ( ClockKind (..),+ SampleHeader (..),+ SampleRecord (..),+ SampleWriter,+ SampleFileReport (..),+ openSampleWriter,+ writeSampleBlock,+ closeSampleWriter,+ sampleWriterBackpressure,+ readSamples,+ )+where++import Control.Concurrent.Async (Async, async, wait)+import Control.Concurrent.MVar+import Control.Concurrent.STM+import Data.Bits+import Data.ByteString (ByteString)+import Data.ByteString qualified as ByteString+import Data.ByteString.Builder+import Data.ByteString.Lazy qualified as LazyByteString+import Data.IORef+import Data.Int (Int64)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.Encoding qualified as Text+import Data.Word+import Kenshou.Measure.Clock (Origin (..))+import System.IO++data ClockKind = MonotonicClock | WallClock+ deriving stock (Eq, Show)++data SampleHeader = SampleHeader+ { origin :: !Origin,+ clockKind :: !ClockKind,+ operation :: !Text,+ processLabel :: !Text+ }+ deriving stock (Eq, Show)++data SampleRecord = SampleRecord+ { intendedStartNs :: !Word64,+ actualStartNs :: !Word64,+ endNs :: !Word64,+ outcomeCode :: !Word8,+ units :: !Word64+ }+ deriving stock (Eq, Show)++data SampleCommand = WriteBlock !Word16 ![SampleRecord] | Stop !(MVar ())++data SampleWriter = SampleWriter+ { queue :: !(TBQueue SampleCommand),+ worker :: !(Async ()),+ backpressure :: !(IORef Word64)+ }++data SampleFileReport = SampleFileReport+ { header :: !SampleHeader,+ recordsRead :: !Word64,+ ignoredBytes :: !Word64+ }+ deriving stock (Eq, Show)++openSampleWriter :: FilePath -> SampleHeader -> IO SampleWriter+openSampleWriter path header = do+ handle <- openBinaryFile path WriteMode+ ByteString.hPut handle (builderBytes (headerBuilder header))+ queue <- newTBQueueIO 256+ backpressure <- newIORef 0+ worker <- async (writerLoop handle queue)+ pure SampleWriter {queue, worker, backpressure}++writeSampleBlock :: SampleWriter -> Word16 -> [SampleRecord] -> IO ()+writeSampleBlock _ _ [] = pure ()+writeSampleBlock writer workerId records = do+ wasFull <- atomically do+ full <- isFullTBQueue writer.queue+ writeTBQueue writer.queue (WriteBlock workerId records)+ pure full+ whenFull wasFull do+ modifyIORef' writer.backpressure (+ 1)+ where+ whenFull True action = action+ whenFull False _ = pure ()++closeSampleWriter :: SampleWriter -> IO ()+closeSampleWriter writer = do+ stopped <- newEmptyMVar+ atomically (writeTBQueue writer.queue (Stop stopped))+ takeMVar stopped+ wait writer.worker++sampleWriterBackpressure :: SampleWriter -> IO Word64+sampleWriterBackpressure = readIORef . (.backpressure)++writerLoop :: Handle -> TBQueue SampleCommand -> IO ()+writerLoop handle queue = do+ command <- atomically (readTBQueue queue)+ case command of+ WriteBlock workerId records -> ByteString.hPut handle (encodeBlock workerId records) >> writerLoop handle queue+ Stop stopped -> hFlush handle >> hClose handle >> putMVar stopped ()++encodeBlock :: Word16 -> [SampleRecord] -> ByteString+encodeBlock _ [] = ByteString.empty+encodeBlock workerId records@(firstRecord : _) =+ builderBytes $+ word16LE workerId+ <> word32LE (fromIntegral (length records))+ <> word32LE (fromIntegral (ByteString.length payload))+ <> word64LE first+ <> byteString payload+ where+ first = firstRecord.intendedStartNs+ payload = builderBytes (recordsBuilder first records)++recordsBuilder :: Word64 -> [SampleRecord] -> Builder+recordsBuilder _ [] = mempty+recordsBuilder previous (record : rest) =+ putVarWord (zigZag (wordDelta record.intendedStartNs previous))+ <> putVarWord (record.actualStartNs - min record.actualStartNs record.intendedStartNs)+ <> putVarWord (record.endNs - min record.endNs record.actualStartNs)+ <> word8 record.outcomeCode+ <> putVarWord record.units+ <> recordsBuilder record.intendedStartNs rest++readSamples :: FilePath -> (SampleRecord -> IO ()) -> IO SampleFileReport+readSamples path consume = do+ bytes <- ByteString.readFile path+ case parseHeader bytes of+ Left message -> ioError (userError (Text.unpack message))+ Right (header, bodyOffset) -> do+ (count, ignored) <- readBlocks bytes bodyOffset consume+ pure (SampleFileReport header count ignored)++readBlocks :: ByteString -> Int -> (SampleRecord -> IO ()) -> IO (Word64, Word64)+readBlocks bytes = go 0+ where+ totalLength = ByteString.length bytes+ go count offset consume+ | offset == totalLength = pure (count, 0)+ | offset + 18 > totalLength = pure (count, fromIntegral (totalLength - offset))+ | otherwise = case parseBlockHeader bytes offset of+ Left _ -> pure (count, fromIntegral (totalLength - offset))+ Right (recordCount, payloadBytes, first, payloadOffset)+ | payloadOffset + payloadBytes > totalLength -> pure (count, fromIntegral (totalLength - offset))+ | otherwise -> case parseRecords bytes payloadOffset payloadBytes recordCount first of+ Left _ -> pure (count, fromIntegral (totalLength - offset))+ Right records -> mapM_ consume records >> go (count + fromIntegral (length records)) (payloadOffset + payloadBytes) consume++parseBlockHeader :: ByteString -> Int -> Either Text (Int, Int, Word64, Int)+parseBlockHeader bytes offset = do+ _workerId <- word16At bytes offset+ count <- word32At bytes (offset + 2)+ payloadBytes <- word32At bytes (offset + 6)+ first <- word64At bytes (offset + 10)+ pure (fromIntegral count, fromIntegral payloadBytes, first, offset + 18)++parseRecords :: ByteString -> Int -> Int -> Int -> Word64 -> Either Text [SampleRecord]+parseRecords bytes offset payloadLength count first = do+ (records, finalOffset) <- go [] offset first count+ ensure (finalOffset == offset + payloadLength) "sample block payload length mismatch"+ pure (reverse records)+ where+ limit = offset + payloadLength+ go records current _ 0 = Right (records, current)+ go records current previous remaining = do+ ensure (current < limit) "truncated sample block"+ (delta, afterDelta) <- readVarWord bytes current limit+ (lag, afterLag) <- readVarWord bytes afterDelta limit+ (service, afterService) <- readVarWord bytes afterLag limit+ outcome <- word8AtLimit bytes afterService limit+ (units, afterUnits) <- readVarWord bytes (afterService + 1) limit+ let intended = addSigned previous (unZigZag delta)+ actual = intended + lag+ record = SampleRecord intended actual (actual + service) outcome units+ go (record : records) afterUnits intended (remaining - 1)++headerBuilder :: SampleHeader -> Builder+headerBuilder header =+ byteString "KSMP"+ <> word16LE 1+ <> word16LE 0+ <> word64LE header.origin.monoNs+ <> int64LE header.origin.wallUnixNs+ <> word8 (case header.clockKind of MonotonicClock -> 0; WallClock -> 1)+ <> textBuilder header.operation+ <> textBuilder header.processLabel++parseHeader :: ByteString -> Either Text (SampleHeader, Int)+parseHeader bytes = do+ ensure (ByteString.take 4 bytes == "KSMP") "invalid sample magic"+ version <- word16At bytes 4+ ensure (version == 1) "unsupported sample version"+ flags <- word16At bytes 6+ ensure (flags == 0) "unsupported sample flags"+ mono <- word64At bytes 8+ wall <- fromIntegral <$> word64At bytes 16+ clockByte <- word8At bytes 24+ clock <- case clockByte of 0 -> Right MonotonicClock; 1 -> Right WallClock; _ -> Left "unknown sample clock"+ (operation, afterOperation) <- textAt bytes 25+ (processLabel, bodyOffset) <- textAt bytes afterOperation+ pure (SampleHeader (Origin mono wall) clock operation processLabel, bodyOffset)++textBuilder :: Text -> Builder+textBuilder value = word16LE (fromIntegral (ByteString.length encoded)) <> byteString encoded+ where+ encoded = Text.encodeUtf8 value++textAt :: ByteString -> Int -> Either Text (Text, Int)+textAt bytes offset = do+ lengthValue <- word16At bytes offset+ let start = offset + 2+ end = start + fromIntegral lengthValue+ ensure (end <= ByteString.length bytes) "truncated sample text"+ case Text.decodeUtf8' (ByteString.take (fromIntegral lengthValue) (ByteString.drop start bytes)) of+ Left _ -> Left "invalid UTF-8 in sample header"+ Right value -> Right (value, end)++builderBytes :: Builder -> ByteString+builderBytes = LazyByteString.toStrict . toLazyByteString++putVarWord :: Word64 -> Builder+putVarWord value+ | value < 0x80 = word8 (fromIntegral value)+ | otherwise = word8 (fromIntegral (value .&. 0x7f) .|. 0x80) <> putVarWord (value `shiftR` 7)++readVarWord :: ByteString -> Int -> Int -> Either Text (Word64, Int)+readVarWord bytes = go 0 0+ where+ go bitOffset accumulator offset limit+ | bitOffset >= 64 = Left "varint is too long"+ | offset >= limit = Left "truncated varint"+ | otherwise =+ let byte = ByteString.index bytes offset+ value = accumulator .|. (fromIntegral (byte .&. 0x7f) `shiftL` bitOffset)+ in if byte .&. 0x80 == 0 then Right (value, offset + 1) else go (bitOffset + 7) value (offset + 1) limit++wordDelta :: Word64 -> Word64 -> Int64+wordDelta current previous = fromIntegral current - fromIntegral previous++addSigned :: Word64 -> Int64 -> Word64+addSigned value delta = fromIntegral (fromIntegral value + delta :: Int64)++zigZag :: Int64 -> Word64+zigZag value = fromIntegral ((value `shiftL` 1) `xor` (value `shiftR` 63))++unZigZag :: Word64 -> Int64+unZigZag value = fromIntegral (value `shiftR` 1) `xor` negate (fromIntegral (value .&. 1))++word8At :: ByteString -> Int -> Either Text Word8+word8At bytes offset = word8AtLimit bytes offset (ByteString.length bytes)++word8AtLimit :: ByteString -> Int -> Int -> Either Text Word8+word8AtLimit bytes offset limit+ | offset < limit = Right (ByteString.index bytes offset)+ | otherwise = Left "truncated sample file"++word16At :: ByteString -> Int -> Either Text Word16+word16At bytes offset = fromIntegral <$> littleEndianAt bytes offset 2++word32At :: ByteString -> Int -> Either Text Word32+word32At bytes offset = fromIntegral <$> littleEndianAt bytes offset 4++word64At :: ByteString -> Int -> Either Text Word64+word64At bytes offset = littleEndianAt bytes offset 8++littleEndianAt :: ByteString -> Int -> Int -> Either Text Word64+littleEndianAt bytes offset width+ | offset + width > ByteString.length bytes = Left "truncated sample file"+ | otherwise = Right $ foldr (.|.) 0 [fromIntegral (ByteString.index bytes (offset + index)) `shiftL` (8 * index) | index <- [0 .. width - 1]]++ensure :: Bool -> Text -> Either Text ()+ensure condition message = if condition then Right () else Left message
+ src/Kenshou/Measure/Selftest.hs view
@@ -0,0 +1,10 @@+module Kenshou.Measure.Selftest (bundle) where++import Kenshou.Core.Bundle (LayerBundle (..), mkBundle)+import Kenshou.Core.Id (selftestLayer)+import Kenshou.Measure.Selftest.PgInsert (pgInsertScenario)+import Kenshou.Measure.Selftest.RegressionInjected (regressionInjectedScenario)+import Kenshou.Measure.Selftest.SleepService (sleepServiceScenario)++bundle :: LayerBundle+bundle = mkBundle selftestLayer [sleepServiceScenario, pgInsertScenario, regressionInjectedScenario] []
+ src/Kenshou/Measure/Selftest/PgInsert.hs view
@@ -0,0 +1,151 @@+module Kenshou.Measure.Selftest.PgInsert (pgInsertScenario) where++import Control.Exception (bracket)+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Data.Word (Word64)+import Hasql.Connection qualified as Connection+import Hasql.Connection.Settings qualified as Settings+import Hasql.Decoders qualified as Decoders+import Hasql.Encoders qualified as Encoders+import Hasql.Session qualified as Session+import Hasql.Statement qualified as Statement+import Kenshou.Core.Context (RunContext (..), requirePostgres)+import Kenshou.Core.Dimension+import Kenshou.Core.Env+import Kenshou.Core.Env.Postgres (PostgresEnv (..))+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Recorder+import Kenshou.Measure.Sampler.Postgres+import Kenshou.Measure.Session+import System.Directory (doesFileExist)+import System.FilePath ((</>))++pgInsertScenario :: Scenario+pgInsertScenario =+ Scenario+ { id = either (error . show) id (parseScenarioId "selftest/measure/benchmark/pg-insert"),+ revision = 2,+ summary = "Verifies runtime, process, host, and PostgreSQL measurement series.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = loadKnobs (defaultLoadDefaults {model = "closed", workers = 8}) <> measureKnobs Benchmark <> [payloadKnob],+ dimensions = postgresDimensions (PgDurable :| []) (Pg17 :| [Pg18]) telemetryOff,+ phases = Core.PhasePlan 3 15 2,+ requires = pgRequirements,+ knownDefect = Nothing,+ run = runPgInsert+ }++runPgInsert :: RunContext -> IO ScenarioReport+runPgInsert context = case (loadModelFromKnobs context.knobs, measureConfigFromKnobs context (phasePlanFromCore context.phases)) of+ (Left message, _) -> pure (failedWith ["invalid-load-config"] message)+ (_, Left message) -> pure (failedWith ["invalid-measure-config"] message)+ (Right loadModel, Right baseConfig) -> do+ let environment = requirePostgres context+ workers = case loadModel of ClosedLoop closed -> closed.workers; OpenLoop open -> open.executors+ payloadBytes = knobInt context.knobs (name "pg-insert.payload-bytes")+ settings = Settings.connectionString environment.connectionString <> Settings.applicationName "kenshou-selftest-writer"+ config = watchInserts baseConfig+ acquired <- acquireMany workers settings+ case acquired of+ Left message -> pure (infrastructureFailureBecause message)+ Right [] -> pure (infrastructureFailureBecause "load.workers produced no PostgreSQL connections")+ Right connections@(firstConnection : _) -> bracket (pure connections) (mapM_ Connection.release) \openConnections -> do+ setup <- Connection.use firstConnection setupSession+ case setup of+ Left err -> pure (infrastructureFailureBecause (Text.pack (show err)))+ Right () -> do+ let operation = Operation (OpName "insert") (insertOne openConnections payloadBytes)+ (_, report) <- withMeasurement context config (\measurement -> runLoad measurement loadModel operation)+ counted <- Connection.use firstConnection (Session.statement () countRows)+ assess context report counted++insertOne :: [Connection.Connection] -> Int64 -> Int -> Word64 -> IO OpResult+insertOne connections payloadBytes worker sequenceNumber = do+ let connection = connections !! (worker `mod` length connections)+ parameter = Text.intercalate "," [Text.pack (show worker), Text.pack (show sequenceNumber), Text.pack (show payloadBytes)]+ result <- Connection.use connection (Session.statement parameter insertRow)+ pure (either (const (OpFailed (ErrorCause "postgres"))) (const (OpOk 1)) result)++assess :: RunContext -> MeasurementReport -> Either error Int64 -> IO ScenarioReport+assess context report counted = do+ seriesOk <- and <$> traverse hasRows expectedSeries+ let successful = case report.recorder.operations of+ operation : _ -> sum [successes | (successes, _, _) <- Map.elems operation.phaseCounts]+ [] -> 0+ rowCount = either (const Nothing) (Just . fromIntegral) counted+ failures = ["row-count" | rowCount /= Just successful] <> ["series-rows" | not seriesOk]+ scenarioReport = if null failures then passed else failedWith failures ("row count=" <> showText rowCount <> ", recorded successes=" <> showText successful)+ pure (scenarioReport {outcome = measuredOutcome report scenarioReport.outcome})+ where+ expectedSeries = ["sampler.csv", "rts.csv", "proc.csv", "load.csv", "pg-activity.csv", "pg-checkpointer.csv", "pg-wal.csv", "pg-database.csv", "pg-relations.csv", "pg-statements.csv"]+ hasRows file = do+ let path = context.outDir </> "series" </> file+ exists <- doesFileExist path+ if not exists then pure False else (> 2) . length . Text.lines <$> Text.readFile path++acquireMany :: Int -> Settings.Settings -> IO (Either Text [Connection.Connection])+acquireMany count settings = go count []+ where+ go 0 acquired = pure (Right (reverse acquired))+ go remaining acquired = do+ result <- Connection.acquire settings+ case result of+ Left err -> mapM_ Connection.release acquired >> pure (Left (Text.pack (show err)))+ Right connection -> go (remaining - 1) (connection : acquired)++setupSession :: Session.Session ()+setupSession = do+ Session.statement () (unitStatement "CREATE SCHEMA IF NOT EXISTS kenshou_selftest")+ Session.statement () (unitStatement "CREATE TABLE IF NOT EXISTS kenshou_selftest.inserts (id bigserial PRIMARY KEY, worker int NOT NULL, seq bigint NOT NULL, payload bytea NOT NULL)")+ Session.statement () (unitStatement "TRUNCATE kenshou_selftest.inserts")++unitStatement :: Text -> Statement.Statement () ()+unitStatement sql = Statement.unpreparable sql Encoders.noParams Decoders.noResult++insertRow :: Statement.Statement Text ()+insertRow =+ Statement.unpreparable+ "INSERT INTO kenshou_selftest.inserts(worker,seq,payload) VALUES (split_part($1,',',1)::int, split_part($1,',',2)::bigint, decode(repeat('00',split_part($1,',',3)::int),'hex'))"+ (Encoders.param (Encoders.nonNullable Encoders.text))+ Decoders.noResult++countRows :: Statement.Statement () Int64+countRows = Statement.unpreparable "SELECT count(*)::bigint FROM kenshou_selftest.inserts" Encoders.noParams (Decoders.singleRow (Decoders.column (Decoders.nonNullable Decoders.int8)))++payloadKnob :: KnobSpec+payloadKnob = KnobSpec (name "pg-insert.payload-bytes") "Inserted payload size" KnobInt (VInt 256) (IntRange 1 65_536) []++telemetryOff :: DimensionSupport+telemetryOff =+ DimensionSupport+ (Supported (Support (TracingOff :| []) TracingOff))+ (Supported (Support (MetricsOff :| []) MetricsOff))+ NotApplicable+ NotApplicable++pgRequirements :: EnvRequirements+pgRequirements = EnvRequirements (Just (PostgresRequirement [] [("shared_preload_libraries", "'pg_stat_statements'")] False)) [] []++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++showText :: (Show value) => value -> Text+showText = Text.pack . show++watchInserts :: MeasureConfig -> MeasureConfig+watchInserts (MeasureConfig phases histogram rawSamples sampleInterval intervalSeconds postgres extraSamplers healthConfig) =+ MeasureConfig phases histogram rawSamples sampleInterval intervalSeconds (fmap addRelation postgres) extraSamplers healthConfig+ where+ addRelation :: PgSamplerConfig -> PgSamplerConfig+ addRelation pg = pg {relations = ["kenshou_selftest.inserts"]}
+ src/Kenshou/Measure/Selftest/RegressionInjected.hs view
@@ -0,0 +1,86 @@+module Kenshou.Measure.Selftest.RegressionInjected (regressionInjectedScenario) where++import Control.Concurrent (threadDelay)+import Data.Int (Int64)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Word (Word64)+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Dimension+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Id qualified as Id+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Random.SplitMix (mkSMGen, nextWord64)++regressionInjectedScenario :: Scenario+regressionInjectedScenario =+ Scenario+ { id = either (error . show) id (parseScenarioId "selftest/measure/benchmark/regression-injected"),+ revision = 2,+ summary = "Produces controlled slowdown and noise for comparator verification.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = loadKnobs (defaultLoadDefaults {model = "closed", workers = 4}) <> measureKnobs Benchmark <> serviceKnobs,+ dimensions = telemetryOff,+ phases = Core.PhasePlan 2 10 1,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = runRegressionInjected+ }++runRegressionInjected :: RunContext -> IO ScenarioReport+runRegressionInjected context = case (loadModelFromKnobs context.knobs, measureConfigFromKnobs context (phasePlanFromCore context.phases)) of+ (Left message, _) -> pure (failedWith ["invalid-load-config"] message)+ (_, Left message) -> pure (failedWith ["invalid-measure-config"] message)+ (Right model, Right config) -> do+ let base = knobInt context.knobs (name "service.base-latency-us")+ extra = knobInt context.knobs (name "service.extra-latency-us")+ noise = knobText context.knobs (name "service.noise")+ factor = if noise == "high" then noiseFactor (Id.unSeed context.seed) else 1+ delay = max 1 (floor (fromIntegral (base + extra) * factor))+ operation = Operation (OpName "work") (\_ _ -> threadDelay delay >> pure (OpOk 1))+ (_, report) <- withMeasurement context config (\measurement -> runLoad measurement model operation)+ let count = case report.recorder.operations of operationReport : _ -> sum [successes | (successes, _, _) <- Map.elems operationReport.phaseCounts]; [] -> 0+ scenarioReport = if count >= 1_000 then passed else failedWith ["insufficient-samples"] ("steady samples=" <> Text.pack (show count))+ pure (scenarioReport {outcome = measuredOutcome report scenarioReport.outcome})++noiseFactor :: Word64 -> Double+noiseFactor seed =+ let (word, _) = nextWord64 (mkSMGen seed)+ unit = fromIntegral word / fromIntegral (maxBound :: Word64)+ in exp (log 0.6 + unit * (log 1.6 - log 0.6))++serviceKnobs :: [KnobSpec]+serviceKnobs =+ [ intKnob "service.base-latency-us" "Base operation latency" 2_000 1 60_000_000,+ intKnob "service.extra-latency-us" "Injected operation latency" 0 0 60_000_000,+ textKnob "service.noise" "Run-level latency noise" "none" ["none", "high"],+ textKnob "service.label" "Behaviour-neutral comparison label" "a" ["a", "b"]+ ]++telemetryOff :: DimensionSupport+telemetryOff =+ DimensionSupport+ (Supported (Support (TracingOff :| []) TracingOff))+ (Supported (Support (MetricsOff :| []) MetricsOff))+ NotApplicable+ NotApplicable++intKnob :: Text -> Text -> Int64 -> Int64 -> Int64 -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt def) (IntRange low high) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []+textKnob knobName _ _ [] = error ("no values for " <> Text.unpack knobName)++name :: Text -> KnobName+name = either (error . show) id . mkKnobName
+ src/Kenshou/Measure/Selftest/SleepService.hs view
@@ -0,0 +1,142 @@+module Kenshou.Measure.Selftest.SleepService (sleepServiceScenario) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.Async (async)+import Control.Monad (void)+import Data.Aeson (encode, object, (.=))+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List (find)+import Data.List.NonEmpty (NonEmpty (..))+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Time (getCurrentTime)+import Data.Word (Word64)+import Kenshou.Core.Context (RunContext (..))+import Kenshou.Core.Dimension+import Kenshou.Core.Env (noEnvironment)+import Kenshou.Core.Id (Kind (Benchmark), parseScenarioId)+import Kenshou.Core.Knob+import Kenshou.Core.Phase qualified as Core+import Kenshou.Core.Scenario+import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Knobs+import Kenshou.Measure.Load+import Kenshou.Measure.Recorder+import Kenshou.Measure.Session+import System.Environment (lookupEnv)+import System.Posix.Process (getProcessID)+import System.Posix.Signals (sigSTOP, signalProcess)+import System.Process (createProcess, proc, waitForProcess)++sleepServiceScenario :: Scenario+sleepServiceScenario =+ Scenario+ { id = either (error . show) id (parseScenarioId "selftest/measure/benchmark/sleep-service"),+ revision = 2,+ summary = "Demonstrates intended-start latency and coordinated-omission correction.",+ tier = TierSmoke,+ placement = PlaceEither,+ knobs = loadKnobs defaults <> measureKnobs Benchmark <> serviceKnobs,+ dimensions = telemetryOff,+ phases = Core.PhasePlan 5 30 5,+ requires = noEnvironment,+ knownDefect = Nothing,+ run = runSleepService+ }+ where+ defaults = defaultLoadDefaults {model = "open-constant", workers = 1, ratePerSecond = 100, executors = 1, maxLagMs = 5_000}++runSleepService :: RunContext -> IO ScenarioReport+runSleepService context = case (loadModelFromKnobs context.knobs, measureConfigFromKnobs context (phasePlanFromCore context.phases)) of+ (Left message, _) -> pure (failedWith ["invalid-load-config"] message)+ (_, Left message) -> pure (failedWith ["invalid-measure-config"] message)+ (Right configuredModel, Right config) -> do+ serviceStart <- nowNs+ let injection = knobText context.knobs (name "selftest.inject")+ model = injectedModel injection configuredModel+ baseLatencyUs = knobInt context.knobs (name "service.base-latency-us")+ stallMs = knobInt context.knobs (name "service.stall-ms")+ stallEveryMs = knobInt context.knobs (name "service.stall-every-ms")+ operation = Operation (OpName "request") (\_ _ -> serviceCall serviceStart (fromIntegral baseLatencyUs) (fromIntegral stallMs) (fromIntegral stallEveryMs))+ (_, report) <- withMeasurement context config \measurement -> do+ startInjection injection+ runLoad measurement model operation+ let scenarioReport = if injection == "none" then assess model (fromIntegral baseLatencyUs * 1_000) report else passed+ pure (scenarioReport {outcome = measuredOutcome report scenarioReport.outcome})++serviceCall :: Word64 -> Word64 -> Word64 -> Word64 -> IO OpResult+serviceCall serviceStart baseLatencyUs stallMs stallEveryMs = do+ current <- nowNs+ let intervalNs = stallEveryMs * 1_000_000+ stallNs = stallMs * 1_000_000+ elapsed = current - min current serviceStart+ offset = elapsed `mod` intervalNs+ if offset < stallNs then sleepUntilNs (current + stallNs - offset) else pure ()+ threadDelay (fromIntegral baseLatencyUs)+ pure (OpOk 1)++assess :: LoadModel -> Word64 -> MeasurementReport -> ScenarioReport+assess model baseLatencyNs report = case find ((== OpName "request") . (.name)) report.recorder.operations of+ Nothing -> failedWith ["missing-request-report"] "request operation was not recorded"+ Just operation ->+ let latencyP99 = Histogram.valueAtQuantile operation.latency 0.99+ serviceP99 = Histogram.valueAtQuantile operation.service 0.99+ maximumLatency = Histogram.maxValue operation.latency+ failures = case model of+ OpenLoop _ -> ["intended-p99" | latencyP99 < 800_000_000 || latencyP99 > 1_000_000_000] <> ["service-p99" | serviceP99 >= 5 * baseLatencyNs]+ ClosedLoop _ -> ["closed-p99" | latencyP99 >= 5 * baseLatencyNs] <> ["closed-max" | maximumLatency <= 900_000_000]+ in if null failures then passed else failedWith failures ("unexpected latency distribution: p99=" <> showText latencyP99 <> " service-p99=" <> showText serviceP99 <> " max=" <> showText maximumLatency)++serviceKnobs :: [KnobSpec]+serviceKnobs =+ [ intKnob "service.base-latency-us" "Base service latency" 1_000 1 60_000_000,+ intKnob "service.stall-ms" "Periodic stall duration" 1_000 1 60_000,+ intKnob "service.stall-every-ms" "Period between stall starts" 10_000 2 600_000,+ textKnob "selftest.inject" "Inject a measurement health condition" "none" ["none", "backlog", "process-pause", "maintenance-notice"]+ ]++injectedModel :: Text -> LoadModel -> LoadModel+injectedModel "backlog" (OpenLoop config) = OpenLoop (config {arrival = ConstantRate 2_000, executors = 1})+injectedModel "backlog" _ = OpenLoop (OpenConfig (ConstantRate 2_000) 1 1 (OverloadConfig 5_000_000_000 3 30_000_000_000))+injectedModel _ model = model++startInjection :: Text -> IO ()+startInjection "process-pause" = void . async $ do+ threadDelay 10_000_000+ processId <- getProcessID+ (_, _, _, processHandle) <- createProcess (proc "sh" ["-c", "sleep 8; kill -CONT " <> show processId])+ signalProcess sigSTOP processId+ void (waitForProcess processHandle)+startInjection "maintenance-notice" = void . async $ do+ threadDelay 8_000_000+ path <- lookupEnv "KENSHOU_HEALTH_NOTICES"+ whenJust path \noticePath -> do+ now <- getCurrentTime+ LazyByteString.appendFile noticePath (encode (object ["schema" .= ("kenshou.health-notice/v1" :: Text), "source" .= ("selftest-maintenance" :: Text), "severity" .= ("hard" :: Text), "at" .= now, "detail" .= ("injected host maintenance notice" :: Text)]) <> "\n")+startInjection _ = pure ()++telemetryOff :: DimensionSupport+telemetryOff =+ DimensionSupport+ (Supported (Support (TracingOff :| []) TracingOff))+ (Supported (Support (MetricsOff :| []) MetricsOff))+ NotApplicable+ NotApplicable++intKnob :: Text -> Text -> Int -> Int -> Int -> KnobSpec+intKnob knobName summary def low high = KnobSpec (name knobName) summary KnobInt (VInt (fromIntegral def)) (IntRange (fromIntegral low) (fromIntegral high)) []++textKnob :: Text -> Text -> Text -> [Text] -> KnobSpec+textKnob knobName summary def (first : rest) = KnobSpec (name knobName) summary KnobText (VText def) (OneOf (VText first :| fmap VText rest)) []+textKnob knobName _ _ [] = error ("no values for " <> Text.unpack knobName)++whenJust :: Maybe value -> (value -> IO ()) -> IO ()+whenJust Nothing _ = pure ()+whenJust (Just value) action = action value++name :: Text -> KnobName+name = either (error . show) id . mkKnobName++showText :: (Show value) => value -> Text+showText = Text.pack . show
+ src/Kenshou/Measure/Session.hs view
@@ -0,0 +1,218 @@+module Kenshou.Measure.Session+ ( MeasureEnv (..),+ MeasureConfig (..),+ Measurement,+ MeasurementReport (..),+ measureEnvFromRunContext,+ phasePlanFromCore,+ measureConfigFromKnobs,+ withMeasurement,+ measurementPhaseClock,+ measurementRecorder,+ measurementConfig,+ measurementEnv,+ appendLoadReport,+ measuredOutcome,+ )+where++import Control.Exception (SomeException, mask, throwIO, try)+import Control.Monad (forM_, when)+import Data.Aeson (Value, toJSON)+import Data.IORef+import Data.Text (Text)+import Data.Word (Word64)+import Kenshou.Core.Context qualified as Core+import Kenshou.Core.Dimension (Dimensions (..), renderDurability)+import Kenshou.Core.Env.Postgres (PgSettingsSnapshot (..), PostgresEnv (..))+import Kenshou.Core.Id (Kind, ScenarioId (..), unSeed)+import Kenshou.Core.Knob (ResolvedKnobs)+import Kenshou.Core.Knob qualified as Knob+import Kenshou.Core.Log qualified as Log+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Core.Phase qualified as Core+import Kenshou.Measure.Clock+import Kenshou.Measure.Health+import Kenshou.Measure.Histogram (HistogramConfig (..), defaultHistogramConfig)+import Kenshou.Measure.Load.Types (LoadReport)+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Sampler+import Kenshou.Measure.Sampler.Postgres+import Kenshou.Measure.Summary+import System.FilePath (makeRelative)++data MeasureEnv = MeasureEnv+ { runDir :: FilePath,+ seed :: Word64,+ origin :: Origin,+ processLabel :: Maybe Text,+ scenarioKind :: Kind,+ pgDurability :: Maybe Text,+ specPhases :: Maybe PhasePlan,+ knobs :: ResolvedKnobs,+ dimensions :: Dimensions,+ postgres :: Maybe PostgresEnv,+ onPhase :: Phase -> Origin -> IO (),+ registerSection :: Text -> Value -> IO (),+ declareArtifact :: FilePath -> Text -> IO (),+ logLine :: Text -> IO ()+ }++data MeasureConfig = MeasureConfig+ { defaultPhases :: PhasePlan,+ histogram :: HistogramConfig,+ rawSamples :: RawSamplePolicy,+ sampleIntervalMs :: Int,+ intervalHistogramSeconds :: Word64,+ postgres :: Maybe PgSamplerConfig,+ extraSamplers :: [Sampler],+ healthConfig :: HealthConfig+ }++data Measurement = Measurement MeasureEnv PhaseClock Recorder MeasureConfig (IORef [LoadReport])++data MeasurementReport = MeasurementReport+ { recorder :: RecorderReport,+ loads :: [LoadReport],+ samplers :: SamplerReport,+ summary :: MeasurementSummary,+ health :: [HealthObservation]+ }++measureEnvFromRunContext :: Core.RunContext -> IO MeasureEnv+measureEnvFromRunContext context = do+ origin <- captureOrigin+ let ScenarioId _ _ scenarioKind _ = context.scenario+ Dimensions _ _ durability _ = context.dimensions+ pure+ MeasureEnv+ { runDir = context.outDir,+ seed = unSeed context.seed,+ origin,+ processLabel = Nothing,+ scenarioKind,+ pgDurability = renderDurability <$> durability,+ specPhases = Just (phasePlanFromCore context.phases),+ knobs = context.knobs,+ dimensions = context.dimensions,+ postgres = context.env.postgres,+ onPhase = \phase _ -> Log.logAt context.logger Log.Debug ("measurement phase " <> renderPhase phase) [],+ registerSection = Core.putSummary context Core.Measurements,+ declareArtifact = \path mediaType -> Core.declareMediaType context (makeRelative context.outDir path) mediaType,+ logLine = \message -> Log.logAt context.logger Log.Info message []+ }++phasePlanFromCore :: Core.PhasePlan -> PhasePlan+phasePlanFromCore plan =+ PhasePlan+ { warmUp = seconds plan.warmUpSeconds,+ steady = SteadyFor (seconds plan.steadySeconds),+ drain = seconds plan.drainSeconds+ }+ where+ seconds value = Nanos (floor (value * 1_000_000_000))++measureConfigFromKnobs :: Core.RunContext -> PhasePlan -> Either Text MeasureConfig+measureConfigFromKnobs context defaultPhases = do+ rawSamples <- case Knob.knobText context.knobs (knobName "measure.raw-samples") of+ "full" -> Right RawFull+ "sampled" -> Right (RawSampled (fromIntegral (Knob.knobInt context.knobs (knobName "measure.raw-sample-one-in"))))+ "off" -> Right RawOff+ value -> Left ("unknown measure.raw-samples " <> value)+ pure+ MeasureConfig+ { defaultPhases,+ histogram = defaultHistogramConfig {significantDigits = fromIntegral (Knob.knobInt context.knobs (knobName "measure.histogram-digits"))},+ rawSamples,+ sampleIntervalMs = fromIntegral (Knob.knobInt context.knobs (knobName "measure.sample-interval-ms")),+ intervalHistogramSeconds = fromIntegral (Knob.knobInt context.knobs (knobName "measure.interval-histogram-seconds")),+ postgres = postgresConfig,+ extraSamplers = [],+ healthConfig = defaultHealthConfig+ }+ where+ postgresConfig =+ fmap+ ( \environment ->+ PgSamplerConfig+ { connectionString = environment.connectionString,+ relations = [],+ statements = case Knob.knobText context.knobs (knobName "measure.pg-statements") of+ "off" -> PgStatementsOff+ "periodic" -> PgStatementsPeriodic+ _ -> PgStatementsSnapshots,+ serverVersionNum = environment.snapshot.serverVersionNum+ }+ )+ context.env.postgres++withMeasurement :: Core.RunContext -> MeasureConfig -> (Measurement -> IO value) -> IO (value, MeasurementReport)+withMeasurement context config action = mask \restore -> do+ env <- measureEnvFromRunContext context+ phaseClock <- newPhaseClock env.onPhase config.defaultPhases+ recorder <-+ newRecorder+ RecorderConfig+ { runDir = env.runDir,+ origin = env.origin,+ processLabel = env.processLabel,+ histogram = config.histogram,+ rawSamples = config.rawSamples,+ intervalHistogramSeconds = config.intervalHistogramSeconds,+ declareArtifact = env.declareArtifact+ }+ phaseClock+ loadReports <- newIORef []+ let measurement = Measurement env phaseClock recorder config loadReports+ sampling <-+ startSampling+ SamplerConfig+ { runDir = env.runDir,+ origin = env.origin,+ intervalMs = config.sampleIntervalMs,+ postgres = config.postgres,+ extraSamplers = config.extraSamplers,+ phaseClock,+ declareArtifact = env.declareArtifact,+ logLine = env.logLine+ }+ result <- try (restore (action measurement))+ phase <- currentPhase phaseClock+ when (phase /= Done) (enterPhase phaseClock Done)+ samplerReport <- stopSampling sampling+ recorderReport <- finishRecorder recorder+ loads <- readIORef loadReports+ case result of+ Left exception -> throwIO (exception :: SomeException)+ Right value -> do+ capturedNotices <- captureHealthNotices env.runDir+ forM_ capturedNotices (\path -> env.declareArtifact path "application/x-ndjson")+ summarized <- summarizeRunDirWithHealth config.healthConfig env.runDir+ summary <- either (ioError . userError . show) pure summarized+ env.registerSection "measurements" (toJSON summary)+ pure (value, MeasurementReport recorderReport loads samplerReport summary summary.health)++measurementPhaseClock :: Measurement -> PhaseClock+measurementPhaseClock (Measurement _ phaseClock _ _ _) = phaseClock++measurementRecorder :: Measurement -> Recorder+measurementRecorder (Measurement _ _ recorder _ _) = recorder++measurementConfig :: Measurement -> MeasureConfig+measurementConfig (Measurement _ _ _ config _) = config++measurementEnv :: Measurement -> MeasureEnv+measurementEnv (Measurement env _ _ _ _) = env++appendLoadReport :: Measurement -> LoadReport -> IO ()+appendLoadReport (Measurement _ _ _ _ loadReports) report = modifyIORef' loadReports (<> [report])++measuredOutcome :: MeasurementReport -> Outcome -> Outcome+measuredOutcome report scenarioOutcome = case healthOutcome report.health of+ Just InfrastructureFailure -> InfrastructureFailure+ Just Inconclusive | scenarioOutcome `elem` [Passed, Failed] -> Inconclusive+ _ -> scenarioOutcome++knobName :: Text -> Knob.KnobName+knobName = either (error . show) id . Knob.mkKnobName
+ src/Kenshou/Measure/Stats.hs view
@@ -0,0 +1,85 @@+module Kenshou.Measure.Stats+ ( Interval (..),+ exactQuantile,+ arithmeticMean,+ geometricMean,+ coefficientOfVariation,+ bootstrapInterval,+ studentTInterval,+ intervalEnvelope,+ )+where++import Data.Aeson (ToJSON (..), object, (.=))+import Data.List (sort)+import Data.Word (Word64)+import System.Random.SplitMix (mkSMGen, nextWord64)++data Interval = Interval {low :: Double, estimate :: Double, high :: Double}+ deriving stock (Eq, Show)++instance ToJSON Interval where+ toJSON value = object ["low" .= value.low, "estimate" .= value.estimate, "high" .= value.high]++exactQuantile :: Double -> [Double] -> Double+exactQuantile _ [] = 0+exactQuantile quantile values =+ let ordered = sort values+ position = max 0 (min 1 quantile) * fromIntegral (length ordered - 1)+ lower = floor position+ upper = ceiling position+ fraction = position - fromIntegral lower+ in ordered !! lower * (1 - fraction) + ordered !! upper * fraction++arithmeticMean :: [Double] -> Double+arithmeticMean [] = 0+arithmeticMean values = sum values / fromIntegral (length values)++geometricMean :: [Double] -> Double+geometricMean [] = 0+geometricMean values = exp (arithmeticMean (fmap log values))++coefficientOfVariation :: [Double] -> Double+coefficientOfVariation values =+ let mean = arithmeticMean values+ variance = if length values < 2 then 0 else sum [square (value - mean) | value <- values] / fromIntegral (length values - 1)+ in if mean == 0 then 0 else sqrt variance / abs mean+ where+ square value = value * value++bootstrapInterval :: Word64 -> Int -> Double -> ([Double] -> Double) -> [Double] -> Interval+bootstrapInterval _ _ _ _ [] = Interval 0 0 0+bootstrapInterval seed iterations confidence statistic values =+ let estimates = take (max 1 iterations) (resamples (mkSMGen seed))+ alpha = (1 - confidence) / 2+ in Interval (exactQuantile alpha estimates) (statistic values) (exactQuantile (1 - alpha) estimates)+ where+ count = length values+ resamples generator =+ let (sample, next) = draw count generator []+ in statistic sample : resamples next+ draw 0 generator accumulated = (accumulated, generator)+ draw remaining generator accumulated =+ let (word, next) = nextWord64 generator+ value = values !! fromIntegral (word `mod` fromIntegral count)+ in draw (remaining - 1) next (value : accumulated)++studentTInterval :: Double -> [Double] -> Interval+studentTInterval _ [] = Interval 0 0 0+studentTInterval _ values =+ let mean = arithmeticMean values+ count = length values+ variance = if count < 2 then 0 else sum [(value - mean) ^ (2 :: Int) | value <- values] / fromIntegral (count - 1)+ margin = critical (count - 1) * sqrt variance / sqrt (fromIntegral count)+ in Interval (mean - margin) mean (mean + margin)++intervalEnvelope :: Interval -> Interval -> Interval+intervalEnvelope left right = Interval (min left.low right.low) left.estimate (max left.high right.high)++critical :: Int -> Double+critical degrees+ | degrees <= 0 = 0+ | degrees <= length table = table !! (degrees - 1)+ | otherwise = 1.96+ where+ table = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045]
+ src/Kenshou/Measure/Summary.hs view
@@ -0,0 +1,287 @@+module Kenshou.Measure.Summary+ ( SummaryError (..),+ SummaryWindow (..),+ MeasurementSummary (..),+ summarizeRunDir,+ summarizeRunDirWithHealth,+ )+where++import Control.Exception (SomeException, try)+import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.List (isSuffixOf)+import Data.Map.Strict (Map)+import Data.Map.Strict qualified as Map+import Data.Maybe (catMaybes, mapMaybe)+import Data.Text (Text)+import Data.Text qualified as Text+import Data.Text.IO qualified as Text+import Data.Word (Word64)+import Kenshou.Measure.Health+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (decodeHistogram)+import Kenshou.Measure.Metrics+import Kenshou.Measure.Samples+import Kenshou.Measure.Stats (arithmeticMean)+import System.Directory (doesDirectoryExist, doesFileExist, listDirectory)+import System.FilePath ((</>))+import Text.Read (readMaybe)++newtype SummaryError = SummaryError Text deriving stock (Eq, Show)++data SummaryWindow = SummaryWindow+ { steadyStartMonoNs :: Word64,+ steadyEndMonoNs :: Word64,+ steadySeconds :: Double+ }+ deriving stock (Eq, Show)++data MeasurementSummary = MeasurementSummary+ { grade :: Text,+ gradeReasons :: [Text],+ window :: SummaryWindow,+ ops :: Map Text Value,+ metrics :: Map Text Metric,+ health :: [HealthObservation]+ }+ deriving stock (Eq, Show)++instance ToJSON SummaryWindow where+ toJSON value = object ["steadyStartMonoNs" .= value.steadyStartMonoNs, "steadyEndMonoNs" .= value.steadyEndMonoNs, "steadySeconds" .= value.steadySeconds]++instance FromJSON SummaryWindow where+ parseJSON = withObject "SummaryWindow" \value -> SummaryWindow <$> value .: "steadyStartMonoNs" <*> value .: "steadyEndMonoNs" <*> value .: "steadySeconds"++instance ToJSON MeasurementSummary where+ toJSON summary =+ object+ [ "schema" .= ("kenshou.measurements/v1" :: Text),+ "algorithm" .= object ["name" .= ("kenshou-summary" :: Text), "version" .= (1 :: Int)],+ "grade" .= summary.grade,+ "gradeReasons" .= summary.gradeReasons,+ "window" .= summary.window,+ "ops" .= summary.ops,+ "metrics" .= summary.metrics,+ "health" .= summary.health+ ]++instance FromJSON MeasurementSummary where+ parseJSON = withObject "MeasurementSummary" \value -> do+ schema <- value .: "schema"+ if schema /= ("kenshou.measurements/v1" :: Text) then fail "unsupported measurement summary" else pure ()+ MeasurementSummary <$> value .: "grade" <*> value .:? "gradeReasons" .!= [] <*> value .: "window" <*> value .: "ops" <*> value .: "metrics" <*> value .:? "health" .!= []++data Counts = Counts {successes :: Word64, failures :: Word64, units :: Word64}++data SampleMeta = SampleMeta {operation :: Text, latencyBasis :: Text, rawFull :: Bool, steadyCounts :: Counts, errorCauses :: Map Text Word64}++instance FromJSON Counts where+ parseJSON = withObject "Counts" \value -> Counts <$> value .:? "successes" .!= 0 <*> value .:? "failures" .!= 0 <*> value .:? "units" .!= 0++instance FromJSON SampleMeta where+ parseJSON = withObject "SampleMeta" \value -> do+ phaseCounts <- value .: "phaseCounts"+ steady <- withObject "phaseCounts" (\counts -> counts .:? "steady" .!= Counts 0 0 0) phaseCounts+ raw <- value .: "rawSamples"+ SampleMeta <$> value .: "operation" <*> value .:? "latencyBasis" .!= "intended-start" <*> pure (raw == String "full") <*> pure steady <*> value .:? "errorCauses" .!= Map.empty++summarizeRunDir :: FilePath -> IO (Either SummaryError MeasurementSummary)+summarizeRunDir = summarizeRunDirWithHealth defaultHealthConfig++summarizeRunDirWithHealth :: HealthConfig -> FilePath -> IO (Either SummaryError MeasurementSummary)+summarizeRunDirWithHealth healthConfig runDir = do+ exists <- doesDirectoryExist runDir+ if not exists+ then pure (Left (SummaryError "run directory does not exist"))+ else do+ result <- try (summarize healthConfig runDir)+ pure (either (Left . SummaryError . Text.pack . show) id (result :: Either SomeException (Either SummaryError MeasurementSummary)))++summarize :: HealthConfig -> FilePath -> IO (Either SummaryError MeasurementSummary)+summarize healthConfig runDir = do+ windowResult <- readWindow (runDir </> "series" </> "load.csv")+ case windowResult of+ Left err -> pure (Left err)+ Right window -> do+ files <- listDirectory (runDir </> "samples")+ operationResults <- traverse (summarizeOperation runDir window) [file | file <- files, ".meta.json" `isSuffixOf` file]+ case sequence operationResults of+ Left err -> pure (Left err)+ Right operations -> do+ let operationMap = Map.fromList [(name, value) | (name, value, _, _) <- operations]+ operationMetrics = Map.unions [values | (_, _, values, _) <- operations]+ totalOps = sum [metric.value | (name, metric) <- Map.toList operationMetrics, ".throughput" `Text.isSuffixOf` name] * window.steadySeconds+ runtimeMetrics <- summarizeSeries runDir window totalOps+ reasons <- gradeReasonsFromRun runDir (all (\(_, _, _, full) -> full) operations)+ health <- evaluateHealth healthConfig runDir+ let healthReasons = ["health:" <> item.gate | item <- health, item.severity /= Info]+ allReasons = reasons <> healthReasons+ pure (Right (MeasurementSummary (if null allReasons then "benchmark" else "exploratory") allReasons window operationMap (operationMetrics <> runtimeMetrics) health))++summarizeOperation :: FilePath -> SummaryWindow -> FilePath -> IO (Either SummaryError (Text, Value, Map Text Metric, Bool))+summarizeOperation runDir window metaFile = do+ decoded <- eitherDecodeFileStrict' (runDir </> "samples" </> metaFile) :: IO (Either String SampleMeta)+ case decoded of+ Left message -> pure (Left (SummaryError (Text.pack message)))+ Right meta -> do+ let base = Text.unpack meta.operation+ histogramPath = runDir </> "samples" </> base <> ".hist"+ servicePath = runDir </> "samples" </> base <> ".service.hist"+ rawPath = runDir </> "samples" </> base <> ".raw"+ latency <- decodeHistogram <$> ByteString.readFile histogramPath+ service <- decodeHistogram <$> ByteString.readFile servicePath+ rawOk <-+ if meta.rawFull+ then do+ rawExists <- doesFileExist rawPath+ if not rawExists then pure False else (== 0) . (.ignoredBytes) <$> readSamples rawPath (const (pure ()))+ else pure True+ case (latency, service, rawOk) of+ (_, _, False) -> pure (Left (SummaryError ("raw sample file is missing or truncated for " <> meta.operation)))+ (Left message, _, _) -> pure (Left (SummaryError message))+ (_, Left message, _) -> pure (Left (SummaryError message))+ (Right latencyHistogram, Right serviceHistogram, _) -> do+ let count = Histogram.totalCount latencyHistogram+ failures = meta.steadyCounts.failures+ throughput = fromIntegral count / window.steadySeconds+ unitsPerSecond = fromIntegral meta.steadyCounts.units / window.steadySeconds+ errorRate = if count + failures == 0 then 0 else fromIntegral failures / fromIntegral (count + failures)+ quantile histogram q = fromIntegral (Histogram.valueAtQuantile histogram q) :: Double+ latencyValues = metricSet ("op." <> meta.operation <> ".latency") latencyHistogram+ serviceP99 = quantile serviceHistogram 0.99+ metrics =+ Map.fromList+ [ ("op." <> meta.operation <> ".throughput", Metric throughput "ops/s"),+ ("op." <> meta.operation <> ".units-throughput", Metric unitsPerSecond "units/s"),+ ("op." <> meta.operation <> ".service.p99", Metric serviceP99 "ns"),+ ("op." <> meta.operation <> ".error-rate", Metric errorRate "ratio")+ ]+ <> latencyValues+ value =+ object+ [ "latencyBasis" .= meta.latencyBasis,+ "coordinatedOmissionRisk" .= (meta.latencyBasis == "actual-start"),+ "count" .= count,+ "reportedSuccesses" .= meta.steadyCounts.successes,+ "failures" .= failures,+ "errors" .= meta.errorCauses,+ "throughput" .= throughput,+ "unitsPerSecond" .= unitsPerSecond,+ "latencyNs" .= distribution latencyHistogram,+ "serviceNs" .= distribution serviceHistogram+ ]+ pure (Right (meta.operation, value, metrics, meta.rawFull))+ where+ distribution histogram =+ object+ [ "p50" .= Histogram.valueAtQuantile histogram 0.50,+ "p90" .= Histogram.valueAtQuantile histogram 0.90,+ "p99" .= Histogram.valueAtQuantile histogram 0.99,+ "p999" .= Histogram.valueAtQuantile histogram 0.999,+ "max" .= Histogram.maxValue histogram,+ "mean" .= Histogram.meanValue histogram+ ]+ metricSet prefix histogram =+ Map.fromList+ [ (prefix <> ".p50", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.50)) "ns"),+ (prefix <> ".p90", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.90)) "ns"),+ (prefix <> ".p95", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.95)) "ns"),+ (prefix <> ".p99", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.99)) "ns"),+ (prefix <> ".p999", Metric (fromIntegral (Histogram.valueAtQuantile histogram 0.999)) "ns"),+ (prefix <> ".max", Metric (fromIntegral (Histogram.maxValue histogram)) "ns"),+ (prefix <> ".mean", Metric (Histogram.meanValue histogram) "ns")+ ]++readWindow :: FilePath -> IO (Either SummaryError SummaryWindow)+readWindow path = do+ rows <- readCsv path+ let steady = [readWord (row !! 0) | row <- rows, length row > 2, row !! 2 == "steady"]+ drain = [readWord (row !! 0) | row <- rows, length row > 2, row !! 2 == "drain"]+ pure case (catMaybes steady, catMaybes drain) of+ (start : _, end : _) | end > start -> Right (SummaryWindow start end (fromIntegral (end - start) / 1e9))+ _ -> Left (SummaryError "load.csv has no complete steady window")++summarizeSeries :: FilePath -> SummaryWindow -> Double -> IO (Map Text Metric)+summarizeSeries runDir window totalOps = do+ rts <- numericRows (runDir </> "series" </> "rts.csv") "phase" "steady"+ proc <- numericRows (runDir </> "series" </> "proc.csv") "phase" "steady"+ wal <- numericRows (runDir </> "series" </> "pg-wal.csv") "phase" "steady"+ checkpointer <- numericRows (runDir </> "series" </> "pg-checkpointer.csv") "phase" "steady"+ let delta column rows = (-) <$> (Map.lookup column =<< lastMay rows) <*> (Map.lookup column =<< headMay rows)+ maximumOf column rows = maximumMaybe (mapMaybe (Map.lookup column) rows)+ meanOf column rows = case mapMaybe (Map.lookup column) rows of [] -> Nothing; values -> Just (arithmeticMean values)+ elapsed = fromIntegral (window.steadyEndMonoNs - window.steadyStartMonoNs)+ capabilities = max 1 (maybe 1 id (Map.lookup "capabilities" =<< lastMay rts))+ metrics =+ [ metric "rts.alloc-rate" "bytes/s" ((/ window.steadySeconds) <$> delta "allocated_bytes" rts),+ metric "rts.alloc-bytes-per-op" "bytes/op" ((/ max 1 totalOps) <$> delta "allocated_bytes" rts),+ metric "rts.gc-productivity" "ratio" ((\gc total -> 1 - gc / max 1 total) <$> delta "gc_elapsed_ns" rts <*> delta "elapsed_ns" rts),+ metric "rts.max-live-bytes" "bytes" (maximumOf "max_live_bytes" rts),+ metric "rts.live-bytes-major-mean" "bytes" (meanOf "live_bytes_major_mean" rts),+ metric "proc.cpu-utilisation" "ratio" ((/ (max 1 elapsed * capabilities)) <$> delta "cpu_total_ns" proc),+ metric "proc.rss-max" "bytes" (maximumOf "rss_bytes" proc),+ metric "pg.wal-bytes-per-op" "bytes/op" ((/ max 1 totalOps) <$> delta "wal_bytes" wal),+ metric "pg.checkpoints-in-window" "count" ((+) <$> delta "num_timed" checkpointer <*> delta "num_requested" checkpointer)+ ]+ pure (Map.fromList (catMaybes metrics))+ where+ metric name unit value = (name,) . (`Metric` unit) <$> value++gradeReasonsFromRun :: FilePath -> Bool -> IO [Text]+gradeReasonsFromRun runDir rawFull = do+ decoded <- eitherDecodeFileStrict' (runDir </> "run-spec.json") :: IO (Either String Value)+ let durability = case decoded of+ Right (Object root) -> do+ Object dimensions <- KeyMap.lookup "dimensions" root+ String value <- KeyMap.lookup "pg.durability" dimensions+ pure value+ _ -> Nothing+ pure (["raw samples are not retained in full" | not rawFull] <> ["pg.durability=fsync-off" | durability == Just "fsync-off"])++readCsv :: FilePath -> IO [[Text]]+readCsv path = do+ exists <- doesFileExist path+ if not exists then pure [] else fmap (Text.splitOn ",") . drop 1 . Text.lines <$> Text.readFile path++numericRows :: FilePath -> Text -> Text -> IO [Map Text Double]+numericRows path filterColumn filterValue = do+ exists <- doesFileExist path+ if not exists+ then pure []+ else do+ linesValue <- Text.lines <$> Text.readFile path+ case linesValue of+ [] -> pure []+ header : rows -> do+ let columns = Text.splitOn "," header+ decoded = fmap (Map.fromList . zip columns . fmap readDouble . Text.splitOn ",") rows+ pure [Map.mapMaybe id row | (raw, row) <- zip rows decoded, field columns filterColumn raw == Just filterValue]++field :: [Text] -> Text -> Text -> Maybe Text+field columns wanted row = do+ index <- lookupIndex wanted columns+ atMay (Text.splitOn "," row) index++lookupIndex :: (Eq value) => value -> [value] -> Maybe Int+lookupIndex wanted = go 0+ where+ go _ [] = Nothing; go index (value : rest) = if value == wanted then Just index else go (index + 1) rest++readDouble :: Text -> Maybe Double+readDouble value | Text.null value = Nothing+readDouble value = readMaybe (Text.unpack value)++readWord :: Text -> Maybe Word64+readWord = readMaybe . Text.unpack++headMay, lastMay :: [value] -> Maybe value+headMay [] = Nothing; headMay (value : _) = Just value+lastMay [] = Nothing; lastMay values = Just (last values)++maximumMaybe :: (Ord value) => [value] -> Maybe value+maximumMaybe [] = Nothing; maximumMaybe values = Just (maximum values)++atMay :: [value] -> Int -> Maybe value+atMay values index = case drop index values of value : _ -> Just value; [] -> Nothing
+ test/Kenshou/Measure/ClockSpec.hs view
@@ -0,0 +1,18 @@+module Kenshou.Measure.ClockSpec (spec) where++import Kenshou.Measure.Clock+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Clock" do+ it "sleepUntilNs never returns before the deadline" do+ start <- nowNs+ let deadline = start + 2_000_000+ sleepUntilNs deadline+ finished <- nowNs+ finished `shouldSatisfy` (>= deadline)++ it "captures paired monotonic and wall-clock origins" do+ origin <- captureOrigin+ origin.monoNs `shouldSatisfy` (> 0)+ origin.wallUnixNs `shouldSatisfy` (> 0)
+ test/Kenshou/Measure/CompareSpec.hs view
@@ -0,0 +1,141 @@+module Kenshou.Measure.CompareSpec (spec) where++import Data.Aeson (Value (Null), encode, object, (.=))+import Data.ByteString.Char8 qualified as ByteString+import Data.ByteString.Lazy qualified as LazyByteString+import Data.List.NonEmpty (NonEmpty (..))+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Core.Outcome (Outcome (..))+import Kenshou.Measure.Compare+import Kenshou.Measure.Compare.Compatibility+import Kenshou.Measure.Compare.Ordering+import Kenshou.Measure.Compare.Policy+import Kenshou.Measure.Health+import Kenshou.Measure.Stats+import Kenshou.Measure.Summary (MeasurementSummary (..), SummaryWindow (..))+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = do+ describe "comparison source references" do+ it "retains the baseline and candidate run IDs in pair order" do+ withSystemTempDirectory "kenshou-comparison-arms" $ \root -> do+ let baselineIds = ["baseline-1", "baseline-2", "baseline-3"] :: [Text]+ candidateIds = ["candidate-1", "candidate-2", "candidate-3"] :: [Text]+ baselineDirs = [root </> show index </> "baseline" | index <- [1 :: Int .. 3]]+ candidateDirs = [root </> show index </> "candidate" | index <- [1 :: Int .. 3]]+ sequence_ [writeRun dir runId "released" | (dir, runId) <- zip baselineDirs baselineIds]+ sequence_ [writeRun dir runId "head" | (dir, runId) <- zip candidateDirs candidateIds]+ compared <- compareRuns testPolicy (VaryCohort :| []) baselineDirs candidateDirs+ case compared of+ Left err -> expectationFailure (show err)+ Right result -> do+ result.baselineRuns `shouldBe` baselineIds+ result.candidateRuns `shouldBe` candidateIds+ result.startedAt `shouldSatisfy` (<= result.finishedAt)+ result.design `shouldBe` "sequential"++ describe "pairedSchedule" do+ it "assigns one shared seed to each pair in ABBA order" do+ let schedule = pairedSchedule ABBA 3 17+ fmap (.arm) schedule `shouldBe` [Baseline, Candidate, Candidate, Baseline, Baseline, Candidate]+ fmap (.pairIndex) schedule `shouldBe` [0, 0, 1, 1, 2, 2]+ fmap (.pairSeed) schedule `shouldSatisfy` pairSeedsMatch++ describe "statistics" do+ it "interpolates exact quantiles" do+ exactQuantile 0.25 [10, 0] `shouldBe` 2.5++ it "repeats bootstrap intervals exactly for a fixed seed" do+ let interval = bootstrapInterval 41 1_000 0.95 arithmeticMean [1, 2, 3, 4]+ bootstrapInterval 41 1_000 0.95 arithmeticMean [1, 2, 3, 4] `shouldBe` interval++ describe "comparison policy" do+ it "parses the explicit A/A control axis" do+ parseVaryingAxis "control" `shouldBe` Right VaryControl++ it "rejects fewer than three pairs" do+ decodePolicy (policyJson 2 1_000) `shouldSatisfy` isLeft++ it "rejects fewer than one thousand bootstrap iterations" do+ decodePolicy (policyJson 3 999) `shouldSatisfy` isLeft++ it "accepts the minimum supported policy" do+ decodePolicy (policyJson 3 1_000) `shouldSatisfy` isRight++ describe "paired metric classification" do+ it "keeps a clear regression when another metric is inconclusive" do+ decideVerdict False [] [MetricRegression, MetricInconclusive] `shouldBe` VerdictRegression++ it "reports a clear regression only after both gates are exceeded" do+ let result = compareMetricPairs testPolicy "latency" "ns" latencyRule [(100, 130), (100, 130), (100, 130)]+ result.status `shouldBe` MetricRegression++ it "passes stable identical measurements" do+ let result = compareMetricPairs testPolicy "latency" "ns" latencyRule [(100, 100), (101, 101), (99, 99)]+ result.status `shouldBe` MetricPass++ it "reports noisy mixed measurements as inconclusive" do+ let result = compareMetricPairs testPolicy "latency" "ns" latencyRule [(100, 140), (100, 60), (100, 100)]+ result.status `shouldBe` MetricInconclusive++ it "lets a hard health observation override a statistical regression" do+ decideVerdict True [] [MetricRegression] `shouldBe` VerdictInfrastructureFailure++ it "lets checkpoint asymmetry make a statistical regression inconclusive" do+ decideVerdict False ["checkpoint overlap differs"] [MetricRegression] `shouldBe` VerdictInconclusive++ describe "health outcomes" do+ it "maps soft observations to inconclusive and hard observations to infrastructure failure" do+ healthOutcome [health Soft] `shouldBe` Just Inconclusive+ healthOutcome [health Hard] `shouldBe` Just InfrastructureFailure++pairSeedsMatch :: (Eq a) => [a] -> Bool+pairSeedsMatch [firstA, firstB, secondA, secondB, thirdA, thirdB] = firstA == firstB && secondA == secondB && thirdA == thirdB+pairSeedsMatch _ = False++testPolicy :: Policy+testPolicy = Policy "test" 3 0.95 1_000 42 False "benchmark" 0.5 1 []++latencyRule :: MetricRule+latencyRule = MetricRule "latency" LowerIsBetter 0.1 5++health :: Severity -> HealthObservation+health severity = HealthObservation "test" severity 0 1 "test observation" Null++policyJson :: Int -> Int -> ByteString.ByteString+policyJson pairs iterations =+ ByteString.pack+ ( concat+ [ "{\"schema\":\"kenshou.comparison-policy/v1\",\"name\":\"test\",\"minimumPairs\":",+ show pairs,+ ",\"confidenceLevel\":0.95,\"bootstrapIterations\":",+ show iterations,+ ",\"resamplingSeed\":1,\"requireInterleaving\":false,\"requireGrade\":\"benchmark\",\"maxCiRelativeWidth\":0.5,\"maxCheckpointAsymmetry\":1,\"metrics\":[]}"+ ]+ )++writeRun :: FilePath -> Text -> Text -> IO ()+writeRun directory runId cohort = do+ createDirectoryIfMissing True directory+ let summary = MeasurementSummary "benchmark" [] (SummaryWindow 0 1 1) Map.empty Map.empty []+ document =+ object+ [ "runId" .= runId,+ "outcome" .= ("passed" :: Text),+ "compatibility" .= object ["inputs" .= object ["cohortPlanHash" .= cohort]],+ "fingerprint" .= object ["host" .= object ["os" .= ("darwin" :: Text)]],+ "summaries" .= object ["measurements" .= object ["measurements" .= summary]]+ ]+ LazyByteString.writeFile (directory </> "run-result.json") (encode document)++isLeft :: Either left right -> Bool+isLeft (Left _) = True+isLeft (Right _) = False++isRight :: Either left right -> Bool+isRight = not . isLeft
+ test/Kenshou/Measure/HistogramSpec.hs view
@@ -0,0 +1,74 @@+module Kenshou.Measure.HistogramSpec (spec) where++import Control.Monad (replicateM_)+import Data.ByteString qualified as ByteString+import Data.Char (digitToInt, isHexDigit)+import Data.Either (isLeft)+import Data.Time.Clock (diffUTCTime, getCurrentTime)+import Data.Word (Word64)+import Hedgehog+import Hedgehog.Gen qualified as Gen+import Hedgehog.Range qualified as Range+import Kenshou.Measure.Histogram+import Kenshou.Measure.Histogram.Codec+import Test.Hspec+import Test.Hspec.Hedgehog (hedgehog)++spec :: Spec+spec = describe "Kenshou.Measure.Histogram" do+ it "keeps exact aggregates and bounded quantiles" do+ histogram <- build [1_000, 2_048, 2_000_000, 3_600_000_000_000]+ totalCount histogram `shouldBe` 4+ minValue histogram `shouldBe` 1_000+ maxValue histogram `shouldBe` 3_600_000_000_000+ valueAtQuantile histogram 0.5 `shouldSatisfy` \value -> value >= 2_048 && value <= 2_050+ valueAtQuantile histogram 1 `shouldBe` 3_600_000_000_000++ it "stays within the configured 0.1 percent equivalent range" $ hedgehog do+ value <- forAll (Gen.word64 (Range.linear 1 3_600_000_000_000))+ (low, high) <- evalEither (equivalentRange defaultHistogramConfig value)+ assert (low <= value && high >= value)+ assert (high - low <= max 1 (value `div` 1_000))++ it "merges without changing the recorded distribution" do+ left <- build [1, 2, 3]+ right <- build [4, 5, 6]+ together <- build [1, 2, 3, 4, 5, 6]+ merge left right `shouldBe` Right together+ merge right left `shouldBe` Right together++ it "round-trips the KHST version 1 codec" do+ histogram <- build ([0 .. 10_000] <> [2_000_000, 3_600_000_000_001])+ decodeHistogram (encodeHistogram histogram) `shouldBe` Right histogram++ it "keeps the KHST version 1 golden bytes stable" do+ histogram <- build [1, 1_000, 2_000_000]+ golden <- decodeHex . filter isHexDigit <$> readFile "test/golden/histogram-v1.hex"+ encodeHistogram histogram `shouldBe` golden++ it "records ten million samples inside the cost budget" do+ mutable <- newHistogram defaultHistogramConfig+ started <- getCurrentTime+ replicateM_ 10_000_000 (recordValue mutable 1_000)+ finished <- getCurrentTime+ total <- totalCount <$> freeze mutable+ total `shouldBe` 10_000_000+ realToFrac (diffUTCTime finished started) `shouldSatisfy` (< (2.5 :: Double))++ it "rejects a mismatched merge configuration" do+ left <- build [1]+ mutable <- newHistogram (HistogramConfig 1 10_000 2)+ recordValue mutable 1+ right <- freeze mutable+ merge left right `shouldSatisfy` isLeft++build :: [Word64] -> IO Histogram+build values = do+ mutable <- newHistogram defaultHistogramConfig+ mapM_ (recordValue mutable) values+ freeze mutable++decodeHex :: String -> ByteString.ByteString+decodeHex [] = ByteString.empty+decodeHex (high : low : rest) = ByteString.cons (fromIntegral (digitToInt high * 16 + digitToInt low)) (decodeHex rest)+decodeHex _ = error "golden hex has an odd number of digits"
+ test/Kenshou/Measure/LoadSpec.hs view
@@ -0,0 +1,21 @@+module Kenshou.Measure.LoadSpec (spec) where++import Kenshou.Measure.Load.Arrival+import System.Random.SplitMix (mkSMGen)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Load.Arrival" do+ it "uses exact constant-rate spacing" do+ constantSchedule 100 1_000 5 `shouldBe` [1_000, 10_001_000, 20_001_000, 30_001_000, 40_001_000]++ it "replays Poisson schedules from the same seed" do+ let first = poissonSchedule 100 (mkSMGen 42) 0 1_000+ second = poissonSchedule 100 (mkSMGen 42) 0 1_000+ first `shouldBe` second++ it "keeps the Poisson mean near the requested rate" do+ let arrivals = poissonSchedule 100 (mkSMGen 42) 0 20_000+ gaps = zipWith (-) arrivals (0 : arrivals)+ meanGap = fromIntegral (sum gaps) / fromIntegral (length gaps) :: Double+ meanGap `shouldSatisfy` \value -> value > 9_700_000 && value < 10_300_000
+ test/Kenshou/Measure/MethodologySpec.hs view
@@ -0,0 +1,15 @@+module Kenshou.Measure.MethodologySpec (spec) where++import Kenshou.Core.Bundle (LayerBundle (..), mkBundle, mkRegistry)+import Kenshou.Core.Id (mkLayerId, selftestLayer)+import Kenshou.Measure.Methodology (PoolMethodology (..), poolMethodologyFor)+import Test.Hspec++spec :: Spec+spec = describe "pool methodology" do+ it "reads the methodology a layer's bundle contributes" do+ let methodology = PoolMethodology 12 (10, 13)+ other = either (error . show) id (mkLayerId "acme")+ registry <- either (fail . show) pure (mkRegistry [(mkBundle selftestLayer [] []) {methodology = Just methodology}, mkBundle other [] []])+ poolMethodologyFor registry selftestLayer `shouldBe` Just methodology+ poolMethodologyFor registry other `shouldBe` Nothing
+ test/Kenshou/Measure/PerfSpec.hs view
@@ -0,0 +1,168 @@+module Kenshou.Measure.PerfSpec (spec) where++import Data.Aeson+import Data.Aeson.KeyMap qualified as KeyMap+import Data.ByteString qualified as ByteString+import Data.Map.Strict qualified as Map+import Data.Text (Text)+import Kenshou.Core.Id (newRunId)+import Kenshou.Core.Manifest (writeManifest)+import Kenshou.Measure.Clock (Origin (..))+import Kenshou.Measure.Compare.Policy qualified as Policy+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (encodeHistogram)+import Kenshou.Measure.Metrics (Metric (..))+import Kenshou.Measure.Perf+import Kenshou.Measure.Samples+import Kenshou.Measure.Summary+import Kenshou.Perf.Plan (PerfProfile (..), ProfileCatalog (..), RequiredMetric (..))+import System.Directory (createDirectoryIfMissing, doesFileExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "performance metric admission" do+ it "cannot accept missing p99 while throughput remains present" do+ requiredMetricProblems required summary {metrics = Map.delete p99 summary.metrics}+ `shouldBe` [MissingRequiredMetric p99]+ it "classifies a changed metric unit as invalid evidence" do+ requiredMetricProblems required summary {metrics = Map.insert p99 (Metric 10 "us") summary.metrics}+ `shouldBe` [InvalidPerformanceEvidence ("metric unit mismatch: " <> p99)]+ it "keeps an undersampled required metric inconclusive" do+ let under = summary {ops = Map.singleton "publish-to-finalize" (object ["count" .= (999 :: Int)])}+ requiredMetricProblems required under `shouldBe` [InsufficientSteadySamples p99 1000 999]+ it "requires a policy rule for every required metric" do+ profile <- loadProfile+ policy <- loadPolicy+ validateProfilePolicy profile policy {Policy.metricRules = filter (not . (\rule -> Policy.matchesRule rule.match p99)) policy.metricRules}+ `shouldBe` Left ("required metric has no policy rule: " <> p99)+ it "accepts a sealed run only after raw histogram and summary reconstruction" do+ withSystemTempDirectory "kenshou-perf-admission" \directory -> do+ profile <- loadProfile+ writeRun directory+ admitted <- verifyPerformanceRun profile directory+ admitted `shouldSatisfy` isRight+ it "retains a known runtime failure even when it is nonblocking and exits zero" do+ withSystemTempDirectory "kenshou-perf-known-defect" \directory -> do+ profile <- loadProfile+ writeRun directory+ document <- eitherDecodeFileStrict' (directory </> "run-result.json") >>= either fail pure+ case document of+ Object fields -> encodeFile (directory </> "run-result.json") (Object (KeyMap.insert "exitCode" (Number 0) (KeyMap.insert "blocking" (Bool False) (KeyMap.insert "outcome" (String "failed") fields))))+ _ -> fail "fixture is not an object"+ seal directory+ admitted <- verifyPerformanceRun profile directory+ admitted `shouldBe` Left [RuntimeWorkFailure "runtime assertions failed (including nonblocking known defects)"]+ it "does not admit absent, unknown or infrastructure runtime outcomes" do+ runtimeResultProblems (object []) `shouldBe` [InvalidPerformanceEvidence "runtime outcome is missing or unsupported"]+ runtimeResultProblems (object ["outcome" .= ("infrastructure-failure" :: Text)]) `shouldBe` [InvalidPerformanceEvidence "runtime infrastructure failed"]+ it "detects storage tampering before using a retained summary" do+ withSystemTempDirectory "kenshou-perf-tamper" \directory -> do+ profile <- loadProfile+ writeRun directory+ ByteString.appendFile (directory </> "samples/publish-to-finalize.raw") "tampered"+ admitted <- verifyPerformanceRun profile directory+ admitted `shouldSatisfy` isInvalid+ it "rejects a corrupt histogram even if someone reseals its digest" do+ withSystemTempDirectory "kenshou-perf-histogram" \directory -> do+ profile <- loadProfile+ writeRun directory+ changed <- Histogram.newHistogram Histogram.defaultHistogramConfig+ mapM_ (const (Histogram.recordValue changed 20)) [1 :: Int .. 1000]+ histogram <- Histogram.freeze changed+ ByteString.writeFile (directory </> "samples/publish-to-finalize.hist") (encodeHistogram histogram)+ seal directory+ admitted <- verifyPerformanceRun profile directory+ admitted `shouldSatisfy` isInvalid+ it "rejects a missing retained required metric even with a resealed summary" do+ withSystemTempDirectory "kenshou-perf-summary" \directory -> do+ profile <- loadProfile+ writeRun directory+ recomputed <- summarizeRunDir directory >>= either (fail . show) pure+ let incomplete = recomputed {metrics = Map.delete p99 recomputed.metrics}+ writeResult directory incomplete+ seal directory+ admitted <- verifyPerformanceRun profile directory+ admitted `shouldSatisfy` isInvalid+ where+ isRight (Right _) = True+ isRight _ = False+ isInvalid (Left problems) = any invalid problems+ isInvalid _ = False+ invalid (InvalidPerformanceEvidence _) = True+ invalid _ = False++p99 :: Text+p99 = "op.publish-to-finalize.latency.p99"++required :: [RequiredMetric]+required = [RequiredMetric p99 "ns" 1000]++summary :: MeasurementSummary+summary =+ MeasurementSummary+ "benchmark"+ []+ (SummaryWindow 1000 4000 0.000003)+ (Map.singleton "publish-to-finalize" (object ["count" .= (1000 :: Int)]))+ (Map.fromList [(p99, Metric 10 "ns"), ("op.publish-to-finalize.throughput", Metric 200 "ops/s")])+ []++writeRun :: FilePath -> IO ()+writeRun directory = do+ createDirectoryIfMissing True (directory </> "samples")+ createDirectoryIfMissing True (directory </> "series")+ ByteString.writeFile (directory </> "series/load.csv") "mono_ns,wall,phase\n1000,0,steady\n4000,0,drain\n"+ encodeFile (directory </> "run-spec.json") (object ["dimensions" .= object []])+ mutable <- Histogram.newHistogram Histogram.defaultHistogramConfig+ mapM_ (const (Histogram.recordValue mutable 10)) [1 :: Int .. 1000]+ histogram <- Histogram.freeze mutable+ let base = directory </> "samples/publish-to-finalize"+ ByteString.writeFile (base <> ".hist") (encodeHistogram histogram)+ ByteString.writeFile (base <> ".service.hist") (encodeHistogram histogram)+ writer <- openSampleWriter (base <> ".raw") (SampleHeader (Origin 0 0) MonotonicClock "publish-to-finalize" "fixture")+ writeSampleBlock writer 0 [SampleRecord (1000 + index) (1000 + index) (1010 + index) 0 1 | index <- [0 .. 999]]+ closeSampleWriter writer+ encodeFile+ (base <> ".meta.json")+ ( object+ [ "operation" .= ("publish-to-finalize" :: Text),+ "latencyBasis" .= ("intended-start" :: Text),+ "rawSamples" .= ("full" :: Text),+ "phaseCounts" .= object ["steady" .= object ["successes" .= (1000 :: Int), "failures" .= (0 :: Int), "units" .= (1000 :: Int)]]+ ]+ )+ recomputed <- summarizeRunDir directory >>= either (fail . show) pure+ writeResult directory recomputed+ seal directory++writeResult :: FilePath -> MeasurementSummary -> IO ()+writeResult directory measurements =+ encodeFile+ (directory </> "run-result.json")+ (object ["outcome" .= ("passed" :: Text), "summaries" .= object ["measurements" .= object ["measurements" .= measurements]]])++seal :: FilePath -> IO ()+seal directory = do+ runId <- newRunId+ manifest <- writeManifest directory runId Map.empty+ encodeFile (directory </> "manifest.json") manifest++loadProfile :: IO PerfProfile+loadProfile = do+ path <- repositoryFile "fixtures/consumer-baseline/policies/performance/catalog.v1.json"+ catalog <- eitherDecodeFileStrict' path >>= either fail pure+ case (catalog :: ProfileCatalog).profiles of+ profile : _ -> pure profile+ [] -> fail "empty profile catalog"++loadPolicy :: IO Policy.Policy+loadPolicy = do+ path <- repositoryFile "policies/default.json"+ ByteString.readFile path >>= either (fail . show) pure . Policy.decodePolicy++repositoryFile :: FilePath -> IO FilePath+repositoryFile path = do+ exists <- doesFileExist path+ pure (if exists then path else "../" <> path)
+ test/Kenshou/Measure/RecorderSpec.hs view
@@ -0,0 +1,88 @@+module Kenshou.Measure.RecorderSpec (spec) where++import Data.ByteString qualified as ByteString+import Data.IORef+import Kenshou.Measure.Clock+import Kenshou.Measure.Histogram qualified as Histogram+import Kenshou.Measure.Histogram.Codec (decodeHistogram)+import Kenshou.Measure.Phase+import Kenshou.Measure.Recorder+import Kenshou.Measure.Samples (SampleFileReport (..), SampleRecord (..), readSamples)+import System.Directory (doesFileExist)+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Recorder" do+ it "retains warm-up raw samples while excluding them from steady histograms" $ withSystemTempDirectory "kenshou-recorder" \directory -> do+ origin <- captureOrigin+ clock <- newPhaseClock (\_ _ -> pure ()) (PhasePlan (Nanos 1) (SteadyFor (Nanos 1)) (Nanos 1))+ recorder <-+ newRecorder+ RecorderConfig+ { runDir = directory,+ origin,+ processLabel = Nothing,+ histogram = Histogram.defaultHistogramConfig,+ rawSamples = RawFull,+ intervalHistogramSeconds = 10,+ declareArtifact = \_ _ -> pure ()+ }+ clock+ operation <- registerOp recorder (OpName "request")+ worker <- newWorkerRecorder operation 1+ enterPhase clock WarmUp+ warm <- nowNs+ recordOp worker warm warm (warm + 10) (OpOk 1)+ enterPhase clock Steady+ steady <- nowNs+ recordOp worker steady steady (steady + 20) (OpOk 1)+ recordOp worker (steady + 30) (steady + 35) (steady + 60) (OpFailed (ErrorCause "store"))+ report <- finishRecorder recorder+ case report.operations of+ [operationReport] -> do+ Histogram.totalCount operationReport.latency `shouldBe` 2+ Histogram.valueAtQuantile operationReport.latency 1 `shouldBe` 30+ doesFileExist (directory </> "samples/request.hist") `shouldReturn` True+ encoded <- ByteString.readFile (directory </> "samples/request.hist")+ fmap Histogram.totalCount (decodeHistogram encoded) `shouldBe` Right 2+ rawCount <- newIORef (0 :: Int)+ recomputedMutable <- Histogram.newHistogram Histogram.defaultHistogramConfig+ rawReport <- readSamples (directory </> "samples/request.raw") \(SampleRecord intended _ end _ _) -> do+ modifyIORef' rawCount (+ 1)+ if intended >= steady then Histogram.recordValue recomputedMutable (end - intended) else pure ()+ readIORef rawCount `shouldReturn` 3+ recomputed <- Histogram.freeze recomputedMutable+ recomputed `shouldBe` operationReport.latency+ let SampleFileReport _ _ ignored = rawReport+ ignored `shouldBe` 0+ unexpected -> expectationFailure ("expected one operation report, got " <> show (length unexpected))++ it "writes interval histograms when full raw retention is disabled" $ withSystemTempDirectory "kenshou-intervals" \directory -> do+ origin <- captureOrigin+ clock <- newPhaseClock (\_ _ -> pure ()) (PhasePlan (Nanos 1) (SteadyFor (Nanos 1)) (Nanos 1))+ recorder <-+ newRecorder+ RecorderConfig+ { runDir = directory,+ origin,+ processLabel = Nothing,+ histogram = Histogram.defaultHistogramConfig,+ rawSamples = RawOff,+ intervalHistogramSeconds = 10,+ declareArtifact = \_ _ -> pure ()+ }+ clock+ operation <- registerOp recorder (OpName "request")+ worker <- newWorkerRecorder operation 1+ enterPhase clock Steady+ started <- nowNs+ recordOp worker started started (started + 50) (OpOk 1)+ report <- finishRecorder recorder+ case report.operations of+ [operationReport] -> do+ operationReport.rawFile `shouldBe` Nothing+ operationReport.intervalHistogramFile `shouldBe` Just (directory </> "samples/request.ihist")+ doesFileExist (directory </> "samples/request.ihist") `shouldReturn` True+ unexpected -> expectationFailure ("expected one operation report, got " <> show (length unexpected))
+ test/Kenshou/Measure/SamplerSpec.hs view
@@ -0,0 +1,31 @@+module Kenshou.Measure.SamplerSpec (spec) where++import Data.Text.IO qualified as Text+import Kenshou.Measure.Sampler.Host+import Kenshou.Measure.Sampler.Process+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Sampler parsers" do+ it "parses Linux process statistics after a command name containing spaces and parentheses" do+ stat <- Text.readFile "test/fixtures/proc/stat.txt"+ status <- Text.readFile "test/fixtures/proc/status.txt"+ let sample = parseProcSample stat status 17+ sample.rssBytes `shouldBe` Just (1_024 * 1_024)+ sample.rssMaxBytes `shouldBe` Just (2_048 * 1_024)+ sample.osThreads `shouldBe` Just 9+ sample.openFds `shouldBe` Just 17+ sample.cpuUserNs `shouldBe` Just 1_100_000_000+ sample.cpuSystemNs `shouldBe` Just 220_000_000+ sample.voluntaryContextSwitches `shouldBe` Just 31+ sample.nonvoluntaryContextSwitches `shouldBe` Just 7++ it "parses aggregate Linux host counters" do+ stat <- Text.readFile "test/fixtures/proc/host-stat.txt"+ loadavg <- Text.readFile "test/fixtures/proc/loadavg.txt"+ meminfo <- Text.readFile "test/fixtures/proc/meminfo.txt"+ let sample = parseHostSample stat loadavg meminfo+ sample.cpuUser `shouldBe` Just 100+ sample.cpuSteal `shouldBe` Just 8+ sample.loadAverage1 `shouldBe` Just 1.25+ sample.memAvailableBytes `shouldBe` Just (4_096 * 1_024)
+ test/Kenshou/Measure/SamplesSpec.hs view
@@ -0,0 +1,27 @@+module Kenshou.Measure.SamplesSpec (spec) where++import Data.IORef+import Kenshou.Measure.Clock+import Kenshou.Measure.Samples+import System.FilePath ((</>))+import System.IO.Temp (withSystemTempDirectory)+import Test.Hspec++spec :: Spec+spec = describe "Kenshou.Measure.Samples" do+ it "round-trips worker blocks through KSMP version 1" $ withSystemTempDirectory "kenshou-samples" \directory -> do+ origin <- captureOrigin+ let path = directory </> "request.raw"+ records =+ [ SampleRecord (origin.monoNs + 10) (origin.monoNs + 12) (origin.monoNs + 22) 0 1,+ SampleRecord (origin.monoNs + 30) (origin.monoNs + 35) (origin.monoNs + 50) 1 0+ ]+ writer <- openSampleWriter path (SampleHeader origin MonotonicClock "request" "harness")+ writeSampleBlock writer 3 records+ closeSampleWriter writer+ observed <- newIORef []+ report <- readSamples path (\record -> modifyIORef' observed (<> [record]))+ readIORef observed `shouldReturn` records+ let SampleFileReport _ recordsRead ignoredBytes = report+ recordsRead `shouldBe` 2+ ignoredBytes `shouldBe` 0
+ test/Main.hs view
@@ -0,0 +1,24 @@+module Main (main) where++import Kenshou.Measure.ClockSpec qualified+import Kenshou.Measure.CompareSpec qualified+import Kenshou.Measure.HistogramSpec qualified+import Kenshou.Measure.LoadSpec qualified+import Kenshou.Measure.MethodologySpec qualified+import Kenshou.Measure.PerfSpec qualified+import Kenshou.Measure.RecorderSpec qualified+import Kenshou.Measure.SamplerSpec qualified+import Kenshou.Measure.SamplesSpec qualified+import Test.Hspec (hspec)++main :: IO ()+main = hspec do+ Kenshou.Measure.ClockSpec.spec+ Kenshou.Measure.CompareSpec.spec+ Kenshou.Measure.PerfSpec.spec+ Kenshou.Measure.HistogramSpec.spec+ Kenshou.Measure.LoadSpec.spec+ Kenshou.Measure.MethodologySpec.spec+ Kenshou.Measure.SamplesSpec.spec+ Kenshou.Measure.RecorderSpec.spec+ Kenshou.Measure.SamplerSpec.spec
+ test/fixtures/health-notice.json view
@@ -0,0 +1,7 @@+{+ "schema": "kenshou.health-notice/v1",+ "source": "selftest-maintenance",+ "severity": "hard",+ "at": "2026-09-21T16:00:00Z",+ "detail": "fixture host-maintenance notice"+}
+ test/fixtures/proc/host-stat.txt view
@@ -0,0 +1,2 @@+cpu 100 2 30 400 5 6 7 8 0 0+cpu0 50 1 15 200 2 3 4 4 0 0
+ test/fixtures/proc/loadavg.txt view
@@ -0,0 +1,1 @@+1.25 0.75 0.50 2/100 123
+ test/fixtures/proc/meminfo.txt view
@@ -0,0 +1,2 @@+MemTotal: 8192 kB+MemAvailable: 4096 kB
+ test/fixtures/proc/stat.txt view
@@ -0,0 +1,1 @@+123 (kenshou worker) S 1 2 3 4 5 6 7 8 9 10 110 22 0 0 20 0 9 0 1000 2000 300 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+ test/fixtures/proc/status.txt view
@@ -0,0 +1,6 @@+Name: kenshou worker+VmHWM: 2048 kB+VmRSS: 1024 kB+Threads: 9+voluntary_ctxt_switches: 31+nonvoluntary_ctxt_switches: 7
+ test/golden/histogram-v1.hex view
@@ -0,0 +1,1 @@+4b48535401000000010000000000000000a0b83046030000030300000000000000010000000000000080841e000000000069881e00000000000000000000000000070000000102cb0f02efae0102bbd102