packages feed

kenshou-measure-0.1.0.0: test/Kenshou/Measure/PerfSpec.hs

module Kenshou.Measure.PerfSpec (spec) where

import Data.Aeson
import Data.Aeson.KeyMap qualified as KeyMap
import Data.ByteString qualified as ByteString
import Data.Map.Strict qualified as Map
import Data.Text (Text)
import Kenshou.Core.Id (newRunId)
import Kenshou.Core.Manifest (writeManifest)
import Kenshou.Measure.Clock (Origin (..))
import Kenshou.Measure.Compare.Policy qualified as Policy
import Kenshou.Measure.Histogram qualified as Histogram
import Kenshou.Measure.Histogram.Codec (encodeHistogram)
import Kenshou.Measure.Metrics (Metric (..))
import Kenshou.Measure.Perf
import Kenshou.Measure.Samples
import Kenshou.Measure.Summary
import Kenshou.Perf.Plan (PerfProfile (..), ProfileCatalog (..), RequiredMetric (..))
import System.Directory (createDirectoryIfMissing, doesFileExist)
import System.FilePath ((</>))
import System.IO.Temp (withSystemTempDirectory)
import Test.Hspec

spec :: Spec
spec = describe "performance metric admission" do
  it "cannot accept missing p99 while throughput remains present" do
    requiredMetricProblems required summary {metrics = Map.delete p99 summary.metrics}
      `shouldBe` [MissingRequiredMetric p99]
  it "classifies a changed metric unit as invalid evidence" do
    requiredMetricProblems required summary {metrics = Map.insert p99 (Metric 10 "us") summary.metrics}
      `shouldBe` [InvalidPerformanceEvidence ("metric unit mismatch: " <> p99)]
  it "keeps an undersampled required metric inconclusive" do
    let under = summary {ops = Map.singleton "publish-to-finalize" (object ["count" .= (999 :: Int)])}
    requiredMetricProblems required under `shouldBe` [InsufficientSteadySamples p99 1000 999]
  it "requires a policy rule for every required metric" do
    profile <- loadProfile
    policy <- loadPolicy
    validateProfilePolicy profile policy {Policy.metricRules = filter (not . (\rule -> Policy.matchesRule rule.match p99)) policy.metricRules}
      `shouldBe` Left ("required metric has no policy rule: " <> p99)
  it "accepts a sealed run only after raw histogram and summary reconstruction" do
    withSystemTempDirectory "kenshou-perf-admission" \directory -> do
      profile <- loadProfile
      writeRun directory
      admitted <- verifyPerformanceRun profile directory
      admitted `shouldSatisfy` isRight
  it "retains a known runtime failure even when it is nonblocking and exits zero" do
    withSystemTempDirectory "kenshou-perf-known-defect" \directory -> do
      profile <- loadProfile
      writeRun directory
      document <- eitherDecodeFileStrict' (directory </> "run-result.json") >>= either fail pure
      case document of
        Object fields -> encodeFile (directory </> "run-result.json") (Object (KeyMap.insert "exitCode" (Number 0) (KeyMap.insert "blocking" (Bool False) (KeyMap.insert "outcome" (String "failed") fields))))
        _ -> fail "fixture is not an object"
      seal directory
      admitted <- verifyPerformanceRun profile directory
      admitted `shouldBe` Left [RuntimeWorkFailure "runtime assertions failed (including nonblocking known defects)"]
  it "does not admit absent, unknown or infrastructure runtime outcomes" do
    runtimeResultProblems (object []) `shouldBe` [InvalidPerformanceEvidence "runtime outcome is missing or unsupported"]
    runtimeResultProblems (object ["outcome" .= ("infrastructure-failure" :: Text)]) `shouldBe` [InvalidPerformanceEvidence "runtime infrastructure failed"]
  it "detects storage tampering before using a retained summary" do
    withSystemTempDirectory "kenshou-perf-tamper" \directory -> do
      profile <- loadProfile
      writeRun directory
      ByteString.appendFile (directory </> "samples/publish-to-finalize.raw") "tampered"
      admitted <- verifyPerformanceRun profile directory
      admitted `shouldSatisfy` isInvalid
  it "rejects a corrupt histogram even if someone reseals its digest" do
    withSystemTempDirectory "kenshou-perf-histogram" \directory -> do
      profile <- loadProfile
      writeRun directory
      changed <- Histogram.newHistogram Histogram.defaultHistogramConfig
      mapM_ (const (Histogram.recordValue changed 20)) [1 :: Int .. 1000]
      histogram <- Histogram.freeze changed
      ByteString.writeFile (directory </> "samples/publish-to-finalize.hist") (encodeHistogram histogram)
      seal directory
      admitted <- verifyPerformanceRun profile directory
      admitted `shouldSatisfy` isInvalid
  it "rejects a missing retained required metric even with a resealed summary" do
    withSystemTempDirectory "kenshou-perf-summary" \directory -> do
      profile <- loadProfile
      writeRun directory
      recomputed <- summarizeRunDir directory >>= either (fail . show) pure
      let incomplete = recomputed {metrics = Map.delete p99 recomputed.metrics}
      writeResult directory incomplete
      seal directory
      admitted <- verifyPerformanceRun profile directory
      admitted `shouldSatisfy` isInvalid
  where
    isRight (Right _) = True
    isRight _ = False
    isInvalid (Left problems) = any invalid problems
    isInvalid _ = False
    invalid (InvalidPerformanceEvidence _) = True
    invalid _ = False

p99 :: Text
p99 = "op.publish-to-finalize.latency.p99"

required :: [RequiredMetric]
required = [RequiredMetric p99 "ns" 1000]

summary :: MeasurementSummary
summary =
  MeasurementSummary
    "benchmark"
    []
    (SummaryWindow 1000 4000 0.000003)
    (Map.singleton "publish-to-finalize" (object ["count" .= (1000 :: Int)]))
    (Map.fromList [(p99, Metric 10 "ns"), ("op.publish-to-finalize.throughput", Metric 200 "ops/s")])
    []

writeRun :: FilePath -> IO ()
writeRun directory = do
  createDirectoryIfMissing True (directory </> "samples")
  createDirectoryIfMissing True (directory </> "series")
  ByteString.writeFile (directory </> "series/load.csv") "mono_ns,wall,phase\n1000,0,steady\n4000,0,drain\n"
  encodeFile (directory </> "run-spec.json") (object ["dimensions" .= object []])
  mutable <- Histogram.newHistogram Histogram.defaultHistogramConfig
  mapM_ (const (Histogram.recordValue mutable 10)) [1 :: Int .. 1000]
  histogram <- Histogram.freeze mutable
  let base = directory </> "samples/publish-to-finalize"
  ByteString.writeFile (base <> ".hist") (encodeHistogram histogram)
  ByteString.writeFile (base <> ".service.hist") (encodeHistogram histogram)
  writer <- openSampleWriter (base <> ".raw") (SampleHeader (Origin 0 0) MonotonicClock "publish-to-finalize" "fixture")
  writeSampleBlock writer 0 [SampleRecord (1000 + index) (1000 + index) (1010 + index) 0 1 | index <- [0 .. 999]]
  closeSampleWriter writer
  encodeFile
    (base <> ".meta.json")
    ( object
        [ "operation" .= ("publish-to-finalize" :: Text),
          "latencyBasis" .= ("intended-start" :: Text),
          "rawSamples" .= ("full" :: Text),
          "phaseCounts" .= object ["steady" .= object ["successes" .= (1000 :: Int), "failures" .= (0 :: Int), "units" .= (1000 :: Int)]]
        ]
    )
  recomputed <- summarizeRunDir directory >>= either (fail . show) pure
  writeResult directory recomputed
  seal directory

writeResult :: FilePath -> MeasurementSummary -> IO ()
writeResult directory measurements =
  encodeFile
    (directory </> "run-result.json")
    (object ["outcome" .= ("passed" :: Text), "summaries" .= object ["measurements" .= object ["measurements" .= measurements]]])

seal :: FilePath -> IO ()
seal directory = do
  runId <- newRunId
  manifest <- writeManifest directory runId Map.empty
  encodeFile (directory </> "manifest.json") manifest

loadProfile :: IO PerfProfile
loadProfile = do
  path <- repositoryFile "fixtures/consumer-baseline/policies/performance/catalog.v1.json"
  catalog <- eitherDecodeFileStrict' path >>= either fail pure
  case (catalog :: ProfileCatalog).profiles of
    profile : _ -> pure profile
    [] -> fail "empty profile catalog"

loadPolicy :: IO Policy.Policy
loadPolicy = do
  path <- repositoryFile "policies/default.json"
  ByteString.readFile path >>= either (fail . show) pure . Policy.decodePolicy

repositoryFile :: FilePath -> IO FilePath
repositoryFile path = do
  exists <- doesFileExist path
  pure (if exists then path else "../" <> path)