golds-gym 0.5.0.0 → 0.6.0.0

raw patch · 8 files changed

+442/−47 lines, 8 filesdep −benchpressPVP ok

version bump matches the API change (PVP)

Dependencies removed: benchpress

API changes (from Hackage documentation)

Files

• CHANGELOG.md +21/−0
• README.md +2/−0
• example/Spec.hs +24/−0
• golds-gym.cabal +5/−4
• src/Test/Hspec/BenchGolden.hs +128/−2
• src/Test/Hspec/BenchGolden/CSV.hs +128/−0
• src/Test/Hspec/BenchGolden/Runner.hs +132/−41
• src/Test/Hspec/BenchGolden/Types.hs +2/−0

@@ -1,5 +1,26 @@ # Changelog +## [0.6.0]++### Added++- **Parameter sweep support** for benchmarking scaling behaviour+  - `benchGoldenSweep` combinator for simple parameter sweeps with default config and `benchGoldenSweepWith` combinator for sweeps with custom configuration+  - Individual golden files created per parameter value (e.g., `sort-scaling_n_1000.golden`)+  - Automatic regression detection per sweep point using existing tolerance logic+  - NB: Each timing sample now runs multiple inner iterations (leveraging the SPEC trick in `nf`/`nfIO` combinators) and divides by the iteration count+  - Unified timing path: `runBenchmark` now always uses `runBenchmarkWithRawTimings` for consistent, correct measurements++- **CSV export** for analysis and plotting+  - New `Test.Hspec.BenchGolden.CSV` module+  - Single CSV file per sweep with architecture in filename (e.g., `sort-scaling-aarch64-darwin-Apple_M1.csv`)+  - Columns: timestamp, parameter value, mean, stddev, median, min, max, trimmed_mean, mad, iqr+  - Built with `Text.Builder` for efficient serialization++### Removed++- **benchpress dependency** - timing is now done via `getCPUTime` from base, without `benchpress`+ ## [0.5.0]  ### Added

@@ -4,6 +4,8 @@  Golden testing for performance benchmarks. Save timing baselines on first run, compare against them on subsequent runs. +The library was originally inspired by (and dependent on) `benchpress`, but now has independent measurement routines.+ **Key Features:** - Architecture-specific baselines (different hardware = different golden files) - Hybrid tolerance (handles both fast <1ms and slow operations)

@@ -1,3 +1,5 @@+{-# LANGUAGE OverloadedStrings #-}+ -- | Example benchmark golden tests demonstrating golds-gym usage. module Main (main) where @@ -140,6 +142,28 @@       (defaultBenchConfig { useRobustStatistics = True, iterations = 1000 })       [expect _statsTrimmedMean (Percent 25.0)]       $ nf fib 26++  -- ============================================================================+  -- Parameter Sweeps+  -- ============================================================================+  +  describe "Parameter Sweeps" $ do+    -- Simple parameter sweep: measure how sort scales with input size+    -- This produces:+    --   - Individual golden files: .golden/<arch>/sort-scaling_n=500.golden, etc.+    --   - Single CSV file: .golden/sort-scaling-<arch>.csv+    benchGoldenSweep "sort-scaling" "n" [500, 2000, 5000] $+      \n -> nf sort [n, n-1..1 :: Int]+    +    -- Sweep with custom configuration+    benchGoldenSweepWith+      defaultBenchConfig+        { iterations = 500+        , tolerancePercent = 20.0+        , warmupIterations = 10+        }+      "fib-scaling" "depth" [10, 20, 30] $+      \depth -> nf fib depth   

@@ -1,6 +1,6 @@ cabal-version:      3.0 name:               golds-gym-version:            0.5.0.0+version:            0.6.0.0 synopsis:           Golden testing framework for performance benchmarks description:     A Haskell framework for golden testing of timing benchmarks.@@ -8,7 +8,7 @@     against on subsequent runs. Golden files are architecture-specific     to account for hardware differences.     .-    Based on hspec and benchpress.+    Built on hspec.  license:            MIT license-file:       LICENSE@@ -26,11 +26,11 @@         Test.Hspec.BenchGolden.Arch         Test.Hspec.BenchGolden.Runner         Test.Hspec.BenchGolden.Lenses+        Test.Hspec.BenchGolden.CSV      build-depends:         base >= 4.14 && < 5,         hspec-core >= 2.10 && < 3,-        benchpress >= 0.2 && < 0.3,         aeson >= 2.0 && < 3,         directory >= 1.3 && < 2,         filepath >= 1.4 && < 2,@@ -57,7 +57,8 @@     build-depends:         base >= 4.14 && < 5,         golds-gym,-        hspec >= 2.10 && < 3+        hspec >= 2.10 && < 3,+        text >= 1.2 && < 3  test-suite golds-gym-properties     type:             exitcode-stdio-1.0

@@ -14,13 +14,16 @@ -- = Overview -- -- @golds-gym@ is a framework for golden testing of performance benchmarks.--- It integrates with hspec and uses benchpress for lightweight timing measurements.+-- It integrates with hspec and uses CPU time measurements for benchmarking. -- -- Benchmarks can use robust statistics to mitigate the impact of outliers. -- -- The library can be used both to assert that performance does not regress, and to set expectations -- for improvements across project versions (see `benchGoldenWithExpectation`). --+-- There are also combinators for parameter sweep benchmarks that generate CSV files for analysis and plotting, +-- see `benchGoldenSweep` and `benchGoldenSweepWith`.+-- -- = Quick Start -- -- @@@ -178,6 +181,10 @@   , benchGoldenWith   , benchGoldenWithExpectation +    -- * Parameter Sweeps+  , benchGoldenSweep+  , benchGoldenSweepWith+     -- * Configuration   , BenchConfig(..)   , defaultBenchConfig@@ -218,7 +225,7 @@ import Test.Hspec.BenchGolden.Arch import qualified Test.Hspec.BenchGolden.Lenses as L import Test.Hspec.BenchGolden.Lenses hiding (Expectation)-import Test.Hspec.BenchGolden.Runner (runBenchGolden, setAcceptGoldens, setSkipBenchmarks, nf, nfIO, nfAppIO, io)+import Test.Hspec.BenchGolden.Runner (runBenchGolden, runSweep, setAcceptGoldens, setSkipBenchmarks, nf, nfIO, nfAppIO, io) import Test.Hspec.BenchGolden.Types  -- | Create a benchmark golden test with default configuration.@@ -284,6 +291,125 @@     , benchAction = action     , benchConfig = config     }++-- | Create a parameter sweep benchmark with default configuration.+--+-- This combinator runs the same benchmark with multiple parameter values,+-- saving individual golden files for each point and producing a single CSV+-- file for analysis and plotting.+--+-- Example:+--+-- @+-- describe "Scaling Tests" $ do+--   benchGoldenSweep "sort-scaling" "n" [1000, 5000, 10000, 50000] $+--     \\n -> nf sort [n, n-1..1]+-- @+--+-- This produces:+--+-- * Golden files: @.golden\/\<arch\>\/sort-scaling_n=1000.golden@, etc.+-- * CSV file: @.golden\/sort-scaling-\<arch\>.csv@+benchGoldenSweep ::+     Show a+  => String         -- ^ Sweep name (used for CSV filename and golden file prefix)+  -> T.Text         -- ^ Parameter name (for CSV column header)+  -> [a]            -- ^ Parameter values to sweep over+  -> (a -> BenchAction)  -- ^ Action parameterized by sweep value+  -> Spec+benchGoldenSweep = benchGoldenSweepWith defaultBenchConfig++-- | Create a parameter sweep benchmark with custom configuration.+--+-- Example:+--+-- @+-- describe "Performance Scaling" $ do+--   benchGoldenSweepWith+--     defaultBenchConfig { iterations = 500, tolerancePercent = 10.0 }+--     "algorithm-scaling" "size" [100, 500, 1000, 5000] $+--     \\size -> nf myAlgorithm (generateInput size)+-- @+--+-- The CSV file includes columns for timestamp, parameter value, and all+-- standard statistics (mean, stddev, median, min, max, etc.).+benchGoldenSweepWith ::+     Show a+  => BenchConfig    -- ^ Configuration parameters+  -> String         -- ^ Sweep name+  -> T.Text         -- ^ Parameter name (for CSV column header)+  -> [a]            -- ^ Parameter values to sweep over+  -> (a -> BenchAction)  -- ^ Action parameterized by sweep value+  -> Spec+benchGoldenSweepWith config sweepName paramName paramValues mkAction =+  it (sweepName ++ " [sweep]") $ BenchGoldenSweep sweepName config paramName paramValues mkAction++-- | Internal type for sweep benchmarks.+data BenchGoldenSweep a = BenchGoldenSweep+  !String            -- Sweep name+  !BenchConfig       -- Config+  !T.Text            -- Parameter name+  ![a]               -- Parameter values+  !(a -> BenchAction) -- Action generator++-- | Instance for sweep benchmarks.+instance Show a => Example (BenchGoldenSweep a) where+  type Arg (BenchGoldenSweep a) = ()+  evaluateExample (BenchGoldenSweep sweepName config paramName paramValues mkAction) _params hook _progress = do+    -- Read environment variables to determine accept/skip flags+    acceptEnv <- lookupEnv "GOLDS_GYM_ACCEPT"+    skipEnv <- lookupEnv "GOLDS_GYM_SKIP"+    +    let shouldAccept = case acceptEnv of+          Just "1"    -> True+          Just "true" -> True+          Just "yes"  -> True+          _           -> False+        shouldSkip = case skipEnv of+          Just "1"    -> True+          Just "true" -> True+          Just "yes"  -> True+          _           -> False+    +    -- Store the flags so Runner can access them+    setAcceptGoldens shouldAccept+    setSkipBenchmarks shouldSkip+    +    ref <- newIORef (Result "" Success)+    hook $ \() -> do+      results <- runSweep sweepName config paramName paramValues mkAction+      writeIORef ref (fromSweepResults sweepName paramName paramValues results)+    readIORef ref++-- | Convert sweep results to hspec Result.+fromSweepResults :: Show a => String -> T.Text -> [a] -> [(a, BenchResult, GoldenStats)] -> Result+fromSweepResults sweepName paramName paramValues results =+  let -- Check if any point regressed+      regressions = [(pv, r) | (pv, r@(Regression _ _ _ _ _), _) <- results]+      firstRuns = [pv | (pv, FirstRun _, _) <- results]+      +      nPoints = length paramValues+      +  in case regressions of+     ((pv, Regression golden actual pct tol absTol) : _) ->+       let toleranceDesc :: String+           toleranceDesc = case absTol of+             Nothing -> printf "tolerance: %.1f%%" tol+             Just absMs -> printf "tolerance: %.1f%% or %.3f ms" tol absMs+           message = printf "Sweep '%s': regression at %s=%s\nMean time increased by %.1f%% (%s)\n\n%s"+                       sweepName (T.unpack paramName) (show pv) pct toleranceDesc+                       (formatRegression golden actual)+       in Result message (Failure Nothing (Reason message))+     _ ->+       if not (null firstRuns)+       then+         let info = printf "Sweep '%s': baselines created for %d point(s)\nCSV written with %d row(s)"+                      sweepName (length firstRuns) nPoints+         in Result info Success+       else+         let info = printf "Sweep '%s': all %d point(s) passed\nCSV written with %d row(s)"+                      sweepName nPoints nPoints+         in Result info Success   -- | Create a benchmark golden test with custom lens-based expectations.

@@ -0,0 +1,128 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RecordWildCards #-}++-- |+-- Module      : Test.Hspec.BenchGolden.CSV+-- Description : CSV export for parameter sweep benchmark results+-- Copyright   : (c) 2026+-- License     : MIT+-- Maintainer  : @ocramz+--+-- This module provides CSV export functionality for parameter sweep benchmarks.+-- CSV files are written alongside golden files for convenient analysis and plotting.+--+-- = CSV Format+--+-- The CSV includes columns for:+--+-- * @timestamp@ - When the benchmark was run (ISO 8601 format)+-- * @param_name@ - The name of the sweep parameter+-- * @param_value@ - The value of the parameter for this row+-- * @mean_ms@ - Mean execution time in milliseconds+-- * @stddev_ms@ - Standard deviation in milliseconds+-- * @median_ms@ - Median execution time in milliseconds+-- * @min_ms@ - Minimum execution time in milliseconds+-- * @max_ms@ - Maximum execution time in milliseconds+-- * @trimmed_mean_ms@ - Trimmed mean (if robust statistics enabled)+-- * @mad_ms@ - Median absolute deviation (if robust statistics enabled)+-- * @iqr_ms@ - Interquartile range (if robust statistics enabled)+--+-- = Usage+--+-- CSV files are automatically generated when using 'benchGoldenSweep' or+-- 'benchGoldenSweepWith'. The file is written to:+--+-- @+-- \<outputDir\>/\<sweep-name\>-\<arch-id\>.csv+-- @+--+-- For example: @.golden/sort-scaling-aarch64-darwin-Apple_M1.csv@++module Test.Hspec.BenchGolden.CSV+  ( -- * CSV Generation+    csvHeader+  , csvRow+  , buildCSV++    -- * File I/O+  , writeSweepCSV+  , getSweepCSVPath+  ) where++import Data.Text (Text)+import qualified Data.Text as T+import Data.Text.Lazy.Builder (Builder)+import qualified Data.Text.Lazy.Builder as B+import qualified Data.Text.Lazy.Builder.RealFloat as B+import qualified Data.Text.Lazy.IO as TL+import Data.Time.Format.ISO8601 (iso8601Show)+import System.Directory (createDirectoryIfMissing)+import System.FilePath ((</>), (<.>))++import Test.Hspec.BenchGolden.Arch (sanitizeForFilename)+import Test.Hspec.BenchGolden.Types (GoldenStats(..))++-- | Generate CSV header row.+--+-- The header includes all standard statistics columns plus a parameter column.+csvHeader :: Text -> Builder+csvHeader paramName = mconcat+  [ "timestamp,"+  , B.fromText paramName, ","+  , "mean_ms,stddev_ms,median_ms,min_ms,max_ms,"+  , "trimmed_mean_ms,mad_ms,iqr_ms\n"+  ]++-- | Generate a single CSV data row from benchmark stats.+--+-- The timestamp is taken from the 'GoldenStats', and the parameter value+-- is provided separately as a string representation.+csvRow :: Text -> GoldenStats -> Builder+csvRow paramValue GoldenStats{..} = mconcat+  [ B.fromString (iso8601Show statsTimestamp), ","+  , B.fromText paramValue, ","+  , B.realFloat statsMean, ","+  , B.realFloat statsStddev, ","+  , B.realFloat statsMedian, ","+  , B.realFloat statsMin, ","+  , B.realFloat statsMax, ","+  , B.realFloat statsTrimmedMean, ","+  , B.realFloat statsMAD, ","+  , B.realFloat statsIQR, "\n"+  ]++-- | Build complete CSV content from a list of (param value, stats) pairs.+--+-- Example:+--+-- @+-- let results = [("1000", stats1), ("5000", stats2), ("10000", stats3)]+-- let csv = buildCSV "n" results+-- @+buildCSV :: Text -> [(Text, GoldenStats)] -> Builder+buildCSV paramName rows =+  csvHeader paramName <> mconcat (map (uncurry csvRow) rows)++-- | Get the path for a sweep CSV file.+--+-- The file is placed in the output directory with the architecture ID+-- included in the filename (not as a subdirectory) for easy comparison+-- across architectures.+--+-- Example: @.golden/sort-scaling-aarch64-darwin-Apple_M1.csv@+getSweepCSVPath :: FilePath -> Text -> String -> FilePath+getSweepCSVPath outDir archId sweepName =+  let sanitizedName = T.unpack $ sanitizeForFilename (T.pack sweepName)+      sanitizedArch = T.unpack $ sanitizeForFilename archId+  in outDir </> (sanitizedName <> "-" <> sanitizedArch) <.> "csv"++-- | Write sweep results to a CSV file.+--+-- This creates the output directory if it doesn't exist and writes+-- the complete CSV content atomically.+writeSweepCSV :: FilePath -> Text -> String -> Text -> [(Text, GoldenStats)] -> IO ()+writeSweepCSV outDir archId sweepName paramName rows = do+  createDirectoryIfMissing True outDir+  let path = getSweepCSVPath outDir archId sweepName+      content = B.toLazyText $ buildCSV paramName rows+  TL.writeFile path content

@@ -38,6 +38,10 @@   , runBenchmark   , runBenchmarkWithRawTimings +    -- * Parameter Sweeps+  , runSweepPoint+  , runSweep+     -- * Golden File Operations   , readGoldenFile   , writeGoldenFile@@ -71,7 +75,7 @@  import Control.DeepSeq (NFData, rnf) import Control.Exception (evaluate)-import Control.Monad (when, replicateM_)+import Control.Monad (when, replicateM_, forM) import Data.Aeson (eitherDecodeFileStrict, encodeFile) import Data.IORef (IORef, newIORef, readIORef, writeIORef) import Data.List (sort)@@ -88,9 +92,8 @@  import Lens.Micro ((^.)) -import qualified Test.BenchPress as BP- import Test.Hspec.BenchGolden.Arch (detectArchitecture, sanitizeForFilename)+import Test.Hspec.BenchGolden.CSV (writeSweepCSV) import Test.Hspec.BenchGolden.Lenses (metricFor, varianceFor) import Test.Hspec.BenchGolden.Types @@ -255,42 +258,35 @@               return result  -- | Run a benchmark and collect statistics.+--+-- Uses raw timing collection with proper inner iteration counts to ensure+-- the SPEC trick in nf/nfIO prevents thunk sharing. runBenchmark :: String -> BenchAction -> BenchConfig -> ArchConfig -> IO GoldenStats-runBenchmark _name action config arch = do-  if useRobustStatistics config-    then runBenchmarkWithRawTimings _name action config arch-    else do-      -- Use benchpress for standard statistics-      (cpuStats, _wallStats) <- BP.benchmark-        (iterations config)-        (pure ())                    -- setup-        (const $ runBenchAction action 1)  -- action: run 1 iteration-        (const $ pure ())            -- teardown--      now <- getCurrentTime--      return GoldenStats-        { statsMean        = BP.mean cpuStats-        , statsStddev      = BP.stddev cpuStats-        , statsMedian      = BP.median cpuStats-        , statsMin         = BP.min cpuStats-        , statsMax         = BP.max cpuStats-        , statsPercentiles = BP.percentiles cpuStats-        , statsArch        = archId arch-        , statsTimestamp   = now-        , statsTrimmedMean = 0.0  -- Not calculated in non-robust mode-        , statsMAD         = 0.0-        , statsIQR         = 0.0-        , statsOutliers    = []-        }+runBenchmark name action config arch =+  -- Always use the raw timing path since it correctly handles SPEC+  runBenchmarkWithRawTimings name action config arch  -- | Run a benchmark with raw timing collection for robust statistics.+--+-- This function times running all iterations in a single batch, then+-- divides to get per-iteration timing. The SPEC trick in nf/nfIO+-- prevents sharing within the batch.+--+-- We collect multiple samples by running the full batch multiple times,+-- ensuring accurate measurements even with GHC's -O2 optimizations. runBenchmarkWithRawTimings :: String -> BenchAction -> BenchConfig -> ArchConfig -> IO GoldenStats runBenchmarkWithRawTimings _name action config arch = do-  -- Collect raw CPU timings-  timings <- mapM (const $ measureCPUTimeForAction action) [1 .. iterations config]+  let iters = fromIntegral (iterations config) :: Word64+      numSamples = 10 :: Int  -- Number of timing samples to collect   -  let sortedTimings = sort timings+  -- Collect raw CPU timings (each sample runs all iterations)+  rawTimings <- forM [1 .. numSamples] $ \_ -> do+    startCpu <- getCPUTime+    runBenchAction action iters  -- SPEC trick prevents sharing within this call+    endCpu <- getCPUTime+    pure $ picosToMillis (endCpu - startCpu) / fromIntegral iters+  +  let sortedTimings = sort rawTimings       vec = V.fromList sortedTimings              -- Standard statistics@@ -303,7 +299,7 @@       min' = V.minimum vec       max' = V.maximum vec       -      -- Percentiles (matching benchpress format)+      -- Percentiles       percentiles' = [(p, quantile p vec) | p <- [50, 66, 75, 80, 90, 95, 98, 99, 100]]              -- Robust statistics@@ -326,13 +322,6 @@     , statsOutliers    = outliers'     }   where-    measureCPUTimeForAction act = do-      startCpu <- getCPUTime-      runBenchAction act 1  -- Run the action once-      endCpu <- getCPUTime-      let cpuTime = picosToMillis (endCpu - startCpu)-      return cpuTime-         picosToMillis :: Integer -> Double     picosToMillis t = realToFrac t / (10^(9 :: Int))     @@ -562,3 +551,105 @@ -- @ shouldSkipBenchmarks :: IO Bool shouldSkipBenchmarks = readIORef skipBenchmarksRef++-- -----------------------------------------------------------------------------+-- Parameter Sweeps+-- -----------------------------------------------------------------------------++-- | Run a single point of a parameter sweep.+--+-- This is similar to 'runBenchGolden' but returns the 'GoldenStats' along+-- with the 'BenchResult', allowing the caller to accumulate stats for CSV export.+--+-- Each point is saved to its own golden file with the parameter value+-- included in the filename (e.g., @sort-scaling_n=1000.golden@).+runSweepPoint ::+     Show a+  => String       -- ^ Base sweep name+  -> BenchConfig+  -> T.Text       -- ^ Parameter name+  -> a            -- ^ Parameter value+  -> BenchAction+  -> IO (BenchResult, GoldenStats)+runSweepPoint sweepName config paramName paramValue action = do+  let pointName = sweepName ++ "_" ++ T.unpack paramName ++ "=" ++ show paramValue++  -- Run the benchmark (this writes golden/actual files via runBenchGolden logic)+  skip <- shouldSkipBenchmarks+  if skip+    then do+      now <- getCurrentTime+      arch <- detectArchitecture+      let dummyStats = GoldenStats 0 0 0 0 0 [] (archId arch) now 0 0 0 []+      return (Pass dummyStats dummyStats [], dummyStats)+    else do+      arch <- detectArchitecture+      let archDir = T.unpack $ archId arch++      -- Create output directory+      let dir = outputDir config </> archDir+      createDirectoryIfMissing True dir++      -- Run warm-up iterations+      when (warmupIterations config > 0) $+        runBenchAction action (fromIntegral $ warmupIterations config)++      -- Run the actual benchmark+      actualStats <- runBenchmark pointName action config arch++      -- Write actual results+      writeActualFile (outputDir config) archDir pointName actualStats++      -- Check if we should force update+      update <- shouldUpdateGolden++      -- Read or create golden file+      let goldenPath = getGoldenPath (outputDir config) archDir pointName+      goldenExists <- doesFileExist goldenPath++      if update || not goldenExists+        then do+          -- First run or forced update: create/update golden+          writeGoldenFile (outputDir config) archDir pointName actualStats+          return (FirstRun actualStats, actualStats)+        else do+          -- Compare against existing golden+          goldenResult <- readGoldenFile goldenPath+          case goldenResult of+            Left err -> error $ "Failed to read golden file: " ++ err+            Right goldenStats -> do+              let result = compareStats config goldenStats actualStats+              return (result, actualStats)++-- | Run a full parameter sweep and write CSV output.+--+-- This runs benchmarks for all parameter values, saves individual golden+-- files, and writes a single CSV file with all results for analysis.+--+-- The CSV file is placed at:+--+-- @+-- \<outputDir\>/\<sweep-name\>-\<arch-id\>.csv+-- @+runSweep ::+     Show a+  => String           -- ^ Sweep name+  -> BenchConfig+  -> T.Text           -- ^ Parameter name (for CSV column header)+  -> [a]              -- ^ Parameter values to sweep over+  -> (a -> BenchAction)  -- ^ Action generator+  -> IO [(a, BenchResult, GoldenStats)]+runSweep sweepName config paramName paramValues mkAction = do+  arch <- detectArchitecture+  +  -- Run each parameter value+  results <- forM paramValues $ \paramVal -> do+    let action = mkAction paramVal+    (result, stats) <- runSweepPoint sweepName config paramName paramVal action+    return (paramVal, result, stats)+  +  -- Write CSV with all results+  let csvRows = [(T.pack (show pv), stats) | (pv, _, stats) <- results]+  writeSweepCSV (outputDir config) (archId arch) sweepName paramName csvRows+  +  return results

@@ -239,3 +239,5 @@     <*> v .: "os"     <*> v .: "cpu"     <*> v .:? "model"++