diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [0.6.0] + +### Added + +- **Parameter sweep support** for benchmarking scaling behaviour + - `benchGoldenSweep` combinator for simple parameter sweeps with default config and `benchGoldenSweepWith` combinator for sweeps with custom configuration + - Individual golden files created per parameter value (e.g., `sort-scaling_n_1000.golden`) + - Automatic regression detection per sweep point using existing tolerance logic + - NB: Each timing sample now runs multiple inner iterations (leveraging the SPEC trick in `nf`/`nfIO` combinators) and divides by the iteration count + - Unified timing path: `runBenchmark` now always uses `runBenchmarkWithRawTimings` for consistent, correct measurements + +- **CSV export** for analysis and plotting + - New `Test.Hspec.BenchGolden.CSV` module + - Single CSV file per sweep with architecture in filename (e.g., `sort-scaling-aarch64-darwin-Apple_M1.csv`) + - Columns: timestamp, parameter value, mean, stddev, median, min, max, trimmed_mean, mad, iqr + - Built with `Text.Builder` for efficient serialization + +### Removed + +- **benchpress dependency** - timing is now done via `getCPUTime` from base, without `benchpress` + ## [0.5.0] ### Added diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ Golden testing for performance benchmarks. Save timing baselines on first run, compare against them on subsequent runs. +The library was originally inspired by (and dependent on) `benchpress`, but now has independent measurement routines. + **Key Features:** - Architecture-specific baselines (different hardware = different golden files) - Hybrid tolerance (handles both fast <1ms and slow operations) diff --git a/example/Spec.hs b/example/Spec.hs --- a/example/Spec.hs +++ b/example/Spec.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE OverloadedStrings #-} + -- | Example benchmark golden tests demonstrating golds-gym usage. module Main (main) where @@ -140,6 +142,28 @@ (defaultBenchConfig { useRobustStatistics = True, iterations = 1000 }) [expect _statsTrimmedMean (Percent 25.0)] $ nf fib 26 + + -- ============================================================================ + -- Parameter Sweeps + -- ============================================================================ + + describe "Parameter Sweeps" $ do + -- Simple parameter sweep: measure how sort scales with input size + -- This produces: + -- - Individual golden files: .golden//sort-scaling_n=500.golden, etc. + -- - Single CSV file: .golden/sort-scaling-.csv + benchGoldenSweep "sort-scaling" "n" [500, 2000, 5000] $ + \n -> nf sort [n, n-1..1 :: Int] + + -- Sweep with custom configuration + benchGoldenSweepWith + defaultBenchConfig + { iterations = 500 + , tolerancePercent = 20.0 + , warmupIterations = 10 + } + "fib-scaling" "depth" [10, 20, 30] $ + \depth -> nf fib depth diff --git a/golds-gym.cabal b/golds-gym.cabal --- a/golds-gym.cabal +++ b/golds-gym.cabal @@ -1,6 +1,6 @@ cabal-version: 3.0 name: golds-gym -version: 0.5.0.0 +version: 0.6.0.0 synopsis: Golden testing framework for performance benchmarks description: A Haskell framework for golden testing of timing benchmarks. @@ -8,7 +8,7 @@ against on subsequent runs. Golden files are architecture-specific to account for hardware differences. . - Based on hspec and benchpress. + Built on hspec. license: MIT license-file: LICENSE @@ -26,11 +26,11 @@ Test.Hspec.BenchGolden.Arch Test.Hspec.BenchGolden.Runner Test.Hspec.BenchGolden.Lenses + Test.Hspec.BenchGolden.CSV build-depends: base >= 4.14 && < 5, hspec-core >= 2.10 && < 3, - benchpress >= 0.2 && < 0.3, aeson >= 2.0 && < 3, directory >= 1.3 && < 2, filepath >= 1.4 && < 2, @@ -57,7 +57,8 @@ build-depends: base >= 4.14 && < 5, golds-gym, - hspec >= 2.10 && < 3 + hspec >= 2.10 && < 3, + text >= 1.2 && < 3 test-suite golds-gym-properties type: exitcode-stdio-1.0 diff --git a/src/Test/Hspec/BenchGolden.hs b/src/Test/Hspec/BenchGolden.hs --- a/src/Test/Hspec/BenchGolden.hs +++ b/src/Test/Hspec/BenchGolden.hs @@ -14,13 +14,16 @@ -- = Overview -- -- @golds-gym@ is a framework for golden testing of performance benchmarks. --- It integrates with hspec and uses benchpress for lightweight timing measurements. +-- It integrates with hspec and uses CPU time measurements for benchmarking. -- -- Benchmarks can use robust statistics to mitigate the impact of outliers. -- -- The library can be used both to assert that performance does not regress, and to set expectations -- for improvements across project versions (see `benchGoldenWithExpectation`). -- +-- There are also combinators for parameter sweep benchmarks that generate CSV files for analysis and plotting, +-- see `benchGoldenSweep` and `benchGoldenSweepWith`. +-- -- = Quick Start -- -- @ @@ -178,6 +181,10 @@ , benchGoldenWith , benchGoldenWithExpectation + -- * Parameter Sweeps + , benchGoldenSweep + , benchGoldenSweepWith + -- * Configuration , BenchConfig(..) , defaultBenchConfig @@ -218,7 +225,7 @@ import Test.Hspec.BenchGolden.Arch import qualified Test.Hspec.BenchGolden.Lenses as L import Test.Hspec.BenchGolden.Lenses hiding (Expectation) -import Test.Hspec.BenchGolden.Runner (runBenchGolden, setAcceptGoldens, setSkipBenchmarks, nf, nfIO, nfAppIO, io) +import Test.Hspec.BenchGolden.Runner (runBenchGolden, runSweep, setAcceptGoldens, setSkipBenchmarks, nf, nfIO, nfAppIO, io) import Test.Hspec.BenchGolden.Types -- | Create a benchmark golden test with default configuration. @@ -284,6 +291,125 @@ , benchAction = action , benchConfig = config } + +-- | Create a parameter sweep benchmark with default configuration. +-- +-- This combinator runs the same benchmark with multiple parameter values, +-- saving individual golden files for each point and producing a single CSV +-- file for analysis and plotting. +-- +-- Example: +-- +-- @ +-- describe "Scaling Tests" $ do +-- benchGoldenSweep "sort-scaling" "n" [1000, 5000, 10000, 50000] $ +-- \\n -> nf sort [n, n-1..1] +-- @ +-- +-- This produces: +-- +-- * Golden files: @.golden\/\\/sort-scaling_n=1000.golden@, etc. +-- * CSV file: @.golden\/sort-scaling-\.csv@ +benchGoldenSweep :: + Show a + => String -- ^ Sweep name (used for CSV filename and golden file prefix) + -> T.Text -- ^ Parameter name (for CSV column header) + -> [a] -- ^ Parameter values to sweep over + -> (a -> BenchAction) -- ^ Action parameterized by sweep value + -> Spec +benchGoldenSweep = benchGoldenSweepWith defaultBenchConfig + +-- | Create a parameter sweep benchmark with custom configuration. +-- +-- Example: +-- +-- @ +-- describe "Performance Scaling" $ do +-- benchGoldenSweepWith +-- defaultBenchConfig { iterations = 500, tolerancePercent = 10.0 } +-- "algorithm-scaling" "size" [100, 500, 1000, 5000] $ +-- \\size -> nf myAlgorithm (generateInput size) +-- @ +-- +-- The CSV file includes columns for timestamp, parameter value, and all +-- standard statistics (mean, stddev, median, min, max, etc.). +benchGoldenSweepWith :: + Show a + => BenchConfig -- ^ Configuration parameters + -> String -- ^ Sweep name + -> T.Text -- ^ Parameter name (for CSV column header) + -> [a] -- ^ Parameter values to sweep over + -> (a -> BenchAction) -- ^ Action parameterized by sweep value + -> Spec +benchGoldenSweepWith config sweepName paramName paramValues mkAction = + it (sweepName ++ " [sweep]") $ BenchGoldenSweep sweepName config paramName paramValues mkAction + +-- | Internal type for sweep benchmarks. +data BenchGoldenSweep a = BenchGoldenSweep + !String -- Sweep name + !BenchConfig -- Config + !T.Text -- Parameter name + ![a] -- Parameter values + !(a -> BenchAction) -- Action generator + +-- | Instance for sweep benchmarks. +instance Show a => Example (BenchGoldenSweep a) where + type Arg (BenchGoldenSweep a) = () + evaluateExample (BenchGoldenSweep sweepName config paramName paramValues mkAction) _params hook _progress = do + -- Read environment variables to determine accept/skip flags + acceptEnv <- lookupEnv "GOLDS_GYM_ACCEPT" + skipEnv <- lookupEnv "GOLDS_GYM_SKIP" + + let shouldAccept = case acceptEnv of + Just "1" -> True + Just "true" -> True + Just "yes" -> True + _ -> False + shouldSkip = case skipEnv of + Just "1" -> True + Just "true" -> True + Just "yes" -> True + _ -> False + + -- Store the flags so Runner can access them + setAcceptGoldens shouldAccept + setSkipBenchmarks shouldSkip + + ref <- newIORef (Result "" Success) + hook $ \() -> do + results <- runSweep sweepName config paramName paramValues mkAction + writeIORef ref (fromSweepResults sweepName paramName paramValues results) + readIORef ref + +-- | Convert sweep results to hspec Result. +fromSweepResults :: Show a => String -> T.Text -> [a] -> [(a, BenchResult, GoldenStats)] -> Result +fromSweepResults sweepName paramName paramValues results = + let -- Check if any point regressed + regressions = [(pv, r) | (pv, r@(Regression _ _ _ _ _), _) <- results] + firstRuns = [pv | (pv, FirstRun _, _) <- results] + + nPoints = length paramValues + + in case regressions of + ((pv, Regression golden actual pct tol absTol) : _) -> + let toleranceDesc :: String + toleranceDesc = case absTol of + Nothing -> printf "tolerance: %.1f%%" tol + Just absMs -> printf "tolerance: %.1f%% or %.3f ms" tol absMs + message = printf "Sweep '%s': regression at %s=%s\nMean time increased by %.1f%% (%s)\n\n%s" + sweepName (T.unpack paramName) (show pv) pct toleranceDesc + (formatRegression golden actual) + in Result message (Failure Nothing (Reason message)) + _ -> + if not (null firstRuns) + then + let info = printf "Sweep '%s': baselines created for %d point(s)\nCSV written with %d row(s)" + sweepName (length firstRuns) nPoints + in Result info Success + else + let info = printf "Sweep '%s': all %d point(s) passed\nCSV written with %d row(s)" + sweepName nPoints nPoints + in Result info Success -- | Create a benchmark golden test with custom lens-based expectations. diff --git a/src/Test/Hspec/BenchGolden/CSV.hs b/src/Test/Hspec/BenchGolden/CSV.hs new file mode 100644 --- /dev/null +++ b/src/Test/Hspec/BenchGolden/CSV.hs @@ -0,0 +1,128 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} + +-- | +-- Module : Test.Hspec.BenchGolden.CSV +-- Description : CSV export for parameter sweep benchmark results +-- Copyright : (c) 2026 +-- License : MIT +-- Maintainer : @ocramz +-- +-- This module provides CSV export functionality for parameter sweep benchmarks. +-- CSV files are written alongside golden files for convenient analysis and plotting. +-- +-- = CSV Format +-- +-- The CSV includes columns for: +-- +-- * @timestamp@ - When the benchmark was run (ISO 8601 format) +-- * @param_name@ - The name of the sweep parameter +-- * @param_value@ - The value of the parameter for this row +-- * @mean_ms@ - Mean execution time in milliseconds +-- * @stddev_ms@ - Standard deviation in milliseconds +-- * @median_ms@ - Median execution time in milliseconds +-- * @min_ms@ - Minimum execution time in milliseconds +-- * @max_ms@ - Maximum execution time in milliseconds +-- * @trimmed_mean_ms@ - Trimmed mean (if robust statistics enabled) +-- * @mad_ms@ - Median absolute deviation (if robust statistics enabled) +-- * @iqr_ms@ - Interquartile range (if robust statistics enabled) +-- +-- = Usage +-- +-- CSV files are automatically generated when using 'benchGoldenSweep' or +-- 'benchGoldenSweepWith'. The file is written to: +-- +-- @ +-- \/\-\.csv +-- @ +-- +-- For example: @.golden/sort-scaling-aarch64-darwin-Apple_M1.csv@ + +module Test.Hspec.BenchGolden.CSV + ( -- * CSV Generation + csvHeader + , csvRow + , buildCSV + + -- * File I/O + , writeSweepCSV + , getSweepCSVPath + ) where + +import Data.Text (Text) +import qualified Data.Text as T +import Data.Text.Lazy.Builder (Builder) +import qualified Data.Text.Lazy.Builder as B +import qualified Data.Text.Lazy.Builder.RealFloat as B +import qualified Data.Text.Lazy.IO as TL +import Data.Time.Format.ISO8601 (iso8601Show) +import System.Directory (createDirectoryIfMissing) +import System.FilePath ((), (<.>)) + +import Test.Hspec.BenchGolden.Arch (sanitizeForFilename) +import Test.Hspec.BenchGolden.Types (GoldenStats(..)) + +-- | Generate CSV header row. +-- +-- The header includes all standard statistics columns plus a parameter column. +csvHeader :: Text -> Builder +csvHeader paramName = mconcat + [ "timestamp," + , B.fromText paramName, "," + , "mean_ms,stddev_ms,median_ms,min_ms,max_ms," + , "trimmed_mean_ms,mad_ms,iqr_ms\n" + ] + +-- | Generate a single CSV data row from benchmark stats. +-- +-- The timestamp is taken from the 'GoldenStats', and the parameter value +-- is provided separately as a string representation. +csvRow :: Text -> GoldenStats -> Builder +csvRow paramValue GoldenStats{..} = mconcat + [ B.fromString (iso8601Show statsTimestamp), "," + , B.fromText paramValue, "," + , B.realFloat statsMean, "," + , B.realFloat statsStddev, "," + , B.realFloat statsMedian, "," + , B.realFloat statsMin, "," + , B.realFloat statsMax, "," + , B.realFloat statsTrimmedMean, "," + , B.realFloat statsMAD, "," + , B.realFloat statsIQR, "\n" + ] + +-- | Build complete CSV content from a list of (param value, stats) pairs. +-- +-- Example: +-- +-- @ +-- let results = [("1000", stats1), ("5000", stats2), ("10000", stats3)] +-- let csv = buildCSV "n" results +-- @ +buildCSV :: Text -> [(Text, GoldenStats)] -> Builder +buildCSV paramName rows = + csvHeader paramName <> mconcat (map (uncurry csvRow) rows) + +-- | Get the path for a sweep CSV file. +-- +-- The file is placed in the output directory with the architecture ID +-- included in the filename (not as a subdirectory) for easy comparison +-- across architectures. +-- +-- Example: @.golden/sort-scaling-aarch64-darwin-Apple_M1.csv@ +getSweepCSVPath :: FilePath -> Text -> String -> FilePath +getSweepCSVPath outDir archId sweepName = + let sanitizedName = T.unpack $ sanitizeForFilename (T.pack sweepName) + sanitizedArch = T.unpack $ sanitizeForFilename archId + in outDir (sanitizedName <> "-" <> sanitizedArch) <.> "csv" + +-- | Write sweep results to a CSV file. +-- +-- This creates the output directory if it doesn't exist and writes +-- the complete CSV content atomically. +writeSweepCSV :: FilePath -> Text -> String -> Text -> [(Text, GoldenStats)] -> IO () +writeSweepCSV outDir archId sweepName paramName rows = do + createDirectoryIfMissing True outDir + let path = getSweepCSVPath outDir archId sweepName + content = B.toLazyText $ buildCSV paramName rows + TL.writeFile path content diff --git a/src/Test/Hspec/BenchGolden/Runner.hs b/src/Test/Hspec/BenchGolden/Runner.hs --- a/src/Test/Hspec/BenchGolden/Runner.hs +++ b/src/Test/Hspec/BenchGolden/Runner.hs @@ -38,6 +38,10 @@ , runBenchmark , runBenchmarkWithRawTimings + -- * Parameter Sweeps + , runSweepPoint + , runSweep + -- * Golden File Operations , readGoldenFile , writeGoldenFile @@ -71,7 +75,7 @@ import Control.DeepSeq (NFData, rnf) import Control.Exception (evaluate) -import Control.Monad (when, replicateM_) +import Control.Monad (when, replicateM_, forM) import Data.Aeson (eitherDecodeFileStrict, encodeFile) import Data.IORef (IORef, newIORef, readIORef, writeIORef) import Data.List (sort) @@ -88,9 +92,8 @@ import Lens.Micro ((^.)) -import qualified Test.BenchPress as BP - import Test.Hspec.BenchGolden.Arch (detectArchitecture, sanitizeForFilename) +import Test.Hspec.BenchGolden.CSV (writeSweepCSV) import Test.Hspec.BenchGolden.Lenses (metricFor, varianceFor) import Test.Hspec.BenchGolden.Types @@ -255,42 +258,35 @@ return result -- | Run a benchmark and collect statistics. +-- +-- Uses raw timing collection with proper inner iteration counts to ensure +-- the SPEC trick in nf/nfIO prevents thunk sharing. runBenchmark :: String -> BenchAction -> BenchConfig -> ArchConfig -> IO GoldenStats -runBenchmark _name action config arch = do - if useRobustStatistics config - then runBenchmarkWithRawTimings _name action config arch - else do - -- Use benchpress for standard statistics - (cpuStats, _wallStats) <- BP.benchmark - (iterations config) - (pure ()) -- setup - (const $ runBenchAction action 1) -- action: run 1 iteration - (const $ pure ()) -- teardown - - now <- getCurrentTime - - return GoldenStats - { statsMean = BP.mean cpuStats - , statsStddev = BP.stddev cpuStats - , statsMedian = BP.median cpuStats - , statsMin = BP.min cpuStats - , statsMax = BP.max cpuStats - , statsPercentiles = BP.percentiles cpuStats - , statsArch = archId arch - , statsTimestamp = now - , statsTrimmedMean = 0.0 -- Not calculated in non-robust mode - , statsMAD = 0.0 - , statsIQR = 0.0 - , statsOutliers = [] - } +runBenchmark name action config arch = + -- Always use the raw timing path since it correctly handles SPEC + runBenchmarkWithRawTimings name action config arch -- | Run a benchmark with raw timing collection for robust statistics. +-- +-- This function times running all iterations in a single batch, then +-- divides to get per-iteration timing. The SPEC trick in nf/nfIO +-- prevents sharing within the batch. +-- +-- We collect multiple samples by running the full batch multiple times, +-- ensuring accurate measurements even with GHC's -O2 optimizations. runBenchmarkWithRawTimings :: String -> BenchAction -> BenchConfig -> ArchConfig -> IO GoldenStats runBenchmarkWithRawTimings _name action config arch = do - -- Collect raw CPU timings - timings <- mapM (const $ measureCPUTimeForAction action) [1 .. iterations config] + let iters = fromIntegral (iterations config) :: Word64 + numSamples = 10 :: Int -- Number of timing samples to collect - let sortedTimings = sort timings + -- Collect raw CPU timings (each sample runs all iterations) + rawTimings <- forM [1 .. numSamples] $ \_ -> do + startCpu <- getCPUTime + runBenchAction action iters -- SPEC trick prevents sharing within this call + endCpu <- getCPUTime + pure $ picosToMillis (endCpu - startCpu) / fromIntegral iters + + let sortedTimings = sort rawTimings vec = V.fromList sortedTimings -- Standard statistics @@ -303,7 +299,7 @@ min' = V.minimum vec max' = V.maximum vec - -- Percentiles (matching benchpress format) + -- Percentiles percentiles' = [(p, quantile p vec) | p <- [50, 66, 75, 80, 90, 95, 98, 99, 100]] -- Robust statistics @@ -326,13 +322,6 @@ , statsOutliers = outliers' } where - measureCPUTimeForAction act = do - startCpu <- getCPUTime - runBenchAction act 1 -- Run the action once - endCpu <- getCPUTime - let cpuTime = picosToMillis (endCpu - startCpu) - return cpuTime - picosToMillis :: Integer -> Double picosToMillis t = realToFrac t / (10^(9 :: Int)) @@ -562,3 +551,105 @@ -- @ shouldSkipBenchmarks :: IO Bool shouldSkipBenchmarks = readIORef skipBenchmarksRef + +-- ----------------------------------------------------------------------------- +-- Parameter Sweeps +-- ----------------------------------------------------------------------------- + +-- | Run a single point of a parameter sweep. +-- +-- This is similar to 'runBenchGolden' but returns the 'GoldenStats' along +-- with the 'BenchResult', allowing the caller to accumulate stats for CSV export. +-- +-- Each point is saved to its own golden file with the parameter value +-- included in the filename (e.g., @sort-scaling_n=1000.golden@). +runSweepPoint :: + Show a + => String -- ^ Base sweep name + -> BenchConfig + -> T.Text -- ^ Parameter name + -> a -- ^ Parameter value + -> BenchAction + -> IO (BenchResult, GoldenStats) +runSweepPoint sweepName config paramName paramValue action = do + let pointName = sweepName ++ "_" ++ T.unpack paramName ++ "=" ++ show paramValue + + -- Run the benchmark (this writes golden/actual files via runBenchGolden logic) + skip <- shouldSkipBenchmarks + if skip + then do + now <- getCurrentTime + arch <- detectArchitecture + let dummyStats = GoldenStats 0 0 0 0 0 [] (archId arch) now 0 0 0 [] + return (Pass dummyStats dummyStats [], dummyStats) + else do + arch <- detectArchitecture + let archDir = T.unpack $ archId arch + + -- Create output directory + let dir = outputDir config archDir + createDirectoryIfMissing True dir + + -- Run warm-up iterations + when (warmupIterations config > 0) $ + runBenchAction action (fromIntegral $ warmupIterations config) + + -- Run the actual benchmark + actualStats <- runBenchmark pointName action config arch + + -- Write actual results + writeActualFile (outputDir config) archDir pointName actualStats + + -- Check if we should force update + update <- shouldUpdateGolden + + -- Read or create golden file + let goldenPath = getGoldenPath (outputDir config) archDir pointName + goldenExists <- doesFileExist goldenPath + + if update || not goldenExists + then do + -- First run or forced update: create/update golden + writeGoldenFile (outputDir config) archDir pointName actualStats + return (FirstRun actualStats, actualStats) + else do + -- Compare against existing golden + goldenResult <- readGoldenFile goldenPath + case goldenResult of + Left err -> error $ "Failed to read golden file: " ++ err + Right goldenStats -> do + let result = compareStats config goldenStats actualStats + return (result, actualStats) + +-- | Run a full parameter sweep and write CSV output. +-- +-- This runs benchmarks for all parameter values, saves individual golden +-- files, and writes a single CSV file with all results for analysis. +-- +-- The CSV file is placed at: +-- +-- @ +-- \/\-\.csv +-- @ +runSweep :: + Show a + => String -- ^ Sweep name + -> BenchConfig + -> T.Text -- ^ Parameter name (for CSV column header) + -> [a] -- ^ Parameter values to sweep over + -> (a -> BenchAction) -- ^ Action generator + -> IO [(a, BenchResult, GoldenStats)] +runSweep sweepName config paramName paramValues mkAction = do + arch <- detectArchitecture + + -- Run each parameter value + results <- forM paramValues $ \paramVal -> do + let action = mkAction paramVal + (result, stats) <- runSweepPoint sweepName config paramName paramVal action + return (paramVal, result, stats) + + -- Write CSV with all results + let csvRows = [(T.pack (show pv), stats) | (pv, _, stats) <- results] + writeSweepCSV (outputDir config) (archId arch) sweepName paramName csvRows + + return results diff --git a/src/Test/Hspec/BenchGolden/Types.hs b/src/Test/Hspec/BenchGolden/Types.hs --- a/src/Test/Hspec/BenchGolden/Types.hs +++ b/src/Test/Hspec/BenchGolden/Types.hs @@ -239,3 +239,5 @@ <*> v .: "os" <*> v .: "cpu" <*> v .:? "model" + +