diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ main :: IO () main = defaultMain - [ bgroup "fibonacci numbers" + [ bgroup "Fibonacci numbers" [ bench "fifth" $ nf fibo 5 , bench "tenth" $ nf fibo 10 , bench "twentieth" $ nf fibo 20 @@ -125,7 +125,7 @@ ``` All - fibonacci numbers + Fibonacci numbers fifth: OK (2.13s) 63 ns ± 3.4 ns tenth: OK (1.71s) @@ -216,7 +216,7 @@ ``` All - fibonacci numbers + Fibonacci numbers fifth: OK (2.13s) 63 ns ± 3.4 ns, 223 B allocated, 0 B copied, 2.0 MB peak memory tenth: OK (1.71s) @@ -340,6 +340,30 @@ [`tasty` documentation](https://github.com/UnkindPartition/tasty#patterns) for details and consider using `locateBenchmark`. +* When seeing + + ``` + This benchmark takes more than 100 seconds. Consider setting --timeout, if this is unexpected (or to silence this warning). + ``` + + do follow the advice: abort benchmarks and pass `-t100` or similar. Unless you are + benchmarking a very computationally expensive function, a single benchmark should + stabilize after a couple of seconds. This warning is a sign that your environment + is too noisy, in which case `tasty-bench` will continue trying with exponentially + longer intervals, often unproductively. + +* The following error can be thrown when benchmarks are built with + `ghc-options: -threaded`: + + ``` + Benchmarks must not be run concurrently. Please pass -j1 and/or avoid +RTS -N. + ``` + + The underlying cause is that `tasty` runs tests concurrently, which is harmful + for reliable performance measurements. Make sure to use `tasty-bench >= 0.3.4` and invoke + `Test.Tasty.Bench.defaultMain` and not `Test.Tasty.defaultMain`. Note that + `localOption (NumThreads 1)` quashes the warning, but does not eliminate the cause. + ## Isolating interfering benchmarks One difficulty of benchmarking in Haskell is that it is @@ -412,9 +436,9 @@ ``` Name,Mean (ps),2*Stdev (ps) -All.fibonacci numbers.fifth,48453,4060 -All.fibonacci numbers.tenth,637152,46744 -All.fibonacci numbers.twentieth,81369531,3342646 +All.Fibonacci numbers.fifth,48453,4060 +All.Fibonacci numbers.tenth,637152,46744 +All.Fibonacci numbers.twentieth,81369531,3342646 ``` Now modify implementation and rerun benchmarks @@ -422,7 +446,7 @@ ``` All - fibonacci numbers + Fibonacci numbers fifth: OK (0.44s) 53 ns ± 2.7 ns, 8% more than baseline tenth: OK (0.33s) @@ -476,7 +500,7 @@ main :: IO () main = defaultMain - [ bgroup "fibonacci numbers" + [ bgroup "Fibonacci numbers" [ bcompare "tenth" $ bench "fifth" $ nf fibo 5 , bench "tenth" $ nf fibo 10 , bcompare "tenth" $ bench "twentieth" $ nf fibo 20 @@ -488,7 +512,7 @@ ``` All - fibonacci numbers + Fibonacci numbers fifth: OK (16.56s) 121 ns ± 2.6 ns, 0.08x tenth: OK (6.84s) @@ -531,7 +555,7 @@ ## Command-line options -Use `--help` to list command-line options. +Use `--help` to list all command-line options. * `-p`, `--pattern` diff --git a/bench/bench-fibo.hs b/bench/bench-fibo.hs --- a/bench/bench-fibo.hs +++ b/bench/bench-fibo.hs @@ -7,7 +7,7 @@ main :: IO () main = defaultMain - [ bgroup "fibonacci numbers" + [ bgroup "Fibonacci numbers" [ bench "fifth" $ nf fibo 5 , bench "tenth" $ nf fibo 10 , bench "twentieth" $ nf fibo 20 diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,8 @@ +# 0.3.4 + +* Force single-threaded execution in `defaultMain`. +* Expose `measureCpuTimeAndStDev` helper to analyse benchmarks manually. + # 0.3.3 * Drop support of `tasty < 1.2.3`. diff --git a/compare_benches.sh b/compare_benches.sh --- a/compare_benches.sh +++ b/compare_benches.sh @@ -10,7 +10,6 @@ NEW="$1" shift - HEADREF=$(git rev-parse --verify HEAD) OLDREF=$(git rev-parse --verify "$OLD") NEWREF=$(git rev-parse --verify "$NEW") @@ -22,7 +21,7 @@ cabal run -v0 benchmarks -- --csv "$OLDCSV" "$@" && \ git checkout -q "$NEWREF" && \ cabal run -v0 benchmarks -- --baseline "$OLDCSV" --csv "$NEWCSV" "$@" && \ - git checkout -q "$HEADREF" && \ + git checkout -q "@{-2}" && \ awk 'BEGIN{FS=",";OFS=",";print "Name,'"$OLD"','"$NEW"',Ratio"}FNR==1{trueNF=NF;next}NF0)print "Geometric mean,,",exp(gs/gc)}' "$OLDCSV" "$NEWCSV" > "$OLDVSNEWCSV" } diff --git a/src/Test/Tasty/Bench.hs b/src/Test/Tasty/Bench.hs --- a/src/Test/Tasty/Bench.hs +++ b/src/Test/Tasty/Bench.hs @@ -1,7 +1,7 @@ {- | Module: Test.Tasty.Bench Copyright: (c) 2021 Andrew Lelechenko -Licence: MIT +License: MIT Featherlight benchmark framework (only one file!) for performance measurement with API @@ -87,7 +87,7 @@ > > main :: IO () > main = defaultMain -> [ bgroup "fibonacci numbers" +> [ bgroup "Fibonacci numbers" > [ bench "fifth" $ nf fibo 5 > , bench "tenth" $ nf fibo 10 > , bench "twentieth" $ nf fibo 20 @@ -105,7 +105,7 @@ the following output: > All -> fibonacci numbers +> Fibonacci numbers > fifth: OK (2.13s) > 63 ns ± 3.4 ns > tenth: OK (1.71s) @@ -192,7 +192,7 @@ report memory usage: > All -> fibonacci numbers +> Fibonacci numbers > fifth: OK (2.13s) > 63 ns ± 3.4 ns, 223 B allocated, 0 B copied, 2.0 MB peak memory > tenth: OK (1.71s) @@ -279,9 +279,9 @@ over and over again. - If benchmark results look malformed like below, make sure that you - are invoking 'Test.Tasty.Bench.defaultMain' and not - 'Test.Tasty.defaultMain' (the difference is 'consoleBenchReporter' - vs. 'consoleTestReporter'): + are invoking @Test.Tasty.Bench.@'Test.Tasty.Bench.defaultMain' and not + @Test.Tasty.@'Test.Tasty.defaultMain' (the difference is 'consoleBenchReporter' + vs. 'consoleTestReporter'): > All > fibo 20: OK (1.46s) @@ -308,18 +308,39 @@ [@tasty@ documentation](https://github.com/UnkindPartition/tasty#patterns) for details and consider using 'locateBenchmark'. +- When seeing + + > This benchmark takes more than 100 seconds. Consider setting --timeout, if this is unexpected (or to silence this warning). + + do follow the advice: abort benchmarks and pass @-t100@ or similar. Unless you are + benchmarking a very computationally expensive function, a single benchmark should + stabilize after a couple of seconds. This warning is a sign that your environment + is too noisy, in which case @tasty-bench@ will continue trying with exponentially + longer intervals, often unproductively. + +- The following error can be thrown when benchmarks are built with + @ghc-options: -threaded@: + + > Benchmarks must not be run concurrently. Please pass -j1 and/or avoid +RTS -N. + + The underlying cause is that @tasty@ runs tests concurrently, which is harmful + for reliable performance measurements. Make sure to use @tasty-bench >= 0.3.4@ + and invoke @Test.Tasty.Bench.@'Test.Tasty.Bench.defaultMain' and not + @Test.Tasty.@`Test.Tasty.defaultMain`. Note that 'localOption' ('NumThreads' 1) + quashes the warning, but does not eliminate the cause. + === Isolating interfering benchmarks One difficulty of benchmarking in Haskell is that it is hard to isolate benchmarks so that they do not interfere. Changing the order of -benchmarks or skipping some of them has an effect on heap’s layout and +benchmarks or skipping some of them has an effect on heap's layout and thus affects garbage collection. This issue is well attested in and . Usually (but not always) skipping some benchmarks speeds up remaining -ones. That’s because once a benchmark allocated heap which for some +ones. That's because once a benchmark allocated heap which for some reason was not promptly released afterwards (e. g., it forced a top-level thunk in an underlying library), all further benchmarks are slowed down by garbage collector processing this additional amount of @@ -373,15 +394,15 @@ (it could be a good idea to set smaller @--stdev@, if possible): > Name,Mean (ps),2*Stdev (ps) -> All.fibonacci numbers.fifth,48453,4060 -> All.fibonacci numbers.tenth,637152,46744 -> All.fibonacci numbers.twentieth,81369531,3342646 +> All.Fibonacci numbers.fifth,48453,4060 +> All.Fibonacci numbers.tenth,637152,46744 +> All.Fibonacci numbers.twentieth,81369531,3342646 Now modify implementation and rerun benchmarks with @--baseline@ @FILE@ key. This produces a report as follows: > All -> fibonacci numbers +> Fibonacci numbers > fifth: OK (0.44s) > 53 ns ± 2.7 ns, 8% more than baseline > tenth: OK (0.33s) @@ -425,7 +446,7 @@ > > main :: IO () > main = defaultMain -> [ bgroup "fibonacci numbers" +> [ bgroup "Fibonacci numbers" > [ bcompare "tenth" $ bench "fifth" $ nf fibo 5 > , bench "tenth" $ nf fibo 10 > , bcompare "tenth" $ bench "twentieth" $ nf fibo 20 @@ -436,7 +457,7 @@ to @tenth@: > All -> fibonacci numbers +> Fibonacci numbers > fifth: OK (16.56s) > 121 ns ± 2.6 ns, 0.08x > tenth: OK (6.84s) @@ -478,7 +499,7 @@ === Command-line options -Use @--help@ to list command-line options. +Use @--help@ to list all command-line options. [@-p@, @--pattern@]: @@ -603,6 +624,7 @@ , nfAppIO , whnfAppIO , measureCpuTime + , measureCpuTimeAndStDev #ifdef MIN_VERSION_tasty -- * Ingredients , benchIngredients @@ -616,7 +638,7 @@ , BaselinePath(..) , SvgPath(..) , TimeMode(..) - -- * Utils + -- * Utilities , locateBenchmark , mapLeafBenchmarks #else @@ -920,7 +942,7 @@ data Estimate = Estimate { estMean :: !Measurement - , estStdev :: !Word64 -- ^ stdev in picoseconds + , estStdev :: !Word64 -- ^ standard deviation in picoseconds } deriving (Show, Read) #ifdef MIN_VERSION_tasty @@ -1074,8 +1096,18 @@ -- -- @since 0.3 measureCpuTime :: Timeout -> RelStDev -> Benchmarkable -> IO Double -measureCpuTime - = ((fmap ((/ 1e12) . word64ToDouble . measTime . estMean) .) .) +measureCpuTime = ((fmap fst .) .) . measureCpuTimeAndStDev + +-- | Same as 'measureCpuTime', but returns both CPU execution time +-- and its standard deviation. +-- +-- @since 0.3.4 +measureCpuTimeAndStDev :: Timeout -> RelStDev -> Benchmarkable -> IO (Double, Double) +measureCpuTimeAndStDev + = ((fmap (\x -> + ( word64ToDouble (measTime (estMean x)) / 1e12 + , word64ToDouble (estStdev x) / 1e12 + )) .) .) . measureUntil CpuTime False #ifdef MIN_VERSION_tasty @@ -1133,7 +1165,7 @@ -- > -- > main :: IO () -- > main = defaultMain --- > [ bgroup "fibonacci numbers" +-- > [ bgroup "Fibonacci numbers" -- > [ bcompare "tenth" $ bench "fifth" $ nf fibo 5 -- > , bench "tenth" $ nf fibo 10 -- > , bcompare "tenth" $ bench "twentieth" $ nf fibo 20 @@ -1142,9 +1174,9 @@ -- -- More complex examples: -- --- * https://hackage.haskell.org/package/chimera-0.3.2.0/src/bench/Bench.hs +-- * https://hackage.haskell.org/package/chimera-0.3.3.0/src/bench/Bench.hs -- * https://hackage.haskell.org/package/fast-digits-0.3.1.0/src/bench/Bench.hs --- * https://hackage.haskell.org/package/unicode-data-0.3.0/src/bench/Main.hs +-- * https://hackage.haskell.org/package/unicode-data-0.4.0.1/src/bench/Main.hs -- -- @since 0.2.4 bcompare @@ -1170,7 +1202,7 @@ -- @since 0.3.1 bcompareWithin :: Double -- ^ Lower bound of relative speed up. - -> Double -- ^ Upper bound of relative spped up. + -> Double -- ^ Upper bound of relative speed up. -> String -- ^ @tasty@ pattern to locate a baseline benchmark. -> Benchmark -- ^ Benchmark to compare against baseline. -> Benchmark @@ -1196,7 +1228,7 @@ -- @since 0.1 defaultMain :: [Benchmark] -> IO () defaultMain bs = do - let act = Test.Tasty.defaultMainWithIngredients benchIngredients $ testGroup "All" bs + let act = defaultMain' bs #if MIN_VERSION_base(4,5,0) setLocaleEncoding utf8 #endif @@ -1207,6 +1239,15 @@ act #endif +defaultMain' :: [Benchmark] -> IO () +defaultMain' bs = do + installSignalHandlers + let b = testGroup "All" bs + opts <- parseOptions benchIngredients b + case tryIngredients benchIngredients (setOption (NumThreads 1) opts) b of + Nothing -> exitFailure + Just act -> act >>= \x -> if x then exitSuccess else exitFailure + -- | List of default benchmark ingredients. This is what 'defaultMain' runs. -- -- @since 0.2 @@ -1283,6 +1324,10 @@ -- in memory until it is fully evaluated, while the latter allows -- evaluated parts of the result to be garbage-collected immediately. -- +-- For users of @{-# LANGUAGE LinearTypes #-}@: if @f@ is a linear function, +-- then 'nf' @f@ @x@ is ill-typed, but you can use 'nf' @(\\y -> f y)@ @x@ +-- instead. +-- -- Drop-in replacement for @Criterion.@'Criterion.nf' and -- @Gauge.@'Gauge.nf'. -- @@ -1999,7 +2044,7 @@ -- Real world examples: -- -- * https://hackage.haskell.org/package/chimera-0.3.3.0/src/bench/Bench.hs --- * https://hackage.haskell.org/package/text-builder-linear-0.1/src/bench/Main.hs +-- * https://hackage.haskell.org/package/text-builder-linear-0.1.1/src/bench/Main.hs -- -- @since 0.3.2 mapLeafBenchmarks :: ([String] -> Benchmark -> Benchmark) -> Benchmark -> Benchmark @@ -2025,7 +2070,7 @@ -- Real world examples: -- -- * https://hackage.haskell.org/package/chimera-0.3.3.0/src/bench/Bench.hs --- * https://hackage.haskell.org/package/text-builder-linear-0.1/src/bench/Main.hs +-- * https://hackage.haskell.org/package/text-builder-linear-0.1.1/src/bench/Main.hs -- -- @since 0.3.2 locateBenchmark :: [String] -> Expr diff --git a/tasty-bench.cabal b/tasty-bench.cabal --- a/tasty-bench.cabal +++ b/tasty-bench.cabal @@ -1,5 +1,5 @@ name: tasty-bench -version: 0.3.3 +version: 0.3.4 cabal-version: 1.18 build-type: Simple license: MIT @@ -19,13 +19,13 @@ regular @tasty@ tests. extra-source-files: - changelog.md - README.md compare_benches.sh extra-doc-files: + changelog.md example.svg + README.md -tested-with: GHC == 9.6.1 GHC == 9.4.4, GHC == 9.2.5, GHC==9.0.2, GHC==8.10.7, GHC==8.8.4, GHC==8.6.5, GHC==8.4.4, GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4, GHC==7.6.3, GHC==7.4.2, GHC==7.2.2, GHC==7.0.4 +tested-with: GHC == 9.6.1 GHC == 9.4.4, GHC == 9.2.7, GHC==9.0.2, GHC==8.10.7, GHC==8.8.4, GHC==8.6.5, GHC==8.4.4, GHC==8.2.2, GHC==8.0.2, GHC==7.10.3, GHC==7.8.4, GHC==7.6.3, GHC==7.4.2, GHC==7.2.2, GHC==7.0.4 source-repository head type: git @@ -56,15 +56,15 @@ build-depends: base >= 4.3 && < 5, - deepseq >= 1.1, - ghc-prim + deepseq >= 1.1 && < 1.5, + ghc-prim < 0.11 if flag(tasty) build-depends: - containers >= 0.4, - tasty >= 1.2.3 + containers >= 0.4 && < 0.7, + tasty >= 1.2.3 && < 1.5 if impl(ghc < 7.8) build-depends: - tagged >= 0.2 + tagged >= 0.2 && < 0.9 if flag(debug) cpp-options: -DDEBUG