tasty-bench 0.2 → 0.2.1
raw patch · 4 files changed
+99/−53 lines, 4 files
Files
- README.md +3/−3
- Test/Tasty/Bench.hs +91/−48
- changelog.md +4/−1
- tasty-bench.cabal +1/−1
README.md view
@@ -40,7 +40,7 @@ build-depends: tasty-bench mixins:- tasty-bench (Test.Tasty.Bench as Criterion)+ tasty-bench (Test.Tasty.Bench as Criterion, Test.Tasty.Bench as Criterion.Main) ``` This works vice versa as well: if you use `tasty-bench`, but at some point@@ -261,7 +261,7 @@ ``` All fibo 20: OK (1.46s)- Response {respEstimate = Estimate {estMean = Measurement {measTime = 87496728, measAllocs = 0, measCopied = 0}, estSigma = 694487}, respIfSlower = FailIfSlower {unFailIfSlower = Infinity}, respIfFaster = FailIfFaster {unFailIfFaster = Infinity}}+ Response {respEstimate = Estimate {estMean = Measurement {measTime = 87496728, measAllocs = 0, measCopied = 0}, estStdev = 694487}, respIfSlower = FailIfSlower Infinity, respIfFaster = FailIfFaster Infinity} ``` ## Comparison against baseline@@ -327,7 +327,7 @@ * `--stdev` - Target relative standard deviation of measurements in percents (1% by default).+ Target relative standard deviation of measurements in percents (5% by default). Large values correspond to fast and loose benchmarks, and small ones to long and precise. If it takes far too long, consider setting `--timeout`, which will interrupt benchmarks, potentially before reaching the target deviation.
Test/Tasty/Bench.hs view
@@ -46,7 +46,7 @@ > build-depends: > tasty-bench > mixins:-> tasty-bench (Test.Tasty.Bench as Criterion)+> tasty-bench (Test.Tasty.Bench as Criterion, Test.Tasty.Bench as Criterion.Main) This works vice versa as well: if you use @tasty-bench@, but at some point need a more comprehensive statistical analysis, it is easy to@@ -248,7 +248,7 @@ > All > fibo 20: OK (1.46s)-> Response {respEstimate = Estimate {estMean = Measurement {measTime = 87496728, measAllocs = 0, measCopied = 0}, estSigma = 694487}, respIfSlower = FailIfSlower {unFailIfSlower = Infinity}, respIfFaster = FailIfFaster {unFailIfFaster = Infinity}}+> Response {respEstimate = Estimate {estMean = Measurement {measTime = 87496728, measAllocs = 0, measCopied = 0}, estStdev = 694487}, respIfSlower = FailIfSlower Infinity, respIfFaster = FailIfFaster Infinity} === Comparison against baseline @@ -311,7 +311,7 @@ [@--stdev@]: - Target relative standard deviation of measurements in percents (1%+ Target relative standard deviation of measurements in percents (5% by default). Large values correspond to fast and loose benchmarks, and small ones to long and precise. If it takes far too long, consider setting @--timeout@, which will interrupt benchmarks,@@ -418,10 +418,10 @@ deriving (Show, Read, Typeable) instance IsOption RelStDev where- defaultValue = RelStDev 0.01+ defaultValue = RelStDev 0.05 parseValue = fmap RelStDev . parsePositivePercents optionName = pure "stdev"- optionHelp = pure "Target relative standard deviation of measurements in percents (1 by default). Large values correspond to fast and loose benchmarks, and small ones to long and precise. If it takes far too long, consider setting --timeout, which will interrupt benchmarks, potentially before reaching the target deviation."+ optionHelp = pure "Target relative standard deviation of measurements in percents (5 by default). Large values correspond to fast and loose benchmarks, and small ones to long and precise. If it takes far too long, consider setting --timeout, which will interrupt benchmarks, potentially before reaching the target deviation." -- | In addition to @--fail-if-slower@ command-line option, -- one can adjust an upper bound of acceptable slow down@@ -484,39 +484,41 @@ | t < 995e6 = printf "%3.0f μs" (t / 1e6) | t < 995e7 = printf "%3.1f ms" (t / 1e9) | t < 995e9 = printf "%3.0f ms" (t / 1e9)- | otherwise = printf "%.1f s" (t / 1e12)+ | otherwise = printf "%4.2f s" (t / 1e12) where t :: Double t = fromIntegral i showBytes :: Word64 -> String showBytes i- | t < 1000 = printf "%3.0f B " t- | t < 10189 = printf "%3.1f KB" (t / 1024)- | t < 1023488 = printf "%3.0f KB" (t / 1024)- | t < 10433332 = printf "%3.1f MB" (t / 1048576)- | t < 1048051712 = printf "%3.0f MB" (t / 1048576)- | t < 10683731149 = printf "%3.1f GB" (t / 1073741824)- | t < 1073204953088 = printf "%3.0f GB" (t / 1073741824)- | otherwise = printf "%.1f TB" (t / 1099511627776)+ | t < 1000 = printf "%3.0f B " t+ | t < 10189 = printf "%3.1f KB" (t / 1024)+ | t < 1023488 = printf "%3.0f KB" (t / 1024)+ | t < 10433332 = printf "%3.1f MB" (t / 1048576)+ | t < 1048051712 = printf "%3.0f MB" (t / 1048576)+ | t < 10683731149 = printf "%3.1f GB" (t / 1073741824)+ | t < 1073204953088 = printf "%3.0f GB" (t / 1073741824)+ | t < 10940140696372 = printf "%3.1f TB" (t / 1099511627776)+ | t < 1098961871962112 = printf "%3.0f TB" (t / 1099511627776)+ | t < 11202704073084108 = printf "%3.1f PB" (t / 1125899906842624)+ | t < 1125336956889202624 = printf "%3.0f PB" (t / 1125899906842624)+ | t < 11471568970838126592 = printf "%3.1f EB" (t / 1152921504606846976)+ | otherwise = printf "%3.0f EB" (t / 1152921504606846976) where t :: Double t = fromIntegral i --- | It is crucial for precision to make all fields strict and unboxable. data Measurement = Measurement { measTime :: !Word64 -- ^ time in picoseconds , measAllocs :: !Word64 -- ^ allocations in bytes , measCopied :: !Word64 -- ^ copied bytes } deriving (Show, Read) --- | It is crucial for precision to make all fields strict and unboxable. data Estimate = Estimate { estMean :: !Measurement- , estSigma :: !Word64 -- ^ stdev in picoseconds+ , estStdev :: !Word64 -- ^ stdev in picoseconds } deriving (Show, Read) --- | It is crucial for precision to make all fields strict and unboxable. data Response = Response { respEstimate :: !Estimate , respIfSlower :: !FailIfSlower -- ^ saved value of --fail-if-slower@@ -524,22 +526,20 @@ } deriving (Show, Read) prettyEstimate :: Estimate -> String-prettyEstimate (Estimate m sigma) =- -- Two sigmas correspond to 95% probability,- showPicos (measTime m) ++ " ± " ++ showPicos (2 * sigma)+prettyEstimate (Estimate m stdev) =+ showPicos (measTime m) ++ " ± " ++ showPicos (2 * stdev) prettyEstimateWithGC :: Estimate -> String-prettyEstimateWithGC (Estimate m sigma) =- -- Two sigmas correspond to 95% probability,- showPicos (measTime m) ++ " ± " ++ showPicos (2 * sigma)+prettyEstimateWithGC (Estimate m stdev) =+ showPicos (measTime m) ++ " ± " ++ showPicos (2 * stdev) ++ ", " ++ showBytes (measAllocs m) ++ " allocated, " ++ showBytes (measCopied m) ++ " copied" csvEstimate :: Estimate -> String-csvEstimate (Estimate m sigma) = show (measTime m) ++ "," ++ show (2 * sigma)+csvEstimate (Estimate m stdev) = show (measTime m) ++ "," ++ show (2 * stdev) csvEstimateWithGC :: Estimate -> String-csvEstimateWithGC (Estimate m sigma) = show (measTime m) ++ "," ++ show (2 * sigma)+csvEstimateWithGC (Estimate m stdev) = show (measTime m) ++ "," ++ show (2 * stdev) ++ "," ++ show (measAllocs m) ++ "," ++ show (measCopied m) predict@@ -547,22 +547,22 @@ -> Measurement -- ^ time for two runs -> Estimate predict (Measurement t1 a1 c1) (Measurement t2 a2 c2) = Estimate- { estMean = Measurement t a c- , estSigma = truncate (sqrt (fromIntegral d) :: Double)+ { estMean = Measurement t (fit a1 a2) (fit c1 c2)+ , estStdev = truncate (sqrt d :: Double) } where+ fit x1 x2 = x1 `quot` 5 + 2 * (x2 `quot` 5)+ t = fit t1 t2 sqr x = x * x- d = sqr (t1 - t) + sqr (t2 - 2 * t)- t = (t1 + 2 * t2) `quot` 5- a = (a1 + 2 * a2) `quot` 5- c = (c1 + 2 * c2) `quot` 5+ d = sqr (fromIntegral t1 - fromIntegral t)+ + sqr (fromIntegral t2 - 2 * fromIntegral t) predictPerturbed :: Measurement -> Measurement -> Estimate predictPerturbed t1 t2 = Estimate { estMean = estMean (predict t1 t2)- , estSigma = max- (estSigma (predict (lo t1) (hi t2)))- (estSigma (predict (hi t1) (lo t2)))+ , estStdev = max+ (estStdev (predict (lo t1) (hi t2)))+ (estStdev (predict (hi t1) (lo t2))) } where prec = max (fromInteger cpuTimePrecision) 1000000000 -- 1 ms@@ -613,16 +613,16 @@ go n t1 sumOfTs = do t2 <- measureTime (2 * n) b - let Estimate (Measurement meanN allocN copiedN) sigmaN = predictPerturbed t1 t2+ let Estimate (Measurement meanN allocN copiedN) stdevN = predictPerturbed t1 t2 isTimeoutSoon = case timeout of NoTimeout -> False- -- multiplying by 1.2 helps to avoid accidental timeouts- Timeout micros _ -> (sumOfTs + measTime t1 + 3 * measTime t2) * 12 >= fromInteger micros * 1000000 * 10- isStDevInTargetRange = sigmaN < truncate (max 0 targetRelStDev * fromIntegral meanN)+ -- multiplying by 12/10 helps to avoid accidental timeouts+ Timeout micros _ -> (sumOfTs + measTime t1 + 3 * measTime t2) `quot` (1000000 * 10 `quot` 12) >= fromInteger micros+ isStDevInTargetRange = stdevN < truncate (max 0 targetRelStDev * fromIntegral meanN) scale = (`quot` fromIntegral n) if isStDevInTargetRange || isTimeoutSoon- then pure $ Estimate (Measurement (scale meanN) (scale allocN) (scale copiedN)) (scale sigmaN)+ then pure $ Estimate (Measurement (scale meanN) (scale allocN) (scale copiedN)) (scale stdevN) else go (2 * n) t2 (sumOfTs + measTime t1) instance IsTest Benchmarkable where@@ -685,9 +685,26 @@ {-# INLINE funcToBench #-} -- | 'nf' @f@ @x@ measures time to compute--- a normal form (by means of 'rnf') of an application of @f@ to @x@.+-- a normal form (by means of 'force') of an application of @f@ to @x@. -- This does not include time to evaluate @f@ or @x@ themselves.+-- Ideally @x@ should be a primitive data type like 'Data.Int.Int'. --+-- Here is a textbook antipattern: 'nf' 'sum' @[1..1000000]@.+-- Since an input list is shared by multiple invocations of 'sum',+-- it will be allocated in memory in full, putting immense pressure+-- on garbage collector. Also no list fusion will happen.+-- A better approach is 'nf' (@\\n@ @->@ 'sum' @[1..n]@) @1000000@.+--+-- If you are measuring an inlinable function,+-- it is prudent to ensure that its invocation is fully saturated,+-- otherwise inlining will not happen. That's why one can often+-- see 'nf' (@\\n@ @->@ @f@ @n@) @x@ instead of 'nf' @f@ @x@.+-- Same applies to rewrite rules.+--+-- While @tasty-bench@ is capable to perform micro- and even nanobenchmarks,+-- such measurements are noisy and involve an overhead. Results are more reliable+-- when @f@ @x@ takes at least several milliseconds.+-- -- Note that forcing a normal form requires an additional -- traverse of the structure. In certain scenarios (imagine benchmarking 'tail'), -- especially when 'NFData' instance is badly written,@@ -696,18 +713,25 @@ -- Drop-in replacement for 'Criterion.nf' and 'Gauge.nf'. -- nf :: NFData b => (a -> b) -> a -> Benchmarkable-nf = funcToBench rnf+nf = funcToBench force {-# INLINE nf #-} -- | 'whnf' @f@ @x@ measures time to compute -- a weak head normal form of an application of @f@ to @x@. -- This does not include time to evaluate @f@ or @x@ themselves.+-- Ideally @x@ should be a primitive data type like 'Data.Int.Int'. -- -- Computing only a weak head normal form is -- rarely what intuitively is meant by "evaluation".+-- Beware that many educational materials contain examples with 'whnf':+-- this is a wrong default. -- Unless you understand precisely, what is measured, -- it is recommended to use 'nf' instead. --+-- Here is a textbook antipattern: 'whnf' ('Data.List.replicate' @1000000@) @1@.+-- This will succeed in a matter of nanoseconds, because weak head+-- normal form forces only the first element of the list.+-- -- Drop-in replacement for 'Criterion.whnf' and 'Gauge.whnf'. -- whnf :: (a -> b) -> a -> Benchmarkable@@ -726,7 +750,7 @@ {-# INLINE ioToBench #-} -- | 'nfIO' @x@ measures time to evaluate side-effects of @x@--- and compute its normal form (by means of 'rnf').+-- and compute its normal form (by means of 'force'). -- -- Pure subexpression of an effectful computation @x@ -- may be evaluated only once and get cached; use 'nfAppIO'@@ -737,10 +761,14 @@ -- especially when 'NFData' instance is badly written, -- this traversal may take non-negligible time and affect results. --+-- A typical use case is 'nfIO' ('readFile' @"foo.txt"@).+-- However, if you need I\/O only to read input data from a file,+-- consider using 'env'.+-- -- Drop-in replacement for 'Criterion.nfIO' and 'Gauge.nfIO'. -- nfIO :: NFData a => IO a -> Benchmarkable-nfIO = ioToBench rnf+nfIO = ioToBench force {-# INLINE nfIO #-} -- | 'whnfIO' @x@ measures time to evaluate side-effects of @x@@@ -755,6 +783,9 @@ -- Unless you understand precisely, what is measured, -- it is recommended to use 'nfIO' instead. --+-- Lazy I\/O is treacherous. If you need I\/O only+-- to read input data from a file, consider using 'env'.+-- -- Drop-in replacement for 'Criterion.whnfIO' and 'Gauge.whnfIO'. -- whnfIO :: NFData a => IO a -> Benchmarkable@@ -774,30 +805,39 @@ -- | 'nfAppIO' @f@ @x@ measures time to evaluate side-effects of -- an application of @f@ to @x@.--- and compute its normal form (by means of 'rnf').+-- and compute its normal form (by means of 'force'). -- This does not include time to evaluate @f@ or @x@ themselves.+-- Ideally @x@ should be a primitive data type like 'Data.Int.Int'. -- -- Note that forcing a normal form requires an additional -- traverse of the structure. In certain scenarios, -- especially when 'NFData' instance is badly written, -- this traversal may take non-negligible time and affect results. --+-- A typical use case is 'nfAppIO' 'readFile' @"foo.txt"@.+-- However, if you need I\/O only to read input data from a file,+-- consider using 'env'.+-- -- Drop-in replacement for 'Criterion.nfAppIO' and 'Gauge.nfAppIO'. -- nfAppIO :: NFData b => (a -> IO b) -> a -> Benchmarkable-nfAppIO = ioFuncToBench rnf+nfAppIO = ioFuncToBench force {-# INLINE nfAppIO #-} -- | 'whnfAppIO' @f@ @x@ measures time to evaluate side-effects of -- an application of @f@ to @x@. -- and compute its weak head normal form. -- This does not include time to evaluate @f@ or @x@ themselves.+-- Ideally @x@ should be a primitive data type like 'Data.Int.Int'. -- -- Computing only a weak head normal form is -- rarely what intuitively is meant by "evaluation". -- Unless you understand precisely, what is measured, -- it is recommended to use 'nfAppIO' instead. --+-- Lazy I\/O is treacherous. If you need I\/O only+-- to read input data from a file, consider using 'env'.+-- -- Drop-in replacement for 'Criterion.whnfAppIO' and 'Gauge.whnfAppIO'. -- whnfAppIO :: (a -> IO b) -> a -> Benchmarkable@@ -915,13 +955,16 @@ = fromIntegral slowDown <= 100 * ifSlow && fromIntegral slowDown >= -100 * ifFast +-- | Return slow down in percents. compareVsBaseline :: S.Set TestName -> TestName -> Estimate -> Int64-compareVsBaseline baseline name (Estimate m sigma) = case mOld of+compareVsBaseline baseline name (Estimate m stdev) = case mOld of Nothing -> 0 Just (oldTime, oldDoubleSigma)- | abs (time - oldTime) < max (2 * fromIntegral sigma) oldDoubleSigma -> 0+ -- time and oldTime must be signed integers to use 'abs'+ | abs (time - oldTime) < max (2 * fromIntegral stdev) oldDoubleSigma -> 0 | otherwise -> 100 * (time - oldTime) `quot` oldTime where+ time :: Int64 time = fromIntegral $ measTime m mOld = do let prefix = encodeCsv name ++ ","
changelog.md view
@@ -1,3 +1,7 @@+# 0.2.1++* Fix integer overflow in stdev computations.+ # 0.2 * Add `env` and `envWithCleanup`.@@ -6,7 +10,6 @@ * Add comparison against baseline and relevant options. * Export `RelStDev` option. * Export `benchIngredients`.-* Improve stability of measurements. # 0.1
tasty-bench.cabal view
@@ -1,5 +1,5 @@ name: tasty-bench-version: 0.2+version: 0.2.1 cabal-version: >=1.10 build-type: Simple license: MIT