diff --git a/Criterion.hs b/Criterion.hs --- a/Criterion.hs +++ b/Criterion.hs @@ -59,4 +59,5 @@ initializeTime withConfig cfg $ do _ <- note "benchmarking...\n" - runAndAnalyseOne 0 "function" bm + Analysed rpt <- runAndAnalyseOne 0 "function" bm + return rpt diff --git a/Criterion/IO.hs b/Criterion/IO.hs --- a/Criterion/IO.hs +++ b/Criterion/IO.hs @@ -13,13 +13,12 @@ module Criterion.IO ( header - , hGetReports - , hPutReports - , readReports - , writeReports + , hGetRecords + , hPutRecords + , readRecords + , writeRecords ) where -import Criterion.Types (Report(..)) import Data.Binary (Binary(..), encode) #if MIN_VERSION_binary(0, 6, 3) import Data.Binary.Get (runGetOrFail) @@ -41,27 +40,27 @@ putByteString "criterio" mapM_ (putWord16be . fromIntegral) (versionBranch version) --- | Read all reports from the given 'Handle'. -hGetReports :: Handle -> IO (Either String [Report]) -hGetReports handle = do +-- | Read all records from the given 'Handle'. +hGetRecords :: Binary a => Handle -> IO (Either String [a]) +hGetRecords handle = do bs <- L.hGet handle (fromIntegral (L.length header)) if bs == header then Right `fmap` readAll handle else return $ Left "unexpected header" --- | Write reports to the given 'Handle'. -hPutReports :: Handle -> [Report] -> IO () -hPutReports handle rs = do +-- | Write records to the given 'Handle'. +hPutRecords :: Binary a => Handle -> [a] -> IO () +hPutRecords handle rs = do L.hPut handle header mapM_ (L.hPut handle . encode) rs --- | Read all reports from the given file. -readReports :: FilePath -> IO (Either String [Report]) -readReports path = withFile path ReadMode hGetReports +-- | Read all records from the given file. +readRecords :: Binary a => FilePath -> IO (Either String [a]) +readRecords path = withFile path ReadMode hGetRecords --- | Write reports to the given file. -writeReports :: FilePath -> [Report] -> IO () -writeReports path rs = withFile path WriteMode (flip hPutReports rs) +-- | Write records to the given file. +writeRecords :: Binary a => FilePath -> [a] -> IO () +writeRecords path rs = withFile path WriteMode (flip hPutRecords rs) #if MIN_VERSION_binary(0, 6, 3) readAll :: Binary a => Handle -> IO [a] diff --git a/Criterion/IO/Printf.hs b/Criterion/IO/Printf.hs --- a/Criterion/IO/Printf.hs +++ b/Criterion/IO/Printf.hs @@ -25,7 +25,7 @@ import Criterion.Monad (Criterion) import Criterion.Types (Config(csvFile, verbosity), Verbosity(..)) import Data.Foldable (forM_) -import System.IO (Handle, stderr, stdout) +import System.IO (Handle, hFlush, stderr, stdout) import Text.Printf (PrintfArg) import qualified Data.ByteString.Lazy as B import qualified Data.Csv as Csv @@ -34,7 +34,7 @@ -- First item is the action to print now, given all the arguments -- gathered together so far. The second item is the function that -- will take a further argument and give back a new PrintfCont. -data PrintfCont = PrintfCont (IO ()) (PrintfArg a => a -> PrintfCont) +data PrintfCont = PrintfCont (IO ()) (forall a. PrintfArg a => a -> PrintfCont) -- | An internal class that acts like Printf/HPrintf. -- @@ -47,12 +47,12 @@ instance CritHPrintfType (Criterion a) where chPrintfImpl check (PrintfCont final _) = do x <- ask - when (check x) (liftIO final) + when (check x) (liftIO (final >> hFlush stderr >> hFlush stdout)) return undefined instance CritHPrintfType (IO a) where chPrintfImpl _ (PrintfCont final _) - = final >> return undefined + = final >> hFlush stderr >> hFlush stdout >> return undefined instance (CritHPrintfType r, PrintfArg a) => CritHPrintfType (a -> r) where chPrintfImpl check (PrintfCont _ anotherArg) x diff --git a/Criterion/Internal.hs b/Criterion/Internal.hs --- a/Criterion/Internal.hs +++ b/Criterion/Internal.hs @@ -14,39 +14,48 @@ ( runAndAnalyse , runAndAnalyseOne - , runNotAnalyse + , runOne + , runFixedIters ) where +import Control.Applicative ((<$>)) import Control.DeepSeq (rnf) import Control.Exception (evaluate) import Control.Monad (foldM, forM_, void, when) import Control.Monad.Reader (ask, asks) -import Control.Monad.Trans (liftIO) +import Control.Monad.Trans (MonadIO, liftIO) import Control.Monad.Trans.Except import Data.Binary (encode) import Data.Int (Int64) import qualified Data.ByteString.Lazy as L import Criterion.Analysis (analyseSample, noteOutliers) -import Criterion.IO (header, hGetReports) +import Criterion.IO (header, hGetRecords) import Criterion.IO.Printf (note, printError, prolix, writeCsv) import Criterion.Measurement (runBenchmark, secs) import Criterion.Monad (Criterion) import Criterion.Report (report) import Criterion.Types hiding (measure) import qualified Data.Map as Map +import Data.Vector (Vector) import Statistics.Resampling.Bootstrap (Estimate(..)) import System.Directory (getTemporaryDirectory, removeFile) import System.IO (IOMode(..), SeekMode(..), hClose, hSeek, openBinaryFile, openBinaryTempFile) import Text.Printf (printf) --- | Run a single benchmark and analyse its performance. -runAndAnalyseOne :: Int -> String -> Benchmarkable -> Criterion Report -runAndAnalyseOne i desc bm = do +-- | Run a single benchmark. +runOne :: Int -> String -> Benchmarkable -> Criterion DataRecord +runOne i desc bm = do Config{..} <- ask (meas,timeTaken) <- liftIO $ runBenchmark bm timeLimit when (timeTaken > timeLimit * 1.25) . void $ prolix "measurement took %s\n" (secs timeTaken) + return (Measurement i desc meas) + +-- | Analyse a single benchmark. +analyseOne :: Int -> String -> Vector Measured -> Criterion DataRecord +analyseOne i desc meas = do + Config{..} <- ask _ <- prolix "analysing with %d resamples\n" resamples erp <- runExceptT $ analyseSample i desc meas case erp of @@ -81,7 +90,7 @@ (round (ovFraction * 100) :: Int) wibble return () _ <- note "\n" - return rpt + return (Analysed rpt) where bs :: (Double -> String) -> String -> Estimate -> Criterion () bs f metric Estimate{..} = note "%-20s %-10s (%s .. %s%s)\n" metric @@ -89,10 +98,11 @@ (if estConfidenceLevel == 0.95 then "" else printf ", ci %.3f" estConfidenceLevel) --- | Determine whether an Environment benchmark should be run. -shouldRunEnv :: (String -> Bool) -> String -> (s -> Benchmark) -> Bool -shouldRunEnv p pfx mkbench = - any (p . addPrefix pfx) . benchNames . mkbench $ undefined +-- | Run a single benchmark and analyse its performance. +runAndAnalyseOne :: Int -> String -> Benchmarkable -> Criterion DataRecord +runAndAnalyseOne i desc bm = do + Measurement _ _ meas <- runOne i desc bm + analyseOne i desc meas -- | Run, and analyse, one or more benchmarks. runAndAnalyse :: (String -> Bool) -- ^ A predicate that chooses @@ -100,7 +110,7 @@ -- name. -> Benchmark -> Criterion () -runAndAnalyse p bs' = do +runAndAnalyse select bs = do mbRawFile <- asks rawDataFile (rawFile, handle) <- liftIO $ case mbRawFile of @@ -112,28 +122,14 @@ return (file, handle) liftIO $ L.hPut handle header - let go !k (pfx, Environment mkenv mkbench) - | shouldRunEnv p pfx mkbench = do - e <- liftIO $ do - ee <- mkenv - evaluate (rnf ee) - return ee - go k (pfx, mkbench e) - | otherwise = return (k :: Int) - go !k (pfx, Benchmark desc b) - | p desc' = do _ <- note "benchmarking %s\n" desc' - rpt <- runAndAnalyseOne k desc' b - liftIO $ L.hPut handle (encode rpt) - return $! k + 1 - | otherwise = return (k :: Int) - where desc' = addPrefix pfx desc - go !k (pfx, BenchGroup desc bs) = - foldM go k [(addPrefix pfx desc, b) | b <- bs] - _ <- go 0 ("", bs') + for select bs $ \idx desc bm -> do + _ <- note "benchmarking %s\n" desc + rpt <- runAndAnalyseOne idx desc bm + liftIO $ L.hPut handle (encode rpt) rpts <- (either fail return =<<) . liftIO $ do hSeek handle AbsoluteSeek 0 - rs <- hGetReports handle + rs <- fmap (map (\(Analysed r) -> r)) <$> hGetRecords handle hClose handle case mbRawFile of Just _ -> return rs @@ -143,28 +139,40 @@ junit rpts -- | Run a benchmark without analysing its performance. -runNotAnalyse :: Int64 -- ^ Number of loop iterations to run. +runFixedIters :: Int64 -- ^ Number of loop iterations to run. -> (String -> Bool) -- ^ A predicate that chooses -- whether to run a benchmark by its -- name. -> Benchmark -> Criterion () -runNotAnalyse iters p bs' = goQuickly "" bs' - where goQuickly :: String -> Benchmark -> Criterion () - goQuickly pfx (Environment mkenv mkbench) - | shouldRunEnv p pfx mkbench = do - e <- liftIO mkenv - goQuickly pfx (mkbench e) - | otherwise = return () - goQuickly pfx (Benchmark desc b) - | p desc' = do _ <- note "benchmarking %s\n" desc' - runOne b - | otherwise = return () - where desc' = addPrefix pfx desc - goQuickly pfx (BenchGroup desc bs) = - mapM_ (goQuickly (addPrefix pfx desc)) bs +runFixedIters iters select bs = + for select bs $ \_idx desc bm -> do + _ <- note "benchmarking %s\n" desc + liftIO $ runRepeatedly bm iters - runOne (Benchmarkable run) = liftIO (run iters) +-- | Iterate over benchmarks. +for :: MonadIO m => (String -> Bool) -> Benchmark + -> (Int -> String -> Benchmarkable -> m ()) -> m () +for select bs0 handle = go (0::Int) ("", bs0) >> return () + where + go !idx (pfx, Environment mkenv mkbench) + | shouldRun pfx mkbench = do + e <- liftIO $ do + ee <- mkenv + evaluate (rnf ee) + return ee + go idx (pfx, mkbench e) + | otherwise = return idx + go idx (pfx, Benchmark desc b) + | select desc' = do handle idx desc' b; return $! idx + 1 + | otherwise = return idx + where desc' = addPrefix pfx desc + go idx (pfx, BenchGroup desc bs) = + foldM go idx [(addPrefix pfx desc, b) | b <- bs] + + shouldRun pfx mkbench = + any (select . addPrefix pfx) . benchNames . mkbench $ + error "Criterion.env could not determine the list of your benchmarks since they force the environment (see the documentation for details)" -- | Write summary JUnit file (if applicable) junit :: [Report] -> Criterion () diff --git a/Criterion/Main.hs b/Criterion/Main.hs --- a/Criterion/Main.hs +++ b/Criterion/Main.hs @@ -42,12 +42,13 @@ , defaultConfig -- * Other useful code , makeMatcher + , runMode ) where import Control.Monad (unless) import Control.Monad.Trans (liftIO) import Criterion.IO.Printf (printError, writeCsv) -import Criterion.Internal (runAndAnalyse, runNotAnalyse) +import Criterion.Internal (runAndAnalyse, runFixedIters) import Criterion.Main.Options (MatchType(..), Mode(..), defaultConfig, describe, versionInfo) import Criterion.Measurement (initializeTime) @@ -130,14 +131,21 @@ -> IO () defaultMainWith defCfg bs = do wat <- execParser (describe defCfg) - let bsgroup = BenchGroup "" bs + runMode wat bs + +-- | Run a set of 'Benchmark's with the given 'Mode'. +-- +-- This can be useful if you have a 'Mode' from some other source (e.g. from a +-- one in your benchmark driver's command-line parser). +runMode :: Mode -> [Benchmark] -> IO () +runMode wat bs = case wat of List -> mapM_ putStrLn . sort . concatMap benchNames $ bs Version -> putStrLn versionInfo - OnlyRun iters matchType benches -> do + RunIters iters matchType benches -> do shouldRun <- selectBenches matchType benches bsgroup withConfig defaultConfig $ - runNotAnalyse iters shouldRun bsgroup + runFixedIters iters shouldRun bsgroup Run cfg matchType benches -> do shouldRun <- selectBenches matchType benches bsgroup withConfig cfg $ do @@ -145,6 +153,7 @@ "StddevUB") liftIO initializeTime runAndAnalyse shouldRun bsgroup + where bsgroup = BenchGroup "" bs -- | Display an error message from a command line parsing failure, and -- exit. diff --git a/Criterion/Main/Options.hs b/Criterion/Main/Options.hs --- a/Criterion/Main/Options.hs +++ b/Criterion/Main/Options.hs @@ -54,7 +54,7 @@ -- ^ List all benchmarks. | Version -- ^ Print the version. - | OnlyRun Int64 MatchType [String] + | RunIters Int64 MatchType [String] -- ^ Run the given benchmarks, without collecting or -- analysing performance numbers. | Run Config MatchType [String] @@ -84,51 +84,51 @@ -> Parser Mode parseWith cfg = (matchNames (Run <$> config cfg)) <|> - onlyRun <|> - (List <$ switch (long "list" <> short 'l' <> help "list benchmarks")) <|> - (Version <$ switch (long "version" <> help "show version info")) + runIters <|> + (List <$ switch (long "list" <> short 'l' <> help "List benchmarks")) <|> + (Version <$ switch (long "version" <> help "Show version info")) where - onlyRun = matchNames $ - OnlyRun <$> option auto - (long "only-run" <> short 'n' <> metavar "ITERS" <> - help "run benchmarks, don't analyse") + runIters = matchNames $ + RunIters <$> option auto + (long "iters" <> short 'n' <> metavar "ITERS" <> + help "Run benchmarks, don't analyse") matchNames wat = wat <*> option match (long "match" <> short 'm' <> metavar "MATCH" <> value Prefix <> - help "how to match benchmark names") + help "How to match benchmark names (\"prefix\" or \"glob\")") <*> many (argument str (metavar "NAME...")) config :: Config -> Parser Config config Config{..} = Config <$> option (range 0.001 0.999) (long "ci" <> short 'I' <> metavar "CI" <> value confInterval <> - help "confidence interval") + help "Confidence interval") <*> (not <$> switch (long "no-gc" <> short 'G' <> - help "do not collect garbage between iterations")) + help "Do not collect garbage between iterations")) <*> option (range 0.1 86400) (long "time-limit" <> short 'L' <> metavar "SECS" <> value timeLimit <> - help "time limit to run a benchmark") + help "Time limit to run a benchmark") <*> option (range 1 1000000) (long "resamples" <> metavar "COUNT" <> value resamples <> - help "number of bootstrap resamples to perform") + help "Number of bootstrap resamples to perform") <*> many (option regressParams (long "regress" <> metavar "RESP:PRED.." <> - help "regressions to perform")) + help "Regressions to perform")) <*> outputOption rawDataFile (long "raw" <> - help "file to write raw data to") + help "File to write raw data to") <*> outputOption reportFile (long "output" <> short 'o' <> - help "file to write report to") + help "File to write report to") <*> outputOption csvFile (long "csv" <> - help "file to write CSV summary to") + help "File to write CSV summary to") <*> outputOption junitFile (long "junit" <> - help "file to write JUnit summary to") + help "File to write JUnit summary to") <*> (toEnum <$> option (range 0 2) (long "verbosity" <> short 'v' <> metavar "LEVEL" <> value (fromEnum verbosity) <> - help "verbosity level")) + help "Verbosity level")) <*> strOption (long "template" <> short 't' <> metavar "FILE" <> value template <> - help "template to use for report") + help "Template to use for report") outputOption :: Maybe String -> Mod OptionFields String -> Parser (Maybe String) outputOption file m = @@ -151,8 +151,9 @@ mm | mm `isPrefixOf` "pfx" -> return Prefix | mm `isPrefixOf` "prefix" -> return Prefix | mm `isPrefixOf` "glob" -> return Glob - | otherwise -> readerError $ - show m ++ " is not a known match type" + | otherwise -> + readerError $ show m ++ " is not a known match type. " + ++ "Try \"prefix\" or \"glob\"." regressParams :: ReadM ([String], String) regressParams = do diff --git a/Criterion/Types.hs b/Criterion/Types.hs --- a/Criterion/Types.hs +++ b/Criterion/Types.hs @@ -60,6 +60,7 @@ , KDE(..) , Report(..) , SampleAnalysis(..) + , DataRecord(..) ) where import Control.Applicative ((<$>), (<*>)) @@ -119,7 +120,7 @@ -- | A pure function or impure action that can be benchmarked. The -- 'Int64' parameter indicates the number of times to run the given -- function or action. -newtype Benchmarkable = Benchmarkable (Int64 -> IO ()) +newtype Benchmarkable = Benchmarkable { runRepeatedly :: Int64 -> IO () } -- | A collection of measurements made while benchmarking. -- @@ -384,13 +385,13 @@ -- benchmarks to be run. -- -- > setupEnv = do --- > let small = replicate 1000 1 --- > big <- readFile "/usr/dict/words" +-- > let small = replicate 1000 (1 :: Int) +-- > big <- map length . words <$> readFile "/usr/dict/words" -- > return (small, big) -- > -- > main = defaultMain [ -- > -- notice the lazy pattern match here! --- > env setupEnv $ \ ~(small,big) -> +-- > env setupEnv $ \ ~(small,big) -> bgroup "main" [ -- > bgroup "small" [ -- > bench "length" $ whnf length small -- > , bench "length . filter" $ whnf (length . filter (==1)) small @@ -399,7 +400,7 @@ -- > bench "length" $ whnf length big -- > , bench "length . filter" $ whnf (length . filter (==1)) big -- > ] --- > ] +-- > ] ] -- -- __Discussion.__ The environment created in the example above is -- intentionally /not/ ideal. As Haskell's scoping rules suggest, the @@ -636,3 +637,25 @@ rnf reportNumber `seq` rnf reportName `seq` rnf reportKeys `seq` rnf reportMeasured `seq` rnf reportAnalysis `seq` rnf reportOutliers `seq` rnf reportKDEs + +data DataRecord = Measurement Int String (V.Vector Measured) + | Analysed Report + deriving (Eq, Read, Show, Typeable, Data, Generic) + +instance Binary DataRecord where + put (Measurement i n v) = putWord8 0 >> put i >> put n >> put v + put (Analysed r) = putWord8 1 >> put r + + get = do + w <- getWord8 + case w of + 0 -> Measurement <$> get <*> get <*> get + 1 -> Analysed <$> get + _ -> error ("bad tag " ++ show w) + +instance NFData DataRecord where + rnf (Measurement i n v) = rnf i `seq` rnf n `seq` rnf v + rnf (Analysed r) = rnf r + +instance FromJSON DataRecord +instance ToJSON DataRecord diff --git a/app/App.hs b/app/App.hs --- a/app/App.hs +++ b/app/App.hs @@ -1,5 +1,10 @@ module Main (main) where +import Options + main :: IO () main = do - return () + cmd <- parseCommandLine + case cmd of + Version -> putStrLn versionInfo + _ -> print cmd diff --git a/app/Options.hs b/app/Options.hs new file mode 100644 --- /dev/null +++ b/app/Options.hs @@ -0,0 +1,39 @@ +{-# LANGUAGE DeriveDataTypeable, DeriveGeneric, RecordWildCards #-} + +module Options + ( + CommandLine(..) + , commandLine + , parseCommandLine + , versionInfo + ) where + +import Data.Version (showVersion) +import Data.Data (Data, Typeable) +import GHC.Generics (Generic) +import Paths_criterion (version) +import Options.Applicative + +data CommandLine = Analyse [FilePath] + | Version + deriving (Eq, Read, Show, Typeable, Data, Generic) + +analyseOptions :: Parser CommandLine +analyseOptions = Analyse <$> some (argument str (metavar "FILE [...]")) + +parseCommand :: Parser CommandLine +parseCommand = + (Version <$ switch (long "version" <> help "Show version info")) <|> + (subparser $ + command "analyse" (info analyseOptions (progDesc "Analyse measurements"))) + +commandLine :: ParserInfo CommandLine +commandLine = info (helper <*> parseCommand) $ + header versionInfo <> + fullDesc + +parseCommandLine :: IO CommandLine +parseCommandLine = execParser commandLine + +versionInfo :: String +versionInfo = "criterion " ++ showVersion version diff --git a/cbits/cycles.c b/cbits/cycles.c --- a/cbits/cycles.c +++ b/cbits/cycles.c @@ -1,8 +1,57 @@ #include "Rts.h" +#if x86_64_HOST_ARCH + StgWord64 criterion_rdtsc(void) { StgWord32 hi, lo; __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); return ((StgWord64) lo) | (((StgWord64) hi)<<32); } + +#elif linux_HOST_OS + +/* + * This should work on all Linux. + * + * Technique by Austin Seipp found here: + * + * http://neocontra.blogspot.com/2013/05/user-mode-performance-counters-for.html + */ + +#include +#include +#include + +static int fddev = -1; +__attribute__((constructor)) +static void +init(void) +{ + static struct perf_event_attr attr; + attr.type = PERF_TYPE_HARDWARE; + attr.config = PERF_COUNT_HW_CPU_CYCLES; + fddev = syscall (__NR_perf_event_open, &attr, 0, -1, -1, 0); +} + +__attribute__((destructor)) +static void +fini(void) +{ + close(fddev); +} + +StgWord64 +criterion_rdtsc (void) +{ + StgWord64 result = 0; + if (read (fddev, &result, sizeof(result)) < sizeof(result)) + return 0; + return result; +} + +#else + +#error Unsupported OS/architecture/compiler! + +#endif diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,35 @@ +1.1.1.0 + +* If a benchmark uses `Criterion.env` in a non-lazy way, and you try + to use `--list` to list benchmark names, you'll now get an + understandable error message instead of something cryptic. + +* We now flush stdout and stderr after printing messages, so that + output is printed promptly even when piped (e.g. into a pager). + +* A new function `runMode` allows custom benchmarking applications to + run benchmarks with control over the `Mode` used. + +* Added support for Linux on non-Intel CPUs. + +* This version supports GHC 8. + +* The `--only-run` option for benchmarks is renamed to `--iters`. + 1.1.0.0 -* +* The dependency on the either package has been dropped in favour of a + dependency on transformers-compat. This greatly reduces the number + of packages criterion depends on. This shouldn't affect the + user-visible API. + +* The documentation claimed that environments were created only when + needed, but this wasn't implemented. (gh-76) + +* The package now compiles with GHC 7.10. + +* On Windows with a non-Unicode code page, printing results used to + cause a crash. (gh-55) 1.0.2.0 diff --git a/criterion.cabal b/criterion.cabal --- a/criterion.cabal +++ b/criterion.cabal @@ -1,5 +1,5 @@ name: criterion -version: 1.1.0.0 +version: 1.1.1.0 synopsis: Robust, reliable performance measurement and analysis license: BSD3 license-file: LICENSE @@ -104,16 +104,18 @@ ghc-options: -fwarn-tabs executable criterion - buildable: False hs-source-dirs: app main-is: App.hs + other-modules: + Options ghc-options: -Wall -threaded -rtsopts build-depends: base, - criterion + criterion, + optparse-applicative test-suite sanity type: exitcode-stdio-1.0 diff --git a/examples/fibber.html b/examples/fibber.html deleted file mode 100644 --- a/examples/fibber.html +++ /dev/null @@ -1,726 +0,0 @@ - - - - - criterion report - - - - - - - -
-
-

criterion performance measurements

- -

overview

- -

want to understand this report?

- -
- -

fib/1

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time1.534933170419988e-81.559289558335549e-81.5955027539726543e-8
Standard deviation6.920776075641315e-109.603540851522639e-101.5315630337882154e-9
- - -

Outlying measurements have severe - (0.8088974150289796%) - effect on estimated standard deviation.

- -

fib/5

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time2.0708698124838526e-72.1107084459065708e-72.1648984992614215e-7
Standard deviation1.0816855360797646e-81.509440180252194e-82.2611667067984586e-8
- - -

Outlying measurements have severe - (0.8225875438214796%) - effect on estimated standard deviation.

- -

fib/9

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time1.5092983334102336e-61.531476067375206e-61.5561257644394457e-6
Standard deviation6.251710322835121e-88.358628655460513e-81.1570124318539241e-7
- - -

Outlying measurements have severe - (0.6916115394455409%) - effect on estimated standard deviation.

- -

fib/11

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time4.0381903380782345e-64.1056747646400756e-64.177267574309266e-6
Standard deviation1.967404221672586e-72.3166650962027132e-72.8018472343096967e-7
- - -

Outlying measurements have severe - (0.6846724652533784%) - effect on estimated standard deviation.

- - -

understanding this report

- -

In this report, each function benchmarked by criterion is assigned - a section of its own. The charts in each section are active; if - you hover your mouse over data points and annotations, you will see - more details.

- -
    -
  • The chart on the left is a - kernel - density estimate (also known as a KDE) of time - measurements. This graphs the probability of any given time - measurement occurring. A spike indicates that a measurement of a - particular time occurred; its height indicates how often that - measurement was repeated.
    • - -
  • The chart on the right is the raw data from which the kernel - density estimate is built. The x axis indicates the - number of loop iterations, while the y axis shows measured - execution time for the given number of loop iterations. The - line behind the values is the linear regression prediction of - execution time for a given number of iterations. Ideally, all - measurements will be on (or very near) this line.
    • -
- -

Under the charts is a small table. - The first two rows are the results of a linear regression run - on the measurements displayed in the right-hand chart.

- -
    -
  • OLS regression indicates the - time estimated for a single loop iteration using an ordinary - least-squares regression model. This number is more accurate - than the mean estimate below it, as it more effectively - eliminates measurement overhead and other constant factors.
    • -
  • R² goodness-of-fit is a measure of how - accurately the linear regression model fits the observed - measurements. If the measurements are not too noisy, R² - should lie between 0.99 and 1, indicating an excellent fit. If - the number is below 0.99, something is confounding the accuracy - of the linear model.
    • -
  • Mean execution time and standard deviation are - statistics calculated from execution time - divided by number of iterations.
    • -
- -

We use a statistical technique called - the bootstrap - to provide confidence intervals on our estimates. The - bootstrap-derived upper and lower bounds on estimates let you see - how accurate we believe those estimates to be. (Hover the mouse - over the table headers to see the confidence levels.)

- -

A noisy benchmarking environment can cause some or many - measurements to fall far from the mean. These outlying - measurements can have a significant inflationary effect on the - estimate of the standard deviation. We calculate and display an - estimate of the extent to which the standard deviation has been - inflated by outliers.

- - - -
-
- - - diff --git a/examples/maps.html b/examples/maps.html deleted file mode 100644 --- a/examples/maps.html +++ /dev/null @@ -1,552 +0,0 @@ - - - - - criterion report - - - - - - - -
-
-

criterion performance measurements

- -

overview

- -

want to understand this report?

- -
- -

ByteString/HashMap/random

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
OLS regressionxxxxxxxxx
R² goodness-of-fitxxxxxxxxx
Mean execution time5.54613319607341e-35.621667703915931e-35.713526073454561e-3
Standard deviation1.972562820146363e-42.494938876886024e-43.1487131555210624e-4
- - -

Outlying measurements have moderate - (0.21993690418913867%) - effect on estimated standard deviation.

- - -

understanding this report

- -

In this report, each function benchmarked by criterion is assigned - a section of its own. The charts in each section are active; if - you hover your mouse over data points and annotations, you will see - more details.

- -
    -
  • The chart on the left is a - kernel - density estimate (also known as a KDE) of time - measurements. This graphs the probability of any given time - measurement occurring. A spike indicates that a measurement of a - particular time occurred; its height indicates how often that - measurement was repeated.
    • - -
  • The chart on the right is the raw data from which the kernel - density estimate is built. The x axis indicates the - number of loop iterations, while the y axis shows measured - execution time for the given number of loop iterations. The - line behind the values is the linear regression prediction of - execution time for a given number of iterations. Ideally, all - measurements will be on (or very near) this line.
    • -
- -

Under the charts is a small table. - The first two rows are the results of a linear regression run - on the measurements displayed in the right-hand chart.

- -
    -
  • OLS regression indicates the - time estimated for a single loop iteration using an ordinary - least-squares regression model. This number is more accurate - than the mean estimate below it, as it more effectively - eliminates measurement overhead and other constant factors.
    • -
  • R² goodness-of-fit is a measure of how - accurately the linear regression model fits the observed - measurements. If the measurements are not too noisy, R² - should lie between 0.99 and 1, indicating an excellent fit. If - the number is below 0.99, something is confounding the accuracy - of the linear model.
    • -
  • Mean execution time and standard deviation are - statistics calculated from execution time - divided by number of iterations.
    • -
- -

We use a statistical technique called - the bootstrap - to provide confidence intervals on our estimates. The - bootstrap-derived upper and lower bounds on estimates let you see - how accurate we believe those estimates to be. (Hover the mouse - over the table headers to see the confidence levels.)

- -

A noisy benchmarking environment can cause some or many - measurements to fall far from the mean. These outlying - measurements can have a significant inflationary effect on the - estimate of the standard deviation. We calculate and display an - estimate of the extent to which the standard deviation has been - inflated by outliers.

- - - -
-
- - - diff --git a/templates/default2.tpl b/templates/default2.tpl deleted file mode 100644 --- a/templates/default2.tpl +++ /dev/null @@ -1,296 +0,0 @@ - - - - - criterion report - - - - - - - - -
-
-

criterion performance measurements

- -

overview

- -

want to understand this report?

- -
- -{{#report}} -

{{name}}

- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - -
lower boundestimateupper bound
Mean execution time{{anMean.estLowerBound}}{{anMean.estPoint}}{{anMean.estUpperBound}}
Standard deviation{{anStdDev.estLowerBound}}{{anStdDev.estPoint}}{{anStdDev.estUpperBound}}
- - -

Outlying measurements have {{anOutlierVar.ovDesc}} - ({{anOutlierVar.ovFraction}}%) - effect on estimated standard deviation.

- -{{/report}} - -

understanding this report

- -

In this report, each function benchmarked by criterion is assigned - a section of its own. In each section, we display two charts, each - with an x axis that represents measured execution time. - These charts are active; if you hover your mouse over data points - and annotations, you will see more details.

- -
    -
  • The chart on the left is a - kernel - density estimate (also known as a KDE) of time - measurements. This graphs the probability of any given time - measurement occurring. A spike indicates that a measurement of a - particular time occurred; its height indicates how often that - measurement was repeated.
    • - -
  • The chart on the right is the raw data from which the kernel - density estimate is built. Measurements are displayed on - the x axis in the order in which they occurred. The - number of iterations of the measurement loop increases with each - successive measurement.
    • -
- -

Under the charts is a small table displaying the mean and standard - deviation of the measurements. We use a statistical technique - called - the bootstrap - to provide confidence intervals on our estimates of these values. - The bootstrap-derived upper and lower bounds on the mean and - standard deviation let you see how accurate we believe those - estimates to be. (Hover the mouse over the table headers to see - the confidence levels.)

- -

A noisy benchmarking environment can cause some or many - measurements to fall far from the mean. These outlying - measurements can have a significant inflationary effect on the - estimate of the standard deviation. We calculate and display an - estimate of the extent to which the standard deviation has been - inflated by outliers.

- - - -
-
- - -