criterion 0.3.0 → 0.4.0

raw patch · 13 files changed

+125/−131 lines, 13 filesdep +deepseqPVP ok

version bump matches the API change (PVP)

Dependencies added: deepseq

API changes (from Hackage documentation)

Files

• Criterion.hs +11/−8
• Criterion/Analysis.hs +4/−4
• Criterion/Config.hs +1/−1
• Criterion/Environment.hs +2/−2
• Criterion/IO.hs +3/−3
• Criterion/Main.hs +43/−68
• Criterion/Monad.hs +7/−7
• Criterion/Plot.hs +2/−2
• Criterion/Types.hs +27/−12
• criterion.cabal +2/−1
• examples/Fibber.hs +10/−10
• examples/Judy.hs +8/−8
• examples/Tiny.hs +5/−5

@@ -13,7 +13,9 @@     (       Benchmarkable(..)     , Benchmark-    , B(..)+    , Pure+    , nf+    , whnf     , bench     , bgroup     , runBenchmark@@ -28,9 +30,10 @@ import Criterion.Environment (Environment(..)) import Criterion.IO (note, prolix, summary) import Criterion.Measurement (getTime, runForAtLeast, secs, time_)-import Criterion.Monad (ConfigM, getConfig, getConfigItem)+import Criterion.Monad (Criterion, getConfig, getConfigItem) import Criterion.Plot (plotWith, plotKDE, plotTiming)-import Criterion.Types (Benchmarkable(..), Benchmark(..), B(..), bench, bgroup)+import Criterion.Types (Benchmarkable(..), Benchmark(..), Pure,+                        bench, bgroup, nf, whnf) import Data.Array.Vector ((:*:)(..), concatU, lengthU, mapU) import Statistics.Function (createIO, minMax) import Statistics.KernelDensity (epanechnikovPDF)@@ -44,7 +47,7 @@  -- | Run a single benchmark, and return timings measured when -- executing it.-runBenchmark :: Benchmarkable b => Environment -> b -> ConfigM Sample+runBenchmark :: Benchmarkable b => Environment -> b -> Criterion Sample runBenchmark env b = do   liftIO $ runForAtLeast 0.1 10000 (`replicateM_` getTime)   let minTime = envClockResolution env * 1000@@ -67,7 +70,7 @@  -- | Run a single benchmark and analyse its performance. runAndAnalyseOne :: Benchmarkable b => Environment -> String -> b-                 -> ConfigM Sample+                 -> Criterion Sample runAndAnalyseOne env _desc b = do   times <- runBenchmark env b   let numSamples = lengthU times@@ -91,7 +94,7 @@   note "variance introduced by outliers: %.3f%%\n" (v * 100)   note "variance is %s by outliers\n" wibble   return times-  where bs :: String -> Estimate -> ConfigM ()+  where bs :: String -> Estimate -> Criterion ()         bs d e = do note "%s: %s, lb %s, ub %s, ci %.3f\n" d                       (secs $ estPoint e)                       (secs $ estLowerBound e) (secs $ estUpperBound e)@@ -100,7 +103,7 @@                       (estPoint e)                       (estLowerBound e) (estUpperBound e) -plotAll :: [(String, Sample)] -> ConfigM ()+plotAll :: [(String, Sample)] -> Criterion () plotAll descTimes = forM_ descTimes $ \(desc,times) -> do   plotWith Timing $ \o -> plotTiming o desc times   plotWith KernelDensity $ \o -> uncurry (plotKDE o desc extremes)@@ -120,7 +123,7 @@                                   -- name.               -> Environment               -> Benchmark-              -> ConfigM ()+              -> Criterion () runAndAnalyse p env = plotAll <=< go ""   where go pfx (Benchmark desc b)             | p desc'   = do note "\nbenchmarking %s\n" desc'

@@ -23,7 +23,7 @@ import Control.Monad (when) import Criterion.IO (note) import Criterion.Measurement (secs)-import Criterion.Monad (ConfigM)+import Criterion.Monad (Criterion) import Data.Array.Vector (foldlU) import Data.Int (Int64) import Data.Monoid (Monoid(..))@@ -125,7 +125,7 @@ analyseMean :: Sample             -> Int              -- ^ Number of iterations used to                                 -- compute the sample.-            -> ConfigM Double+            -> Criterion Double analyseMean a iters = do   let µ = mean a   note "mean is %s (%d iterations)\n" (secs µ) iters@@ -133,10 +133,10 @@   return µ  -- | Display a report of the 'Outliers' present in a 'Sample'.-noteOutliers :: Outliers -> ConfigM ()+noteOutliers :: Outliers -> Criterion () noteOutliers o = do   let frac n = (100::Double) * fromIntegral n / fromIntegral (samplesSeen o)-      check :: Int64 -> Double -> String -> ConfigM ()+      check :: Int64 -> Double -> String -> Criterion ()       check k t d = when (frac k > t) $                     note "  %d (%.1g%%) %s\n" k (frac k) d       outCount = countOutliers o

@@ -69,7 +69,7 @@     , cfgPrintExit    :: PrintExit   -- ^ Whether to print information and exit.     , cfgResamples    :: Last Int    -- ^ Number of resamples to perform.     , cfgSamples      :: Last Int    -- ^ Number of samples to collect.-    , cfgSummaryFile  :: Last String -- ^ Filename of summary CSV+    , cfgSummaryFile  :: Last FilePath -- ^ Filename of summary CSV     , cfgVerbosity    :: Last Verbosity -- ^ Whether to run verbosely.     } deriving (Eq, Read, Show, Typeable) 

@@ -22,7 +22,7 @@ import Criterion.Analysis (analyseMean) import Criterion.IO (note) import Criterion.Measurement (getTime, runForAtLeast, time_)-import Criterion.Monad (ConfigM)+import Criterion.Monad (Criterion) import Data.Array.Vector import Data.Typeable (Typeable) import Statistics.Function (createIO)@@ -36,7 +36,7 @@     } deriving (Eq, Read, Show, Typeable)  -- | Measure the execution environment.-measureEnvironment :: ConfigM Environment+measureEnvironment :: Criterion Environment measureEnvironment = do   note "warming up\n"   (_ :*: seed :*: _) <- liftIO $ runForAtLeast 0.1 10000 resolution

@@ -21,7 +21,7 @@ import Control.Monad (when) import Control.Monad.Trans (liftIO) import Criterion.Config (Config, Verbosity(..), cfgSummaryFile, cfgVerbosity, fromLJ)-import Criterion.Monad (ConfigM, getConfig, getConfigItem)+import Criterion.Monad (Criterion, getConfig, getConfigItem) import Data.Monoid (getLast) import System.IO (Handle, stderr, stdout) import qualified Text.Printf (HPrintfType, hPrintf)@@ -39,7 +39,7 @@   chPrintfImpl :: (Config -> Bool) -> PrintfCont -> a  -instance CritHPrintfType (ConfigM a) where+instance CritHPrintfType (Criterion a) where   chPrintfImpl check (PrintfCont final _)     = do x <- getConfig          when (check x) (liftIO final)@@ -85,7 +85,7 @@ printError = chPrintf (const True) stderr  -- | Add to summary CSV (if applicable)-summary :: String -> ConfigM ()+summary :: String -> Criterion () summary msg   = do sumOpt <- getConfigItem (getLast . cfgSummaryFile)        case sumOpt of

@@ -27,10 +27,12 @@     -- * Types       Benchmarkable(..)     , Benchmark-    , B(..)+    , Pure     -- * Constructing benchmarks     , bench     , bgroup+    , nf+    , whnf     -- * Running benchmarks     , defaultMain     , defaultMainWith@@ -46,9 +48,9 @@ import Criterion.Environment (measureEnvironment) import Criterion.IO (note, printError) import Criterion.MultiMap (singleton)-import Criterion.Monad (withConfig)-import Criterion.Types (Benchmarkable(..), Benchmark(..), B(..), bench,-                        benchNames, bgroup)+import Criterion.Monad (Criterion, withConfig)+import Criterion.Types (Benchmarkable(..), Benchmark(..), Pure, bench,+                        benchNames, bgroup, nf, whnf) import Data.List (isPrefixOf, sort) import Data.Monoid (Monoid(..), Last(..)) import System.Console.GetOpt@@ -121,11 +123,11 @@  , Option ['I'] ["ci"] (ReqArg ci "CI")           "bootstrap confidence interval"  , Option ['l'] ["list"] (noArg mempty { cfgPrintExit = List })-          "print a list of all benchmark names, then exit"+          "print only a list of benchmark names"  , Option ['k'] ["plot-kde"] (ReqArg (plot KernelDensity) "TYPE")           "plot kernel density estimate of probabilities"  , Option [] ["kde-same-axis"] (noArg mempty {cfgPlotSameAxis = ljust True })-          "plot all KDE graphs with the same X axis range (useful for comparison)"+          "plot all KDE graphs with identical X axes"  , Option ['q'] ["quiet"] (noArg mempty { cfgVerbosity = ljust Quiet })           "print less output"  , Option [] ["resamples"]@@ -137,7 +139,7 @@  , Option ['t'] ["plot-timing"] (ReqArg (plot Timing) "TYPE")           "plot timings"  , Option ['u'] ["summary"] (ReqArg (\s -> return $ mempty { cfgSummaryFile = ljust s }) "FILENAME")-          "produce a summary CSV file of all the benchmark means and standard deviations"+          "produce a summary CSV file of all results"  , Option ['V'] ["version"] (noArg mempty { cfgPrintExit = Version })           "display version, then exit"  , Option ['v'] ["verbose"] (noArg mempty { cfgVerbosity = ljust Verbose })@@ -200,7 +202,7 @@ -- >                     ] -- >                    ] defaultMain :: [Benchmark] -> IO ()-defaultMain = defaultMainWith defaultConfig+defaultMain = defaultMainWith defaultConfig (return ())  -- | An entry point that can be used as a @main@ function, with -- configurable defaults.@@ -215,7 +217,7 @@ -- >              cfgPlot = M.singleton KernelDensity (Window 800 600) -- >            } -- > --- > main = defaultMainWith myConfig [+-- > main = defaultMainWith myConfig (return ()) [ -- >          bench "fib 30" $ B fib 30 -- >        ] --@@ -226,8 +228,11 @@ -- -- Run @\"Fib --help\"@ on the command line to get a list of command -- line options.-defaultMainWith :: Config -> [Benchmark] -> IO ()-defaultMainWith defCfg bs = do+defaultMainWith :: Config+                -> Criterion () -- ^ Prepare data prior to executing the first benchmark.+                -> [Benchmark]+                -> IO ()+defaultMainWith defCfg prep bs = do   (cfg, args) <- parseArgs defCfg defaultOptions =<< getArgs   withConfig cfg $    if cfgPrintExit cfg == List@@ -240,6 +245,7 @@         Nothing -> return ()       env <- measureEnvironment       let shouldRun b = null args || any (`isPrefixOf` b) args+      prep       runAndAnalyse shouldRun env $ BenchGroup "" bs  -- | Display an error message from a command line parsing failure, and@@ -280,17 +286,22 @@ -- only be evaluated once, for which all but the first iteration of -- the timing loop will be timing the cost of doing nothing. ----- To work around this, we provide two types for benchmarking pure--- code.  The first is a specialised tuple:+-- To work around this, we provide a special type, 'Pure', for+-- benchmarking pure code.  Values of this type are constructed using+-- one of two functions. --+-- The first is a function which will cause results to be evaluated to+-- head normal form (NF):+-- -- @--- data 'B' a = forall b. 'B' (a -> b) a+-- 'nf' :: 'NFData' b => (a -> b) -> a -> 'Pure' -- @ ----- The second is a specialised tuple named 'B':+-- The second will cause results to be evaluated to weak head normal+-- form (the Haskell default): -- -- @--- (a -> b, a)+-- 'whnf' :: (a -> b) -> a -> 'Pure' -- @ -- -- As both of these types suggest, when you want to benchmark a@@ -299,15 +310,10 @@ -- * The first element is the function, saturated with all but its --   last argument. ----- * The second is the last argument to the function.+-- * The second element is the last argument to the function. ----- In practice, it is much easier to use the 'B' tuple than a normal--- tuple.  Using 'B', the type checker can see when the function type--- @a -> b@ and its argument type @a@ are the same, whereas code may--- require an explicit type annotation to make this connection--- explicit for a regular tuple.  Here is an example that makes the--- distinction clearer.  Suppose we want to benchmark the following--- function:+-- Here is an example that makes the use of these functions clearer.+-- Suppose we want to benchmark the following function: -- -- @ -- firstN :: Int -> [Int]@@ -317,58 +323,27 @@ -- So in the easy case, we construct a benchmark as follows: -- -- @--- 'B' firstN 1000+-- 'nf' firstN 1000 -- @ -- -- The compiler will correctly infer that the number 1000 must have--- the type 'Int', and the type of the expression is------ @--- 'B' ['Int'] 'Int'--- @------ However, say we try to construct a benchmark using a tuple, as--- follows:------ @--- (firstN, 1000)--- @------ Since we have written a numeric literal with no explicit type, the--- compiler will correctly infer a rather general type for this--- expression:------ @--- ('Num' a) => ('Int' -> ['Int'], a)--- @------ This does not match the type @(a -> b, a)@, so we would have to--- explicitly annotate the number @1000@ as having the type @'Int'@--- for the typechecker to accept this as a valid benchmarkable--- expression.+-- the type 'Int', and the type of the expression is 'Pure'.  -- $rnf ----- The harness for evaluating a pure function only evaluates the--- result to weak head normal form (WHNF).  If you need the result--- evaluated all the way to normal form, use the @rnf@ function from--- the Control.Parallel.Strategies module to force its complete--- evaluation.+-- The 'whnf' harness for evaluating a pure function only evaluates+-- the result to weak head normal form (WHNF).  If you need the result+-- evaluated all the way to normal form, use the 'nf' function to+-- force its complete evaluation. ----- Using the @firstN@ example from earlier, to naive eyes it /appears/--- that the following code ought to benchmark the production of the--- first 1000 list elements:+-- Using the @firstN@ example from earlier, to naive eyes it might+-- /appear/ that the following code ought to benchmark the production+-- of the first 1000 list elements: -- -- @--- B firstN 1000+-- 'whnf' firstN 1000 -- @ ----- Because the result is only forced until WHNF is reached, what this--- /actually/ benchmarks is merely the production of the first list--- element!  Here is a corrected version:------ @--- B (rnf . firstN) 1000--- @--+-- Because in this case the result will only be forced until it+-- reaches WHNF, what this would /actually/ benchmark is merely the+-- production of the first list element!

@@ -11,7 +11,7 @@ -- The environment in which most criterion code executes. module Criterion.Monad     (-      ConfigM+      Criterion     , getConfig     , getConfigItem     , withConfig@@ -22,15 +22,15 @@ import Criterion.Config (Config)  -- | The monad in which most criterion code executes.-newtype ConfigM a = ConfigM {-      runConfigM :: ReaderT Config IO a+newtype Criterion a = Criterion {+      runCriterion :: ReaderT Config IO a     } deriving (Functor, Monad, MonadReader Config, MonadIO) -getConfig :: ConfigM Config+getConfig :: Criterion Config getConfig = ask -getConfigItem :: (Config -> a) -> ConfigM a+getConfigItem :: (Config -> a) -> Criterion a getConfigItem f = f `fmap` getConfig -withConfig :: Config -> ConfigM a -> IO a-withConfig = flip (runReaderT . runConfigM)+withConfig :: Config -> Criterion a -> IO a+withConfig = flip (runReaderT . runCriterion)

@@ -20,7 +20,7 @@  import Control.Monad.Trans (liftIO) import Criterion.Config-import Criterion.Monad (ConfigM, getConfigItem)+import Criterion.Monad (Criterion, getConfigItem) import Data.Array.Vector import Data.Char (isSpace, toLower) import Data.Foldable (forM_)@@ -40,7 +40,7 @@ import Criterion.IO (printError) #endif -plotWith :: Plot -> (PlotOutput -> IO ()) -> ConfigM ()+plotWith :: Plot -> (PlotOutput -> IO ()) -> Criterion () plotWith p plot = getConfigItem (M.lookup p . cfgPlot)                     >>= maybe (return ()) (liftIO . flip forM_ plot) 

@@ -1,4 +1,5 @@ {-# LANGUAGE ExistentialQuantification, FlexibleInstances, GADTs #-}+{-# OPTIONS_GHC -fno-warn-incomplete-patterns #-}  -- | -- Module      : Criterion.Types@@ -14,7 +15,7 @@ -- The core class is 'Benchmarkable', which admits both pure functions -- and 'IO' actions. ----- For a pure function of type @Int -> a@, the benchmarking harness+-- For a pure function of type @a -> b@, the benchmarking harness -- calls this function repeatedly, each time with a different 'Int' -- argument, and reduces the result the function returns to weak head -- normal form.  If you need the result reduced to normal form, that@@ -27,12 +28,15 @@     (       Benchmarkable(..)     , Benchmark(..)-    , B(..)+    , Pure+    , whnf+    , nf     , bench     , bgroup     , benchNames     ) where +import Control.DeepSeq (NFData, rnf) import Control.Exception (evaluate)  -- | A benchmarkable function or action.@@ -44,18 +48,29 @@  -- | A container for a pure function to benchmark, and an argument to -- supply to it each time it is evaluated.-data B a = forall b. B (a -> b) a+data Pure where+    WHNF :: (a -> b) -> a -> Pure+    NF :: NFData b => (a -> b) -> a -> Pure -instance Benchmarkable (a -> b, a) where-    run fx@(f,x) n-        | n <= 0    = return ()-        | otherwise = evaluate (f x) >> run fx (n-1)-    {-# INLINE run #-}+whnf :: (a -> b) -> a -> Pure+whnf = WHNF+{-# INLINE whnf #-} -instance Benchmarkable (B a) where-    run fx@(B f x) n-        | n <= 0    = return ()-        | otherwise = evaluate (f x) >> run fx (n-1)+nf :: NFData b => (a -> b) -> a -> Pure+nf = NF+{-# INLINE nf #-}++instance Benchmarkable Pure where+    run p@(WHNF _ _) = go p+      where+        go fx@(WHNF f x) n+            | n <= 0    = return ()+            | otherwise = evaluate (f x) >> go fx (n-1)+    run p@(NF _ _) = go p+      where+        go fx@(NF f x) n+            | n <= 0    = return ()+            | otherwise = evaluate (rnf (f x)) >> go fx (n-1)     {-# INLINE run #-}  instance Benchmarkable (IO a) where

@@ -1,5 +1,5 @@ name:           criterion-version:        0.3.0+version:        0.4.0 synopsis:       Robust, reliable performance measurement and analysis license:        BSD3 license-file:   LICENSE@@ -46,6 +46,7 @@     base < 5,     bytestring >= 0.9 && < 1.0,     containers,+    deepseq >= 1.1.0.0,     filepath,     mtl,     parallel,

@@ -25,18 +25,18 @@             return $! i * j  main = defaultMain [-        bgroup "tiny" [ bench "fib 10" $ B fib 10-                      , bench "fib 15" $ B fib 15-                      , bench "fib 20" $ B fib 20-                      , bench "fib 25" $ B fib 25+        bgroup "tiny" [ bench "fib 10" $ whnf fib 10+                      , bench "fib 15" $ whnf fib 15+                      , bench "fib 20" $ whnf fib 20+                      , bench "fib 25" $ whnf fib 25                       ],-        bgroup "fib" [ bench "fib 10" $ B fib 10-                     , bench "fib 35" $ B fib 35-                     , bench "fib 37" $ B fib 37+        bgroup "fib" [ bench "fib 10" $ whnf fib 10+                     , bench "fib 35" $ whnf fib 35+                     , bench "fib 37" $ whnf fib 37                      ],-        bgroup "fact" [ bench "fact 100"  $ B fact 100-                      , bench "fact 1000" $ B fact 1000-                      , bench "fact 3000" $ B fact 3000+        bgroup "fact" [ bench "fact 100"  $ whnf fact 100+                      , bench "fact 1000" $ whnf fact 1000+                      , bench "fact 3000" $ whnf fact 3000                       ],         bgroup "fio" [ bench "fio 100"  (fio 100)                      , bench "fio 1000" (fio 1000)

@@ -17,19 +17,19 @@ -- enough for us. myConfig = defaultConfig { cfgPerformGC = ljust True } -main = defaultMainWith myConfig [+main = defaultMainWith myConfig (return ()) [         bgroup "judy" [-                     bench "insert 1M"   $ B testit 1000000-                   , bench "insert 10M"  $ B testit 10000000-                   , bench "insert 100M" $ B testit 100000000+                     bench "insert 1M"   $ whnf testit 1000000+                   , bench "insert 10M"  $ whnf testit 10000000+                   , bench "insert 100M" $ whnf testit 100000000                    ],         bgroup "map" [-                     bench "insert 100k" $ B testmap 100000-                   , bench "insert 1M"   $ B testmap 1000000+                     bench "insert 100k" $ whnf testmap 100000+                   , bench "insert 1M"   $ whnf testmap 1000000                    ],         bgroup "intmap" [-                     bench "insert 100k" $ B testintmap 100000-                   , bench "insert 1M"   $ B testintmap 1000000+                     bench "insert 100k" $ whnf testintmap 100000+                   , bench "insert 1M"   $ whnf testintmap 1000000                    ]     ] 

@@ -12,11 +12,11 @@              cfgPlot = M.singleton KernelDensity (Window 800 600)            } -main = defaultMainWith myConfig [-         bench "fib 10" $ B fib 10-       , bench "fib 30" $ B fib 30-       , bench "intmap 50k" $ B intmap 50000-       , bench "intmap 75k" $ B intmap 75000+main = defaultMainWith myConfig (return ()) [+         bench "fib 10" $ whnf fib 10+       , bench "fib 30" $ whnf fib 30+       , bench "intmap 50k" $ whnf intmap 50000+       , bench "intmap 75k" $ whnf intmap 75000        ]          fib :: Int -> Int