diff --git a/Criterion/Report.hs b/Criterion/Report.hs --- a/Criterion/Report.hs +++ b/Criterion/Report.hs @@ -28,7 +28,7 @@ ) where import Control.Exception (Exception, IOException, throwIO) -import Control.Monad (mplus) +import Control.Monad (mplus, unless) import Control.Monad.IO.Class (MonadIO(liftIO)) import Control.Monad.Reader (ask) import Criterion.Monad (Criterion) @@ -40,9 +40,12 @@ import GHC.Generics (Generic) import Paths_criterion (getDataFileName) import Statistics.Function (minMax) +import Statistics.Types (confidenceInterval, confidenceLevel, confIntCL, estError) import System.Directory (doesFileExist) import System.FilePath ((), (<.>), isPathSeparator) -import Text.Microstache (Key (..), Template (..), Node (..), compileMustacheText, renderMustache) +import System.IO (hPutStrLn, stderr) +import Text.Microstache (Key (..), MustacheWarning (..), Node (..), Template (..), + compileMustacheText, displayMustacheWarning, renderMustacheW) import Prelude () import Prelude.Compat import qualified Control.Exception as E @@ -106,7 +109,25 @@ , "js-flot" .= flot ] - return (renderMustache template context) + let (warnings, formatted) = renderMustacheW template context + -- If there were any issues during mustache template rendering, make sure + -- to inform the user. See #127. + forM_ warnings $ \warning -> do + -- The one thing we choose not to warn about is substituting in the `json` + -- key. The reason is that `json` is used in: + -- + -- var reports = {{{json}}}; + -- + -- So `json` represents a raw JavaScript array. This is a bit skeevy by + -- mustache conventions, but redesigning the template to avoid this + -- warning would be more work than just substituting the array directly. + unless (warning == MustacheDirectlyRenderedValue (Key ["json"])) $ + mapM_ (hPutStrLn stderr) + [ "criterion: warning:" + , " " ++ displayMustacheWarning warning + , "" + ] + return formatted where includeTemplate :: (FilePath -> IO T.Text) -> Template -> IO Template includeTemplate f Template {..} = fmap @@ -127,27 +148,34 @@ _ -> y inner r@Report {..} = merge reportAnalysis $ merge reportOutliers $ object - [ "name" .= reportName - , "json" .= TLE.decodeUtf8 (encode r) - , "number" .= reportNumber - , "iters" .= vector "x" iters - , "times" .= vector "x" times - , "cycles" .= vector "x" cycles - , "kdetimes" .= vector "x" kdeValues - , "kdepdf" .= vector "x" kdePDF - , "kde" .= vector2 "time" "pdf" kdeValues kdePDF + [ "name" .= reportName + , "json" .= TLE.decodeUtf8 (encode r) + , "number" .= reportNumber + , "iters" .= vector "x" iters + , "times" .= vector "x" times + , "cycles" .= vector "x" cycles + , "kdetimes" .= vector "x" kdeValues + , "kdepdf" .= vector "x" kdePDF + , "kde" .= vector2 "time" "pdf" kdeValues kdePDF + , "anMeanConfidenceLevel" .= anMeanConfidenceLevel + , "anMeanLowerBound" .= anMeanLowerBound + , "anMeanUpperBound" .= anMeanUpperBound + , "anStdDevLowerBound" .= anStdDevLowerBound + , "anStdDevUpperBound" .= anStdDevUpperBound ] where - [KDE{..}] = reportKDEs - iters = measure measIters reportMeasured - times = measure measTime reportMeasured - cycles = measure measCycles reportMeasured -{- - ('a':'n':_)-> mkGenericContext reportAnalysis $ - H.encodeStr nym - _ -> mkGenericContext reportOutliers $ - H.encodeStr nym --} + [KDE{..}] = reportKDEs + SampleAnalysis{..} = reportAnalysis + + iters = measure measIters reportMeasured + times = measure measTime reportMeasured + cycles = measure measCycles reportMeasured + anMeanConfidenceLevel + = confidenceLevel $ confIntCL $ estError anMean + (anMeanLowerBound, anMeanUpperBound) + = confidenceInterval anMean + (anStdDevLowerBound, anStdDevUpperBound) + = confidenceInterval anStdDev -- | Render the elements of a vector. -- diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,12 @@ +1.2.2.0 + +* Important bugfix: versions 1.2.0.0 and 1.2.1.0 were incorrectly displaying + the lower and upper bounds for measured values on HTML reports. + +* Have `criterion` emit warnings if suspicious things happen during mustache + template substitution when creating HTML reports. This can be useful when + using custom templates with the `--template` flag. + 1.2.1.0 * Add `GCStatistics`, `getGCStatistics`, and `applyGCStatistics` to @@ -36,8 +45,6 @@ 1.1.4.0 * Unicode output is now correctly printed on Windows. - -1.1.4.0 * Add Safe Haskell annotations. diff --git a/criterion.cabal b/criterion.cabal --- a/criterion.cabal +++ b/criterion.cabal @@ -1,5 +1,5 @@ name: criterion -version: 1.2.1.0 +version: 1.2.2.0 synopsis: Robust, reliable performance measurement and analysis license: BSD3 license-file: LICENSE @@ -94,7 +94,7 @@ exceptions >= 0.8.2 && < 0.9, filepath, Glob >= 0.7.2, - microstache >= 1 && < 1.1, + microstache >= 1.0.1 && < 1.1, js-flot, js-jquery, mtl >= 2, diff --git a/templates/default.tpl b/templates/default.tpl --- a/templates/default.tpl +++ b/templates/default.tpl @@ -53,10 +53,10 @@ lower bound + title="{{anMeanConfidenceLevel}} confidence level">lower bound estimate upper bound + title="{{anMeanConfidenceLevel}} confidence level">upper bound @@ -73,15 +73,15 @@ Mean execution time - {{anMean.estError.confIntLDX}} + {{anMeanLowerBound}} {{anMean.estPoint}} - {{anMean.estError.confIntUDX}} + {{anMeanUpperBound}} Standard deviation - {{anStdDev.estError.confIntLDX}} + {{anStdDevLowerBound}} {{anStdDev.estPoint}} - {{anStdDev.estError.confIntUDX}} + {{anStdDevUpperBound}} @@ -160,6 +160,12 @@ var idx = rpt.reportKeys.indexOf(key); return rpt.reportMeasured.map(function(r) { return r[idx]; }); }; + var lowerBound = function(est) { + return est.estPoint - est.estError.confIntLDX; + }; + var upperBound = function(est) { + return est.estPoint + est.estError.confIntUDX; + }; var number = rpt.reportNumber; var name = rpt.reportName; var mean = rpt.reportAnalysis.anMean.estPoint; @@ -177,19 +183,19 @@ return $.renderTime(olsTime.estPoint); }); $(".olstimelb" + number).text(function() { - return $.renderTime(olsTime.estError.confIntLDX); + return $.renderTime(lowerBound(olsTime)); }); $(".olstimeub" + number).text(function() { - return $.renderTime(olsTime.estError.confIntUDX); + return $.renderTime(upperBound(olsTime)); }); $(".olsr2pt" + number).text(function() { return rgrs.regRSquare.estPoint.toFixed(3); }); $(".olsr2lb" + number).text(function() { - return rgrs.regRSquare.estError.confIntLDX.toFixed(3); + return lowerBound(rgrs.regRSquare).toFixed(3); }); $(".olsr2ub" + number).text(function() { - return rgrs.regRSquare.estError.confIntUDX.toFixed(3); + return upperBound(rgrs.regRSquare).toFixed(3); }); mean *= scale; kdetimes = $.scaleBy(scale, kdetimes);