diff --git a/Benchmarks.hs b/Benchmarks.hs new file mode 100644 --- /dev/null +++ b/Benchmarks.hs @@ -0,0 +1,42 @@ +-- | +-- Module : Main +-- Copyright : (c) 2018 Harendra Kumar +-- +-- License : BSD3 +-- Maintainer : harendra.kumar@gmail.com + +import Gauge +import Streamly +import System.Random (randomRIO) +import Control.Concurrent.Async + +count :: Int +count = 100000 + +------------------------------------------------------------------------------- +-- Append +------------------------------------------------------------------------------- + +{-# INLINE append #-} +append + :: (Monoid (t IO Int), Monad (t IO)) + => (t IO Int -> SerialT IO Int) -> IO () +append t = randomRIO (1,1) >>= \x -> runStream $ t $ foldMap return [x..count] + +main :: IO () +main = do + defaultMain + [ + bgroup "streamly" + [ bench "serial" $ nfIO $ append serially + , bench "wSerial" $ nfIO $ append wSerially + , bench "ahead" $ nfIO $ append aheadly + , bench "async" $ nfIO $ append asyncly + , bench "wAsync" $ nfIO $ append wAsyncly + , bench "parallel" $ nfIO $ append parallely + ] + , bgroup "async" + [ bench "mapConcurrently_" $ nfIO $ mapConcurrently_ return [1..count] + , bench "mapConcurrently" $ nfIO $ mapConcurrently return [1..count] + ] + ] diff --git a/Changelog.md b/Changelog.md new file mode 100644 --- /dev/null +++ b/Changelog.md @@ -0,0 +1,3 @@ +## 0.1.0 + +* Initial release diff --git a/Charts.hs b/Charts.hs new file mode 100644 --- /dev/null +++ b/Charts.hs @@ -0,0 +1,57 @@ +{-# LANGUAGE FlexibleContexts #-} + +module Main where + +import Data.Char (isSpace) +import BenchGraph (bgraph, defaultConfig, Config(..), ComparisonStyle(..)) +import WithCli (withCli) +import Data.List + +------------------------------------------------------------------------------- + +benchmarks :: [String] +benchmarks = + [ "streamly/async" + , "streamly/ahead" + , "streamly/parallel" + , "async/mapConcurrently_" + ] + +createCharts :: String -> IO () +createCharts input = do + let cfg title = defaultConfig + { chartTitle = Just title + , outputDir = "charts" + , comparisonStyle = CompareFull + , classifyBenchmark = \bm -> + case bm `elem` benchmarks of + True -> Just ("Operations", bm) + False -> Nothing + , sortBenchmarks = \bs -> + let i = intersect benchmarks bs + in i ++ (bs \\ i) + } + + -- links in README.rst eat up the space so we match the same + let toOutfile title field = + (filter (not . isSpace) (takeWhile (/= '(') title)) + ++ "-" + ++ field + + makeOneGraph infile field title = do + let title' = + title + ++ " (" ++ field ++ ")" + ++ " (Lower is Better)" + bgraph infile (toOutfile title field) field (cfg title') + + putStrLn "Creating time charts..." + makeOneGraph input "time" "Concurrency overhead (100,000 threads)" + putStrLn "\nCreating allocation charts..." + makeOneGraph input "allocated" "Concurrency overhead (100,000 threads)" + putStrLn "\nCreating maxrss charts..." + makeOneGraph input "maxrss" "Concurrency overhead (100,000 threads)" + +-- Pass +main :: IO () +main = withCli createCharts diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +Copyright © 2018 Harendra Kumar + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Please also see the licenses for other contributions in the "licenses" +directory. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,48 @@ +# concurrency-benchmarks + +[![Hackage](https://img.shields.io/hackage/v/concurrency-benchmarks.svg?style=flat)](https://hackage.haskell.org/package/concurrency-benchmarks) +[![Build Status](https://travis-ci.org/composewell/concurrency-benchmarks.svg?branch=master)](https://travis-ci.org/composewell/concurrency-benchmarks) +[![Windows Build status](https://ci.appveyor.com/api/projects/status/wqban615v9f21xqi?svg=true)](https://ci.appveyor.com/project/harendra-kumar/concurrency-benchmarks) + +Benchmarks to compare the pure concurrency overhead of various flavors of +concurrent [streamly](https://github.com/composewell/streamly) streams and the +[async](https://hackage.haskell.org/package/async) package. + +Use `cabal new-bench` or `stack bench` to run the benchmarks. To generate +charts, run the benchmarks with `--csv-raw=results.csv` option and then run +`makecharts results.csv`. Charts are generated in the `charts` directory. + +## Methodology + +A total of 100,000 tasks are run for each concurrency mechanism being compared. +Each task is a noop i.e. it does nothing, just returns. Therefore the benchmark +measures the pure overhead of concurrency for the tiniest possible tasks. + +Streamly's `async` and `ahead` style streams may automatically run the tasks in +less number of threads than the actual number of tasks i.e. they may run +multiple tasks per thread, if the tasks do not block and have a very low +latency. Therefore these streams are much faster on this benchmark. However, +the streamly `parallel` style stream guarantees that each task runs in a +separate thread however small it is. For the `async` package, +`mapConcurrently_` is used, which runs the tasks but does not collect the +results. + +## Results + +These charts compare the [streamly master +branch](https://github.com/composewell/streamly/commit/d73041c957d4211a6dc89624f0ebff54178bda6a) +and `async-2.2.1` on a MacBook Pro with a 2.2 GHz Intel Core i7 processor. + +This chart shows the time taken for the benchmark completion. + +[![Comparison of time](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-time.svg)](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-time.svg) + +This chart shows the maximum rss (resident set size), in other words peak +memory consumed during the benchmark run. + +[![Comparison of maxrss](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-maxrss.svg)](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-maxrss.svg) + +## Feedback + +Feedback is welcome. Please raise an issue, send a PR or send an email to the +author. diff --git a/concurrency-benchmarks.cabal b/concurrency-benchmarks.cabal new file mode 100644 --- /dev/null +++ b/concurrency-benchmarks.cabal @@ -0,0 +1,78 @@ +name: concurrency-benchmarks +category: Benchmark +version: 0.1.0 +license: MIT +license-file: LICENSE +author: Harendra Kumar +maintainer: Harendra Kumar +stability: provisional +homepage: http://github.com/composewell/concurrency-benchmarks +bug-reports: http://github.com/composewell/concurrency-benchmarks/issues +copyright: Copyright (c) 2018 Harendra Kumar +synopsis: Benchmarks to compare concurrency APIs +description: + Benchmarks to compare the pure concurrency overhead of various flavors of + concurrent @streamly@ streams and the @async@ package. + . + Use @cabal new-bench@ or @stack bench@ to run the benchmarks. To generate + charts, run the benchmarks with @--csv-raw=results.csv@ option and then run + @makecharts results.csv@. Charts are generated in the @charts@ directory. + +cabal-version: >= 1.10 +tested-with: GHC==8.2.2, GHC==8.4.3 +build-type: Simple +extra-source-files: + Changelog.md + README.md + stack.yaml + +source-repository head + type: git + location: git://github.com/composewell/concurrency-benchmarks.git + +benchmark benchmarks + default-language: Haskell2010 + type: exitcode-stdio-1.0 + hs-source-dirs: . + main-is: Benchmarks.hs + ghc-options: -O2 -Wall -with-rtsopts "-T" + if impl(ghc >= 8.0) + ghc-options: -Wcompat + -Wunrecognised-warning-flags + -Widentities + -Wincomplete-record-updates + -Wincomplete-uni-patterns + -Wredundant-constraints + -Wnoncanonical-monad-instances + -Wnoncanonical-monadfail-instances + + build-depends: + base == 4.*, + deepseq >= 1.4.0 && < 1.5, + gauge >= 0.2.1 && < 0.3, + mtl >= 2 && < 2.3, + transformers >= 0.4 && < 0.6, + async, + random, + streamly + +executable makecharts + default-language: Haskell2010 + default-extensions: OverloadedStrings + hs-source-dirs: . + main-is: Charts.hs + ghc-options: -Wall + + build-depends: + base == 4.* + , bench-graph >= 0.1 && < 0.2 + , bytestring >= 0.9 && < 0.11 + , Chart >= 1.6 && < 2 + , Chart-diagrams >= 1.6 && < 2 + , csv >= 0.1 && < 0.2 + , directory >= 1.2 && < 1.4 + , split >= 0.2 && < 0.3 + , text >= 1.1.1 && < 1.3 + , transformers >= 0.4 && < 0.6 + , typed-process >= 0.1.0.0 && < 0.3 + , getopt-generics >= 0.11 && < 0.14 diff --git a/stack.yaml b/stack.yaml new file mode 100644 --- /dev/null +++ b/stack.yaml @@ -0,0 +1,12 @@ +resolver: lts-12.5 +packages: +- '.' +extra-deps: + - streamly-0.5.0 + + # for lts-12.0 + - bench-graph-0.1.3 + - Chart-1.9 + - Chart-diagrams-1.9 + - SVGFonts-1.6.0.3 +rebuild-ghc-options: true