concurrency-benchmarks 0.1.0 → 0.1.1
raw patch · 6 files changed
+160/−63 lines, 6 filesdep ~asyncdep ~randomdep ~streamly
Dependency ranges changed: async, random, streamly
Files
- Benchmarks.hs +54/−22
- Changelog.md +4/−0
- Charts.hs +28/−15
- README.md +68/−20
- concurrency-benchmarks.cabal +5/−5
- stack.yaml +1/−1
Benchmarks.hs view
@@ -5,38 +5,70 @@ -- License : BSD3 -- Maintainer : harendra.kumar@gmail.com -import Gauge-import Streamly-import System.Random (randomRIO)+import Control.Concurrent import Control.Concurrent.Async+import Control.Monad (when)+import System.Random (randomRIO) -count :: Int-count = 100000+import Gauge+import Streamly+import qualified Streamly.Prelude as S ------------------------------------------------------------------------------- -- Append ------------------------------------------------------------------------------- {-# INLINE append #-}-append- :: (Monoid (t IO Int), Monad (t IO))- => (t IO Int -> SerialT IO Int) -> IO ()-append t = randomRIO (1,1) >>= \x -> runStream $ t $ foldMap return [x..count]+append :: IsStream t => Int -> Int -> (t IO Int -> SerialT IO Int) -> IO ()+append tcount d t = randomRIO (1,1) >>= \x ->+ let work = (\i -> when (d /= 0) (threadDelay d) >> return i)+ in runStream+ $ t+ $ maxBuffer (-1)+ $ maxThreads (-1)+ $ S.fromFoldableM $ map work [x..tcount] +mkBgroup :: Int -> Int -> [Benchmark]+mkBgroup d n =+ let work = (\i -> when (d /= 0) (threadDelay d) >> return i)+ in [ bgroup "streamly"+ [ bench "ahead" $ nfIO $ append n d aheadly+ , bench "async" $ nfIO $ append n d asyncly+ -- , bench "wAsync" $ nfIO $ append n d wAsyncly+ -- , bench "parallel" $ nfIO $ append n d parallely+ ]+ , bgroup "async"+ [ bench "mapConcurrently_" $ nfIO $ mapConcurrently_ work [1..n]+ , bench "mapConcurrently" $ nfIO $ mapConcurrently work [1..n]+ ]+ ]+ main :: IO () main = do defaultMain- [- bgroup "streamly"- [ bench "serial" $ nfIO $ append serially- , bench "wSerial" $ nfIO $ append wSerially- , bench "ahead" $ nfIO $ append aheadly- , bench "async" $ nfIO $ append asyncly- , bench "wAsync" $ nfIO $ append wAsyncly- , bench "parallel" $ nfIO $ append parallely- ]- , bgroup "async"- [ bench "mapConcurrently_" $ nfIO $ mapConcurrently_ return [1..count]- , bench "mapConcurrently" $ nfIO $ mapConcurrently return [1..count]- ]+ [ -- bgroup "delay-0ms-1k" (mkBgroup 0 1000)+ bgroup "delay-0ms-10k" (mkBgroup 0 10000)+ {-+ , bgroup "delay-0ms-100k" (mkBgroup 0 100000)++ , bgroup "delay-1ms-1k" (mkBgroup 1000 1000)+ , bgroup "delay-1ms-10k" (mkBgroup 1000 10000)+ , bgroup "delay-1ms-100k" (mkBgroup 1000 100000)++ , bgroup "delay-10ms-1k" (mkBgroup 10000 1000)+ , bgroup "delay-10ms-10k" (mkBgroup 10000 10000)+ , bgroup "delay-10ms-100k" (mkBgroup 10000 100000)++ , bgroup "delay-100ms-1k" (mkBgroup 100000 1000)+ , bgroup "delay-100ms-10k" (mkBgroup 100000 10000)+ , bgroup "delay-100ms-100k" (mkBgroup 100000 100000)++ , bgroup "delay-1000ms-1k" (mkBgroup 1000000 1000)+ , bgroup "delay-1000ms-10k" (mkBgroup 1000000 10000)+ , bgroup "delay-1000ms-100k" (mkBgroup 1000000 100000)++ , bgroup "delay-5000ms-1k" (mkBgroup 5000000 1000)+ -}+ , bgroup "delay-5000ms-10k" (mkBgroup 5000000 10000)+ -- , bgroup "delay-5000ms-100k" (mkBgroup 5000000 100000) ]
Changelog.md view
@@ -1,3 +1,7 @@+## 0.1.1++* Update version bounds+ ## 0.1.0 * Initial release
Charts.hs view
@@ -9,26 +9,35 @@ ------------------------------------------------------------------------------- -benchmarks :: [String]-benchmarks =- [ "streamly/async"- , "streamly/ahead"- , "streamly/parallel"- , "async/mapConcurrently_"+benchmarks0ms :: [String]+benchmarks0ms =+ [ "delay-0ms-10k/streamly/async"+ , "delay-0ms-10k/streamly/ahead"+ , "delay-0ms-10k/async/mapConcurrently" ] -createCharts :: String -> IO ()-createCharts input = do+benchmarks5s :: [String]+benchmarks5s =+ [ "delay-5000ms-10k/streamly/async"+ , "delay-5000ms-10k/streamly/ahead"+ , "delay-5000ms-10k/async/mapConcurrently"+ ]++dropPrefix :: String -> String+dropPrefix = drop 1 . dropWhile (/= '/')++createCharts :: String -> [String] -> String -> IO ()+createCharts desc benchmarks input = do let cfg title = defaultConfig { chartTitle = Just title , outputDir = "charts" , comparisonStyle = CompareFull , classifyBenchmark = \bm -> case bm `elem` benchmarks of- True -> Just ("Operations", bm)+ True -> Just ("Operations", dropPrefix bm) False -> Nothing , sortBenchmarks = \bs ->- let i = intersect benchmarks bs+ let i = intersect (map dropPrefix benchmarks) bs in i ++ (bs \\ i) } @@ -45,13 +54,17 @@ ++ " (Lower is Better)" bgraph infile (toOutfile title field) field (cfg title') + let t = "10,000 tasks, " ++ desc putStrLn "Creating time charts..."- makeOneGraph input "time" "Concurrency overhead (100,000 threads)"- putStrLn "\nCreating allocation charts..."- makeOneGraph input "allocated" "Concurrency overhead (100,000 threads)"+ makeOneGraph input "time" t putStrLn "\nCreating maxrss charts..."- makeOneGraph input "maxrss" "Concurrency overhead (100,000 threads)"+ makeOneGraph input "maxrss" t +createAllCharts :: String -> IO ()+createAllCharts input = do+ createCharts "0 sec delay" benchmarks0ms input+ createCharts "5 sec delay" benchmarks5s input+ -- Pass <input file> main :: IO ()-main = withCli createCharts+main = withCli createAllCharts
README.md view
@@ -8,39 +8,87 @@ concurrent [streamly](https://github.com/composewell/streamly) streams and the [async](https://hackage.haskell.org/package/async) package. -Use `cabal new-bench` or `stack bench` to run the benchmarks. To generate+Run the `run.sh` script to run the benchmarks and create the charts. You can+use `cabal new-bench` or `stack bench` to run the benchmarks. To generate charts, run the benchmarks with `--csv-raw=results.csv` option and then run `makecharts results.csv`. Charts are generated in the `charts` directory. ## Methodology -A total of 100,000 tasks are run for each concurrency mechanism being compared.-Each task is a noop i.e. it does nothing, just returns. Therefore the benchmark-measures the pure overhead of concurrency for the tiniest possible tasks.+A total of 10,000 tasks are run for each concurrency mechanism being compared.+Two independent experiments are performed: -Streamly's `async` and `ahead` style streams may automatically run the tasks in-less number of threads than the actual number of tasks i.e. they may run-multiple tasks per thread, if the tasks do not block and have a very low-latency. Therefore these streams are much faster on this benchmark. However,-the streamly `parallel` style stream guarantees that each task runs in a-separate thread however small it is. For the `async` package,-`mapConcurrently_` is used, which runs the tasks but does not collect the-results.+1. In the first experiment, each task is just a noop i.e. it takes almost 0 time+ to execute.+2. In the second experiment, each task introduces a 5 second delay +The first case shows streamly's smart scheduling to automatically run the tasks+in less number of threads than the actual number of tasks. When the tasks do+not block and have a very low latency, streamly may run multiple tasks per+thread. Therefore streamly is much faster on this benchmark.++In the second case a 5 second delay is introduced to make sure that streamly+uses one thread per task which is similar to what `async` does and therefore a+fair comparison. For the `async` package, `mapConcurrently` is used which can+be compared with streamly's `ahead` style stream.++For streamly this is the code that is benchmarked, by default streamly has a+limit on the buffer size and the number of threads, we set those limits to `-1`+which means there is no limit:++```haskell+ let work = (\i -> threadDelay 5000000 >> return i)+ in runStream+ $ aheadly+ $ maxBuffer (-1)+ $ maxThreads (-1)+ $ S.fromFoldableM $ map work [1..10000]+```++For `async` this is the code that is benchmarked:++```haskell+ let work = (\i -> threadDelay 5000000 >> return i)+ mapConcurrently work [1..10000]+```+ ## Results -These charts compare the [streamly master-branch](https://github.com/composewell/streamly/commit/d73041c957d4211a6dc89624f0ebff54178bda6a)-and `async-2.2.1` on a MacBook Pro with a 2.2 GHz Intel Core i7 processor.+These charts compare+[streamly-0.5.1](https://hackage.haskell.org/package/streamly) and+`async-2.2.1` on a MacBook Pro with a 2.2 GHz Intel Core i7 processor. -This chart shows the time taken for the benchmark completion.+When compiling, `-threaded -with-rtsopts "-N"` GHC options were used to enable+the use of multiple processor cores in parallel. -[](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-time.svg)+For streamly, results for both `async` and `ahead` style streams are shown. -This chart shows the maximum rss (resident set size), in other words peak-memory consumed during the benchmark run.+### Zero delay case -[](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-maxrss.svg)+#### Peak Memory Consumed++<img src="https://github.com/composewell/concurrency-benchmarks/blob/master/charts/10,000tasks,0secdelay-maxrss.svg" alt="Comparison of maxrss" width="640"/>++#### Time Taken++<img src="https://github.com/composewell/concurrency-benchmarks/blob/master/charts/10,000tasks,0secdelay-time.svg" alt="Comparison of time" width="640"/>++### 5 second delay case++#### Peak Memory Consumed++<img src="https://github.com/composewell/concurrency-benchmarks/blob/master/charts/10,000tasks,5secdelay-maxrss.svg" alt="Comparison of maxrss" width="640"/>++#### Time Taken++Note, this time shows the overhead only and not the full time taken by the+benchmark. For example the actual time taken by the `async` benchmark is+`5.135` seconds, but since 5 second in this is the delay introduced by each+parallel task, we compute the overhead of concurrency by deducting the 5+seconds from the actual time taken, so the overhead is `135 ms` in case of+`async`.++<img src="https://github.com/composewell/concurrency-benchmarks/blob/master/charts/10,000tasks,5secdelay-time.svg" alt="Comparison of time" width="640"/> ## Feedback
concurrency-benchmarks.cabal view
@@ -1,6 +1,6 @@ name: concurrency-benchmarks category: Benchmark-version: 0.1.0+version: 0.1.1 license: MIT license-file: LICENSE author: Harendra Kumar@@ -35,7 +35,7 @@ type: exitcode-stdio-1.0 hs-source-dirs: . main-is: Benchmarks.hs- ghc-options: -O2 -Wall -with-rtsopts "-T"+ ghc-options: -O2 -Wall -threaded -with-rtsopts "-T -N" if impl(ghc >= 8.0) ghc-options: -Wcompat -Wunrecognised-warning-flags@@ -52,9 +52,9 @@ gauge >= 0.2.1 && < 0.3, mtl >= 2 && < 2.3, transformers >= 0.4 && < 0.6,- async,- random,- streamly+ async >= 2.1.1 && < 2.3,+ random >= 1.0 && < 2,+ streamly >= 0.4.0 && < 0.6 executable makecharts default-language: Haskell2010
stack.yaml view
@@ -2,7 +2,7 @@ packages: - '.' extra-deps:- - streamly-0.5.0+ - streamly-0.5.1 # for lts-12.0 - bench-graph-0.1.3