diff --git a/Benchmarks.hs b/Benchmarks.hs --- a/Benchmarks.hs +++ b/Benchmarks.hs @@ -5,38 +5,70 @@ -- License : BSD3 -- Maintainer : harendra.kumar@gmail.com -import Gauge -import Streamly -import System.Random (randomRIO) +import Control.Concurrent import Control.Concurrent.Async +import Control.Monad (when) +import System.Random (randomRIO) -count :: Int -count = 100000 +import Gauge +import Streamly +import qualified Streamly.Prelude as S ------------------------------------------------------------------------------- -- Append ------------------------------------------------------------------------------- {-# INLINE append #-} -append - :: (Monoid (t IO Int), Monad (t IO)) - => (t IO Int -> SerialT IO Int) -> IO () -append t = randomRIO (1,1) >>= \x -> runStream $ t $ foldMap return [x..count] +append :: IsStream t => Int -> Int -> (t IO Int -> SerialT IO Int) -> IO () +append tcount d t = randomRIO (1,1) >>= \x -> + let work = (\i -> when (d /= 0) (threadDelay d) >> return i) + in runStream + $ t + $ maxBuffer (-1) + $ maxThreads (-1) + $ S.fromFoldableM $ map work [x..tcount] +mkBgroup :: Int -> Int -> [Benchmark] +mkBgroup d n = + let work = (\i -> when (d /= 0) (threadDelay d) >> return i) + in [ bgroup "streamly" + [ bench "ahead" $ nfIO $ append n d aheadly + , bench "async" $ nfIO $ append n d asyncly + -- , bench "wAsync" $ nfIO $ append n d wAsyncly + -- , bench "parallel" $ nfIO $ append n d parallely + ] + , bgroup "async" + [ bench "mapConcurrently_" $ nfIO $ mapConcurrently_ work [1..n] + , bench "mapConcurrently" $ nfIO $ mapConcurrently work [1..n] + ] + ] + main :: IO () main = do defaultMain - [ - bgroup "streamly" - [ bench "serial" $ nfIO $ append serially - , bench "wSerial" $ nfIO $ append wSerially - , bench "ahead" $ nfIO $ append aheadly - , bench "async" $ nfIO $ append asyncly - , bench "wAsync" $ nfIO $ append wAsyncly - , bench "parallel" $ nfIO $ append parallely - ] - , bgroup "async" - [ bench "mapConcurrently_" $ nfIO $ mapConcurrently_ return [1..count] - , bench "mapConcurrently" $ nfIO $ mapConcurrently return [1..count] - ] + [ -- bgroup "delay-0ms-1k" (mkBgroup 0 1000) + bgroup "delay-0ms-10k" (mkBgroup 0 10000) + {- + , bgroup "delay-0ms-100k" (mkBgroup 0 100000) + + , bgroup "delay-1ms-1k" (mkBgroup 1000 1000) + , bgroup "delay-1ms-10k" (mkBgroup 1000 10000) + , bgroup "delay-1ms-100k" (mkBgroup 1000 100000) + + , bgroup "delay-10ms-1k" (mkBgroup 10000 1000) + , bgroup "delay-10ms-10k" (mkBgroup 10000 10000) + , bgroup "delay-10ms-100k" (mkBgroup 10000 100000) + + , bgroup "delay-100ms-1k" (mkBgroup 100000 1000) + , bgroup "delay-100ms-10k" (mkBgroup 100000 10000) + , bgroup "delay-100ms-100k" (mkBgroup 100000 100000) + + , bgroup "delay-1000ms-1k" (mkBgroup 1000000 1000) + , bgroup "delay-1000ms-10k" (mkBgroup 1000000 10000) + , bgroup "delay-1000ms-100k" (mkBgroup 1000000 100000) + + , bgroup "delay-5000ms-1k" (mkBgroup 5000000 1000) + -} + , bgroup "delay-5000ms-10k" (mkBgroup 5000000 10000) + -- , bgroup "delay-5000ms-100k" (mkBgroup 5000000 100000) ] diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,7 @@ +## 0.1.1 + +* Update version bounds + ## 0.1.0 * Initial release diff --git a/Charts.hs b/Charts.hs --- a/Charts.hs +++ b/Charts.hs @@ -9,26 +9,35 @@ ------------------------------------------------------------------------------- -benchmarks :: [String] -benchmarks = - [ "streamly/async" - , "streamly/ahead" - , "streamly/parallel" - , "async/mapConcurrently_" +benchmarks0ms :: [String] +benchmarks0ms = + [ "delay-0ms-10k/streamly/async" + , "delay-0ms-10k/streamly/ahead" + , "delay-0ms-10k/async/mapConcurrently" ] -createCharts :: String -> IO () -createCharts input = do +benchmarks5s :: [String] +benchmarks5s = + [ "delay-5000ms-10k/streamly/async" + , "delay-5000ms-10k/streamly/ahead" + , "delay-5000ms-10k/async/mapConcurrently" + ] + +dropPrefix :: String -> String +dropPrefix = drop 1 . dropWhile (/= '/') + +createCharts :: String -> [String] -> String -> IO () +createCharts desc benchmarks input = do let cfg title = defaultConfig { chartTitle = Just title , outputDir = "charts" , comparisonStyle = CompareFull , classifyBenchmark = \bm -> case bm `elem` benchmarks of - True -> Just ("Operations", bm) + True -> Just ("Operations", dropPrefix bm) False -> Nothing , sortBenchmarks = \bs -> - let i = intersect benchmarks bs + let i = intersect (map dropPrefix benchmarks) bs in i ++ (bs \\ i) } @@ -45,13 +54,17 @@ ++ " (Lower is Better)" bgraph infile (toOutfile title field) field (cfg title') + let t = "10,000 tasks, " ++ desc putStrLn "Creating time charts..." - makeOneGraph input "time" "Concurrency overhead (100,000 threads)" - putStrLn "\nCreating allocation charts..." - makeOneGraph input "allocated" "Concurrency overhead (100,000 threads)" + makeOneGraph input "time" t putStrLn "\nCreating maxrss charts..." - makeOneGraph input "maxrss" "Concurrency overhead (100,000 threads)" + makeOneGraph input "maxrss" t +createAllCharts :: String -> IO () +createAllCharts input = do + createCharts "0 sec delay" benchmarks0ms input + createCharts "5 sec delay" benchmarks5s input + -- Pass main :: IO () -main = withCli createCharts +main = withCli createAllCharts diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -8,39 +8,87 @@ concurrent [streamly](https://github.com/composewell/streamly) streams and the [async](https://hackage.haskell.org/package/async) package. -Use `cabal new-bench` or `stack bench` to run the benchmarks. To generate +Run the `run.sh` script to run the benchmarks and create the charts. You can +use `cabal new-bench` or `stack bench` to run the benchmarks. To generate charts, run the benchmarks with `--csv-raw=results.csv` option and then run `makecharts results.csv`. Charts are generated in the `charts` directory. ## Methodology -A total of 100,000 tasks are run for each concurrency mechanism being compared. -Each task is a noop i.e. it does nothing, just returns. Therefore the benchmark -measures the pure overhead of concurrency for the tiniest possible tasks. +A total of 10,000 tasks are run for each concurrency mechanism being compared. +Two independent experiments are performed: -Streamly's `async` and `ahead` style streams may automatically run the tasks in -less number of threads than the actual number of tasks i.e. they may run -multiple tasks per thread, if the tasks do not block and have a very low -latency. Therefore these streams are much faster on this benchmark. However, -the streamly `parallel` style stream guarantees that each task runs in a -separate thread however small it is. For the `async` package, -`mapConcurrently_` is used, which runs the tasks but does not collect the -results. +1. In the first experiment, each task is just a noop i.e. it takes almost 0 time + to execute. +2. In the second experiment, each task introduces a 5 second delay +The first case shows streamly's smart scheduling to automatically run the tasks +in less number of threads than the actual number of tasks. When the tasks do +not block and have a very low latency, streamly may run multiple tasks per +thread. Therefore streamly is much faster on this benchmark. + +In the second case a 5 second delay is introduced to make sure that streamly +uses one thread per task which is similar to what `async` does and therefore a +fair comparison. For the `async` package, `mapConcurrently` is used which can +be compared with streamly's `ahead` style stream. + +For streamly this is the code that is benchmarked, by default streamly has a +limit on the buffer size and the number of threads, we set those limits to `-1` +which means there is no limit: + +```haskell + let work = (\i -> threadDelay 5000000 >> return i) + in runStream + $ aheadly + $ maxBuffer (-1) + $ maxThreads (-1) + $ S.fromFoldableM $ map work [1..10000] +``` + +For `async` this is the code that is benchmarked: + +```haskell + let work = (\i -> threadDelay 5000000 >> return i) + mapConcurrently work [1..10000] +``` + ## Results -These charts compare the [streamly master -branch](https://github.com/composewell/streamly/commit/d73041c957d4211a6dc89624f0ebff54178bda6a) -and `async-2.2.1` on a MacBook Pro with a 2.2 GHz Intel Core i7 processor. +These charts compare +[streamly-0.5.1](https://hackage.haskell.org/package/streamly) and +`async-2.2.1` on a MacBook Pro with a 2.2 GHz Intel Core i7 processor. -This chart shows the time taken for the benchmark completion. +When compiling, `-threaded -with-rtsopts "-N"` GHC options were used to enable +the use of multiple processor cores in parallel. -[![Comparison of time](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-time.svg)](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-time.svg) +For streamly, results for both `async` and `ahead` style streams are shown. -This chart shows the maximum rss (resident set size), in other words peak -memory consumed during the benchmark run. +### Zero delay case -[![Comparison of maxrss](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-maxrss.svg)](https://github.com/composewell/concurrency-benchmarks/blob/master/charts/Concurrencyoverhead-maxrss.svg) +#### Peak Memory Consumed + +Comparison of maxrss + +#### Time Taken + +Comparison of time + +### 5 second delay case + +#### Peak Memory Consumed + +Comparison of maxrss + +#### Time Taken + +Note, this time shows the overhead only and not the full time taken by the +benchmark. For example the actual time taken by the `async` benchmark is +`5.135` seconds, but since 5 second in this is the delay introduced by each +parallel task, we compute the overhead of concurrency by deducting the 5 +seconds from the actual time taken, so the overhead is `135 ms` in case of +`async`. + +Comparison of time ## Feedback diff --git a/concurrency-benchmarks.cabal b/concurrency-benchmarks.cabal --- a/concurrency-benchmarks.cabal +++ b/concurrency-benchmarks.cabal @@ -1,6 +1,6 @@ name: concurrency-benchmarks category: Benchmark -version: 0.1.0 +version: 0.1.1 license: MIT license-file: LICENSE author: Harendra Kumar @@ -35,7 +35,7 @@ type: exitcode-stdio-1.0 hs-source-dirs: . main-is: Benchmarks.hs - ghc-options: -O2 -Wall -with-rtsopts "-T" + ghc-options: -O2 -Wall -threaded -with-rtsopts "-T -N" if impl(ghc >= 8.0) ghc-options: -Wcompat -Wunrecognised-warning-flags @@ -52,9 +52,9 @@ gauge >= 0.2.1 && < 0.3, mtl >= 2 && < 2.3, transformers >= 0.4 && < 0.6, - async, - random, - streamly + async >= 2.1.1 && < 2.3, + random >= 1.0 && < 2, + streamly >= 0.4.0 && < 0.6 executable makecharts default-language: Haskell2010 diff --git a/stack.yaml b/stack.yaml --- a/stack.yaml +++ b/stack.yaml @@ -2,7 +2,7 @@ packages: - '.' extra-deps: - - streamly-0.5.0 + - streamly-0.5.1 # for lts-12.0 - bench-graph-0.1.3