Commit 8d12d721 authored by Christian Wulf's avatar Christian Wulf

improved benchmark design

parent f7fefc59
package teetime.benchmark.scheduler;
import java.util.stream.IntStream;
import teetime.framework.AbstractProducerStage;
import teetime.framework.Configuration;
import teetime.stage.BlackHoleStage;
import teetime.stage.CollectorSink;
import teetime.stage.StreamProducer;
class BenchmarkConfiguration extends Configuration {
public BenchmarkConfiguration(final int numElements, final int numThreads, final int numTokens) {
IntStream inputElements = IntStream.iterate(0, i -> i + 1).limit(numElements);
AbstractProducerStage<Integer> producer = new StreamProducer<>(inputElements);
if (numThreads >= 1)
producer.declareActive();
BlackHoleStage<Integer> blackHole0 = new BlackHoleStage<>(numTokens);
// passive successor
BlackHoleStage<Integer> blackHole0Successor = new BlackHoleStage<>(numTokens);
BlackHoleStage<Integer> blackHole1 = new BlackHoleStage<>(numTokens);
if (numThreads >= 2)
blackHole1.declareActive();
// passive successor
BlackHoleStage<Integer> blackHole1Successor = new BlackHoleStage<>(numTokens);
BlackHoleStage<Integer> blackHole2 = new BlackHoleStage<>(numTokens);
if (numThreads >= 3)
blackHole2.declareActive();
// passive successor
BlackHoleStage<Integer> blackHole2Successor = new BlackHoleStage<>(numTokens);
CollectorSink<Integer> sink = new CollectorSink<>();
from(producer).to(blackHole0).to(blackHole0Successor).to(blackHole1).to(blackHole1Successor).to(blackHole2).to(blackHole2Successor).end(sink);
}
}
package teetime.benchmark.scheduler; package teetime.benchmark.scheduler;
import java.util.*;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;
import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import teetime.framework.*; import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import teetime.framework.Configuration;
import teetime.framework.Execution;
import teetime.framework.TeeTimeScheduler;
import teetime.framework.scheduling.globaltaskpool.GlobalTaskPoolScheduling; import teetime.framework.scheduling.globaltaskpool.GlobalTaskPoolScheduling;
import teetime.stage.*;
@State(Scope.Benchmark) @State(Scope.Benchmark)
@BenchmarkMode(Mode.SingleShotTime) // do not execute a benchmark method more than once per iteration @BenchmarkMode(Mode.SingleShotTime) // do not execute a benchmark method more than once per iteration
...@@ -49,24 +59,7 @@ public class GlobalTaskPoolBenchmark { ...@@ -49,24 +59,7 @@ public class GlobalTaskPoolBenchmark {
private Execution<Configuration> shouldExecutePipelineCorrectlyManyElements(final int numElements, private Execution<Configuration> shouldExecutePipelineCorrectlyManyElements(final int numElements,
final int numThreads, int numExecutions) { final int numThreads, int numExecutions) {
List<Integer> processedElements = new ArrayList<>(); Configuration config = new BenchmarkConfiguration(numElements, numThreads, numTokens);
IntStream inputElements = IntStream.iterate(0, i -> i + 1).limit(numElements);
AbstractProducerStage<Integer> producer = new StreamProducer<>(inputElements);
if (numThreads >= 1) producer.declareActive();
BlackHoleStage<Integer> blackHole0 = new BlackHoleStage<>(numTokens);
BlackHoleStage<Integer> blackHole1 = new BlackHoleStage<>(numTokens);
if (numThreads >= 2) blackHole1.declareActive();
BlackHoleStage<Integer> blackHole2 = new BlackHoleStage<>(numTokens);
if (numThreads >= 3) blackHole2.declareActive();
CollectorSink<Integer> sink = new CollectorSink<>(processedElements);
Configuration config = new Configuration().from(producer).to(blackHole0).to(blackHole1).to(blackHole2)
.end(sink);
TeeTimeScheduler scheduling = new GlobalTaskPoolScheduling(numThreads, config, numExecutions); TeeTimeScheduler scheduling = new GlobalTaskPoolScheduling(numThreads, config, numExecutions);
return new Execution<>(config, true, scheduling); return new Execution<>(config, true, scheduling);
......
package teetime.benchmark.scheduler; package teetime.benchmark.scheduler;
import java.util.*;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;
import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import teetime.framework.*; import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import teetime.framework.Configuration;
import teetime.framework.Execution;
import teetime.framework.TeeTimeScheduler;
import teetime.framework.scheduling.pushpullmodel.PushPullScheduling; import teetime.framework.scheduling.pushpullmodel.PushPullScheduling;
import teetime.stage.*;
@State(Scope.Benchmark) @State(Scope.Benchmark)
@BenchmarkMode(Mode.SingleShotTime) // do not execute a benchmark method more than once per iteration @BenchmarkMode(Mode.SingleShotTime) // do not execute a benchmark method more than once per iteration
...@@ -18,17 +28,20 @@ import teetime.stage.*; ...@@ -18,17 +28,20 @@ import teetime.stage.*;
@Measurement(iterations = 10) @Measurement(iterations = 10)
public class PushPullBenchmark { public class PushPullBenchmark {
// # Run complete. Total time: 00:10:41 // # Run complete. Total time: 00:10:41
// //
// Benchmark (numElements) (numThreads) (numTokens) Mode Cnt Score Error Units // Benchmark (numElements) (numThreads) (numTokens) Mode Cnt Score Error Units
// PushPullBenchmark.executeBlocking 1000 1 0 ss 30 1,256 ± 0,317 ms/op // PushPullBenchmark.executeBlocking 1000 1 0 ss 30 1,256 ± 0,317 ms/op
// PushPullBenchmark.executeBlocking 1000 1 1000000 ss 30 7810,192 ± 33,942 ms/op // PushPullBenchmark.executeBlocking 1000 1 1000000 ss 30 7810,192 ± 33,942
// PushPullBenchmark.executeBlocking 1000 2 0 ss 30 4,138 ± 2,922 ms/op // ms/op
// PushPullBenchmark.executeBlocking 1000 2 1000000 ss 30 5340,630 ± 16,741 ms/op // PushPullBenchmark.executeBlocking 1000 2 0 ss 30 4,138 ± 2,922 ms/op
// PushPullBenchmark.executeBlocking 1000 3 0 ss 30 7,640 ± 5,637 ms/op // PushPullBenchmark.executeBlocking 1000 2 1000000 ss 30 5340,630 ± 16,741
// PushPullBenchmark.executeBlocking 1000 3 1000000 ss 30 2977,330 ± 36,424 ms/op // ms/op
// PushPullBenchmark.executeBlocking 1000 3 0 ss 30 7,640 ± 5,637 ms/op
@Param(value = { "1000" }) // one million (low variance), TODO ten million (high variance!) // PushPullBenchmark.executeBlocking 1000 3 1000000 ss 30 2977,330 ± 36,424
// ms/op
@Param(value = { "1000" }) // one million (low variance), TODO ten million (high variance!)
private int numElements; private int numElements;
@Param(value = { "3", "2", "1" }) @Param(value = { "3", "2", "1" })
private int numThreads; private int numThreads;
...@@ -42,31 +55,15 @@ public class PushPullBenchmark { ...@@ -42,31 +55,15 @@ public class PushPullBenchmark {
private Execution<Configuration> execution; private Execution<Configuration> execution;
@Setup(Level.Iteration) // iteration lvl: necessary for teetime since a configuration cannot be used more than once @Setup(Level.Iteration) // iteration lvl: necessary for teetime since a configuration cannot be used
// more than once
public void setup() { public void setup() {
execution = shouldExecutePipelineCorrectlyManyElements(numElements, numThreads); execution = shouldExecutePipelineCorrectlyManyElements(numElements, numThreads);
} }
private Execution<Configuration> shouldExecutePipelineCorrectlyManyElements(final int numElements, private Execution<Configuration> shouldExecutePipelineCorrectlyManyElements(final int numElements,
final int numThreads) { final int numThreads) {
List<Integer> processedElements = new ArrayList<>(); Configuration config = new BenchmarkConfiguration(numElements, numThreads, numTokens);
IntStream inputElements = IntStream.iterate(0, i -> i + 1).limit(numElements);
AbstractProducerStage<Integer> producer = new StreamProducer<>(inputElements);
if (numThreads >= 1) producer.declareActive();
BlackHoleStage<Integer> blackHole0 = new BlackHoleStage<>(numTokens);
BlackHoleStage<Integer> blackHole1 = new BlackHoleStage<>(numTokens);
if (numThreads >= 2) blackHole1.declareActive();
BlackHoleStage<Integer> blackHole2 = new BlackHoleStage<>(numTokens);
if (numThreads >= 3) blackHole2.declareActive();
CollectorSink<Integer> sink = new CollectorSink<>(processedElements);
Configuration config = new Configuration().from(producer).to(blackHole0).to(blackHole1).to(blackHole2)
.end(sink);
TeeTimeScheduler scheduling = new PushPullScheduling(config); TeeTimeScheduler scheduling = new PushPullScheduling(config);
return new Execution<>(config, true, scheduling); return new Execution<>(config, true, scheduling);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment