You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2021/07/15 08:01:42 UTC
[flink-benchmarks] 01/01: [FLINK-23392] Exclude startup/shutdown
time from benchmarks
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch 23392
in repository https://gitbox.apache.org/repos/asf/flink-benchmarks.git
commit 94b732e0a25bf105661ef32d200a9194fa67b7bc
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Jul 15 09:59:35 2021 +0200
[FLINK-23392] Exclude startup/shutdown time from benchmarks
---
pom.xml | 7 +++++
.../flink/benchmark/FlinkEnvironmentContext.java | 36 +++++++++++++++++++---
2 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/pom.xml b/pom.xml
index 9c0778d..730e8ac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,6 +190,13 @@ under the License.
</dependency>
<dependency>
+ <!-- required for using a pre-defined MiniCluster -->
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
diff --git a/src/main/java/org/apache/flink/benchmark/FlinkEnvironmentContext.java b/src/main/java/org/apache/flink/benchmark/FlinkEnvironmentContext.java
index f206141..91d5e11 100644
--- a/src/main/java/org/apache/flink/benchmark/FlinkEnvironmentContext.java
+++ b/src/main/java/org/apache/flink/benchmark/FlinkEnvironmentContext.java
@@ -20,12 +20,18 @@ package org.apache.flink.benchmark;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.DeploymentOptions;
import org.apache.flink.configuration.NettyShuffleEnvironmentOptions;
+import org.apache.flink.configuration.RestOptions;
+import org.apache.flink.runtime.minicluster.MiniCluster;
+import org.apache.flink.runtime.minicluster.MiniClusterConfiguration;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.util.MiniClusterPipelineExecutorServiceLoader;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
import java.io.IOException;
@@ -37,14 +43,29 @@ public class FlinkEnvironmentContext {
public static final int NUM_NETWORK_BUFFERS = 1000;
public StreamExecutionEnvironment env;
+ public MiniCluster miniCluster;
protected final int parallelism = 1;
protected final boolean objectReuse = true;
@Setup
public void setUp() throws IOException {
+ if (miniCluster != null) {
+ throw new RuntimeException("setUp was called multiple times!");
+ }
+ final Configuration clusterConfig = createConfiguration();
+ miniCluster = new MiniCluster(new MiniClusterConfiguration.Builder().setConfiguration(clusterConfig).build());
+ try {
+ miniCluster.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
// set up the execution environment
- env = getStreamExecutionEnvironment();
+ env = new StreamExecutionEnvironment(
+ new MiniClusterPipelineExecutorServiceLoader(miniCluster),
+ clusterConfig,
+ null);
+
env.setParallelism(parallelism);
if (objectReuse) {
env.getConfig().enableObjectReuse();
@@ -53,17 +74,22 @@ public class FlinkEnvironmentContext {
env.setStateBackend(new MemoryStateBackend());
}
+ @TearDown
+ public void tearDown() throws Exception {
+ miniCluster.close();
+ miniCluster = null;
+ }
+
public void execute() throws Exception {
env.execute();
}
protected Configuration createConfiguration() {
final Configuration configuration = new Configuration();
+ configuration.setString(RestOptions.BIND_PORT, "0");
configuration.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, NUM_NETWORK_BUFFERS);
+ configuration.set(DeploymentOptions.TARGET, MiniClusterPipelineExecutorServiceLoader.NAME);
+ configuration.set(DeploymentOptions.ATTACHED, true);
return configuration;
}
-
- private StreamExecutionEnvironment getStreamExecutionEnvironment() {
- return StreamExecutionEnvironment.createLocalEnvironment(1, createConfiguration());
- }
}