You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2021/07/29 12:01:55 UTC
[flink] branch master updated (5c475d4 -> 230f659)
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 5c475d4 [FLINK-23354][blob] Limit the size of ShuffleDescriptors in PermanentBlobCache on TaskExecutor
new 3cbdd78 [FLINK-23452][refactor] Extracted PeriodTimer interface for the abstraction of idle/backpressure time management
new fe26c9d [FLINK-23452][runtime] Added extra classes for the ability to calculate throughput
new 45890c0 [FLINK-23452][core] Added specific configuration for the calculation of the throughput
new 29d7bcd [hotfix] Stopping the StreamTask#systemTimerService explicitly after the invoke
new 230f659 [FLINK-23452][streaming] Integration ThroughputCalculator in StreamTask for the calculation of the subtask level throughput
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../generated/all_taskmanager_network_section.html | 12 ++
.../shortcodes/generated/rest_v1_dispatcher.html | 58 +++++++---
.../generated/task_manager_configuration.html | 12 ++
.../flink/configuration/TaskManagerOptions.java | 19 +++
.../state/api/runtime/SavepointEnvironment.java | 9 ++
.../flink/runtime/execution/Environment.java | 8 ++
.../runtime/taskmanager/InputGateWithMetrics.java | 15 ++-
.../runtime/taskmanager/RuntimeEnvironment.java | 12 +-
.../org/apache/flink/runtime/taskmanager/Task.java | 17 ++-
.../runtime/throughput/ThroughputCalculator.java | 74 ++++++++++++
.../flink/runtime/throughput/ThroughputEMA.java | 63 ++++++++++
.../operators/testutils/DummyEnvironment.java | 7 ++
.../operators/testutils/MockEnvironment.java | 12 +-
.../testutils/MockEnvironmentBuilder.java | 12 +-
.../throughput/ThroughputCalculatorTest.java | 128 +++++++++++++++++++++
.../runtime/throughput/ThroughputEMATest.java | 93 +++++++++++++++
.../flink/streaming/runtime/tasks/StreamTask.java | 64 ++++++++++-
.../runtime/tasks/mailbox/GaugePeriodTimer.java | 25 ++--
.../tasks/mailbox/MailboxDefaultAction.java | 7 +-
.../runtime/tasks/mailbox/MailboxProcessor.java | 11 +-
.../runtime/tasks/mailbox/PeriodTimer.java | 20 +---
.../StreamNetworkBenchmarkEnvironment.java | 7 +-
.../runtime/operators/StreamTaskTimerTest.java | 2 +-
.../runtime/tasks/StreamMockEnvironment.java | 16 ++-
.../tasks/StreamTaskMailboxTestHarnessBuilder.java | 18 ++-
.../streaming/runtime/tasks/StreamTaskTest.java | 61 +++++++++-
26 files changed, 709 insertions(+), 73 deletions(-)
create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputCalculator.java
create mode 100644 flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputEMA.java
create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputCalculatorTest.java
create mode 100644 flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputEMATest.java
copy flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/conversion/IdentityConverter.java => flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/GaugePeriodTimer.java (63%)
copy flink-runtime-web/web-dashboard/src/app/pages/job/checkpoints/job-checkpoints.component.less => flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/PeriodTimer.java (80%)
[flink] 03/05: [FLINK-23452][core] Added specific configuration for
the calculation of the throughput
Posted by pn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 45890c0a00171acce52ba41a29fbe5de64537e24
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Tue Jul 27 18:33:10 2021 +0200
[FLINK-23452][core] Added specific configuration for the calculation of the throughput
---
.../generated/all_taskmanager_network_section.html | 12 +++++
.../shortcodes/generated/rest_v1_dispatcher.html | 58 ++++++++++++++++------
.../generated/task_manager_configuration.html | 12 +++++
.../flink/configuration/TaskManagerOptions.java | 19 +++++++
4 files changed, 87 insertions(+), 14 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
index f15895c..8a9264b 100644
--- a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
+++ b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html
@@ -27,6 +27,18 @@
<td>Boolean flag to enable/disable more detailed metrics about inbound/outbound network queue lengths.</td>
</tr>
<tr>
+ <td><h5>taskmanager.network.memory.automatic-buffer-adjustment.period</h5></td>
+ <td style="word-wrap: break-word;">500</td>
+ <td>Integer</td>
+ <td>The minimum period of time after which the buffer size will be automatically adjusted to a new value if required. The low value provides a fast reaction to the load fluctuation but can influence the performance.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.memory.automatic-buffer-adjustment.samples</h5></td>
+ <td style="word-wrap: break-word;">20</td>
+ <td>Integer</td>
+ <td>The number of the last buffer size values that will be taken for the correct calculation of the new one.</td>
+ </tr>
+ <tr>
<td><h5>taskmanager.network.memory.buffers-per-channel</h5></td>
<td style="word-wrap: break-word;">2</td>
<td>Integer</td>
diff --git a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
index 740e00d..88594fd 100644
--- a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
+++ b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
@@ -2083,23 +2083,23 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
"properties" : {
"alignment_buffered" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"end_to_end_duration" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"persisted_data" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"processed_data" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"state_size" : {
"type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto",
"properties" : {
"avg" : {
"type" : "integer"
@@ -2109,6 +2109,21 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
},
"min" : {
"type" : "integer"
+ },
+ "p50" : {
+ "type" : "number"
+ },
+ "p90" : {
+ "type" : "number"
+ },
+ "p95" : {
+ "type" : "number"
+ },
+ "p99" : {
+ "type" : "number"
+ },
+ "p999" : {
+ "type" : "number"
}
}
}
@@ -2497,19 +2512,19 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
"properties" : {
"buffered" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"duration" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"persisted" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"processed" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
}
}
},
@@ -2519,25 +2534,25 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
"properties" : {
"async" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"sync" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
}
}
},
"end_to_end_duration" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"start_delay" : {
"type" : "object",
- "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics"
+ "$ref" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto"
},
"state_size" : {
"type" : "object",
- "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:MinMaxAvgStatistics",
+ "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:StatsSummaryDto",
"properties" : {
"avg" : {
"type" : "integer"
@@ -2547,6 +2562,21 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
},
"min" : {
"type" : "integer"
+ },
+ "p50" : {
+ "type" : "number"
+ },
+ "p90" : {
+ "type" : "number"
+ },
+ "p95" : {
+ "type" : "number"
+ },
+ "p99" : {
+ "type" : "number"
+ },
+ "p999" : {
+ "type" : "number"
}
}
}
diff --git a/docs/layouts/shortcodes/generated/task_manager_configuration.html b/docs/layouts/shortcodes/generated/task_manager_configuration.html
index 18c60d1..8919882 100644
--- a/docs/layouts/shortcodes/generated/task_manager_configuration.html
+++ b/docs/layouts/shortcodes/generated/task_manager_configuration.html
@@ -64,6 +64,18 @@
<ul><li>"name" - uses hostname as binding address</li><li>"ip" - uses host's ip address as binding address</li></ul></td>
</tr>
<tr>
+ <td><h5>taskmanager.network.memory.automatic-buffer-adjustment.period</h5></td>
+ <td style="word-wrap: break-word;">500</td>
+ <td>Integer</td>
+ <td>The minimum period of time after which the buffer size will be automatically adjusted to a new value if required. The low value provides a fast reaction to the load fluctuation but can influence the performance.</td>
+ </tr>
+ <tr>
+ <td><h5>taskmanager.network.memory.automatic-buffer-adjustment.samples</h5></td>
+ <td style="word-wrap: break-word;">20</td>
+ <td>Integer</td>
+ <td>The number of the last buffer size values that will be taken for the correct calculation of the new one.</td>
+ </tr>
+ <tr>
<td><h5>taskmanager.numberOfTaskSlots</h5></td>
<td style="word-wrap: break-word;">1</td>
<td>Integer</td>
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
index 88d373e..2d1e5cb 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java
@@ -512,6 +512,25 @@ public class TaskManagerOptions {
+ " the configured min/max size, the min/max size will be used. The exact size of Network Memory can be"
+ " explicitly specified by setting the min/max size to the same value.");
+ /** The period between recalculation the relevant size of the buffer. */
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
+ public static final ConfigOption<Integer> AUTOMATIC_BUFFER_ADJUSTMENT_PERIOD =
+ ConfigOptions.key("taskmanager.network.memory.automatic-buffer-adjustment.period")
+ .intType()
+ .defaultValue(500)
+ .withDescription(
+ "The minimum period of time after which the buffer size will be automatically adjusted to a new value if required. "
+ + "The low value provides a fast reaction to the load fluctuation but can influence the performance.");
+
+ /** The number of samples requires for the buffer size adjustment. */
+ @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK)
+ public static final ConfigOption<Integer> AUTOMATIC_BUFFER_ADJUSTMENT_SAMPLES =
+ ConfigOptions.key("taskmanager.network.memory.automatic-buffer-adjustment.samples")
+ .intType()
+ .defaultValue(20)
+ .withDescription(
+ "The number of the last buffer size values that will be taken for the correct calculation of the new one.");
+
/**
* Size of direct memory used by blocking shuffle for shuffle data read (currently only used by
* sort-merge shuffle).
[flink] 02/05: [FLINK-23452][runtime] Added extra classes for the
ability to calculate throughput
Posted by pn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit fe26c9d0644ce050b50c1cf14a5c917f07d006ca
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Tue Jul 27 18:30:22 2021 +0200
[FLINK-23452][runtime] Added extra classes for the ability to calculate throughput
---
.../runtime/throughput/ThroughputCalculator.java | 74 ++++++++++++
.../flink/runtime/throughput/ThroughputEMA.java | 63 ++++++++++
.../throughput/ThroughputCalculatorTest.java | 128 +++++++++++++++++++++
.../runtime/throughput/ThroughputEMATest.java | 93 +++++++++++++++
4 files changed, 358 insertions(+)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputCalculator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputCalculator.java
new file mode 100644
index 0000000..83aa408
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputCalculator.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.throughput;
+
+import org.apache.flink.util.clock.Clock;
+
+/** Class for measuring the throughput based on incoming data size and measurement period. */
+public class ThroughputCalculator {
+ private static final long NOT_TRACKED = -1;
+ private final Clock clock;
+ private final ThroughputEMA throughputEMA;
+
+ private long currentAccumulatedDataSize;
+ private long currentMeasurementTime;
+ private long measurementStartTime = NOT_TRACKED;
+ private long lastThroughput;
+
+ public ThroughputCalculator(Clock clock, int numberOfSamples) {
+ this.clock = clock;
+ this.throughputEMA = new ThroughputEMA(numberOfSamples);
+ }
+
+ public void incomingDataSize(long receivedDataSize) {
+ resumeMeasurement();
+ currentAccumulatedDataSize += receivedDataSize;
+ }
+
+ /** Mark when the time should not be taken into account. */
+ public void pauseMeasurement() {
+ if (measurementStartTime != NOT_TRACKED) {
+ currentMeasurementTime += clock.relativeTimeMillis() - measurementStartTime;
+ }
+ measurementStartTime = NOT_TRACKED;
+ }
+
+ /** Mark when the time should be included to the throughput calculation. */
+ public void resumeMeasurement() {
+ if (measurementStartTime == NOT_TRACKED) {
+ measurementStartTime = clock.relativeTimeMillis();
+ }
+ }
+
+ /** @return Calculated throughput based on the collected data for the last period. */
+ public long calculateThroughput() {
+ if (measurementStartTime != NOT_TRACKED) {
+ currentMeasurementTime += clock.relativeTimeMillis() - measurementStartTime;
+ }
+
+ lastThroughput =
+ throughputEMA.calculateThroughput(
+ currentAccumulatedDataSize, currentMeasurementTime);
+
+ measurementStartTime = clock.relativeTimeMillis();
+ currentAccumulatedDataSize = currentMeasurementTime = 0;
+
+ return lastThroughput;
+ }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputEMA.java b/flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputEMA.java
new file mode 100644
index 0000000..30d066f
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/throughput/ThroughputEMA.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.throughput;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/** Implementation of 'Exponential moving average' algorithm. */
+public class ThroughputEMA {
+ private static final long MILLIS_IN_SECOND = 1000;
+ private long currentThroughput;
+
+ /** EMA algorithm specific constant which responsible for speed of reaction. */
+ private final double alpha;
+
+ public ThroughputEMA(long numberOfSamples) {
+ alpha = 2.0 / (numberOfSamples + 1);
+ }
+
+ /**
+ * Calculating the throughput over the size of data which were received in given amount of time.
+ *
+ * @param dataSize Size of the data which were received in given time.
+ * @param time Time during which the data was received.
+ * @return Throughput calculated according to implemented algorithm.
+ */
+ public long calculateThroughput(long dataSize, long time) {
+ checkArgument(dataSize >= 0, "Size of data should be non negative");
+ checkArgument(time >= 0, "Time should be non negative");
+
+ if (time == 0) {
+ return currentThroughput;
+ }
+
+ if (currentThroughput != 0) {
+ currentThroughput += alpha * (instantThroughput(dataSize, time) - currentThroughput);
+ } else {
+ // Warm-up case.
+ currentThroughput = instantThroughput(dataSize, time);
+ }
+
+ return currentThroughput;
+ }
+
+ static long instantThroughput(long dataSize, long time) {
+ return (long) ((double) dataSize / time * MILLIS_IN_SECOND);
+ }
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputCalculatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputCalculatorTest.java
new file mode 100644
index 0000000..dbe537d
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputCalculatorTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.throughput;
+
+import org.apache.flink.util.clock.ManualClock;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+
+import java.time.Duration;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/** Test for {@link ThroughputCalculator}. */
+public class ThroughputCalculatorTest extends TestCase {
+
+ @Test
+ public void testCorrectThroughputCalculation() {
+ ManualClock clock = new ManualClock();
+ ThroughputCalculator throughputCalculator = new ThroughputCalculator(clock, 10);
+ // Start throughput time.
+ throughputCalculator.calculateThroughput();
+
+ throughputCalculator.incomingDataSize(6666);
+ clock.advanceTime(Duration.ofMillis(1));
+ throughputCalculator.incomingDataSize(3333);
+ clock.advanceTime(Duration.ofMillis(33));
+ throughputCalculator.incomingDataSize(1);
+ clock.advanceTime(Duration.ofMillis(66));
+
+ assertThat(throughputCalculator.calculateThroughput(), is(10_000L * 1_000 / 100));
+ }
+
+ @Test
+ public void testResetValueAfterCalculation() {
+ ManualClock clock = new ManualClock();
+ ThroughputCalculator throughputCalculator = new ThroughputCalculator(clock, 10);
+ // Start throughput time.
+ throughputCalculator.calculateThroughput();
+
+ throughputCalculator.incomingDataSize(666);
+ clock.advanceTime(Duration.ofMillis(100));
+
+ assertThat(throughputCalculator.calculateThroughput(), is(6660L));
+ // It should be the same as previous time.
+ assertThat(throughputCalculator.calculateThroughput(), is(6660L));
+
+ clock.advanceTime(Duration.ofMillis(1));
+ assertThat(throughputCalculator.calculateThroughput(), is(5449L));
+ }
+
+ @Test
+ public void testIgnoringIdleTime() {
+ ManualClock clock = new ManualClock();
+ ThroughputCalculator throughputCalculator = new ThroughputCalculator(clock, 10);
+ // Start throughput time.
+ throughputCalculator.calculateThroughput();
+
+ throughputCalculator.incomingDataSize(7);
+ clock.advanceTime(Duration.ofMillis(1));
+ throughputCalculator.pauseMeasurement();
+ // This will be ignored because it is in idle now.
+ clock.advanceTime(Duration.ofMillis(9));
+ // This should resume the measurement time.
+ throughputCalculator.incomingDataSize(3);
+ clock.advanceTime(Duration.ofMillis(1));
+
+ assertThat(throughputCalculator.calculateThroughput(), is(5L * 1_000));
+ }
+
+ @Test
+ public void testCalculationDuringIdleTime() {
+ ManualClock clock = new ManualClock();
+ ThroughputCalculator throughputCalculator = new ThroughputCalculator(clock, 10);
+ // Start throughput time.
+ throughputCalculator.calculateThroughput();
+
+ throughputCalculator.incomingDataSize(10);
+ clock.advanceTime(Duration.ofMillis(1));
+ throughputCalculator.pauseMeasurement();
+ // This will be ignored because it is in idle now.
+ clock.advanceTime(Duration.ofMillis(9));
+
+ assertThat(throughputCalculator.calculateThroughput(), is(10L * 1_000));
+ }
+
+ @Test
+ public void testMultiplyIdleEnd() {
+ ManualClock clock = new ManualClock();
+ ThroughputCalculator throughputCalculator = new ThroughputCalculator(clock, 10);
+
+ // Start throughput time.
+ throughputCalculator.calculateThroughput();
+
+ throughputCalculator.incomingDataSize(10);
+ // It won't be ignored.
+ clock.advanceTime(Duration.ofMillis(3));
+ throughputCalculator.resumeMeasurement();
+ // It won't be ignored.
+ clock.advanceTime(Duration.ofMillis(3));
+ throughputCalculator.resumeMeasurement();
+ // It won't be ignored.
+ clock.advanceTime(Duration.ofMillis(3));
+ throughputCalculator.resumeMeasurement();
+
+ clock.advanceTime(Duration.ofMillis(1));
+
+ // resumeMeasurement should not reset the time because pauseMeasurement was not called.
+ assertThat(throughputCalculator.calculateThroughput(), is(1_000L));
+ }
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputEMATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputEMATest.java
new file mode 100644
index 0000000..94df20a
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/throughput/ThroughputEMATest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.throughput;
+
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/** Test for {@link ThroughputEMA}. */
+public class ThroughputEMATest extends TestLogger {
+ @Test
+ public void testWarmUpThroughputCalculation() {
+ ThroughputEMA calculator = new ThroughputEMA(5);
+
+ // Calculation throughput bytes/second.
+ assertThat(calculator.calculateThroughput(100, 33), is(3030L));
+ assertThat(calculator.calculateThroughput(200, 33), is(4040L));
+ assertThat(calculator.calculateThroughput(100, 9), is(6397L));
+ }
+
+ @Test
+ public void testWarmUpIsFinished() {
+ ThroughputEMA calculator = new ThroughputEMA(2);
+
+ assertThat(calculator.calculateThroughput(100, 33), is(3030L));
+ assertThat(calculator.calculateThroughput(200, 33), is(5050L));
+ }
+
+ @Test
+ public void testCalculationThroughput() {
+ ThroughputEMA calculator = new ThroughputEMA(3);
+
+ // warm-up
+ assertThat(calculator.calculateThroughput(100, 33), is(3030L));
+ assertThat(calculator.calculateThroughput(200, 33), is(4545L));
+ assertThat(calculator.calculateThroughput(100, 10), is(7272L));
+
+ // Calculation of the average
+ assertThat(calculator.calculateThroughput(100, 10), is(8636L));
+
+ // The result value seeks to the upper limit but it will take a while until it reaches it.
+ assertThat(calculator.calculateThroughput(100, 10), is(9318L));
+ assertThat(calculator.calculateThroughput(100, 10), is(9659L));
+ assertThat(calculator.calculateThroughput(100, 10), is(9829L));
+ assertThat(calculator.calculateThroughput(100, 10), is(9914L));
+ assertThat(calculator.calculateThroughput(100, 10), is(9957L));
+
+ // Downgrade
+ // The fast drop for the first value because the last value has the highest weight.
+ assertThat(calculator.calculateThroughput(10, 10), is(5478L));
+ assertThat(calculator.calculateThroughput(10, 10), is(3239L));
+ assertThat(calculator.calculateThroughput(10, 10), is(2119L));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNegativeDataSize() {
+ ThroughputEMA calculator = new ThroughputEMA(2);
+ calculator.calculateThroughput(-1, 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNegativeTime() {
+ ThroughputEMA calculator = new ThroughputEMA(2);
+ calculator.calculateThroughput(1, -1);
+ }
+
+ @Test
+ public void testPreviousValueWhenTimeIsZero() {
+ ThroughputEMA calculator = new ThroughputEMA(2);
+
+ assertThat(calculator.calculateThroughput(100, 33), is(3030L));
+ assertThat(calculator.calculateThroughput(200, 0), is(3030L));
+ }
+}
[flink] 04/05: [hotfix] Stopping the StreamTask#systemTimerService
explicitly after the invoke
Posted by pn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 29d7bcd2bb3347caec25b3aa04b508968ac2a4cd
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Tue Jul 27 18:38:52 2021 +0200
[hotfix] Stopping the StreamTask#systemTimerService explicitly after the invoke
---
.../main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 1ffff16..59759e5 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -715,6 +715,7 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
getCompletionFuture().exceptionally(unused -> null).join();
final CompletableFuture<Void> timersFinishedFuture = new CompletableFuture<>();
+ final CompletableFuture<Void> systemTimersFinishedFuture = new CompletableFuture<>();
// close all operators in a chain effect way
operatorChain.finishOperators(actionExecutor);
@@ -749,6 +750,7 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
// make sure no new timers can come
FutureUtils.forward(timerService.quiesce(), timersFinishedFuture);
+ FutureUtils.forward(systemTimerService.quiesce(), systemTimersFinishedFuture);
// let mailbox execution reject all new letters from this point
mailboxProcessor.prepareClose();
@@ -768,6 +770,7 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
// make sure all timers finish
timersFinishedFuture.get();
+ systemTimersFinishedFuture.get();
LOG.debug("Closed operators for task {}", getName());
[flink] 01/05: [FLINK-23452][refactor] Extracted PeriodTimer
interface for the abstraction of idle/backpressure time management
Posted by pn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3cbdd783e4ea32891b9bfee3a7370822c9c5c6e5
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Tue Jul 27 18:21:13 2021 +0200
[FLINK-23452][refactor] Extracted PeriodTimer interface for the abstraction of idle/backpressure time management
---
.../flink/streaming/runtime/tasks/StreamTask.java | 8 +++--
.../runtime/tasks/mailbox/GaugePeriodTimer.java | 40 ++++++++++++++++++++++
.../tasks/mailbox/MailboxDefaultAction.java | 7 ++--
.../runtime/tasks/mailbox/MailboxProcessor.java | 11 +++---
.../runtime/tasks/mailbox/PeriodTimer.java | 26 ++++++++++++++
5 files changed, 79 insertions(+), 13 deletions(-)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 2f20d0e..1ffff16 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -78,10 +78,12 @@ import org.apache.flink.streaming.runtime.io.checkpointing.CheckpointBarrierHand
import org.apache.flink.streaming.runtime.partitioner.ConfigurableStreamPartitioner;
import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.mailbox.GaugePeriodTimer;
import org.apache.flink.streaming.runtime.tasks.mailbox.MailboxDefaultAction;
import org.apache.flink.streaming.runtime.tasks.mailbox.MailboxDefaultAction.Suspension;
import org.apache.flink.streaming.runtime.tasks.mailbox.MailboxExecutorFactory;
import org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor;
+import org.apache.flink.streaming.runtime.tasks.mailbox.PeriodTimer;
import org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailbox;
import org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailboxImpl;
import org.apache.flink.util.ExceptionUtils;
@@ -449,13 +451,13 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
}
TaskIOMetricGroup ioMetrics = getEnvironment().getMetricGroup().getIOMetricGroup();
- TimerGauge timer;
+ PeriodTimer timer;
CompletableFuture<?> resumeFuture;
if (!recordWriter.isAvailable()) {
- timer = ioMetrics.getBackPressuredTimePerSecond();
+ timer = new GaugePeriodTimer(ioMetrics.getBackPressuredTimePerSecond());
resumeFuture = recordWriter.getAvailableFuture();
} else {
- timer = ioMetrics.getIdleTimeMsPerSecond();
+ timer = new GaugePeriodTimer(ioMetrics.getIdleTimeMsPerSecond());
resumeFuture = inputProcessor.getAvailableFuture();
}
assertNoException(
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/GaugePeriodTimer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/GaugePeriodTimer.java
new file mode 100644
index 0000000..8dc5eff
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/GaugePeriodTimer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.tasks.mailbox;
+
+import org.apache.flink.runtime.metrics.TimerGauge;
+
+/** Implementation of {@link PeriodTimer} based on {@link TimerGauge}. */
+public class GaugePeriodTimer implements PeriodTimer {
+ private final TimerGauge timerGauge;
+
+ public GaugePeriodTimer(TimerGauge timerGauge) {
+ this.timerGauge = timerGauge;
+ }
+
+ @Override
+ public void markStart() {
+ timerGauge.markStart();
+ }
+
+ @Override
+ public void markEnd() {
+ timerGauge.markEnd();
+ }
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxDefaultAction.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxDefaultAction.java
index 85550b7..df3e27a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxDefaultAction.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxDefaultAction.java
@@ -18,7 +18,6 @@
package org.apache.flink.streaming.runtime.tasks.mailbox;
import org.apache.flink.annotation.Internal;
-import org.apache.flink.runtime.metrics.TimerGauge;
/** Interface for the default action that is repeatedly invoked in the mailbox-loop. */
@Internal
@@ -61,14 +60,14 @@ public interface MailboxDefaultAction {
* the default action, e.g. because there is currently no input available. This method must
* be invoked from the mailbox-thread only!
*
- * @param suspensionIdleTimer started (ticking) {@link TimerGauge} that measures how long
+ * @param suspensionPeriodTimer started (ticking) {@link PeriodTimer} that measures how long
* the default action was suspended/idling. If mailbox loop is busy processing mails,
* this timer should be paused for the time required to process the mails.
*/
- Suspension suspendDefaultAction(TimerGauge suspensionIdleTimer);
+ Suspension suspendDefaultAction(PeriodTimer suspensionPeriodTimer);
/**
- * Same as {@link #suspendDefaultAction(TimerGauge)} but without any associated timer
+ * Same as {@link #suspendDefaultAction(PeriodTimer)} but without any associated timer
* measuring the idle time.
*/
Suspension suspendDefaultAction();
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxProcessor.java
index 1a920d0..811d3e9 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/MailboxProcessor.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.runtime.tasks.mailbox;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.mailbox.MailboxExecutor;
-import org.apache.flink.runtime.metrics.TimerGauge;
import org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor;
import org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailbox.MailboxClosedException;
import org.apache.flink.util.ExceptionUtils;
@@ -381,7 +380,7 @@ public class MailboxProcessor implements Closeable {
* default action, e.g. because there is currently no input available.
*/
private MailboxDefaultAction.Suspension suspendDefaultAction(
- @Nullable TimerGauge suspensionTimer) {
+ @Nullable PeriodTimer suspensionTimer) {
checkState(
mailbox.isMailboxThread(),
@@ -434,8 +433,8 @@ public class MailboxProcessor implements Closeable {
@Override
public MailboxDefaultAction.Suspension suspendDefaultAction(
- TimerGauge suspensionIdleTimer) {
- return mailboxProcessor.suspendDefaultAction(suspensionIdleTimer);
+ PeriodTimer suspensionPeriodTimer) {
+ return mailboxProcessor.suspendDefaultAction(suspensionPeriodTimer);
}
@Override
@@ -449,9 +448,9 @@ public class MailboxProcessor implements Closeable {
* resume execution.
*/
private final class DefaultActionSuspension implements MailboxDefaultAction.Suspension {
- @Nullable private final TimerGauge suspensionTimer;
+ @Nullable private final PeriodTimer suspensionTimer;
- public DefaultActionSuspension(@Nullable TimerGauge suspensionTimer) {
+ public DefaultActionSuspension(@Nullable PeriodTimer suspensionTimer) {
this.suspensionTimer = suspensionTimer;
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/PeriodTimer.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/PeriodTimer.java
new file mode 100644
index 0000000..33349c7
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/mailbox/PeriodTimer.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.tasks.mailbox;
+
+/** Interface for the measurement of the period of time. */
+public interface PeriodTimer {
+ void markStart();
+
+ void markEnd();
+}
[flink] 05/05: [FLINK-23452][streaming] Integration
ThroughputCalculator in StreamTask for the calculation of the subtask level
throughput
Posted by pn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 230f659e790785fd6b8b7660a0bfee5807718b0b
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Tue Jul 27 18:41:07 2021 +0200
[FLINK-23452][streaming] Integration ThroughputCalculator in StreamTask for the calculation of the subtask level throughput
---
.../state/api/runtime/SavepointEnvironment.java | 9 ++++
.../flink/runtime/execution/Environment.java | 8 +++
.../runtime/taskmanager/InputGateWithMetrics.java | 15 +++++-
.../runtime/taskmanager/RuntimeEnvironment.java | 12 ++++-
.../org/apache/flink/runtime/taskmanager/Task.java | 17 +++++-
.../operators/testutils/DummyEnvironment.java | 7 +++
.../operators/testutils/MockEnvironment.java | 12 ++++-
.../testutils/MockEnvironmentBuilder.java | 12 ++++-
.../flink/streaming/runtime/tasks/StreamTask.java | 55 +++++++++++++++++--
.../StreamNetworkBenchmarkEnvironment.java | 7 ++-
.../runtime/operators/StreamTaskTimerTest.java | 2 +-
.../runtime/tasks/StreamMockEnvironment.java | 16 +++++-
.../tasks/StreamTaskMailboxTestHarnessBuilder.java | 18 ++++++-
.../streaming/runtime/tasks/StreamTaskTest.java | 61 ++++++++++++++++++++--
14 files changed, 233 insertions(+), 18 deletions(-)
diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java
index e4a331e..73a2756 100644
--- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java
+++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/runtime/SavepointEnvironment.java
@@ -53,10 +53,12 @@ import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.TaskStateManager;
import org.apache.flink.runtime.taskexecutor.GlobalAggregateManager;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.UserCodeClassLoader;
+import org.apache.flink.util.clock.SystemClock;
import java.util.Collections;
import java.util.Map;
@@ -285,6 +287,13 @@ public class SavepointEnvironment implements Environment {
throw new UnsupportedOperationException(ERROR_MSG);
}
+ @Override
+ public ThroughputCalculator getThroughputMeter() {
+ // The throughput calculator doesn't make sense for savepoint but the not null value is
+ // preferable when StreamTask is instantiated.
+ return new ThroughputCalculator(SystemClock.getInstance(), 10);
+ }
+
/** {@link SavepointEnvironment} builder. */
public static class Builder {
private RuntimeContext ctx;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
index 4b465d8..912d3fb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
@@ -45,6 +45,7 @@ import org.apache.flink.runtime.state.TaskStateManager;
import org.apache.flink.runtime.state.internal.InternalKvState;
import org.apache.flink.runtime.taskexecutor.GlobalAggregateManager;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.util.UserCodeClassLoader;
import java.util.Map;
@@ -235,6 +236,13 @@ public interface Environment {
TaskEventDispatcher getTaskEventDispatcher();
+ /**
+ * Returns the throughput meter for calculation the throughput for certain period.
+ *
+ * @return the throughput calculation service.
+ */
+ ThroughputCalculator getThroughputMeter();
+
// --------------------------------------------------------------------------------------------
// Fields set in the StreamTask to provide access to mailbox and other runtime resources
// --------------------------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java
index 6ec8ea3..cbfa594 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java
@@ -27,6 +27,7 @@ import org.apache.flink.runtime.io.network.partition.consumer.IndexedInputGate;
import org.apache.flink.runtime.io.network.partition.consumer.InputChannel;
import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import java.io.IOException;
import java.util.List;
@@ -45,9 +46,15 @@ public class InputGateWithMetrics extends IndexedInputGate {
private final Counter numBytesIn;
- public InputGateWithMetrics(IndexedInputGate inputGate, Counter numBytesIn) {
+ private final ThroughputCalculator throughputCalculator;
+
+ public InputGateWithMetrics(
+ IndexedInputGate inputGate,
+ Counter numBytesIn,
+ ThroughputCalculator throughputCalculator) {
this.inputGate = checkNotNull(inputGate);
this.numBytesIn = checkNotNull(numBytesIn);
+ this.throughputCalculator = throughputCalculator;
}
@Override
@@ -141,7 +148,11 @@ public class InputGateWithMetrics extends IndexedInputGate {
}
private BufferOrEvent updateMetrics(BufferOrEvent bufferOrEvent) {
- numBytesIn.inc(bufferOrEvent.getSize());
+ int incomingDataSize = bufferOrEvent.getSize();
+
+ numBytesIn.inc(incomingDataSize);
+ throughputCalculator.incomingDataSize(incomingDataSize);
+
return bufferOrEvent;
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
index 51ab991..de00978 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
@@ -44,6 +44,7 @@ import org.apache.flink.runtime.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.query.TaskKvStateRegistry;
import org.apache.flink.runtime.state.TaskStateManager;
import org.apache.flink.runtime.taskexecutor.GlobalAggregateManager;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.util.UserCodeClassLoader;
import javax.annotation.Nullable;
@@ -100,6 +101,8 @@ public class RuntimeEnvironment implements Environment {
@Nullable private ExecutorService asyncOperationsThreadPool;
+ private final ThroughputCalculator throughputCalculator;
+
// ------------------------------------------------------------------------
public RuntimeEnvironment(
@@ -128,7 +131,8 @@ public class RuntimeEnvironment implements Environment {
TaskManagerRuntimeInfo taskManagerInfo,
TaskMetricGroup metrics,
Task containingTask,
- ExternalResourceInfoProvider externalResourceInfoProvider) {
+ ExternalResourceInfoProvider externalResourceInfoProvider,
+ ThroughputCalculator throughputCalculator) {
this.jobId = checkNotNull(jobId);
this.jobVertexId = checkNotNull(jobVertexId);
@@ -156,6 +160,7 @@ public class RuntimeEnvironment implements Environment {
this.containingTask = containingTask;
this.metrics = metrics;
this.externalResourceInfoProvider = checkNotNull(externalResourceInfoProvider);
+ this.throughputCalculator = throughputCalculator;
}
// ------------------------------------------------------------------------
@@ -338,4 +343,9 @@ public class RuntimeEnvironment implements Environment {
asyncOperationsThreadPool,
"asyncOperationsThreadPool has not been initialized yet!");
}
+
+ @Override
+ public ThroughputCalculator getThroughputMeter() {
+ return throughputCalculator;
+ }
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index dd30907..8a312f2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -75,6 +75,7 @@ import org.apache.flink.runtime.taskexecutor.GlobalAggregateManager;
import org.apache.flink.runtime.taskexecutor.KvStateService;
import org.apache.flink.runtime.taskexecutor.PartitionProducerStateChecker;
import org.apache.flink.runtime.taskexecutor.slot.TaskSlotPayload;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.types.Either;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FatalExitExceptionHandler;
@@ -85,6 +86,7 @@ import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TaskManagerExceptionUtils;
import org.apache.flink.util.UserCodeClassLoader;
import org.apache.flink.util.WrappingRuntimeException;
+import org.apache.flink.util.clock.SystemClock;
import org.apache.flink.util.concurrent.FutureUtils;
import org.slf4j.Logger;
@@ -112,6 +114,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.function.Consumer;
+import static org.apache.flink.configuration.TaskManagerOptions.AUTOMATIC_BUFFER_ADJUSTMENT_SAMPLES;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
@@ -294,6 +297,9 @@ public class Task
*/
private UserCodeClassLoader userCodeClassLoader;
+ /** The only one throughput meter per subtask. */
+ private ThroughputCalculator throughputCalculator;
+
/**
* <b>IMPORTANT:</b> This constructor may not start any work that would need to be undone in the
* case of a failing task deployment.
@@ -417,11 +423,17 @@ public class Task
.toArray(new IndexedInputGate[0]);
this.inputGates = new IndexedInputGate[gates.length];
+ this.throughputCalculator =
+ new ThroughputCalculator(
+ SystemClock.getInstance(),
+ taskConfiguration.get(AUTOMATIC_BUFFER_ADJUSTMENT_SAMPLES));
int counter = 0;
for (IndexedInputGate gate : gates) {
inputGates[counter++] =
new InputGateWithMetrics(
- gate, metrics.getIOMetricGroup().getNumBytesInCounter());
+ gate,
+ metrics.getIOMetricGroup().getNumBytesInCounter(),
+ throughputCalculator);
}
if (shuffleEnvironment instanceof NettyShuffleEnvironment) {
@@ -714,7 +726,8 @@ public class Task
taskManagerConfig,
metrics,
this,
- externalResourceInfoProvider);
+ externalResourceInfoProvider,
+ throughputCalculator);
// Make sure the user code classloader is accessible thread-locally.
// We are setting the correct context class loader before instantiating the invokable
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
index ab2c4fb..55da6301 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
@@ -49,9 +49,11 @@ import org.apache.flink.runtime.taskexecutor.GlobalAggregateManager;
import org.apache.flink.runtime.taskexecutor.TestGlobalAggregateManager;
import org.apache.flink.runtime.taskmanager.NoOpTaskOperatorEventGateway;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.runtime.util.TestingUserCodeClassLoader;
import org.apache.flink.util.UserCodeClassLoader;
+import org.apache.flink.util.clock.SystemClock;
import java.util.Collections;
import java.util.Map;
@@ -249,6 +251,11 @@ public class DummyEnvironment implements Environment {
throw new UnsupportedOperationException();
}
+ @Override
+ public ThroughputCalculator getThroughputMeter() {
+ return new ThroughputCalculator(SystemClock.getInstance(), 10);
+ }
+
public void setTaskStateManager(TaskStateManager taskStateManager) {
this.taskStateManager = taskStateManager;
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
index 377f64c..80be834 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
@@ -50,6 +50,7 @@ import org.apache.flink.runtime.state.TaskStateManager;
import org.apache.flink.runtime.taskexecutor.GlobalAggregateManager;
import org.apache.flink.runtime.taskmanager.NoOpTaskOperatorEventGateway;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.types.Record;
import org.apache.flink.util.MutableObjectIterator;
import org.apache.flink.util.Preconditions;
@@ -130,6 +131,8 @@ public class MockEnvironment implements Environment, AutoCloseable {
private ExecutorService asyncOperationsThreadPool;
+ private final ThroughputCalculator throughputCalculator;
+
public static MockEnvironmentBuilder builder() {
return new MockEnvironmentBuilder();
}
@@ -152,10 +155,12 @@ public class MockEnvironment implements Environment, AutoCloseable {
TaskMetricGroup taskMetricGroup,
TaskManagerRuntimeInfo taskManagerRuntimeInfo,
MemoryManager memManager,
- ExternalResourceInfoProvider externalResourceInfoProvider) {
+ ExternalResourceInfoProvider externalResourceInfoProvider,
+ ThroughputCalculator throughputCalculator) {
this.jobID = jobID;
this.jobVertexID = jobVertexID;
+ this.throughputCalculator = throughputCalculator;
this.taskInfo = new TaskInfo(taskName, maxParallelism, subtaskIndex, parallelism, 0);
this.jobConfiguration = new Configuration();
@@ -313,6 +318,11 @@ public class MockEnvironment implements Environment, AutoCloseable {
}
@Override
+ public ThroughputCalculator getThroughputMeter() {
+ return throughputCalculator;
+ }
+
+ @Override
public JobVertexID getJobVertexId() {
return jobVertexID;
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java
index d3e723d..32fed2d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java
@@ -34,9 +34,11 @@ import org.apache.flink.runtime.state.TestTaskStateManager;
import org.apache.flink.runtime.taskexecutor.GlobalAggregateManager;
import org.apache.flink.runtime.taskexecutor.TestGlobalAggregateManager;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.runtime.util.TestingUserCodeClassLoader;
import org.apache.flink.util.UserCodeClassLoader;
+import org.apache.flink.util.clock.SystemClock;
public class MockEnvironmentBuilder {
private String taskName = "mock-task";
@@ -61,6 +63,8 @@ public class MockEnvironmentBuilder {
buildMemoryManager(1024 * MemoryManager.DEFAULT_PAGE_SIZE);
private ExternalResourceInfoProvider externalResourceInfoProvider =
ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES;
+ private ThroughputCalculator throughputCalculator =
+ new ThroughputCalculator(SystemClock.getInstance(), 10);
private MemoryManager buildMemoryManager(long memorySize) {
return MemoryManagerBuilder.newBuilder().setMemorySize(memorySize).build();
@@ -159,6 +163,11 @@ public class MockEnvironmentBuilder {
return this;
}
+ public MockEnvironmentBuilder setThroughputMeter(ThroughputCalculator throughputCalculator) {
+ this.throughputCalculator = throughputCalculator;
+ return this;
+ }
+
public MockEnvironment build() {
if (ioManager == null) {
ioManager = new IOManagerAsync();
@@ -181,6 +190,7 @@ public class MockEnvironmentBuilder {
taskMetricGroup,
taskManagerRuntimeInfo,
memoryManager,
- externalResourceInfoProvider);
+ externalResourceInfoProvider,
+ throughputCalculator);
}
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 59759e5..b8afd95 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -62,6 +62,7 @@ import org.apache.flink.runtime.state.StateBackend;
import org.apache.flink.runtime.state.StateBackendLoader;
import org.apache.flink.runtime.state.ttl.TtlTimeProvider;
import org.apache.flink.runtime.taskmanager.DispatcherThreadFactory;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.graph.StreamConfig;
@@ -117,6 +118,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
+import static org.apache.flink.configuration.TaskManagerOptions.AUTOMATIC_BUFFER_ADJUSTMENT_PERIOD;
import static org.apache.flink.util.ExceptionUtils.firstOrSuppressed;
import static org.apache.flink.util.ExceptionUtils.rethrowException;
import static org.apache.flink.util.Preconditions.checkState;
@@ -260,6 +262,8 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
private final CompletableFuture<Void> terminationFuture = new CompletableFuture<>();
+ private final ThroughputCalculator throughputCalculator;
+
// ------------------------------------------------------------------------
/**
@@ -379,6 +383,7 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
injectChannelStateWriterIntoChannels();
environment.getMetricGroup().getIOMetricGroup().setEnableBusyTime(true);
+ this.throughputCalculator = environment.getThroughputMeter();
}
private TimerService createTimerService(String timerThreadName) {
@@ -457,7 +462,9 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
timer = new GaugePeriodTimer(ioMetrics.getBackPressuredTimePerSecond());
resumeFuture = recordWriter.getAvailableFuture();
} else {
- timer = new GaugePeriodTimer(ioMetrics.getIdleTimeMsPerSecond());
+ timer =
+ new ThroughputPeriodTimer(
+ ioMetrics.getIdleTimeMsPerSecond(), throughputCalculator);
resumeFuture = inputProcessor.getAvailableFuture();
}
assertNoException(
@@ -661,6 +668,8 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
// final check to exit early before starting to run
ensureNotCanceled();
+ throughputCalculationSetup();
+
// let the task do its work
runMailboxLoop();
@@ -671,6 +680,19 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
afterInvoke();
}
+ private void throughputCalculationSetup() {
+ systemTimerService.registerTimer(
+ systemTimerService.getCurrentProcessingTime()
+ + getTaskConfiguration().get(AUTOMATIC_BUFFER_ADJUSTMENT_PERIOD),
+ timestamp ->
+ mainMailboxExecutor.submit(
+ () -> {
+ throughputCalculator.calculateThroughput();
+ throughputCalculationSetup();
+ },
+ "Throughput recalculation"));
+ }
+
private void runWithCleanUpOnFail(RunnableWithException run) throws Exception {
try {
run.run();
@@ -1534,9 +1556,9 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
private static class ResumeWrapper implements Runnable {
private final Suspension suspendedDefaultAction;
- private final TimerGauge timer;
+ private final PeriodTimer timer;
- public ResumeWrapper(Suspension suspendedDefaultAction, TimerGauge timer) {
+ public ResumeWrapper(Suspension suspendedDefaultAction, PeriodTimer timer) {
this.suspendedDefaultAction = suspendedDefaultAction;
timer.markStart();
this.timer = timer;
@@ -1549,6 +1571,33 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
}
}
+ /**
+ * Implementation of {@link org.apache.flink.streaming.runtime.tasks.mailbox.PeriodTimer} which
+ * combine signal for metric and the throughput.
+ */
+ private static class ThroughputPeriodTimer implements PeriodTimer {
+ private final TimerGauge idleTimerGauge;
+ private final ThroughputCalculator throughputCalculator;
+
+ private ThroughputPeriodTimer(
+ TimerGauge idleTimerGauge, ThroughputCalculator throughputCalculator) {
+ this.idleTimerGauge = idleTimerGauge;
+ this.throughputCalculator = throughputCalculator;
+ }
+
+ @Override
+ public void markStart() {
+ idleTimerGauge.markStart();
+ throughputCalculator.pauseMeasurement();
+ }
+
+ @Override
+ public void markEnd() {
+ idleTimerGauge.markEnd();
+ throughputCalculator.resumeMeasurement();
+ }
+ }
+
@Override
public boolean isUsingNonBlockingInput() {
return true;
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java
index 7b93464..d50c779 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java
@@ -43,8 +43,10 @@ import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.shuffle.ShuffleDescriptor;
import org.apache.flink.runtime.taskmanager.InputGateWithMetrics;
import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.runtime.util.ConfigurationParserUtils;
import org.apache.flink.runtime.util.NettyShuffleDescriptorBuilder;
+import org.apache.flink.util.clock.SystemClock;
import java.io.IOException;
import java.net.InetAddress;
@@ -271,7 +273,10 @@ public class StreamNetworkBenchmarkEnvironment<T extends IOReadableWritable> {
SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER,
InputChannelTestUtils.newUnregisteredInputChannelMetrics());
- return new InputGateWithMetrics(singleGate, new SimpleCounter());
+ return new InputGateWithMetrics(
+ singleGate,
+ new SimpleCounter(),
+ new ThroughputCalculator(SystemClock.getInstance(), 10));
}
private static ShuffleDescriptor createShuffleDescriptor(
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java
index 3a74109..e126a26 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamTaskTimerTest.java
@@ -74,7 +74,7 @@ public class StreamTaskTimerTest extends TestLogger {
// first one spawns thread
timeService.registerTimer(System.currentTimeMillis(), timestamp -> {});
- assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
+ assertEquals(2, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
@Test
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
index 277bac3..5cad561 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
@@ -56,10 +56,12 @@ import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.NoOpCheckpointResponder;
import org.apache.flink.runtime.taskmanager.NoOpTaskOperatorEventGateway;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
import org.apache.flink.runtime.util.TestingUserCodeClassLoader;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.UserCodeClassLoader;
+import org.apache.flink.util.clock.SystemClock;
import javax.annotation.Nullable;
@@ -125,6 +127,8 @@ public class StreamMockEnvironment implements Environment {
private CheckpointResponder checkpointResponder = NoOpCheckpointResponder.INSTANCE;
+ private ThroughputCalculator throughputCalculator;
+
public StreamMockEnvironment(
Configuration jobConfig,
Configuration taskConfig,
@@ -142,7 +146,8 @@ public class StreamMockEnvironment implements Environment {
memorySize,
inputSplitProvider,
bufferSize,
- taskStateManager);
+ taskStateManager,
+ new ThroughputCalculator(SystemClock.getInstance(), 10));
}
public StreamMockEnvironment(
@@ -154,7 +159,8 @@ public class StreamMockEnvironment implements Environment {
long offHeapMemorySize,
MockInputSplitProvider inputSplitProvider,
int bufferSize,
- TaskStateManager taskStateManager) {
+ TaskStateManager taskStateManager,
+ ThroughputCalculator throughputCalculator) {
this.jobID = jobID;
this.executionAttemptID = executionAttemptID;
@@ -184,6 +190,7 @@ public class StreamMockEnvironment implements Environment {
KvStateRegistry registry = new KvStateRegistry();
this.kvStateRegistry = registry.createTaskRegistry(jobID, getJobVertexId());
+ this.throughputCalculator = throughputCalculator;
}
public StreamMockEnvironment(
@@ -302,6 +309,11 @@ public class StreamMockEnvironment implements Environment {
}
@Override
+ public ThroughputCalculator getThroughputMeter() {
+ return throughputCalculator;
+ }
+
+ @Override
public JobVertexID getJobVertexId() {
return new JobVertexID(new byte[16]);
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskMailboxTestHarnessBuilder.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskMailboxTestHarnessBuilder.java
index 63a2499..ded2d24 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskMailboxTestHarnessBuilder.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskMailboxTestHarnessBuilder.java
@@ -19,12 +19,14 @@
package org.apache.flink.streaming.runtime.tasks;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.checkpoint.TaskStateSnapshot;
import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
import org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate;
import org.apache.flink.runtime.jobgraph.OperatorID;
@@ -37,6 +39,7 @@ import org.apache.flink.runtime.state.TestTaskStateManager;
import org.apache.flink.runtime.state.TestTaskStateManagerBuilder;
import org.apache.flink.runtime.taskmanager.CheckpointResponder;
import org.apache.flink.runtime.taskmanager.TestCheckpointResponder;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.graph.StreamConfig.InputConfig;
@@ -51,6 +54,7 @@ import org.apache.flink.streaming.api.operators.StreamOperatorFactory;
import org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner;
import org.apache.flink.streaming.runtime.partitioner.ForwardPartitioner;
import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer;
+import org.apache.flink.util.clock.SystemClock;
import org.apache.flink.util.function.FunctionWithException;
import javax.annotation.Nullable;
@@ -93,6 +97,9 @@ public class StreamTaskMailboxTestHarnessBuilder<OUT> {
private boolean setupCalled = false;
+ private ThroughputCalculator throughputCalculator =
+ new ThroughputCalculator(SystemClock.getInstance(), 10);
+
public StreamTaskMailboxTestHarnessBuilder(
FunctionWithException<Environment, ? extends StreamTask<OUT, ?>, Exception> taskFactory,
TypeInformation<OUT> outputType) {
@@ -119,6 +126,12 @@ public class StreamTaskMailboxTestHarnessBuilder<OUT> {
return this;
}
+ public <T> StreamTaskMailboxTestHarnessBuilder<OUT> setThroughputMeter(
+ ThroughputCalculator throughputCalculator) {
+ this.throughputCalculator = throughputCalculator;
+ return this;
+ }
+
public StreamTaskMailboxTestHarnessBuilder<OUT> addInput(TypeInformation<?> inputType) {
return addInput(inputType, 1);
}
@@ -191,13 +204,16 @@ public class StreamTaskMailboxTestHarnessBuilder<OUT> {
TestTaskStateManager taskStateManager = taskStateManagerBuilder.build();
StreamMockEnvironment streamMockEnvironment =
new StreamMockEnvironment(
+ new JobID(),
+ new ExecutionAttemptID(),
jobConfig,
streamConfig.getConfiguration(),
executionConfig,
memorySize,
new MockInputSplitProvider(),
bufferSize,
- taskStateManager);
+ taskStateManager,
+ throughputCalculator);
streamMockEnvironment.setCheckpointResponder(taskStateManager.getCheckpointResponder());
initializeInputs(streamMockEnvironment);
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index 42a7a98..dbc2f25 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -90,6 +90,7 @@ import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.runtime.taskmanager.TaskManagerActions;
import org.apache.flink.runtime.taskmanager.TestTaskBuilder;
+import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.runtime.util.NettyShuffleDescriptorBuilder;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
@@ -121,6 +122,7 @@ import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FatalExitExceptionHandler;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.TestLogger;
+import org.apache.flink.util.clock.SystemClock;
import org.apache.flink.util.concurrent.FutureUtils;
import org.apache.flink.util.concurrent.TestingUncaughtExceptionHandler;
import org.apache.flink.util.function.BiConsumerWithException;
@@ -164,12 +166,14 @@ import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO;
import static org.apache.flink.configuration.StateBackendOptions.STATE_BACKEND;
+import static org.apache.flink.configuration.TaskManagerOptions.AUTOMATIC_BUFFER_ADJUSTMENT_PERIOD;
import static org.apache.flink.runtime.checkpoint.CheckpointFailureReason.UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE;
import static org.apache.flink.runtime.checkpoint.StateObjectCollection.singleton;
import static org.apache.flink.runtime.state.CheckpointStorageLocationReference.getDefault;
import static org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailbox.MAX_PRIORITY;
import static org.apache.flink.streaming.util.StreamTaskUtil.waitTaskIsRunning;
import static org.apache.flink.util.Preconditions.checkState;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.sameInstance;
@@ -1233,7 +1237,7 @@ public class StreamTaskTest extends TestLogger {
harness.streamTask.runMailboxLoop();
fail();
} catch (ExpectedTestException expected) {
- // expected exception
+ // expected exceptionestProcessWithUnAvailableInput
}
}
@@ -1495,7 +1499,7 @@ public class StreamTaskTest extends TestLogger {
long totalDuration = System.currentTimeMillis() - startTs;
assertThat(
ioMetricGroup.getBackPressuredTimePerSecond().getCount(),
- Matchers.greaterThanOrEqualTo(sleepTimeOutsideMail));
+ greaterThanOrEqualTo(sleepTimeOutsideMail));
assertThat(
ioMetricGroup.getBackPressuredTimePerSecond().getCount(),
Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
@@ -1512,6 +1516,7 @@ public class StreamTaskTest extends TestLogger {
public void testProcessWithUnAvailableInput() throws Exception {
final long sleepTimeOutsideMail = 42;
final long sleepTimeInsideMail = 44;
+ final int incomingDataSize = 10_000;
@Nullable WaitingThread waitingThread = null;
try (final MockEnvironment environment = setupEnvironment(true, true)) {
@@ -1523,6 +1528,7 @@ public class StreamTaskTest extends TestLogger {
.build();
TaskIOMetricGroup ioMetricGroup =
task.getEnvironment().getMetricGroup().getIOMetricGroup();
+ ThroughputCalculator throughputCalculator = environment.getThroughputMeter();
final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
final RunnableWithException completeFutureTask =
@@ -1546,12 +1552,20 @@ public class StreamTaskTest extends TestLogger {
"Start WaitingThread after Task starts processing input.");
long startTs = System.currentTimeMillis();
+ throughputCalculator.calculateThroughput();
+ throughputCalculator.incomingDataSize(incomingDataSize);
task.invoke();
+ long resultThroughput = throughputCalculator.calculateThroughput();
long totalDuration = System.currentTimeMillis() - startTs;
assertThat(
+ resultThroughput,
+ greaterThanOrEqualTo(
+ incomingDataSize * 1000 / (totalDuration - sleepTimeOutsideMail)));
+
+ assertThat(
ioMetricGroup.getIdleTimeMsPerSecond().getCount(),
- Matchers.greaterThanOrEqualTo(sleepTimeOutsideMail));
+ greaterThanOrEqualTo(sleepTimeOutsideMail));
assertThat(
ioMetricGroup.getIdleTimeMsPerSecond().getCount(),
Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
@@ -1727,6 +1741,47 @@ public class StreamTaskTest extends TestLogger {
assertTrue(OpenFailingOperator.wasClosed);
}
+ /**
+ * This test checks the fact that throughput calculation is started automatically(just to be
+ * sure that the scheduler is configured).
+ */
+ @Test
+ public void testThroughputSchedulerStartsOnInvoke() throws Exception {
+ // given: Throughput meter which finishes the task when calculateThroughput was called.
+ UnAvailableTestInputProcessor inputProcessor = new UnAvailableTestInputProcessor();
+ AtomicInteger desirableNumberOfCalculation = new AtomicInteger(2);
+ try (MockEnvironment mockEnvironment =
+ new MockEnvironmentBuilder()
+ .setTaskConfiguration(
+ new Configuration().set(AUTOMATIC_BUFFER_ADJUSTMENT_PERIOD, 1))
+ .setThroughputMeter(
+ new ThroughputCalculator(SystemClock.getInstance(), 10) {
+ @Override
+ public long calculateThroughput() {
+ if (desirableNumberOfCalculation.decrementAndGet() == 0) {
+ inputProcessor
+ .availabilityProvider
+ .getUnavailableToResetAvailable()
+ .complete(null);
+ }
+ return super.calculateThroughput();
+ }
+ })
+ .build()) {
+
+ StreamTask task =
+ new MockStreamTaskBuilder(mockEnvironment)
+ .setStreamInputProcessor(inputProcessor)
+ .build();
+
+ // when: Starting the task.
+ task.invoke();
+
+ // then: The task successfully finishes because throughput calculation was called from
+ // scheduler which leads to the finish of the task
+ }
+ }
+
private MockEnvironment setupEnvironment(boolean... outputAvailabilities) {
final Configuration configuration = new Configuration();
new MockStreamConfig(configuration, outputAvailabilities.length);