You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2023/09/11 05:13:05 UTC
[flink] branch master updated: [FLINK-33054][datastream] Align the execution result fetching timeout in CollectResultFetcher with akka ask timeout
This is an automated email from the ASF dual-hosted git repository.
guoyangze pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 9c91ea3d355 [FLINK-33054][datastream] Align the execution result fetching timeout in CollectResultFetcher with akka ask timeout
9c91ea3d355 is described below
commit 9c91ea3d35528ddc6a5d5be08407d5b0c71f2e76
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Thu Sep 7 13:30:32 2023 +0800
[FLINK-33054][datastream] Align the execution result fetching timeout in CollectResultFetcher with akka ask timeout
---
.../apache/flink/streaming/api/datastream/DataStream.java | 6 +++++-
.../api/operators/collect/CollectResultFetcher.java | 14 ++++++++------
.../api/operators/collect/CollectResultIterator.java | 15 ++++++++++++---
.../table/planner/connectors/CollectDynamicSink.java | 10 +++++++++-
.../connector/testframe/testsuites/SinkTestSuiteBase.java | 4 +++-
.../testframe/testsuites/SourceTestSuiteBase.java | 4 +++-
6 files changed, 40 insertions(+), 13 deletions(-)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 40dca836e21..d0195ae1c15 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -51,6 +51,7 @@ import org.apache.flink.api.java.io.TextOutputFormat;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.typeutils.InputTypeConfigurable;
import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.core.execution.JobClient;
import org.apache.flink.core.fs.FileSystem.WriteMode;
import org.apache.flink.core.fs.Path;
@@ -1421,12 +1422,15 @@ public class DataStream<T> {
CollectSinkOperatorFactory<T> factory =
new CollectSinkOperatorFactory<>(serializer, accumulatorName);
CollectSinkOperator<T> operator = (CollectSinkOperator<T>) factory.getOperator();
+ long resultFetchTimeout =
+ env.getConfiguration().get(AkkaOptions.ASK_TIMEOUT_DURATION).toMillis();
CollectResultIterator<T> iterator =
new CollectResultIterator<>(
operator.getOperatorIdFuture(),
serializer,
accumulatorName,
- env.getCheckpointConfig());
+ env.getCheckpointConfig(),
+ resultFetchTimeout);
CollectStreamSink<T> sink = new CollectStreamSink<>(this, factory);
sink.name("Data stream collect sink");
env.addOperator(sink.getTransformation());
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java
index a7916502214..e48c3122517 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java
@@ -48,8 +48,6 @@ import java.util.concurrent.TimeoutException;
public class CollectResultFetcher<T> {
private static final int DEFAULT_RETRY_MILLIS = 100;
- private static final long DEFAULT_ACCUMULATOR_GET_MILLIS = 10000;
-
private static final Logger LOG = LoggerFactory.getLogger(CollectResultFetcher.class);
private final AbstractCollectResultBuffer<T> buffer;
@@ -57,6 +55,7 @@ public class CollectResultFetcher<T> {
private final CompletableFuture<OperatorID> operatorIdFuture;
private final String accumulatorName;
private final int retryMillis;
+ private final long resultFetchTimeout;
@Nullable private JobClient jobClient;
@Nullable private CoordinationRequestGateway gateway;
@@ -67,20 +66,23 @@ public class CollectResultFetcher<T> {
public CollectResultFetcher(
AbstractCollectResultBuffer<T> buffer,
CompletableFuture<OperatorID> operatorIdFuture,
- String accumulatorName) {
- this(buffer, operatorIdFuture, accumulatorName, DEFAULT_RETRY_MILLIS);
+ String accumulatorName,
+ long resultFetchTimeout) {
+ this(buffer, operatorIdFuture, accumulatorName, DEFAULT_RETRY_MILLIS, resultFetchTimeout);
}
CollectResultFetcher(
AbstractCollectResultBuffer<T> buffer,
CompletableFuture<OperatorID> operatorIdFuture,
String accumulatorName,
- int retryMillis) {
+ int retryMillis,
+ long resultFetchTimeout) {
this.buffer = buffer;
this.operatorIdFuture = operatorIdFuture;
this.accumulatorName = accumulatorName;
this.retryMillis = retryMillis;
+ this.resultFetchTimeout = resultFetchTimeout;
this.jobTerminated = false;
this.closed = false;
@@ -180,7 +182,7 @@ public class CollectResultFetcher<T> {
executionResult =
jobClient
.getJobExecutionResult()
- .get(DEFAULT_ACCUMULATOR_GET_MILLIS, TimeUnit.MILLISECONDS);
+ .get(resultFetchTimeout, TimeUnit.MILLISECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
throw new IOException("Failed to fetch job execution result", e);
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java
index 8aecf13603e..6769eef8585 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.operators.collect;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.core.execution.JobClient;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.streaming.api.CheckpointingMode;
@@ -55,9 +56,12 @@ public class CollectResultIterator<T> implements CloseableIterator<T> {
CompletableFuture<OperatorID> operatorIdFuture,
TypeSerializer<T> serializer,
String accumulatorName,
- CheckpointConfig checkpointConfig) {
+ CheckpointConfig checkpointConfig,
+ long resultFetchTimeout) {
AbstractCollectResultBuffer<T> buffer = createBuffer(serializer, checkpointConfig);
- this.fetcher = new CollectResultFetcher<>(buffer, operatorIdFuture, accumulatorName);
+ this.fetcher =
+ new CollectResultFetcher<>(
+ buffer, operatorIdFuture, accumulatorName, resultFetchTimeout);
this.bufferedResult = null;
}
@@ -68,7 +72,12 @@ public class CollectResultIterator<T> implements CloseableIterator<T> {
String accumulatorName,
int retryMillis) {
this.fetcher =
- new CollectResultFetcher<>(buffer, operatorIdFuture, accumulatorName, retryMillis);
+ new CollectResultFetcher<>(
+ buffer,
+ operatorIdFuture,
+ accumulatorName,
+ retryMillis,
+ AkkaOptions.ASK_TIMEOUT_DURATION.defaultValue().toMillis());
this.bufferedResult = null;
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java
index cfdadfd07ec..9804d55c2f5 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java
@@ -21,6 +21,7 @@ package org.apache.flink.table.planner.connectors;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.core.execution.JobClient;
import org.apache.flink.streaming.api.datastream.DataStream;
@@ -116,13 +117,20 @@ public final class CollectDynamicSink implements DynamicTableSink {
externalSerializer, accumulatorName, maxBatchSize, socketTimeout);
final CollectSinkOperator<RowData> operator =
(CollectSinkOperator<RowData>) factory.getOperator();
+ final long resultFetchTimeout =
+ inputStream
+ .getExecutionEnvironment()
+ .getConfiguration()
+ .get(AkkaOptions.ASK_TIMEOUT_DURATION)
+ .toMillis();
iterator =
new CollectResultIterator<>(
operator.getOperatorIdFuture(),
externalSerializer,
accumulatorName,
- checkpointConfig);
+ checkpointConfig,
+ resultFetchTimeout);
converter = context.createDataStructureConverter(consumedDataType);
converter.open(RuntimeConverter.Context.create(classLoader));
diff --git a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
index 8499bcfc2d6..faeac88842a 100644
--- a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
+++ b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.connector.sink2.Sink;
import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.testframe.environment.TestEnvironment;
import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
@@ -611,7 +612,8 @@ public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
operator.getOperatorIdFuture(),
serializer,
accumulatorName,
- stream.getExecutionEnvironment().getCheckpointConfig());
+ stream.getExecutionEnvironment().getCheckpointConfig(),
+ AkkaOptions.ASK_TIMEOUT_DURATION.defaultValue().toMillis());
}
private void waitExpectedSizeData(CollectResultIterator<T> iterator, int targetNum) {
diff --git a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java
index 5eb78a48872..4a0069c5335 100644
--- a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java
+++ b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.connector.source.Boundedness;
import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.configuration.AkkaOptions;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.testframe.environment.ClusterControllable;
import org.apache.flink.connector.testframe.environment.TestEnvironment;
@@ -792,7 +793,8 @@ public abstract class SourceTestSuiteBase<T> {
operator.getOperatorIdFuture(),
serializer,
accumulatorName,
- checkpointConfig);
+ checkpointConfig,
+ AkkaOptions.ASK_TIMEOUT_DURATION.defaultValue().toMillis());
iterator.setJobClient(jobClient);
return iterator;
}