You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gu...@apache.org on 2023/09/11 05:13:05 UTC

[flink] branch master updated: [FLINK-33054][datastream] Align the execution result fetching timeout in CollectResultFetcher with akka ask timeout

This is an automated email from the ASF dual-hosted git repository.

guoyangze pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 9c91ea3d355 [FLINK-33054][datastream] Align the execution result fetching timeout in CollectResultFetcher with akka ask timeout
9c91ea3d355 is described below

commit 9c91ea3d35528ddc6a5d5be08407d5b0c71f2e76
Author: Yangze Guo <ka...@gmail.com>
AuthorDate: Thu Sep 7 13:30:32 2023 +0800

    [FLINK-33054][datastream] Align the execution result fetching timeout in CollectResultFetcher with akka ask timeout
---
 .../apache/flink/streaming/api/datastream/DataStream.java |  6 +++++-
 .../api/operators/collect/CollectResultFetcher.java       | 14 ++++++++------
 .../api/operators/collect/CollectResultIterator.java      | 15 ++++++++++++---
 .../table/planner/connectors/CollectDynamicSink.java      | 10 +++++++++-
 .../connector/testframe/testsuites/SinkTestSuiteBase.java |  4 +++-
 .../testframe/testsuites/SourceTestSuiteBase.java         |  4 +++-
 6 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 40dca836e21..d0195ae1c15 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -51,6 +51,7 @@ import org.apache.flink.api.java.io.TextOutputFormat;
 import org.apache.flink.api.java.tuple.Tuple;
 import org.apache.flink.api.java.typeutils.InputTypeConfigurable;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.configuration.AkkaOptions;
 import org.apache.flink.core.execution.JobClient;
 import org.apache.flink.core.fs.FileSystem.WriteMode;
 import org.apache.flink.core.fs.Path;
@@ -1421,12 +1422,15 @@ public class DataStream<T> {
         CollectSinkOperatorFactory<T> factory =
                 new CollectSinkOperatorFactory<>(serializer, accumulatorName);
         CollectSinkOperator<T> operator = (CollectSinkOperator<T>) factory.getOperator();
+        long resultFetchTimeout =
+                env.getConfiguration().get(AkkaOptions.ASK_TIMEOUT_DURATION).toMillis();
         CollectResultIterator<T> iterator =
                 new CollectResultIterator<>(
                         operator.getOperatorIdFuture(),
                         serializer,
                         accumulatorName,
-                        env.getCheckpointConfig());
+                        env.getCheckpointConfig(),
+                        resultFetchTimeout);
         CollectStreamSink<T> sink = new CollectStreamSink<>(this, factory);
         sink.name("Data stream collect sink");
         env.addOperator(sink.getTransformation());
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java
index a7916502214..e48c3122517 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultFetcher.java
@@ -48,8 +48,6 @@ import java.util.concurrent.TimeoutException;
 public class CollectResultFetcher<T> {
 
     private static final int DEFAULT_RETRY_MILLIS = 100;
-    private static final long DEFAULT_ACCUMULATOR_GET_MILLIS = 10000;
-
     private static final Logger LOG = LoggerFactory.getLogger(CollectResultFetcher.class);
 
     private final AbstractCollectResultBuffer<T> buffer;
@@ -57,6 +55,7 @@ public class CollectResultFetcher<T> {
     private final CompletableFuture<OperatorID> operatorIdFuture;
     private final String accumulatorName;
     private final int retryMillis;
+    private final long resultFetchTimeout;
 
     @Nullable private JobClient jobClient;
     @Nullable private CoordinationRequestGateway gateway;
@@ -67,20 +66,23 @@ public class CollectResultFetcher<T> {
     public CollectResultFetcher(
             AbstractCollectResultBuffer<T> buffer,
             CompletableFuture<OperatorID> operatorIdFuture,
-            String accumulatorName) {
-        this(buffer, operatorIdFuture, accumulatorName, DEFAULT_RETRY_MILLIS);
+            String accumulatorName,
+            long resultFetchTimeout) {
+        this(buffer, operatorIdFuture, accumulatorName, DEFAULT_RETRY_MILLIS, resultFetchTimeout);
     }
 
     CollectResultFetcher(
             AbstractCollectResultBuffer<T> buffer,
             CompletableFuture<OperatorID> operatorIdFuture,
             String accumulatorName,
-            int retryMillis) {
+            int retryMillis,
+            long resultFetchTimeout) {
         this.buffer = buffer;
 
         this.operatorIdFuture = operatorIdFuture;
         this.accumulatorName = accumulatorName;
         this.retryMillis = retryMillis;
+        this.resultFetchTimeout = resultFetchTimeout;
 
         this.jobTerminated = false;
         this.closed = false;
@@ -180,7 +182,7 @@ public class CollectResultFetcher<T> {
             executionResult =
                     jobClient
                             .getJobExecutionResult()
-                            .get(DEFAULT_ACCUMULATOR_GET_MILLIS, TimeUnit.MILLISECONDS);
+                            .get(resultFetchTimeout, TimeUnit.MILLISECONDS);
         } catch (InterruptedException | ExecutionException | TimeoutException e) {
             throw new IOException("Failed to fetch job execution result", e);
         }
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java
index 8aecf13603e..6769eef8585 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectResultIterator.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.operators.collect;
 
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.configuration.AkkaOptions;
 import org.apache.flink.core.execution.JobClient;
 import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.streaming.api.CheckpointingMode;
@@ -55,9 +56,12 @@ public class CollectResultIterator<T> implements CloseableIterator<T> {
             CompletableFuture<OperatorID> operatorIdFuture,
             TypeSerializer<T> serializer,
             String accumulatorName,
-            CheckpointConfig checkpointConfig) {
+            CheckpointConfig checkpointConfig,
+            long resultFetchTimeout) {
         AbstractCollectResultBuffer<T> buffer = createBuffer(serializer, checkpointConfig);
-        this.fetcher = new CollectResultFetcher<>(buffer, operatorIdFuture, accumulatorName);
+        this.fetcher =
+                new CollectResultFetcher<>(
+                        buffer, operatorIdFuture, accumulatorName, resultFetchTimeout);
         this.bufferedResult = null;
     }
 
@@ -68,7 +72,12 @@ public class CollectResultIterator<T> implements CloseableIterator<T> {
             String accumulatorName,
             int retryMillis) {
         this.fetcher =
-                new CollectResultFetcher<>(buffer, operatorIdFuture, accumulatorName, retryMillis);
+                new CollectResultFetcher<>(
+                        buffer,
+                        operatorIdFuture,
+                        accumulatorName,
+                        retryMillis,
+                        AkkaOptions.ASK_TIMEOUT_DURATION.defaultValue().toMillis());
         this.bufferedResult = null;
     }
 
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java
index cfdadfd07ec..9804d55c2f5 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/connectors/CollectDynamicSink.java
@@ -21,6 +21,7 @@ package org.apache.flink.table.planner.connectors;
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.configuration.AkkaOptions;
 import org.apache.flink.configuration.MemorySize;
 import org.apache.flink.core.execution.JobClient;
 import org.apache.flink.streaming.api.datastream.DataStream;
@@ -116,13 +117,20 @@ public final class CollectDynamicSink implements DynamicTableSink {
                                 externalSerializer, accumulatorName, maxBatchSize, socketTimeout);
                 final CollectSinkOperator<RowData> operator =
                         (CollectSinkOperator<RowData>) factory.getOperator();
+                final long resultFetchTimeout =
+                        inputStream
+                                .getExecutionEnvironment()
+                                .getConfiguration()
+                                .get(AkkaOptions.ASK_TIMEOUT_DURATION)
+                                .toMillis();
 
                 iterator =
                         new CollectResultIterator<>(
                                 operator.getOperatorIdFuture(),
                                 externalSerializer,
                                 accumulatorName,
-                                checkpointConfig);
+                                checkpointConfig,
+                                resultFetchTimeout);
                 converter = context.createDataStructureConverter(consumedDataType);
                 converter.open(RuntimeConverter.Context.create(classLoader));
 
diff --git a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
index 8499bcfc2d6..faeac88842a 100644
--- a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
+++ b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SinkTestSuiteBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.restartstrategy.RestartStrategies;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.connector.sink2.Sink;
 import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.AkkaOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.connector.testframe.environment.TestEnvironment;
 import org.apache.flink.connector.testframe.environment.TestEnvironmentSettings;
@@ -611,7 +612,8 @@ public abstract class SinkTestSuiteBase<T extends Comparable<T>> {
                 operator.getOperatorIdFuture(),
                 serializer,
                 accumulatorName,
-                stream.getExecutionEnvironment().getCheckpointConfig());
+                stream.getExecutionEnvironment().getCheckpointConfig(),
+                AkkaOptions.ASK_TIMEOUT_DURATION.defaultValue().toMillis());
     }
 
     private void waitExpectedSizeData(CollectResultIterator<T> iterator, int targetNum) {
diff --git a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java
index 5eb78a48872..4a0069c5335 100644
--- a/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java
+++ b/flink-test-utils-parent/flink-connector-test-utils/src/main/java/org/apache/flink/connector/testframe/testsuites/SourceTestSuiteBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.restartstrategy.RestartStrategies;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.connector.source.Boundedness;
 import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.configuration.AkkaOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.connector.testframe.environment.ClusterControllable;
 import org.apache.flink.connector.testframe.environment.TestEnvironment;
@@ -792,7 +793,8 @@ public abstract class SourceTestSuiteBase<T> {
                             operator.getOperatorIdFuture(),
                             serializer,
                             accumulatorName,
-                            checkpointConfig);
+                            checkpointConfig,
+                            AkkaOptions.ASK_TIMEOUT_DURATION.defaultValue().toMillis());
             iterator.setJobClient(jobClient);
             return iterator;
         }