You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2021/08/23 12:25:02 UTC

[flink] 02/02: [FLINK-23811][tests] Handle finished subtasks in CommonTestUtils.waitForAllTaskRunning

This is an automated email from the ASF dual-hosted git repository.

pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 136bb52326922dfe70b3cc79fd463f9d6cfecc32
Author: Roman Khachatryan <kh...@gmail.com>
AuthorDate: Fri Aug 13 01:54:07 2021 +0100

    [FLINK-23811][tests] Handle finished subtasks in CommonTestUtils.waitForAllTaskRunning
    
    CommonTestUtils.waitForAllTaskRunning returns when all the subtasks are running AND
    the job is running and not finished. However, with FLIP-147, subtasks may finish and
    the job will still be running. So the method won't return and instead timeout.
    
    This commit adds a flag to indicate whether to fail or proceed
    when a finished sub-task is encountered.
---
 .../file/sink/writer/FileSinkMigrationITCase.java  |  2 +-
 .../flink/runtime/testutils/CommonTestUtils.java   | 42 +++++++++++++++-------
 .../JobMasterStopWithSavepointITCase.java          |  3 +-
 .../flink/test/checkpointing/RescalingITCase.java  |  8 ++---
 .../flink/test/checkpointing/SavepointITCase.java  |  8 ++---
 .../test/checkpointing/TimersSavepointITCase.java  |  2 +-
 .../UnalignedCheckpointCompatibilityITCase.java    |  3 +-
 7 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java
index 8835145..225b3b2 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java
@@ -185,7 +185,7 @@ public class FileSinkMigrationITCase extends TestLogger {
                     miniCluster.submitJob(jobGraph);
             JobID jobId = jobSubmissionResultFuture.get().getJobID();
 
-            waitForAllTaskRunning(miniCluster, jobId);
+            waitForAllTaskRunning(miniCluster, jobId, false);
 
             CompletableFuture<String> savepointResultFuture =
                     miniCluster.triggerSavepoint(jobId, savepointBasePath, true);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index 671c723..b66ee2a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -22,8 +22,8 @@ import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobStatus;
 import org.apache.flink.api.common.time.Deadline;
 import org.apache.flink.core.execution.JobClient;
-import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
+import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
 import org.apache.flink.runtime.minicluster.MiniCluster;
 import org.apache.flink.util.FileUtils;
 import org.apache.flink.util.Preconditions;
@@ -44,6 +44,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Predicate;
 
 /** This class contains auxiliary methods for unit tests. */
 public class CommonTestUtils {
@@ -158,22 +159,43 @@ public class CommonTestUtils {
         }
     }
 
-    public static void waitForAllTaskRunning(MiniCluster miniCluster, JobID jobId)
-            throws Exception {
-        waitForAllTaskRunning(() -> miniCluster.getExecutionGraph(jobId).get(60, TimeUnit.SECONDS));
+    public static void waitForAllTaskRunning(
+            MiniCluster miniCluster, JobID jobId, boolean allowFinished) throws Exception {
+        waitForAllTaskRunning(
+                () -> miniCluster.getExecutionGraph(jobId).get(60, TimeUnit.SECONDS),
+                allowFinished);
     }
 
     public static void waitForAllTaskRunning(
-            SupplierWithException<AccessExecutionGraph, Exception> executionGraphSupplier)
+            SupplierWithException<AccessExecutionGraph, Exception> executionGraphSupplier,
+            boolean allowFinished)
             throws Exception {
         waitForAllTaskRunning(
-                executionGraphSupplier, Deadline.fromNow(Duration.of(1, ChronoUnit.MINUTES)));
+                executionGraphSupplier,
+                Deadline.fromNow(Duration.of(1, ChronoUnit.MINUTES)),
+                allowFinished);
     }
 
     public static void waitForAllTaskRunning(
             SupplierWithException<AccessExecutionGraph, Exception> executionGraphSupplier,
-            Deadline timeout)
+            Deadline timeout,
+            boolean allowFinished)
             throws Exception {
+        Predicate<AccessExecutionVertex> subtaskPredicate =
+                task -> {
+                    switch (task.getExecutionState()) {
+                        case RUNNING:
+                            return true;
+                        case FINISHED:
+                            if (allowFinished) {
+                                return true;
+                            } else {
+                                throw new RuntimeException("Sub-Task finished unexpectedly" + task);
+                            }
+                        default:
+                            return false;
+                    }
+                };
         waitUntilCondition(
                 () -> {
                     final AccessExecutionGraph graph = executionGraphSupplier.get();
@@ -186,11 +208,7 @@ public class CommonTestUtils {
                                     .allMatch(
                                             jobVertex ->
                                                     Arrays.stream(jobVertex.getTaskVertices())
-                                                            .allMatch(
-                                                                    task ->
-                                                                            task.getExecutionState()
-                                                                                    == ExecutionState
-                                                                                            .RUNNING));
+                                                            .allMatch(subtaskPredicate));
                 },
                 timeout);
     }
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
index 941b40e..399b960 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
@@ -315,7 +315,8 @@ public class JobMasterStopWithSavepointITCase extends AbstractTestBase {
                                 .getMiniCluster()
                                 .getExecutionGraph(jobID)
                                 .get(60, TimeUnit.SECONDS),
-                deadline);
+                deadline,
+                false);
     }
 
     /**
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java
index 9854df2..5511455 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java
@@ -238,7 +238,7 @@ public class RescalingITCase extends TestLogger {
             // clear the CollectionSink set for the restarted job
             CollectionSink.clearElementsSet();
 
-            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
             CompletableFuture<String> savepointPathFuture = client.triggerSavepoint(jobID, null);
 
             final String savepointPath =
@@ -317,7 +317,7 @@ public class RescalingITCase extends TestLogger {
             client.submitJob(jobGraph).get();
 
             // wait until the operator is started
-            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
             // wait until the operator handles some data
             StateSourceBase.workStartedLatch.await();
 
@@ -418,7 +418,7 @@ public class RescalingITCase extends TestLogger {
             // clear the CollectionSink set for the restarted job
             CollectionSink.clearElementsSet();
 
-            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
             CompletableFuture<String> savepointPathFuture = client.triggerSavepoint(jobID, null);
 
             final String savepointPath =
@@ -542,7 +542,7 @@ public class RescalingITCase extends TestLogger {
             client.submitJob(jobGraph).get();
 
             // wait until the operator is started
-            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
             // wait until the operator handles some data
             StateSourceBase.workStartedLatch.await();
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
index 7666472..cb6426a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
@@ -370,7 +370,7 @@ public class SavepointITCase extends TestLogger {
         try {
             client.submitJob(jobGraph).get();
 
-            waitForAllTaskRunning(cluster.getMiniCluster(), jobId);
+            waitForAllTaskRunning(cluster.getMiniCluster(), jobId, false);
             StatefulCounter.getProgressLatch().await();
 
             return client.cancelWithSavepoint(jobId, null).get();
@@ -502,7 +502,7 @@ public class SavepointITCase extends TestLogger {
         try {
             client.submitJob(graph).get();
             // triggerSavepoint is only available after all tasks are running
-            waitForAllTaskRunning(cluster.getMiniCluster(), graph.getJobID());
+            waitForAllTaskRunning(cluster.getMiniCluster(), graph.getJobID(), false);
 
             client.triggerSavepoint(graph.getJobID(), null).get();
 
@@ -993,7 +993,7 @@ public class SavepointITCase extends TestLogger {
             JobID jobID = client.submitJob(originalJobGraph).get();
 
             // wait for the Tasks to be ready
-            waitForAllTaskRunning(cluster.getMiniCluster(), jobID);
+            waitForAllTaskRunning(cluster.getMiniCluster(), jobID, false);
             assertTrue(
                     StatefulCounter.getProgressLatch()
                             .await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
@@ -1334,7 +1334,7 @@ public class SavepointITCase extends TestLogger {
         try {
             client.submitJob(jobGraph).get();
 
-            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+            waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
 
             for (OneShotLatch latch : iterTestSnapshotWait) {
                 latch.await();
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java
index f7d0637..3782496 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java
@@ -120,7 +120,7 @@ public class TimersSavepointITCase {
     private void takeSavepoint(String savepointPath, ClusterClient<?> client) throws Exception {
         JobGraph jobGraph = getJobGraph(EmbeddedRocksDBStateBackend.PriorityQueueStateType.ROCKSDB);
         client.submitJob(jobGraph).get();
-        waitForAllTaskRunning(miniClusterResource.getMiniCluster(), jobGraph.getJobID());
+        waitForAllTaskRunning(miniClusterResource.getMiniCluster(), jobGraph.getJobID(), false);
         CompletableFuture<String> savepointPathFuture =
                 client.triggerSavepoint(jobGraph.getJobID(), null);
 
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java
index 21bfa3f..8a39643 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java
@@ -146,7 +146,8 @@ public class UnalignedCheckpointCompatibilityITCase extends TestLogger {
     private Tuple2<String, Map<String, Object>> runAndTakeSavepoint() throws Exception {
         JobClient jobClient = submitJobInitially(env(startAligned, 0));
         waitForAllTaskRunning(
-                () -> miniCluster.getMiniCluster().getExecutionGraph(jobClient.getJobID()).get());
+                () -> miniCluster.getMiniCluster().getExecutionGraph(jobClient.getJobID()).get(),
+                false);
         Thread.sleep(FIRST_RUN_BACKPRESSURE_MS); // wait for some backpressure from sink
 
         Future<Map<String, Object>> accFuture =