You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2021/08/23 12:25:02 UTC
[flink] 02/02: [FLINK-23811][tests] Handle finished subtasks in
CommonTestUtils.waitForAllTaskRunning
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 136bb52326922dfe70b3cc79fd463f9d6cfecc32
Author: Roman Khachatryan <kh...@gmail.com>
AuthorDate: Fri Aug 13 01:54:07 2021 +0100
[FLINK-23811][tests] Handle finished subtasks in CommonTestUtils.waitForAllTaskRunning
CommonTestUtils.waitForAllTaskRunning returns when all the subtasks are running AND
the job is running and not finished. However, with FLIP-147, subtasks may finish and
the job will still be running. So the method won't return and instead timeout.
This commit adds a flag to indicate whether to fail or proceed
when a finished sub-task is encountered.
---
.../file/sink/writer/FileSinkMigrationITCase.java | 2 +-
.../flink/runtime/testutils/CommonTestUtils.java | 42 +++++++++++++++-------
.../JobMasterStopWithSavepointITCase.java | 3 +-
.../flink/test/checkpointing/RescalingITCase.java | 8 ++---
.../flink/test/checkpointing/SavepointITCase.java | 8 ++---
.../test/checkpointing/TimersSavepointITCase.java | 2 +-
.../UnalignedCheckpointCompatibilityITCase.java | 3 +-
7 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java
index 8835145..225b3b2 100644
--- a/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java
+++ b/flink-connectors/flink-connector-files/src/test/java/org/apache/flink/connector/file/sink/writer/FileSinkMigrationITCase.java
@@ -185,7 +185,7 @@ public class FileSinkMigrationITCase extends TestLogger {
miniCluster.submitJob(jobGraph);
JobID jobId = jobSubmissionResultFuture.get().getJobID();
- waitForAllTaskRunning(miniCluster, jobId);
+ waitForAllTaskRunning(miniCluster, jobId, false);
CompletableFuture<String> savepointResultFuture =
miniCluster.triggerSavepoint(jobId, savepointBasePath, true);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index 671c723..b66ee2a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -22,8 +22,8 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
import org.apache.flink.api.common.time.Deadline;
import org.apache.flink.core.execution.JobClient;
-import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
+import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
import org.apache.flink.runtime.minicluster.MiniCluster;
import org.apache.flink.util.FileUtils;
import org.apache.flink.util.Preconditions;
@@ -44,6 +44,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.function.Predicate;
/** This class contains auxiliary methods for unit tests. */
public class CommonTestUtils {
@@ -158,22 +159,43 @@ public class CommonTestUtils {
}
}
- public static void waitForAllTaskRunning(MiniCluster miniCluster, JobID jobId)
- throws Exception {
- waitForAllTaskRunning(() -> miniCluster.getExecutionGraph(jobId).get(60, TimeUnit.SECONDS));
+ public static void waitForAllTaskRunning(
+ MiniCluster miniCluster, JobID jobId, boolean allowFinished) throws Exception {
+ waitForAllTaskRunning(
+ () -> miniCluster.getExecutionGraph(jobId).get(60, TimeUnit.SECONDS),
+ allowFinished);
}
public static void waitForAllTaskRunning(
- SupplierWithException<AccessExecutionGraph, Exception> executionGraphSupplier)
+ SupplierWithException<AccessExecutionGraph, Exception> executionGraphSupplier,
+ boolean allowFinished)
throws Exception {
waitForAllTaskRunning(
- executionGraphSupplier, Deadline.fromNow(Duration.of(1, ChronoUnit.MINUTES)));
+ executionGraphSupplier,
+ Deadline.fromNow(Duration.of(1, ChronoUnit.MINUTES)),
+ allowFinished);
}
public static void waitForAllTaskRunning(
SupplierWithException<AccessExecutionGraph, Exception> executionGraphSupplier,
- Deadline timeout)
+ Deadline timeout,
+ boolean allowFinished)
throws Exception {
+ Predicate<AccessExecutionVertex> subtaskPredicate =
+ task -> {
+ switch (task.getExecutionState()) {
+ case RUNNING:
+ return true;
+ case FINISHED:
+ if (allowFinished) {
+ return true;
+ } else {
+ throw new RuntimeException("Sub-Task finished unexpectedly" + task);
+ }
+ default:
+ return false;
+ }
+ };
waitUntilCondition(
() -> {
final AccessExecutionGraph graph = executionGraphSupplier.get();
@@ -186,11 +208,7 @@ public class CommonTestUtils {
.allMatch(
jobVertex ->
Arrays.stream(jobVertex.getTaskVertices())
- .allMatch(
- task ->
- task.getExecutionState()
- == ExecutionState
- .RUNNING));
+ .allMatch(subtaskPredicate));
},
timeout);
}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
index 941b40e..399b960 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
@@ -315,7 +315,8 @@ public class JobMasterStopWithSavepointITCase extends AbstractTestBase {
.getMiniCluster()
.getExecutionGraph(jobID)
.get(60, TimeUnit.SECONDS),
- deadline);
+ deadline,
+ false);
}
/**
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java
index 9854df2..5511455 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/RescalingITCase.java
@@ -238,7 +238,7 @@ public class RescalingITCase extends TestLogger {
// clear the CollectionSink set for the restarted job
CollectionSink.clearElementsSet();
- waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+ waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
CompletableFuture<String> savepointPathFuture = client.triggerSavepoint(jobID, null);
final String savepointPath =
@@ -317,7 +317,7 @@ public class RescalingITCase extends TestLogger {
client.submitJob(jobGraph).get();
// wait until the operator is started
- waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+ waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
// wait until the operator handles some data
StateSourceBase.workStartedLatch.await();
@@ -418,7 +418,7 @@ public class RescalingITCase extends TestLogger {
// clear the CollectionSink set for the restarted job
CollectionSink.clearElementsSet();
- waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+ waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
CompletableFuture<String> savepointPathFuture = client.triggerSavepoint(jobID, null);
final String savepointPath =
@@ -542,7 +542,7 @@ public class RescalingITCase extends TestLogger {
client.submitJob(jobGraph).get();
// wait until the operator is started
- waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+ waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
// wait until the operator handles some data
StateSourceBase.workStartedLatch.await();
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
index 7666472..cb6426a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java
@@ -370,7 +370,7 @@ public class SavepointITCase extends TestLogger {
try {
client.submitJob(jobGraph).get();
- waitForAllTaskRunning(cluster.getMiniCluster(), jobId);
+ waitForAllTaskRunning(cluster.getMiniCluster(), jobId, false);
StatefulCounter.getProgressLatch().await();
return client.cancelWithSavepoint(jobId, null).get();
@@ -502,7 +502,7 @@ public class SavepointITCase extends TestLogger {
try {
client.submitJob(graph).get();
// triggerSavepoint is only available after all tasks are running
- waitForAllTaskRunning(cluster.getMiniCluster(), graph.getJobID());
+ waitForAllTaskRunning(cluster.getMiniCluster(), graph.getJobID(), false);
client.triggerSavepoint(graph.getJobID(), null).get();
@@ -993,7 +993,7 @@ public class SavepointITCase extends TestLogger {
JobID jobID = client.submitJob(originalJobGraph).get();
// wait for the Tasks to be ready
- waitForAllTaskRunning(cluster.getMiniCluster(), jobID);
+ waitForAllTaskRunning(cluster.getMiniCluster(), jobID, false);
assertTrue(
StatefulCounter.getProgressLatch()
.await(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
@@ -1334,7 +1334,7 @@ public class SavepointITCase extends TestLogger {
try {
client.submitJob(jobGraph).get();
- waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID());
+ waitForAllTaskRunning(cluster.getMiniCluster(), jobGraph.getJobID(), false);
for (OneShotLatch latch : iterTestSnapshotWait) {
latch.await();
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java
index f7d0637..3782496 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/TimersSavepointITCase.java
@@ -120,7 +120,7 @@ public class TimersSavepointITCase {
private void takeSavepoint(String savepointPath, ClusterClient<?> client) throws Exception {
JobGraph jobGraph = getJobGraph(EmbeddedRocksDBStateBackend.PriorityQueueStateType.ROCKSDB);
client.submitJob(jobGraph).get();
- waitForAllTaskRunning(miniClusterResource.getMiniCluster(), jobGraph.getJobID());
+ waitForAllTaskRunning(miniClusterResource.getMiniCluster(), jobGraph.getJobID(), false);
CompletableFuture<String> savepointPathFuture =
client.triggerSavepoint(jobGraph.getJobID(), null);
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java
index 21bfa3f..8a39643 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/UnalignedCheckpointCompatibilityITCase.java
@@ -146,7 +146,8 @@ public class UnalignedCheckpointCompatibilityITCase extends TestLogger {
private Tuple2<String, Map<String, Object>> runAndTakeSavepoint() throws Exception {
JobClient jobClient = submitJobInitially(env(startAligned, 0));
waitForAllTaskRunning(
- () -> miniCluster.getMiniCluster().getExecutionGraph(jobClient.getJobID()).get());
+ () -> miniCluster.getMiniCluster().getExecutionGraph(jobClient.getJobID()).get(),
+ false);
Thread.sleep(FIRST_RUN_BACKPRESSURE_MS); // wait for some backpressure from sink
Future<Map<String, Object>> accFuture =