You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2019/08/04 07:36:43 UTC
[flink] branch master updated: [FLINK-13476][coordination] Release
pipelined partitions on vertex restart / job failure
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 572efd1 [FLINK-13476][coordination] Release pipelined partitions on vertex restart / job failure
572efd1 is described below
commit 572efd14205232595799addb1919d0dfa13312a1
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Wed Jul 31 13:42:06 2019 +0200
[FLINK-13476][coordination] Release pipelined partitions on vertex restart / job failure
---
.../flink/runtime/executiongraph/Execution.java | 22 ++++++++++++++++++++--
.../ExecutionPartitionLifecycleTest.java | 18 ++++++++++++++++--
2 files changed, 36 insertions(+), 4 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index d8a1b6f..209c490 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -782,8 +782,21 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
// else: fall through the loop
}
- else if (current == FINISHED || current == FAILED) {
- // nothing to do any more. finished/failed before it could be cancelled.
+ else if (current == FINISHED) {
+ // finished before it could be cancelled.
+ // in any case, the task is removed from the TaskManager already
+
+ // a pipelined partition whose consumer has never been deployed could still be buffered on the TM
+ // release it here since pipelined partitions for FINISHED executions aren't handled elsewhere
+ // covers the following cases:
+ // a) restarts of this vertex
+ // b) a global failure (which may result in a FAILED job state)
+ sendReleaseIntermediateResultPartitionsRpcCall();
+
+ return;
+ }
+ else if (current == FAILED) {
+ // failed before it could be cancelled.
// in any case, the task is removed from the TaskManager already
return;
@@ -816,6 +829,11 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
completeCancelling();
break;
case FINISHED:
+ // a pipelined partition whose consumer has never been deployed could still be buffered on the TM
+ // release it here since pipelined partitions for FINISHED executions aren't handled elsewhere
+ // most notably, the TaskExecutor does not release pipelined partitions when disconnecting from the JM
+ sendReleaseIntermediateResultPartitionsRpcCall();
+ break;
case FAILED:
case CANCELED:
break;
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionPartitionLifecycleTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionPartitionLifecycleTest.java
index 21bd002..efa9ad4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionPartitionLifecycleTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionPartitionLifecycleTest.java
@@ -99,6 +99,20 @@ public class ExecutionPartitionLifecycleTest extends TestLogger {
@Test
public void testPartitionReleaseOnFinishWhileCanceling() throws Exception {
+ testPartitionReleaseOnStateTransitionsAfterRunning(Execution::cancel, Execution::markFinished);
+ }
+
+ @Test
+ public void testPartitionReleaseOnCancelWhileFinished() throws Exception {
+ testPartitionReleaseOnStateTransitionsAfterRunning(Execution::markFinished, Execution::cancel);
+ }
+
+ @Test
+ public void testPartitionReleaseOnSuspendWhileFinished() throws Exception {
+ testPartitionReleaseOnStateTransitionsAfterRunning(Execution::markFinished, Execution::suspend);
+ }
+
+ private void testPartitionReleaseOnStateTransitionsAfterRunning(Consumer<Execution> stateTransition1, Consumer<Execution> stateTransition2) throws Exception {
final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsCallFuture = new CompletableFuture<>();
taskManagerGateway.setReleasePartitionsConsumer(((jobID, partitionIds) -> releasePartitionsCallFuture.complete(Tuple2.of(jobID, partitionIds))));
@@ -107,10 +121,10 @@ public class ExecutionPartitionLifecycleTest extends TestLogger {
setupExecutionGraphAndStartRunningJob(ResultPartitionType.PIPELINED, NoOpPartitionTracker.INSTANCE, taskManagerGateway, testingShuffleMaster);
- execution.cancel();
+ stateTransition1.accept(execution);
assertFalse(releasePartitionsCallFuture.isDone());
- execution.markFinished();
+ stateTransition2.accept(execution);
assertTrue(releasePartitionsCallFuture.isDone());
final Tuple2<JobID, Collection<ResultPartitionID>> releasePartitionsCall = releasePartitionsCallFuture.get();