You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2021/08/03 11:22:09 UTC
[flink] 04/04: [FLINK-21080][runtime][checkpoint] Report latest
completed checkpoint id when notifying checkpoint abort
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 2f5655a085adcb7723ccf8045f29cae340e33e81
Author: Yun Gao <ga...@gmail.com>
AuthorDate: Mon Aug 2 19:42:42 2021 +0800
[FLINK-21080][runtime][checkpoint] Report latest completed checkpoint id when notifying checkpoint abort
This closes #16633
---
.../runtime/checkpoint/CheckpointCoordinator.java | 6 +-
.../checkpoint/CompletedCheckpointStore.java | 15 +++++
.../flink/runtime/executiongraph/Execution.java | 10 ++-
.../runtime/jobgraph/tasks/AbstractInvokable.java | 4 +-
.../jobmanager/slots/TaskManagerGateway.java | 7 ++-
.../runtime/jobmaster/RpcTaskManagerGateway.java | 9 ++-
.../flink/runtime/taskexecutor/TaskExecutor.java | 7 ++-
.../runtime/taskexecutor/TaskExecutorGateway.java | 6 +-
.../TaskExecutorGatewayDecoratorBase.java | 7 ++-
.../org/apache/flink/runtime/taskmanager/Task.java | 14 ++---
.../checkpoint/CheckpointCoordinatorTest.java | 72 ++++++++++++++++++++++
.../CheckpointCoordinatorTestingUtils.java | 6 +-
.../checkpoint/CompletedCheckpointStoreTest.java | 15 +++++
.../utils/SimpleAckingTaskManagerGateway.java | 1 +
.../CoordinatorEventsExactlyOnceITCase.java | 3 +-
.../taskexecutor/TestingTaskExecutorGateway.java | 5 +-
.../flink/streaming/runtime/tasks/StreamTask.java | 15 ++++-
.../tasks/StreamTaskFinalCheckpointsTest.java | 2 +-
.../streaming/runtime/tasks/StreamTaskTest.java | 66 +++++++++++++++++++-
.../runtime/tasks/SynchronousCheckpointITCase.java | 3 +-
.../JobMasterStopWithSavepointITCase.java | 3 +-
.../jobmaster/JobMasterTriggerSavepointITCase.java | 3 +-
22 files changed, 246 insertions(+), 33 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 0b15841..05ff7eb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -1314,6 +1314,9 @@ public class CheckpointCoordinator {
private void sendAbortedMessages(
List<ExecutionVertex> tasksToAbort, long checkpointId, long timeStamp) {
+ assert (Thread.holdsLock(lock));
+ long latestCompletedCheckpointId = completedCheckpointStore.getLatestCheckpointId();
+
// send notification of aborted checkpoints asynchronously.
executor.execute(
() -> {
@@ -1321,7 +1324,8 @@ public class CheckpointCoordinator {
for (ExecutionVertex ev : tasksToAbort) {
Execution ee = ev.getCurrentExecutionAttempt();
if (ee != null) {
- ee.notifyCheckpointAborted(checkpointId, timeStamp);
+ ee.notifyCheckpointAborted(
+ checkpointId, latestCompletedCheckpointId, timeStamp);
}
}
});
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
index 10c5c07..8a73f4b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java
@@ -85,6 +85,21 @@ public interface CompletedCheckpointStore {
return lastCompleted;
}
+ /** Returns the id of the latest completed checkpoints. */
+ default long getLatestCheckpointId() {
+ try {
+ List<CompletedCheckpoint> allCheckpoints = getAllCheckpoints();
+ if (allCheckpoints.isEmpty()) {
+ return 0;
+ }
+
+ return allCheckpoints.get(allCheckpoints.size() - 1).getCheckpointID();
+ } catch (Throwable throwable) {
+ LOG.warn("Get the latest completed checkpoints failed", throwable);
+ return 0;
+ }
+ }
+
/**
* Shuts down the store.
*
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index b047229..58756e2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -796,16 +796,22 @@ public class Execution
* Notify the task of this execution about a aborted checkpoint.
*
* @param abortCheckpointId of the subsumed checkpoint
+ * @param latestCompletedCheckpointId of the latest completed checkpoint
* @param timestamp of the subsumed checkpoint
*/
- public void notifyCheckpointAborted(long abortCheckpointId, long timestamp) {
+ public void notifyCheckpointAborted(
+ long abortCheckpointId, long latestCompletedCheckpointId, long timestamp) {
final LogicalSlot slot = assignedResource;
if (slot != null) {
final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
taskManagerGateway.notifyCheckpointAborted(
- attemptId, getVertex().getJobId(), abortCheckpointId, timestamp);
+ attemptId,
+ getVertex().getJobId(),
+ abortCheckpointId,
+ latestCompletedCheckpointId,
+ timestamp);
} else {
LOG.debug(
"The execution has no slot assigned. This indicates that the execution is "
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
index 453bd85..70a9f4a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
@@ -282,9 +282,11 @@ public abstract class AbstractInvokable {
* notification.
*
* @param checkpointId The ID of the checkpoint that is aborted.
+ * @param latestCompletedCheckpointId The ID of the latest completed checkpoint.
* @return future that completes when the notification has been processed by the task.
*/
- public Future<Void> notifyCheckpointAbortAsync(long checkpointId) {
+ public Future<Void> notifyCheckpointAbortAsync(
+ long checkpointId, long latestCompletedCheckpointId) {
throw new UnsupportedOperationException(
String.format(
"notifyCheckpointAbortAsync not supported by %s",
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java
index baef1bb..695f243 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/slots/TaskManagerGateway.java
@@ -102,10 +102,15 @@ public interface TaskManagerGateway extends TaskExecutorOperatorEventGateway {
* @param executionAttemptID identifying the task
* @param jobId identifying the job to which the task belongs
* @param checkpointId of the subsumed checkpoint
+ * @param latestCompletedCheckpointId of the latest completed checkpoint
* @param timestamp of the subsumed checkpoint
*/
void notifyCheckpointAborted(
- ExecutionAttemptID executionAttemptID, JobID jobId, long checkpointId, long timestamp);
+ ExecutionAttemptID executionAttemptID,
+ JobID jobId,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long timestamp);
/**
* Trigger for the given task a checkpoint.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java
index 81879a5..eb16d7d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/RpcTaskManagerGateway.java
@@ -87,8 +87,13 @@ public class RpcTaskManagerGateway implements TaskManagerGateway {
@Override
public void notifyCheckpointAborted(
- ExecutionAttemptID executionAttemptID, JobID jobId, long checkpointId, long timestamp) {
- taskExecutorGateway.abortCheckpoint(executionAttemptID, checkpointId, timestamp);
+ ExecutionAttemptID executionAttemptID,
+ JobID jobId,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long timestamp) {
+ taskExecutorGateway.abortCheckpoint(
+ executionAttemptID, checkpointId, latestCompletedCheckpointId, timestamp);
}
@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
index 2fb744f..d2e3f3e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java
@@ -1000,7 +1000,10 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway {
@Override
public CompletableFuture<Acknowledge> abortCheckpoint(
- ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) {
+ ExecutionAttemptID executionAttemptID,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long checkpointTimestamp) {
log.debug(
"Abort checkpoint {}@{} for {}.",
checkpointId,
@@ -1010,7 +1013,7 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway {
final Task task = taskSlotTable.getTask(executionAttemptID);
if (task != null) {
- task.notifyCheckpointAborted(checkpointId);
+ task.notifyCheckpointAborted(checkpointId, latestCompletedCheckpointId);
return CompletableFuture.completedFuture(Acknowledge.get());
} else {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
index 1ef98a0..6de0fac 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGateway.java
@@ -156,11 +156,15 @@ public interface TaskExecutorGateway
*
* @param executionAttemptID identifying the task
* @param checkpointId unique id for the checkpoint
+ * @param latestCompletedCheckpointId the id of the latest completed checkpoint
* @param checkpointTimestamp is the timestamp when the checkpoint has been initiated
* @return Future acknowledge if the checkpoint has been successfully confirmed
*/
CompletableFuture<Acknowledge> abortCheckpoint(
- ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp);
+ ExecutionAttemptID executionAttemptID,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long checkpointTimestamp);
/**
* Cancel the given task.
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGatewayDecoratorBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGatewayDecoratorBase.java
index 6e333b5..0cbee3d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGatewayDecoratorBase.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutorGatewayDecoratorBase.java
@@ -138,9 +138,12 @@ public class TaskExecutorGatewayDecoratorBase implements TaskExecutorGateway {
@Override
public CompletableFuture<Acknowledge> abortCheckpoint(
- ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) {
+ ExecutionAttemptID executionAttemptID,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long checkpointTimestamp) {
return originalGateway.abortCheckpoint(
- executionAttemptID, checkpointId, checkpointTimestamp);
+ executionAttemptID, checkpointId, latestCompletedCheckpointId, checkpointTimestamp);
}
@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 8a312f2..f2b1ba9e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -23,7 +23,6 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.api.common.state.CheckpointListener;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.TaskManagerOptions;
import org.apache.flink.core.fs.FileSystemSafetyNet;
@@ -137,11 +136,7 @@ import static org.apache.flink.util.Preconditions.checkState;
* <p>Each Task is run by one dedicated thread.
*/
public class Task
- implements Runnable,
- TaskSlotPayload,
- TaskActions,
- PartitionProducerStateProvider,
- CheckpointListener {
+ implements Runnable, TaskSlotPayload, TaskActions, PartitionProducerStateProvider {
/** The class logger. */
private static final Logger LOG = LoggerFactory.getLogger(Task.class);
@@ -1356,7 +1351,6 @@ public class Task
}
}
- @Override
public void notifyCheckpointComplete(final long checkpointID) {
final AbstractInvokable invokable = this.invokable;
@@ -1384,13 +1378,13 @@ public class Task
}
}
- @Override
- public void notifyCheckpointAborted(final long checkpointID) {
+ public void notifyCheckpointAborted(
+ final long checkpointID, final long latestCompletedCheckpointId) {
final AbstractInvokable invokable = this.invokable;
if (executionState == ExecutionState.RUNNING && invokable != null) {
try {
- invokable.notifyCheckpointAbortAsync(checkpointID);
+ invokable.notifyCheckpointAbortAsync(checkpointID, latestCompletedCheckpointId);
} catch (RejectedExecutionException ex) {
// This may happen if the mailbox is closed. It means that the task is shutting
// down, so we just ignore it.
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index a410c60..ba158b1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -104,6 +104,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.flink.runtime.checkpoint.CheckpointFailureReason.CHECKPOINT_ASYNC_EXCEPTION;
@@ -523,6 +524,7 @@ public class CheckpointCoordinatorTest extends TestLogger {
ExecutionAttemptID executionAttemptID,
JobID jobId,
long checkpointId,
+ long latestCompletedCheckpointId,
long timestamp) {
checkpointAborted.set(true);
}
@@ -3514,6 +3516,76 @@ public class CheckpointCoordinatorTest extends TestLogger {
}
}
+ @Test
+ public void testReportLatestCompletedCheckpointIdWithAbort() throws Exception {
+ JobVertexID jobVertexID = new JobVertexID();
+ ExecutionGraph graph =
+ new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder()
+ .addJobVertex(jobVertexID)
+ .setTransitToRunning(false)
+ .build();
+
+ ExecutionVertex task = graph.getJobVertex(jobVertexID).getTaskVertices()[0];
+
+ AtomicLong reportedCheckpointId = new AtomicLong(-1);
+ LogicalSlot slot =
+ new TestingLogicalSlotBuilder()
+ .setTaskManagerGateway(
+ new SimpleAckingTaskManagerGateway() {
+ @Override
+ public void notifyCheckpointAborted(
+ ExecutionAttemptID executionAttemptID,
+ JobID jobId,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long timestamp) {
+ reportedCheckpointId.set(latestCompletedCheckpointId);
+ }
+ })
+ .createTestingLogicalSlot();
+ ExecutionGraphTestUtils.setVertexResource(task, slot);
+ task.getCurrentExecutionAttempt().transitionState(ExecutionState.RUNNING);
+
+ CheckpointCoordinator checkpointCoordinator =
+ new CheckpointCoordinatorBuilder()
+ .setExecutionGraph(graph)
+ .setTimer(manuallyTriggeredScheduledExecutor)
+ .setAllowCheckpointsAfterTasksFinished(true)
+ .build();
+
+ // Trigger a successful checkpoint
+ CompletableFuture<CompletedCheckpoint> result =
+ checkpointCoordinator.triggerCheckpoint(false);
+ manuallyTriggeredScheduledExecutor.triggerAll();
+ long completedCheckpointId =
+ checkpointCoordinator.getPendingCheckpoints().entrySet().iterator().next().getKey();
+ checkpointCoordinator.receiveAcknowledgeMessage(
+ new AcknowledgeCheckpoint(
+ graph.getJobID(),
+ task.getCurrentExecutionAttempt().getAttemptId(),
+ completedCheckpointId,
+ new CheckpointMetrics(),
+ new TaskStateSnapshot()),
+ "localhost");
+ assertTrue(result.isDone());
+ assertFalse(result.isCompletedExceptionally());
+
+ result = checkpointCoordinator.triggerCheckpoint(false);
+ manuallyTriggeredScheduledExecutor.triggerAll();
+ long abortedCheckpointId =
+ checkpointCoordinator.getPendingCheckpoints().entrySet().iterator().next().getKey();
+ checkpointCoordinator.receiveDeclineMessage(
+ new DeclineCheckpoint(
+ graph.getJobID(),
+ task.getCurrentExecutionAttempt().getAttemptId(),
+ abortedCheckpointId,
+ new CheckpointException(CHECKPOINT_EXPIRED)),
+ "localhost");
+ assertTrue(result.isCompletedExceptionally());
+
+ assertEquals(completedCheckpointId, reportedCheckpointId.get());
+ }
+
private CheckpointCoordinator getCheckpointCoordinator(ExecutionGraph graph) throws Exception {
return new CheckpointCoordinatorBuilder()
.setExecutionGraph(graph)
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTestingUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTestingUtils.java
index 23eba74..38f9d2c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTestingUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTestingUtils.java
@@ -501,7 +501,11 @@ public class CheckpointCoordinatorTestingUtils {
@Override
public void notifyCheckpointAborted(
- ExecutionAttemptID attemptId, JobID jobId, long checkpointId, long timestamp) {
+ ExecutionAttemptID attemptId,
+ JobID jobId,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long timestamp) {
notifiedAbortCheckpoints
.computeIfAbsent(attemptId, k -> new ArrayList<>())
.add(new NotifiedCheckpoint(jobId, checkpointId, timestamp));
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
index a337f18..e68c889 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStoreTest.java
@@ -198,6 +198,21 @@ public abstract class CompletedCheckpointStoreTest extends TestLogger {
}
}
+ @Test
+ public void testAcquireLatestCompletedCheckpointId() throws Exception {
+ SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
+ CompletedCheckpointStore checkpoints = createCompletedCheckpoints(1);
+ assertEquals(0, checkpoints.getLatestCheckpointId());
+
+ checkpoints.addCheckpoint(
+ createCheckpoint(2, sharedStateRegistry), new CheckpointsCleaner(), () -> {});
+ assertEquals(2, checkpoints.getLatestCheckpointId());
+
+ checkpoints.addCheckpoint(
+ createCheckpoint(4, sharedStateRegistry), new CheckpointsCleaner(), () -> {});
+ assertEquals(4, checkpoints.getLatestCheckpointId());
+ }
+
// ---------------------------------------------------------------------------------------------
public static TestCompletedCheckpoint createCheckpoint(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java
index d6be8b9..9ea834a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleAckingTaskManagerGateway.java
@@ -137,6 +137,7 @@ public class SimpleAckingTaskManagerGateway implements TaskManagerGateway {
ExecutionAttemptID executionAttemptID,
JobID jobId,
long checkpointId,
+ long latestCompletedCheckpointId,
long timestamp) {}
@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/CoordinatorEventsExactlyOnceITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/CoordinatorEventsExactlyOnceITCase.java
index be87490..0add10e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/CoordinatorEventsExactlyOnceITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/CoordinatorEventsExactlyOnceITCase.java
@@ -604,7 +604,8 @@ public class CoordinatorEventsExactlyOnceITCase extends TestLogger {
}
@Override
- public Future<Void> notifyCheckpointAbortAsync(long checkpointId) {
+ public Future<Void> notifyCheckpointAbortAsync(
+ long checkpointId, long latestCompletedCheckpointId) {
return CompletableFuture.completedFuture(null);
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java
index 2bbcdee..2883445 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TestingTaskExecutorGateway.java
@@ -229,7 +229,10 @@ public class TestingTaskExecutorGateway implements TaskExecutorGateway {
@Override
public CompletableFuture<Acknowledge> abortCheckpoint(
- ExecutionAttemptID executionAttemptID, long checkpointId, long checkpointTimestamp) {
+ ExecutionAttemptID executionAttemptID,
+ long checkpointId,
+ long latestCompletedCheckpointId,
+ long checkpointTimestamp) {
return CompletableFuture.completedFuture(Acknowledge.get());
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 3345913..858b868 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -261,6 +261,8 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
private Long syncSavepointId = null;
private Long activeSyncSavepointId = null;
+ private long latestReportCheckpointId = -1;
+
private long latestAsyncCheckpointStartDelayNanos;
private final CompletableFuture<Void> terminationFuture = new CompletableFuture<>();
@@ -1260,9 +1262,14 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
}
@Override
- public Future<Void> notifyCheckpointAbortAsync(long checkpointId) {
+ public Future<Void> notifyCheckpointAbortAsync(
+ long checkpointId, long latestCompletedCheckpointId) {
return notifyCheckpointOperation(
() -> {
+ if (latestCompletedCheckpointId > 0) {
+ notifyCheckpointComplete(latestCompletedCheckpointId);
+ }
+
resetSynchronousSavepointId(checkpointId, false);
subtaskCheckpointCoordinator.notifyCheckpointAborted(
checkpointId, operatorChain, this::isRunning);
@@ -1290,6 +1297,12 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>> extends Ab
}
private void notifyCheckpointComplete(long checkpointId) throws Exception {
+ if (checkpointId <= latestReportCheckpointId) {
+ return;
+ }
+
+ latestReportCheckpointId = checkpointId;
+
subtaskCheckpointCoordinator.notifyCheckpointComplete(
checkpointId, operatorChain, this::isRunning);
if (isRunning && isSynchronousSavepointId(checkpointId)) {
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskFinalCheckpointsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskFinalCheckpointsTest.java
index 8f342ff..e065820 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskFinalCheckpointsTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskFinalCheckpointsTest.java
@@ -406,7 +406,7 @@ public class StreamTaskFinalCheckpointsTest {
// Checkpoint notification.
harness.streamTask.notifyCheckpointCompleteAsync(2);
- harness.streamTask.notifyCheckpointAbortAsync(3);
+ harness.streamTask.notifyCheckpointAbortAsync(3, 2);
harness.processAll();
// Finish & close operators.
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index dbc2f25..6fd5114 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -94,6 +94,7 @@ import org.apache.flink.runtime.throughput.ThroughputCalculator;
import org.apache.flink.runtime.util.NettyShuffleDescriptorBuilder;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
import org.apache.flink.streaming.api.graph.StreamConfig;
@@ -146,6 +147,7 @@ import java.io.ObjectInputStream;
import java.io.StreamCorruptedException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -267,13 +269,19 @@ public class StreamTaskTest extends TestLogger {
@Test
public void testSavepointSuspendAbortedAsync() throws Exception {
testSyncSavepointWithEndInput(
- StreamTask::notifyCheckpointAbortAsync, CheckpointType.SAVEPOINT_SUSPEND, true);
+ (streamTask, abortCheckpointId) ->
+ streamTask.notifyCheckpointAbortAsync(abortCheckpointId, 0),
+ CheckpointType.SAVEPOINT_SUSPEND,
+ true);
}
@Test
public void testSavepointTerminateAbortedAsync() throws Exception {
testSyncSavepointWithEndInput(
- StreamTask::notifyCheckpointAbortAsync, CheckpointType.SAVEPOINT_TERMINATE, true);
+ (streamTask, abortCheckpointId) ->
+ streamTask.notifyCheckpointAbortAsync(abortCheckpointId, 0),
+ CheckpointType.SAVEPOINT_TERMINATE,
+ true);
}
/**
@@ -1218,7 +1226,8 @@ public class StreamTaskTest extends TestLogger {
@Test
public void testFailToConfirmCheckpointAborted() throws Exception {
- testFailToConfirmCheckpointMessage(streamTask -> streamTask.notifyCheckpointAbortAsync(1L));
+ testFailToConfirmCheckpointMessage(
+ streamTask -> streamTask.notifyCheckpointAbortAsync(1L, 0L));
}
private void testFailToConfirmCheckpointMessage(Consumer<StreamTask<?, ?>> consumer)
@@ -1782,6 +1791,39 @@ public class StreamTaskTest extends TestLogger {
}
}
+ @Test
+ public void testSkipRepeatCheckpointComplete() throws Exception {
+ try (StreamTaskMailboxTestHarness<String> testHarness =
+ new StreamTaskMailboxTestHarnessBuilder<>(
+ OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO)
+ .addInput(BasicTypeInfo.STRING_TYPE_INFO, 3)
+ .modifyStreamConfig(
+ config -> {
+ config.setCheckpointingEnabled(true);
+ config.getConfiguration()
+ .set(
+ ExecutionCheckpointingOptions
+ .ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH,
+ true);
+ })
+ .setupOutputForSingletonOperatorChain(
+ new CheckpointCompleteRecordOperator())
+ .build()) {
+ testHarness.streamTask.notifyCheckpointCompleteAsync(3);
+ testHarness.streamTask.notifyCheckpointAbortAsync(5, 3);
+
+ testHarness.streamTask.notifyCheckpointAbortAsync(10, 8);
+ testHarness.streamTask.notifyCheckpointCompleteAsync(8);
+
+ testHarness.processAll();
+
+ CheckpointCompleteRecordOperator operator =
+ (CheckpointCompleteRecordOperator)
+ (AbstractStreamOperator<?>) testHarness.streamTask.getMainOperator();
+ assertEquals(Arrays.asList(3L, 8L), operator.getNotifiedCheckpoint());
+ }
+ }
+
private MockEnvironment setupEnvironment(boolean... outputAvailabilities) {
final Configuration configuration = new Configuration();
new MockStreamConfig(configuration, outputAvailabilities.length);
@@ -2787,4 +2829,22 @@ public class StreamTaskTest extends TestLogger {
};
}
}
+
+ private static class CheckpointCompleteRecordOperator extends AbstractStreamOperator<Integer>
+ implements OneInputStreamOperator<Integer, Integer> {
+
+ private final List<Long> notifiedCheckpoint = new ArrayList<>();
+
+ @Override
+ public void processElement(StreamRecord<Integer> element) throws Exception {}
+
+ @Override
+ public void notifyCheckpointComplete(long checkpointId) throws Exception {
+ notifiedCheckpoint.add(checkpointId);
+ }
+
+ public List<Long> getNotifiedCheckpoint() {
+ return notifiedCheckpoint;
+ }
+ }
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java
index 0ca5d18..912474c 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SynchronousCheckpointITCase.java
@@ -181,7 +181,8 @@ public class SynchronousCheckpointITCase {
}
@Override
- public Future<Void> notifyCheckpointAbortAsync(long checkpointId) {
+ public Future<Void> notifyCheckpointAbortAsync(
+ long checkpointId, long latestCompletedCheckpointId) {
return CompletableFuture.completedFuture(null);
}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
index c348fc2..23278c8 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterStopWithSavepointITCase.java
@@ -371,7 +371,8 @@ public class JobMasterStopWithSavepointITCase extends AbstractTestBase {
}
@Override
- public Future<Void> notifyCheckpointAbortAsync(long checkpointId) {
+ public Future<Void> notifyCheckpointAbortAsync(
+ long checkpointId, long latestCompletedCheckpointId) {
return CompletableFuture.completedFuture(null);
}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java
index 6cb55a4..99b090a 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterTriggerSavepointITCase.java
@@ -264,7 +264,8 @@ public class JobMasterTriggerSavepointITCase extends AbstractTestBase {
}
@Override
- public Future<Void> notifyCheckpointAbortAsync(long checkpointId) {
+ public Future<Void> notifyCheckpointAbortAsync(
+ long checkpointId, long latestCompletedCheckpointId) {
return CompletableFuture.completedFuture(null);
}
}