You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by jq...@apache.org on 2020/12/12 11:08:28 UTC

[flink] 25/32: [FLINK-20396][checkpointing] Checkpoint Coordinator exposes the checkpoint ID of the restored checkpoint for partial state restores.

This is an automated email from the ASF dual-hosted git repository.

jqin pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git

commit d79590d491e12784ef1b98c0aca6b742a6e98787
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Fri Nov 27 22:34:51 2020 +0100

    [FLINK-20396][checkpointing] Checkpoint Coordinator exposes the checkpoint ID of the restored checkpoint for partial state restores.
    
    This closes #14255
---
 .../runtime/checkpoint/CheckpointCoordinator.java  | 35 ++++++++++++++++------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index cccebf7..1c9f76d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -67,6 +67,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalLong;
 import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -1227,11 +1228,13 @@ public class CheckpointCoordinator {
 			boolean errorIfNoCheckpoint,
 			boolean allowNonRestoredState) throws Exception {
 
-		return restoreLatestCheckpointedStateInternal(
+		final OptionalLong restoredCheckpointId =  restoreLatestCheckpointedStateInternal(
 				new HashSet<>(tasks.values()),
 				OperatorCoordinatorRestoreBehavior.RESTORE_OR_RESET,
 				errorIfNoCheckpoint,
 				allowNonRestoredState);
+
+		return restoredCheckpointId.isPresent();
 	}
 
 	/**
@@ -1242,7 +1245,9 @@ public class CheckpointCoordinator {
 	 * @param tasks Set of job vertices to restore. State for these vertices is
 	 * restored via {@link Execution#setInitialState(JobManagerTaskRestore)}.
 
-	 * @return <code>true</code> if state was restored, <code>false</code> otherwise.
+	 * @return An {@code OptionalLong} with the checkpoint ID, if state was restored,
+	 *         an empty {@code OptionalLong} otherwise.
+	 *
 	 * @throws IllegalStateException If the CheckpointCoordinator is shut down.
 	 * @throws IllegalStateException If no completed checkpoint is available and
 	 *                               the <code>failIfNoCheckpoint</code> flag has been set.
@@ -1255,7 +1260,7 @@ public class CheckpointCoordinator {
 	 *                               that restores <i>non-partitioned</i> state from this
 	 *                               checkpoint.
 	 */
-	public boolean restoreLatestCheckpointedStateToSubtasks(final Set<ExecutionJobVertex> tasks) throws Exception {
+	public OptionalLong restoreLatestCheckpointedStateToSubtasks(final Set<ExecutionJobVertex> tasks) throws Exception {
 		// when restoring subtasks only we accept potentially unmatched state for the
 		// following reasons
 		//   - the set frequently does not include all Job Vertices (only the ones that are part
@@ -1296,11 +1301,13 @@ public class CheckpointCoordinator {
 			final Set<ExecutionJobVertex> tasks,
 			final boolean allowNonRestoredState) throws Exception {
 
-		return restoreLatestCheckpointedStateInternal(
+		final OptionalLong restoredCheckpointId = restoreLatestCheckpointedStateInternal(
 				tasks,
 				OperatorCoordinatorRestoreBehavior.RESTORE_OR_RESET, // global recovery restores coordinators, or resets them to empty
 				false,   // recovery might come before first successful checkpoint
 				allowNonRestoredState);
+
+		return restoredCheckpointId.isPresent();
 	}
 
 	/**
@@ -1314,14 +1321,22 @@ public class CheckpointCoordinator {
 	 * @return True, if a checkpoint was found and its state was restored, false otherwise.
 	 */
 	public boolean restoreInitialCheckpointIfPresent(final Set<ExecutionJobVertex> tasks) throws Exception {
-		return restoreLatestCheckpointedStateInternal(
+		final OptionalLong restoredCheckpointId = restoreLatestCheckpointedStateInternal(
 			tasks,
 			OperatorCoordinatorRestoreBehavior.RESTORE_IF_CHECKPOINT_PRESENT,
 			false,    // initial checkpoints exist only on JobManager failover. ok if not present.
 			false); // JobManager failover means JobGraphs match exactly.
+
+		return restoredCheckpointId.isPresent();
 	}
 
-	private boolean restoreLatestCheckpointedStateInternal(
+	/**
+	 * Performs the actual restore operation to the given tasks.
+	 *
+	 * <p>This method returns the restored checkpoint ID (as an optional) or an empty
+	 * optional, if no checkpoint was restored.
+	 */
+	private OptionalLong restoreLatestCheckpointedStateInternal(
 		final Set<ExecutionJobVertex> tasks,
 		final OperatorCoordinatorRestoreBehavior operatorCoordinatorRestoreBehavior,
 		final boolean errorIfNoCheckpoint,
@@ -1370,7 +1385,7 @@ public class CheckpointCoordinator {
 							Collections.emptyMap());
 				}
 
-				return false;
+				return OptionalLong.empty();
 			}
 
 			LOG.info("Restoring job {} from latest valid checkpoint: {}.", job, latest);
@@ -1411,7 +1426,7 @@ public class CheckpointCoordinator {
 				statsTracker.reportRestoredCheckpoint(restored);
 			}
 
-			return true;
+			return OptionalLong.of(latest.getCheckpointID());
 		}
 	}
 
@@ -1452,11 +1467,13 @@ public class CheckpointCoordinator {
 
 		LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);
 
-		return restoreLatestCheckpointedStateInternal(
+		final OptionalLong restoredCheckpointId = restoreLatestCheckpointedStateInternal(
 				new HashSet<>(tasks.values()),
 				OperatorCoordinatorRestoreBehavior.RESTORE_IF_CHECKPOINT_PRESENT,
 				true,
 				allowNonRestored);
+
+		return restoredCheckpointId.isPresent();
 	}
 
 	// ------------------------------------------------------------------------