You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by jq...@apache.org on 2020/12/12 11:08:24 UTC

[flink] 21/32: [FLINK-20222][checkpointing] Operator Coordinators are reset with null state when no checkpoint or state available.

This is an automated email from the ASF dual-hosted git repository.

jqin pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 03b2e704e3417cefadbc62a61a81efa685e24bea
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Mon Nov 23 17:54:34 2020 +0100

    [FLINK-20222][checkpointing] Operator Coordinators are reset with null state when no checkpoint or state available.
    
    This includes the following cases:
      - JobManager/Scheduler/Coordinator failover happens before the first checkpoint completed.
      - Coordinator has not stored any state in the checkpoint.
    
    For both cases, this is useful to signal to the coordinators that a reset-to-checkpoint happened, even if
    they do not have checkpointed state to restore.
---
 .../runtime/checkpoint/CheckpointCoordinator.java  | 82 +++++++++++++++++-----
 .../OperatorCoordinatorCheckpointContext.java      | 15 +++-
 .../coordination/OperatorCoordinator.java          | 10 ++-
 .../coordination/OperatorCoordinatorHolder.java    |  2 +-
 .../RecreateOnResetOperatorCoordinator.java        |  4 +-
 .../flink/runtime/scheduler/SchedulerBase.java     |  5 +-
 .../source/coordinator/SourceCoordinator.java      |  9 ++-
 .../OperatorCoordinatorSchedulerTest.java          | 12 ++++
 .../coordination/TestingOperatorCoordinator.java   |  8 ++-
 .../collect/CollectSinkOperatorCoordinator.java    | 13 ++--
 10 files changed, 129 insertions(+), 31 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 0e61a11..61d6f22 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -1226,7 +1226,11 @@ public class CheckpointCoordinator {
 			boolean errorIfNoCheckpoint,
 			boolean allowNonRestoredState) throws Exception {
 
-		return restoreLatestCheckpointedStateInternal(new HashSet<>(tasks.values()), true, errorIfNoCheckpoint, allowNonRestoredState);
+		return restoreLatestCheckpointedStateInternal(
+				new HashSet<>(tasks.values()),
+				OperatorCoordinatorRestoreBehavior.RESTORE_OR_RESET,
+				errorIfNoCheckpoint,
+				allowNonRestoredState);
 	}
 
 	/**
@@ -1257,7 +1261,11 @@ public class CheckpointCoordinator {
 		//     of the restarted region), meaning there will be unmatched state by design.
 		//   - because what we might end up restoring from an original savepoint with unmatched
 		//     state, if there is was no checkpoint yet.
-		return restoreLatestCheckpointedStateInternal(tasks, false, false, true);
+		return restoreLatestCheckpointedStateInternal(
+				tasks,
+				OperatorCoordinatorRestoreBehavior.SKIP, // local/regional recovery does not reset coordinators
+				false,         // recovery might come before first successful checkpoint
+				true);       // see explanation above
 	}
 
 	/**
@@ -1287,12 +1295,34 @@ public class CheckpointCoordinator {
 			final Set<ExecutionJobVertex> tasks,
 			final boolean allowNonRestoredState) throws Exception {
 
-		return restoreLatestCheckpointedStateInternal(tasks, true, false, allowNonRestoredState);
+		return restoreLatestCheckpointedStateInternal(
+				tasks,
+				OperatorCoordinatorRestoreBehavior.RESTORE_OR_RESET, // global recovery restores coordinators, or resets them to empty
+				false,   // recovery might come before first successful checkpoint
+				allowNonRestoredState);
+	}
+
+	/**
+	 * Restores the latest checkpointed at the beginning of the job execution.
+	 * If there is a checkpoint, this method acts like a "global restore"-style
+	 * operation where all stateful tasks and coordinators from the given
+	 * set of Job Vertices are restored.
+	 *
+	 * @param tasks Set of job vertices to restore. State for these vertices is
+	 *              restored via {@link Execution#setInitialState(JobManagerTaskRestore)}.
+	 * @return True, if a checkpoint was found and its state was restored, false otherwise.
+	 */
+	public boolean restoreInitialCheckpointIfPresent(final Set<ExecutionJobVertex> tasks) throws Exception {
+		return restoreLatestCheckpointedStateInternal(
+			tasks,
+			OperatorCoordinatorRestoreBehavior.RESTORE_IF_CHECKPOINT_PRESENT,
+			false,    // initial checkpoints exist only on JobManager failover. ok if not present.
+			false); // JobManager failover means JobGraphs match exactly.
 	}
 
 	private boolean restoreLatestCheckpointedStateInternal(
 		final Set<ExecutionJobVertex> tasks,
-		final boolean restoreCoordinators,
+		final OperatorCoordinatorRestoreBehavior operatorCoordinatorRestoreBehavior,
 		final boolean errorIfNoCheckpoint,
 		final boolean allowNonRestoredState) throws Exception {
 
@@ -1321,14 +1351,23 @@ public class CheckpointCoordinator {
 			CompletedCheckpoint latest = completedCheckpointStore.getLatestCheckpoint(isPreferCheckpointForRecovery);
 
 			if (latest == null) {
+				LOG.info("No checkpoint found during restore.");
+
 				if (errorIfNoCheckpoint) {
 					throw new IllegalStateException("No completed checkpoint available");
-				} else {
+				}
+
+				if (operatorCoordinatorRestoreBehavior == OperatorCoordinatorRestoreBehavior.RESTORE_OR_RESET) {
+					// we let the JobManager-side components know that there was a recovery,
+					// even if there was no checkpoint to recover from, yet
 					LOG.debug("Resetting the master hooks.");
 					MasterHooks.reset(masterHooks.values(), LOG);
 
-					return false;
+					LOG.info("Resetting the Coordinators to an empty state.");
+					restoreStateToCoordinators(Collections.emptyMap());
 				}
+
+				return false;
 			}
 
 			LOG.info("Restoring job {} from latest valid checkpoint: {}.", job, latest);
@@ -1352,7 +1391,7 @@ public class CheckpointCoordinator {
 					allowNonRestoredState,
 					LOG);
 
-			if (restoreCoordinators) {
+			if (operatorCoordinatorRestoreBehavior != OperatorCoordinatorRestoreBehavior.SKIP) {
 				restoreStateToCoordinators(operatorStates);
 			}
 
@@ -1410,7 +1449,11 @@ public class CheckpointCoordinator {
 
 		LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);
 
-		return restoreLatestCheckpointedStateInternal(new HashSet<>(tasks.values()), true, true, allowNonRestored);
+		return restoreLatestCheckpointedStateInternal(
+				new HashSet<>(tasks.values()),
+				OperatorCoordinatorRestoreBehavior.RESTORE_IF_CHECKPOINT_PRESENT,
+				true,
+				allowNonRestored);
 	}
 
 	// ------------------------------------------------------------------------
@@ -1567,14 +1610,9 @@ public class CheckpointCoordinator {
 	private void restoreStateToCoordinators(final Map<OperatorID, OperatorState> operatorStates) throws Exception {
 		for (OperatorCoordinatorCheckpointContext coordContext : coordinatorsToCheckpoint) {
 			final OperatorState state = operatorStates.get(coordContext.operatorId());
-			if (state == null) {
-				continue;
-			}
-
-			final ByteStreamStateHandle coordinatorState = state.getCoordinatorState();
-			if (coordinatorState != null) {
-				coordContext.resetToCheckpoint(coordinatorState.getData());
-			}
+			final ByteStreamStateHandle coordinatorState = state == null ? null : state.getCoordinatorState();
+			final byte[] bytes = coordinatorState == null ? null : coordinatorState.getData();
+			coordContext.resetToCheckpoint(bytes);
 		}
 	}
 
@@ -1848,4 +1886,16 @@ public class CheckpointCoordinator {
 			return props.forceCheckpoint();
 		}
 	}
+
+	private enum OperatorCoordinatorRestoreBehavior {
+
+		/** Coordinators are always restored. If there is no checkpoint, they are restored empty. */
+		RESTORE_OR_RESET,
+
+		/** Coordinators are restored if there was a checkpoint. */
+		RESTORE_IF_CHECKPOINT_PRESENT,
+
+		/** Coordinators are not restored during this checkpoint restore. */
+		SKIP;
+	}
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorCoordinatorCheckpointContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorCoordinatorCheckpointContext.java
index f5f72d8..2f14e58 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorCoordinatorCheckpointContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/OperatorCoordinatorCheckpointContext.java
@@ -22,6 +22,8 @@ import org.apache.flink.runtime.operators.coordination.OperatorCoordinator;
 import org.apache.flink.runtime.operators.coordination.OperatorInfo;
 import org.apache.flink.runtime.state.CheckpointListener;
 
+import javax.annotation.Nullable;
+
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -52,5 +54,16 @@ public interface OperatorCoordinatorCheckpointContext extends OperatorInfo, Chec
 	@Override
 	default void notifyCheckpointAborted(long checkpointId) {}
 
-	void resetToCheckpoint(byte[] checkpointData) throws Exception;
+	/**
+	 * Resets the coordinator to the checkpoint with the given state.
+	 *
+	 * <p>This method is called with a null state argument in the following situations:
+	 * <ul>
+	 *   <li>There is a recovery and there was no completed checkpoint yet.</li>
+	 *   <li>There is a recovery from a completed checkpoint/savepoint but it contained no state
+	 *       for the coordinator.</li>
+	 * </ul>
+	 * In both cases, the coordinator should reset to an empty (new) state.
+	 */
+	void resetToCheckpoint(@Nullable byte[] checkpointData) throws Exception;
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java
index 21b67f2..1a6a640 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinator.java
@@ -127,6 +127,14 @@ public interface OperatorCoordinator extends CheckpointListener, AutoCloseable {
 	 * to {@code Context} methods. For example, Events being sent by the Coordinator after this method
 	 * returns are assumed to take place after the checkpoint that was restored.
 	 *
+	 * <p>This method is called with a null state argument in the following situations:
+	 * <ul>
+	 *   <li>There is a recovery and there was no completed checkpoint yet.</li>
+	 *   <li>There is a recovery from a completed checkpoint/savepoint but it contained no state
+	 *       for the coordinator.</li>
+	 * </ul>
+	 * In both cases, the coordinator should reset to an empty (new) state.
+	 *
 	 * <h2>Restoring implicitly notifies of Checkpoint Completion</h2>
 	 *
 	 * <p>Restoring to a checkpoint is a way of confirming that the checkpoint is complete.
@@ -137,7 +145,7 @@ public interface OperatorCoordinator extends CheckpointListener, AutoCloseable {
 	 * complete (for example when a system failure happened directly after committing the checkpoint,
 	 * before calling the {@link #notifyCheckpointComplete(long)} method).
 	 */
-	void resetToCheckpoint(byte[] checkpointData) throws Exception;
+	void resetToCheckpoint(@Nullable byte[] checkpointData) throws Exception;
 
 	// ------------------------------------------------------------------------
 	// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java
index e5844ef..4c7f252 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorHolder.java
@@ -229,7 +229,7 @@ public class OperatorCoordinatorHolder implements OperatorCoordinator, OperatorC
 	}
 
 	@Override
-	public void resetToCheckpoint(byte[] checkpointData) throws Exception {
+	public void resetToCheckpoint(@Nullable byte[] checkpointData) throws Exception {
 		// ideally we would like to check this here, however this method is called early during
 		// execution graph construction, before the main thread executor is set
 
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/RecreateOnResetOperatorCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/RecreateOnResetOperatorCoordinator.java
index ae67868..52b30ed 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/RecreateOnResetOperatorCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/coordination/RecreateOnResetOperatorCoordinator.java
@@ -105,7 +105,7 @@ public class RecreateOnResetOperatorCoordinator implements OperatorCoordinator {
 	}
 
 	@Override
-	public void resetToCheckpoint(byte[] checkpointData) throws Exception {
+	public void resetToCheckpoint(@Nullable byte[] checkpointData) {
 		// First bump up the coordinator epoch to fence out the active coordinator.
 		LOG.info("Resetting coordinator to checkpoint.");
 		// Replace the coordinator variable with a new DeferrableCoordinator instance.
@@ -366,7 +366,7 @@ public class RecreateOnResetOperatorCoordinator implements OperatorCoordinator {
 			internalCoordinator.start();
 		}
 
-		void resetAndStart(byte[] checkpointData, boolean started) {
+		void resetAndStart(@Nullable byte[] checkpointData, boolean started) {
 			if (failed || closed || internalCoordinator == null) {
 				return;
 			}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java
index 42d3e02..f65f3c1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java
@@ -245,9 +245,8 @@ public abstract class SchedulerBase implements SchedulerNG {
 
 		if (checkpointCoordinator != null) {
 			// check whether we find a valid checkpoint
-			if (!checkpointCoordinator.restoreLatestCheckpointedStateToAll(
-				new HashSet<>(newExecutionGraph.getAllVertices().values()),
-				false)) {
+			if (!checkpointCoordinator.restoreInitialCheckpointIfPresent(
+					new HashSet<>(newExecutionGraph.getAllVertices().values()))) {
 
 				// check whether we can restore from a savepoint
 				tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java
index 07ebbdb..3dea157 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java
@@ -223,10 +223,17 @@ public class SourceCoordinator<SplitT extends SourceSplit, EnumChkT> implements
 	}
 
 	@Override
-	public void resetToCheckpoint(byte[] checkpointData) throws Exception {
+	public void resetToCheckpoint(@Nullable byte[] checkpointData) throws Exception {
 		checkState(!started, "The coordinator can only be reset if it was not yet started");
 		assert enumerator == null;
 
+		// the checkpoint data is null if there was no completed checkpoint before
+		// in that case we don't restore here, but let a fresh SplitEnumerator be created
+		// when "start()" is called.
+		if (checkpointData == null) {
+			return;
+		}
+
 		LOG.info("Restoring SplitEnumerator of source {} from checkpoint.", operatorName);
 
 		final ClassLoader userCodeClassLoader = context.getCoordinatorContext().getUserCodeClassloader();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java
index 353b6b9..024cd46 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/OperatorCoordinatorSchedulerTest.java
@@ -78,6 +78,7 @@ import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -322,6 +323,17 @@ public class OperatorCoordinatorSchedulerTest extends TestLogger {
 	}
 
 	@Test
+	public void testGlobalFailureBeforeCheckpointResetsToEmptyState() throws Exception {
+		final DefaultScheduler scheduler = createSchedulerAndDeployTasks();
+		final TestingOperatorCoordinator coordinator = getCoordinator(scheduler);
+
+		failGlobalAndRestart(scheduler, new TestException());
+
+		assertSame("coordinator should have null restored state",
+			TestingOperatorCoordinator.NULL_RESTORE_VALUE, coordinator.getLastRestoredCheckpointState());
+	}
+
+	@Test
 	public void testLocalFailureDoesNotResetToCheckpoint() throws Exception {
 		final DefaultScheduler scheduler = createSchedulerAndDeployTasks();
 		final TestingOperatorCoordinator coordinator = getCoordinator(scheduler);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/TestingOperatorCoordinator.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/TestingOperatorCoordinator.java
index 5c5b91f..9572601 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/TestingOperatorCoordinator.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/coordination/TestingOperatorCoordinator.java
@@ -35,6 +35,8 @@ import java.util.concurrent.LinkedBlockingQueue;
  */
 class TestingOperatorCoordinator implements OperatorCoordinator {
 
+	public static final byte[] NULL_RESTORE_VALUE = new byte[0];
+
 	private final OperatorCoordinator.Context context;
 
 	private final ArrayList<Integer> failedTasks = new ArrayList<>();
@@ -104,8 +106,10 @@ class TestingOperatorCoordinator implements OperatorCoordinator {
 	}
 
 	@Override
-	public void resetToCheckpoint(byte[] checkpointData) {
-		lastRestoredCheckpointState = checkpointData;
+	public void resetToCheckpoint(@Nullable byte[] checkpointData) {
+		lastRestoredCheckpointState = checkpointData == null
+				? NULL_RESTORE_VALUE
+				: checkpointData;
 	}
 
 	// ------------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkOperatorCoordinator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkOperatorCoordinator.java
index 1423415..5454412 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkOperatorCoordinator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/collect/CollectSinkOperatorCoordinator.java
@@ -203,10 +203,15 @@ public class CollectSinkOperatorCoordinator implements OperatorCoordinator, Coor
 	}
 
 	@Override
-	public void resetToCheckpoint(byte[] checkpointData) throws Exception {
-		ByteArrayInputStream bais = new ByteArrayInputStream(checkpointData);
-		ObjectInputStream ois = new ObjectInputStream(bais);
-		address = (InetSocketAddress) ois.readObject();
+	public void resetToCheckpoint(@Nullable byte[] checkpointData) throws Exception {
+		if (checkpointData == null) {
+			// restore before any checkpoint completed
+			closeConnection();
+		} else {
+			ByteArrayInputStream bais = new ByteArrayInputStream(checkpointData);
+			ObjectInputStream ois = new ObjectInputStream(bais);
+			address = (InetSocketAddress) ois.readObject();
+		}
 	}
 
 	/**