You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sr...@apache.org on 2018/02/27 14:37:13 UTC

flink git commit: [hotfix] Improved logging for task local recovery

Repository: flink
Updated Branches:
  refs/heads/master 8d180d5fa -> 56c756040


[hotfix] Improved logging for task local recovery


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/56c75604
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/56c75604
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/56c75604

Branch: refs/heads/master
Commit: 56c756040fc4b3f224c4e7c12d208a3ccf5a7c5e
Parents: 8d180d5
Author: Stefan Richter <s....@data-artisans.com>
Authored: Mon Feb 26 18:03:14 2018 +0100
Committer: Stefan Richter <s....@data-artisans.com>
Committed: Tue Feb 27 15:28:11 2018 +0100

----------------------------------------------------------------------
 .../PrioritizedOperatorSubtaskState.java        | 31 +++++----
 .../TaskExecutorLocalStateStoresManager.java    | 68 ++++++++++++++------
 .../runtime/state/TaskLocalStateStoreImpl.java  | 63 ++++++++++++------
 .../runtime/state/TaskStateManagerImpl.java     | 27 +++++---
 .../PrioritizedOperatorSubtaskStateTest.java    |  4 +-
 .../runtime/state/TaskStateManagerImplTest.java |  4 +-
 .../api/operators/BackendRestorerProcedure.java | 61 ++++++++++++------
 .../StreamTaskStateInitializerImpl.java         | 20 ++++--
 .../operators/BackendRestorerProcedureTest.java | 18 ++----
 9 files changed, 193 insertions(+), 103 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java
index f48d311..512f912 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskState.java
@@ -27,7 +27,6 @@ import javax.annotation.Nonnull;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.function.BiFunction;
 
@@ -79,39 +78,39 @@ public class PrioritizedOperatorSubtaskState {
 	// -----------------------------------------------------------------------------------------------------------------
 
 	/**
-	 * Returns an iterator over all alternative snapshots to restore the managed operator state, in the order in which
-	 * we should attempt to restore.
+	 * Returns an immutable list with all alternative snapshots to restore the managed operator state, in the order in
+	 * which we should attempt to restore.
 	 */
 	@Nonnull
-	public Iterator<StateObjectCollection<OperatorStateHandle>> getPrioritizedManagedOperatorState() {
-		return prioritizedManagedOperatorState.iterator();
+	public List<StateObjectCollection<OperatorStateHandle>> getPrioritizedManagedOperatorState() {
+		return prioritizedManagedOperatorState;
 	}
 
 	/**
-	 * Returns an iterator over all alternative snapshots to restore the raw operator state, in the order in which we
-	 * should attempt to restore.
+	 * Returns an immutable list with all alternative snapshots to restore the raw operator state, in the order in
+	 * which we should attempt to restore.
 	 */
 	@Nonnull
-	public Iterator<StateObjectCollection<OperatorStateHandle>> getPrioritizedRawOperatorState() {
-		return prioritizedRawOperatorState.iterator();
+	public List<StateObjectCollection<OperatorStateHandle>> getPrioritizedRawOperatorState() {
+		return prioritizedRawOperatorState;
 	}
 
 	/**
-	 * Returns an iterator over all alternative snapshots to restore the managed keyed state, in the order in which we
-	 * should attempt to restore.
+	 * Returns an immutable list with all alternative snapshots to restore the managed keyed state, in the order in
+	 * which we should attempt to restore.
 	 */
 	@Nonnull
-	public Iterator<StateObjectCollection<KeyedStateHandle>> getPrioritizedManagedKeyedState() {
-		return prioritizedManagedKeyedState.iterator();
+	public List<StateObjectCollection<KeyedStateHandle>> getPrioritizedManagedKeyedState() {
+		return prioritizedManagedKeyedState;
 	}
 
 	/**
-	 * Returns an iterator over all alternative snapshots to restore the raw keyed state, in the order in which we
+	 * Returns an immutable list with all alternative snapshots to restore the raw keyed state, in the order in which we
 	 * should attempt to restore.
 	 */
 	@Nonnull
-	public Iterator<StateObjectCollection<KeyedStateHandle>> getPrioritizedRawKeyedState() {
-		return prioritizedRawKeyedState.iterator();
+	public List<StateObjectCollection<KeyedStateHandle>> getPrioritizedRawKeyedState() {
+		return prioritizedRawKeyedState;
 	}
 
 	// -----------------------------------------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java
index a940aef..e7a7d8f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskExecutorLocalStateStoresManager.java
@@ -110,38 +110,68 @@ public class TaskExecutorLocalStateStoresManager {
 					"register a new TaskLocalStateStore.");
 			}
 
-			final Map<JobVertexSubtaskKey, TaskLocalStateStoreImpl> taskStateManagers =
-				this.taskStateStoresByAllocationID.computeIfAbsent(allocationID, k -> new HashMap<>());
+			Map<JobVertexSubtaskKey, TaskLocalStateStoreImpl> taskStateManagers =
+				this.taskStateStoresByAllocationID.get(allocationID);
+
+			if (taskStateManagers == null) {
+				taskStateManagers = new HashMap<>();
+				this.taskStateStoresByAllocationID.put(allocationID, taskStateManagers);
+
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Registered new allocation id {} for local state stores for job {}.",
+						allocationID, jobId);
+				}
+			}
 
 			final JobVertexSubtaskKey taskKey = new JobVertexSubtaskKey(jobVertexID, subtaskIndex);
 
-			// create the allocation base dirs, one inside each root dir.
-			File[] allocationBaseDirectories = allocationBaseDirectories(allocationID);
+			TaskLocalStateStoreImpl taskLocalStateStore = taskStateManagers.get(taskKey);
 
-			LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(
-				allocationBaseDirectories,
-				jobId,
-				jobVertexID,
-				subtaskIndex);
+			if (taskLocalStateStore == null) {
+
+				// create the allocation base dirs, one inside each root dir.
+				File[] allocationBaseDirectories = allocationBaseDirectories(allocationID);
+
+				LocalRecoveryDirectoryProviderImpl directoryProvider = new LocalRecoveryDirectoryProviderImpl(
+					allocationBaseDirectories,
+					jobId,
+					jobVertexID,
+					subtaskIndex);
 
-			LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(
-				localRecoveryMode,
-				directoryProvider);
+				LocalRecoveryConfig localRecoveryConfig =
+					new LocalRecoveryConfig(localRecoveryMode, directoryProvider);
 
-			return taskStateManagers.computeIfAbsent(
-				taskKey,
-				k -> new TaskLocalStateStoreImpl(
+				taskLocalStateStore = new TaskLocalStateStoreImpl(
 					jobId,
 					allocationID,
 					jobVertexID,
 					subtaskIndex,
 					localRecoveryConfig,
-					discardExecutor));
+					discardExecutor);
+
+				taskStateManagers.put(taskKey, taskLocalStateStore);
+
+				if (LOG.isTraceEnabled()) {
+					LOG.trace("Registered new local state store with configuration {} for {} - {} - {} under allocation id {}.",
+						localRecoveryConfig, jobId, jobVertexID, subtaskIndex, allocationID);
+				}
+			} else {
+				if (LOG.isTraceEnabled()) {
+					LOG.trace("Found existing local state store for {} - {} - {} under allocation id {}.",
+						jobId, jobVertexID, subtaskIndex, allocationID);
+				}
+			}
+
+			return taskLocalStateStore;
 		}
 	}
 
 	public void releaseLocalStateForAllocationId(@Nonnull AllocationID allocationID) {
 
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Releasing local state under allocation id {}.", allocationID);
+		}
+
 		Map<JobVertexSubtaskKey, TaskLocalStateStoreImpl> cleanupLocalStores;
 
 		synchronized (lock) {
@@ -175,7 +205,7 @@ public class TaskExecutorLocalStateStoresManager {
 
 		ShutdownHookUtil.removeShutdownHook(shutdownHook, getClass().getSimpleName(), LOG);
 
-		LOG.debug("Shutting down TaskExecutorLocalStateStoresManager.");
+		LOG.info("Shutting down TaskExecutorLocalStateStoresManager.");
 
 		for (Map.Entry<AllocationID, Map<JobVertexSubtaskKey, TaskLocalStateStoreImpl>> entry :
 			toRelease.entrySet()) {
@@ -217,7 +247,7 @@ public class TaskExecutorLocalStateStoresManager {
 				try {
 					stateStore.dispose();
 				} catch (Exception disposeEx) {
-					LOG.warn("Exception while disposing local state store " + stateStore, disposeEx);
+					LOG.warn("Exception while disposing local state store {}.", stateStore, disposeEx);
 				}
 			}
 		}
@@ -233,7 +263,7 @@ public class TaskExecutorLocalStateStoresManager {
 			try {
 				FileUtils.deleteFileOrDirectory(directory);
 			} catch (IOException e) {
-				LOG.warn("Exception while deleting local state directory for allocation " + allocationID, e);
+				LOG.warn("Exception while deleting local state directory for allocation id {}.", allocationID, e);
 			}
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java
index 191c109..bb4f011 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskLocalStateStoreImpl.java
@@ -103,15 +103,15 @@ public class TaskLocalStateStoreImpl implements TaskLocalStateStore {
 		@Nonnull LocalRecoveryConfig localRecoveryConfig,
 		@Nonnull Executor discardExecutor) {
 
+		this.lock = new Object();
+		this.storedTaskStateByCheckpointID = new TreeMap<>();
 		this.jobID = jobID;
 		this.allocationID = allocationID;
 		this.jobVertexID = jobVertexID;
 		this.subtaskIndex = subtaskIndex;
 		this.discardExecutor = discardExecutor;
-		this.lock = new Object();
-		this.storedTaskStateByCheckpointID = new TreeMap<>();
-		this.disposed = false;
 		this.localRecoveryConfig = localRecoveryConfig;
+		this.disposed = false;
 	}
 
 	@Override
@@ -123,8 +123,15 @@ public class TaskLocalStateStoreImpl implements TaskLocalStateStore {
 			localState = NULL_DUMMY;
 		}
 
-		LOG.info("Storing local state for checkpoint {}.", checkpointId);
-		LOG.debug("Local state for checkpoint {} is {}.", checkpointId, localState);
+		if (LOG.isTraceEnabled()) {
+			LOG.debug(
+				"Stored local state for checkpoint {} in subtask ({} - {} - {}) : {}.",
+				checkpointId, jobID, jobVertexID, subtaskIndex, localState);
+		} else if (LOG.isDebugEnabled()) {
+			LOG.debug(
+				"Stored local state for checkpoint {} in subtask ({} - {} - {})",
+				checkpointId, jobID, jobVertexID, subtaskIndex);
+		}
 
 		Map<Long, TaskStateSnapshot> toDiscard = new HashMap<>(16);
 
@@ -148,10 +155,21 @@ public class TaskLocalStateStoreImpl implements TaskLocalStateStore {
 	@Override
 	@Nullable
 	public TaskStateSnapshot retrieveLocalState(long checkpointID) {
+
+		TaskStateSnapshot snapshot;
 		synchronized (lock) {
-			TaskStateSnapshot snapshot = storedTaskStateByCheckpointID.get(checkpointID);
-			return snapshot != NULL_DUMMY ? snapshot : null;
+			snapshot = storedTaskStateByCheckpointID.get(checkpointID);
 		}
+
+		if (LOG.isTraceEnabled()) {
+			LOG.trace("Found entry for local state for checkpoint {} in subtask ({} - {} - {}) : {}",
+				checkpointID, jobID, jobVertexID, subtaskIndex, snapshot);
+		} else if (LOG.isDebugEnabled()) {
+			LOG.debug("Found entry for local state for checkpoint {} in subtask ({} - {} - {})",
+				checkpointID, jobID, jobVertexID, subtaskIndex);
+		}
+
+		return snapshot != NULL_DUMMY ? snapshot : null;
 	}
 
 	@Override
@@ -163,7 +181,8 @@ public class TaskLocalStateStoreImpl implements TaskLocalStateStore {
 	@Override
 	public void confirmCheckpoint(long confirmedCheckpointId) {
 
-		LOG.debug("Received confirmation for checkpoint {}. Starting to prune history.", confirmedCheckpointId);
+		LOG.debug("Received confirmation for checkpoint {} in subtask ({} - {} - {}). Starting to prune history.",
+			confirmedCheckpointId, jobID, jobVertexID, subtaskIndex);
 
 		final List<Map.Entry<Long, TaskStateSnapshot>> toRemove = new ArrayList<>();
 
@@ -216,7 +235,8 @@ public class TaskLocalStateStoreImpl implements TaskLocalStateStore {
 					try {
 						deleteDirectory(subtaskBaseDirectory);
 					} catch (IOException e) {
-						LOG.warn("Exception when deleting local recovery subtask base dir: " + subtaskBaseDirectory, e);
+						LOG.warn("Exception when deleting local recovery subtask base directory {} in subtask ({} - {} - {})",
+							subtaskBaseDirectory, jobID, jobVertexID, subtaskIndex, e);
 					}
 				}
 			},
@@ -240,27 +260,32 @@ public class TaskLocalStateStoreImpl implements TaskLocalStateStore {
 	 */
 	private void discardLocalStateForCheckpoint(long checkpointID, TaskStateSnapshot o) {
 
+		if (LOG.isTraceEnabled()) {
+			LOG.trace("Discarding local task state snapshot of checkpoint {} for subtask ({} - {} - {}).",
+				checkpointID, jobID, jobVertexID, subtaskIndex);
+		} else {
+			LOG.debug("Discarding local task state snapshot {} of checkpoint {} for subtask ({} - {} - {}).",
+				o, checkpointID, jobID, jobVertexID, subtaskIndex);
+		}
+
 		try {
-			if (LOG.isTraceEnabled()) {
-				LOG.trace("Discarding local task state snapshot of checkpoint {} for {}/{}/{}.",
-					checkpointID, jobID, jobVertexID, subtaskIndex);
-			} else {
-				LOG.debug("Discarding local task state snapshot {} of checkpoint {} for {}/{}/{}.",
-					o, checkpointID, jobID, jobVertexID, subtaskIndex);
-			}
 			o.discardState();
 		} catch (Exception discardEx) {
-			LOG.warn("Exception while discarding local task state snapshot of checkpoint " + checkpointID + ".", discardEx);
+			LOG.warn("Exception while discarding local task state snapshot of checkpoint {} in subtask ({} - {} - {}).",
+				checkpointID, jobID, jobVertexID, subtaskIndex, discardEx);
 		}
 
 		LocalRecoveryDirectoryProvider directoryProvider = localRecoveryConfig.getLocalStateDirectoryProvider();
 		File checkpointDir = directoryProvider.subtaskSpecificCheckpointDirectory(checkpointID);
-		LOG.debug("Deleting local state directory {} of checkpoint {} for {}/{}/{}/{}.",
+
+		LOG.debug("Deleting local state directory {} of checkpoint {} for subtask ({} - {} - {}).",
 			checkpointDir, checkpointID, jobID, jobVertexID, subtaskIndex);
+
 		try {
 			deleteDirectory(checkpointDir);
 		} catch (IOException ex) {
-			LOG.warn("Exception while deleting local state directory of checkpoint " + checkpointID + ".", ex);
+			LOG.warn("Exception while deleting local state directory of checkpoint {} in subtask ({} - {} - {}).",
+				checkpointID, jobID, jobVertexID, subtaskIndex, ex);
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java
index 3acca7c..e057d19 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/TaskStateManagerImpl.java
@@ -29,10 +29,14 @@ import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.taskmanager.CheckpointResponder;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import java.util.Collections;
+import java.util.List;
 
 /**
  * This class is the default implementation of {@link TaskStateManager} and collaborates with the job manager
@@ -40,13 +44,14 @@ import java.util.Collections;
  * not have to deal with the differences between remote or local state on recovery because this class handles both
  * cases transparently.
  *
- * Reported state is tagged by clients so that this class can properly forward to the right receiver for the
+ * <p>Reported state is tagged by clients so that this class can properly forward to the right receiver for the
  * checkpointed state.
- *
- * TODO: all interaction with local state store must still be implemented! It is currently just a placeholder.
  */
 public class TaskStateManagerImpl implements TaskStateManager {
 
+	/** The logger for this class. */
+	private static final Logger LOG = LoggerFactory.getLogger(TaskStateManagerImpl.class);
+
 	/** The id of the job for which this manager was created, can report, and recover. */
 	private final JobID jobId;
 
@@ -117,21 +122,27 @@ public class TaskStateManagerImpl implements TaskStateManager {
 		TaskStateSnapshot localStateSnapshot =
 			localStateStore.retrieveLocalState(jobManagerTaskRestore.getRestoreCheckpointId());
 
+		List<OperatorSubtaskState> alternativesByPriority = Collections.emptyList();
+
 		if (localStateSnapshot != null) {
 			OperatorSubtaskState localSubtaskState = localStateSnapshot.getSubtaskStateByOperatorID(operatorID);
 
 			if (localSubtaskState != null) {
-				PrioritizedOperatorSubtaskState.Builder builder = new PrioritizedOperatorSubtaskState.Builder(
-					jobManagerSubtaskState,
-					Collections.singletonList(localSubtaskState));
-				return builder.build();
+				alternativesByPriority = Collections.singletonList(localSubtaskState);
 			}
 		}
 
+		if (LOG.isTraceEnabled()) {
+			LOG.trace("Operator {} has remote state {} from job manager and local state alternatives {} from local " +
+					"state store {}.",
+				operatorID, jobManagerSubtaskState, alternativesByPriority, localStateStore);
+		}
+
 		PrioritizedOperatorSubtaskState.Builder builder = new PrioritizedOperatorSubtaskState.Builder(
 			jobManagerSubtaskState,
-			Collections.emptyList(),
+			alternativesByPriority,
 			true);
+
 		return builder.build();
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java
index 09c9efb..82082e0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/PrioritizedOperatorSubtaskStateTest.java
@@ -216,7 +216,7 @@ public class PrioritizedOperatorSubtaskStateTest extends TestLogger {
 
 	private <T extends StateObject> boolean checkResultAsExpected(
 		Function<OperatorSubtaskState, StateObjectCollection<T>> extractor,
-		Function<PrioritizedOperatorSubtaskState, Iterator<StateObjectCollection<T>>> extractor2,
+		Function<PrioritizedOperatorSubtaskState, List<StateObjectCollection<T>>> extractor2,
 		PrioritizedOperatorSubtaskState prioritizedResult,
 		OperatorSubtaskState... expectedOrdered) {
 
@@ -226,7 +226,7 @@ public class PrioritizedOperatorSubtaskStateTest extends TestLogger {
 		}
 
 		return checkRepresentSameOrder(
-			extractor2.apply(prioritizedResult),
+			extractor2.apply(prioritizedResult).iterator(),
 			collector.toArray(new StateObjectCollection[collector.size()]));
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java
index 926c196..f58f3f4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/state/TaskStateManagerImplTest.java
@@ -142,7 +142,7 @@ public class TaskStateManagerImplTest extends TestLogger {
 
 		// checks for operator 1.
 		Iterator<StateObjectCollection<KeyedStateHandle>> prioritizedManagedKeyedState_1 =
-			prioritized_1.getPrioritizedManagedKeyedState();
+			prioritized_1.getPrioritizedManagedKeyedState().iterator();
 
 		Assert.assertTrue(prioritizedManagedKeyedState_1.hasNext());
 		StateObjectCollection<KeyedStateHandle> current = prioritizedManagedKeyedState_1.next();
@@ -158,7 +158,7 @@ public class TaskStateManagerImplTest extends TestLogger {
 
 		// checks for operator 2.
 		Iterator<StateObjectCollection<KeyedStateHandle>> prioritizedRawKeyedState_2 =
-			prioritized_2.getPrioritizedRawKeyedState();
+			prioritized_2.getPrioritizedRawKeyedState().iterator();
 
 		Assert.assertTrue(prioritizedRawKeyedState_2.hasNext());
 		current = prioritizedRawKeyedState_2.next();

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java
index ba27a0a..dd75fb2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedure.java
@@ -36,7 +36,7 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Iterator;
+import java.util.List;
 
 /**
  * This class implements the logic that creates (and potentially restores) a state backend. The restore logic
@@ -62,6 +62,9 @@ public class BackendRestorerProcedure<
 	/** This registry is used so that recovery can participate in the task lifecycle, i.e. can be canceled. */
 	private final CloseableRegistry backendCloseableRegistry;
 
+	/** Description of this instance for logging. */
+	private final String logDescription;
+
 	/**
 	 * Creates a new backend restorer using the given backend supplier and the closeable registry.
 	 *
@@ -70,43 +73,63 @@ public class BackendRestorerProcedure<
 	 */
 	public BackendRestorerProcedure(
 		@Nonnull SupplierWithException<T, Exception> instanceSupplier,
-		@Nonnull CloseableRegistry backendCloseableRegistry) {
+		@Nonnull CloseableRegistry backendCloseableRegistry,
+		@Nonnull String logDescription) {
 
 		this.instanceSupplier = Preconditions.checkNotNull(instanceSupplier);
 		this.backendCloseableRegistry = Preconditions.checkNotNull(backendCloseableRegistry);
+		this.logDescription = logDescription;
 	}
 
 	/**
 	 * Creates a new state backend and restores it from the provided set of state snapshot alternatives.
 	 *
-	 * @param restoreOptions iterator over a prioritized set of state snapshot alternatives for recovery.
+	 * @param restoreOptions list of prioritized state snapshot alternatives for recovery.
 	 * @return the created (and restored) state backend.
 	 * @throws Exception if the backend could not be created or restored.
 	 */
-	public @Nonnull
-	T createAndRestore(@Nonnull Iterator<? extends Collection<S>> restoreOptions) throws Exception {
+	@Nonnull
+	public T createAndRestore(@Nonnull List<? extends Collection<S>> restoreOptions) throws Exception {
+
+		if (restoreOptions.isEmpty()) {
+			restoreOptions = Collections.singletonList(Collections.emptyList());
+		}
+
+		int alternativeIdx = 0;
+
+		Exception collectedException = null;
+
+		while (alternativeIdx < restoreOptions.size()) {
+
+			Collection<S> restoreState = restoreOptions.get(alternativeIdx);
 
-		// This ensures that we always call the restore method even if there is no previous state
-		// (required by some backends).
-		Collection<S> attemptState = restoreOptions.hasNext() ?
-			restoreOptions.next() :
-			Collections.emptyList();
+			++alternativeIdx;
+
+			if (restoreState.isEmpty()) {
+				LOG.debug("Creating {} with empty state.", logDescription);
+			} else {
+				if (LOG.isTraceEnabled()) {
+					LOG.trace("Creating {} and restoring with state {} from alternative ({}/{}).",
+						logDescription, restoreState, alternativeIdx, restoreOptions.size());
+				} else {
+					LOG.debug("Creating {} and restoring with state from alternative ({}/{}).",
+						logDescription, alternativeIdx, restoreOptions.size());
+				}
+			}
 
-		while (true) {
 			try {
-				return attemptCreateAndRestore(attemptState);
+				return attemptCreateAndRestore(restoreState);
 			} catch (Exception ex) {
-				// more attempts?
-				if (restoreOptions.hasNext()) {
 
-					attemptState = restoreOptions.next();
-					LOG.warn("Exception while restoring backend, will retry with another snapshot replica.", ex);
-				} else {
+				collectedException = ExceptionUtils.firstOrSuppressed(ex, collectedException);
 
-					throw new FlinkException("Could not restore from any of the provided restore options.", ex);
-				}
+				LOG.warn("Exception while restoring {} from alternative ({}/{}), will retry while more " +
+					"alternatives are available.", logDescription, alternativeIdx, restoreOptions.size(), ex);
 			}
 		}
+
+		throw new FlinkException("Could not restore " + logDescription + " from any of the " + restoreOptions.size() +
+			" provided restore options.", collectedException);
 	}
 
 	private T attemptCreateAndRestore(Collection<S> restoreState) throws Exception {

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
index 11e2dda..acbc2f8 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamTaskStateInitializerImpl.java
@@ -143,11 +143,11 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
 
 			// -------------- Raw State Streams --------------
 			rawKeyedStateInputs = rawKeyedStateInputs(
-				prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState());
+				prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState().iterator());
 			streamTaskCloseableRegistry.registerCloseable(rawKeyedStateInputs);
 
 			rawOperatorStateInputs = rawOperatorStateInputs(
-				prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState());
+				prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState().iterator());
 			streamTaskCloseableRegistry.registerCloseable(rawOperatorStateInputs);
 
 			// -------------- Internal Timer Service Manager --------------
@@ -226,12 +226,16 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
 		PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates,
 		CloseableRegistry backendCloseableRegistry) throws Exception {
 
+		String logDescription = "operator state backend for " + operatorIdentifierText;
+
 		BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> backendRestorer =
 			new BackendRestorerProcedure<>(
 				() -> stateBackend.createOperatorStateBackend(environment, operatorIdentifierText),
-				backendCloseableRegistry);
+				backendCloseableRegistry,
+				logDescription);
 
-		return backendRestorer.createAndRestore(prioritizedOperatorSubtaskStates.getPrioritizedManagedOperatorState());
+		return backendRestorer.createAndRestore(
+			prioritizedOperatorSubtaskStates.getPrioritizedManagedOperatorState());
 	}
 
 	protected <K> AbstractKeyedStateBackend<K> keyedStatedBackend(
@@ -244,6 +248,8 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
 			return null;
 		}
 
+		String logDescription = "keyed state backend for " + operatorIdentifierText;
+
 		TaskInfo taskInfo = environment.getTaskInfo();
 
 		final KeyGroupRange keyGroupRange = KeyGroupRangeAssignment.computeKeyGroupRangeForOperatorIndex(
@@ -261,9 +267,11 @@ public class StreamTaskStateInitializerImpl implements StreamTaskStateInitialize
 					taskInfo.getMaxNumberOfParallelSubtasks(),
 					keyGroupRange,
 					environment.getTaskKvStateRegistry()),
-					backendCloseableRegistry);
+				backendCloseableRegistry,
+				logDescription);
 
-		return backendRestorer.createAndRestore(prioritizedOperatorSubtaskStates.getPrioritizedManagedKeyedState());
+		return backendRestorer.createAndRestore(
+			prioritizedOperatorSubtaskStates.getPrioritizedManagedKeyedState());
 	}
 
 	protected CloseableIterable<StatePartitionStreamProvider> rawOperatorStateInputs(

http://git-wip-us.apache.org/repos/asf/flink/blob/56c75604/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java
index 2126f70..0f15d11 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/BackendRestorerProcedureTest.java
@@ -104,21 +104,17 @@ public class BackendRestorerProcedureTest extends TestLogger {
 			new StateObjectCollection<>(Collections.singletonList(firstFailHandle)),
 			new StateObjectCollection<>(Collections.singletonList(secondSuccessHandle)),
 			new StateObjectCollection<>(Collections.singletonList(thirdNotUsedHandle)));
-		Iterator<StateObjectCollection<OperatorStateHandle>> iterator = sortedRestoreOptions.iterator();
 
 		BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> restorerProcedure =
-			new BackendRestorerProcedure<>(backendSupplier, closeableRegistry);
+			new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend");
 
-		OperatorStateBackend restoredBackend = restorerProcedure.createAndRestore(iterator);
+		OperatorStateBackend restoredBackend = restorerProcedure.createAndRestore(sortedRestoreOptions);
 		Assert.assertNotNull(restoredBackend);
 
 		try {
-			Assert.assertTrue(iterator.hasNext());
-			Assert.assertTrue(thirdNotUsedHandle == iterator.next().iterator().next());
 			verify(firstFailHandle).openInputStream();
 			verify(secondSuccessHandle).openInputStream();
 			verifyZeroInteractions(thirdNotUsedHandle);
-			Assert.assertFalse(iterator.hasNext());
 
 			ListState<Integer> listState = restoredBackend.getListState(stateDescriptor);
 
@@ -151,13 +147,12 @@ public class BackendRestorerProcedureTest extends TestLogger {
 			new StateObjectCollection<>(Collections.singletonList(firstFailHandle)),
 			new StateObjectCollection<>(Collections.singletonList(secondFailHandle)),
 			new StateObjectCollection<>(Collections.singletonList(thirdFailHandle)));
-		Iterator<StateObjectCollection<OperatorStateHandle>> iterator = sortedRestoreOptions.iterator();
 
 		BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> restorerProcedure =
-			new BackendRestorerProcedure<>(backendSupplier, closeableRegistry);
+			new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend");
 
 		try {
-			restorerProcedure.createAndRestore(iterator);
+			restorerProcedure.createAndRestore(sortedRestoreOptions);
 			Assert.fail();
 		} catch (Exception ignore) {
 		}
@@ -165,7 +160,6 @@ public class BackendRestorerProcedureTest extends TestLogger {
 		verify(firstFailHandle).openInputStream();
 		verify(secondFailHandle).openInputStream();
 		verify(thirdFailHandle).openInputStream();
-		Assert.assertFalse(iterator.hasNext());
 	}
 
 	/**
@@ -183,12 +177,12 @@ public class BackendRestorerProcedureTest extends TestLogger {
 			Collections.singletonList(new StateObjectCollection<>(Collections.singletonList(blockingRestoreHandle)));
 
 		BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> restorerProcedure =
-			new BackendRestorerProcedure<>(backendSupplier, closeableRegistry);
+			new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend");
 
 		AtomicReference<Exception> exceptionReference = new AtomicReference<>(null);
 		Thread restoreThread = new Thread(() -> {
 			try {
-				restorerProcedure.createAndRestore(sortedRestoreOptions.iterator());
+				restorerProcedure.createAndRestore(sortedRestoreOptions);
 			} catch (Exception e) {
 				exceptionReference.set(e);
 			}