You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ar...@apache.org on 2020/11/26 07:19:02 UTC
[flink] branch release-1.12 updated:
[FLINK-20331][checkpointing][task] Don't fail the task if unaligned
checkpoint was subsumed
This is an automated email from the ASF dual-hosted git repository.
arvid pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.12 by this push:
new 2c77c38 [FLINK-20331][checkpointing][task] Don't fail the task if unaligned checkpoint was subsumed
2c77c38 is described below
commit 2c77c38d5504025400110cc1c2aca2c4fc85b88b
Author: Roman Khachatryan <kh...@gmail.com>
AuthorDate: Wed Nov 25 13:26:06 2020 +0100
[FLINK-20331][checkpointing][task] Don't fail the task if unaligned checkpoint was subsumed
---
.../partition/consumer/CheckpointableInput.java | 3 ++-
.../partition/consumer/IndexedInputGate.java | 3 ++-
.../network/partition/consumer/InputChannel.java | 3 ++-
.../partition/consumer/RemoteInputChannel.java | 18 +++++++++------
.../partition/consumer/RemoteInputChannelTest.java | 3 ++-
.../runtime/io/AlternatingController.java | 2 +-
.../io/CheckpointBarrierBehaviourController.java | 2 +-
.../runtime/io/SingleCheckpointBarrierHandler.java | 27 ++++++++++++++--------
.../streaming/runtime/io/UnalignedController.java | 2 +-
9 files changed, 39 insertions(+), 24 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/CheckpointableInput.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/CheckpointableInput.java
index 04fb7d9..2316a90 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/CheckpointableInput.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/CheckpointableInput.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.io.network.partition.consumer;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.channel.InputChannelInfo;
import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
@@ -38,7 +39,7 @@ public interface CheckpointableInput {
int getNumberOfInputChannels();
- void checkpointStarted(CheckpointBarrier barrier);
+ void checkpointStarted(CheckpointBarrier barrier) throws CheckpointException;
void checkpointStopped(long cancelledCheckpointId);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/IndexedInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/IndexedInputGate.java
index af0f3a5..3b8ecb2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/IndexedInputGate.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/IndexedInputGate.java
@@ -17,6 +17,7 @@
package org.apache.flink.runtime.io.network.partition.consumer;
+import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.channel.InputChannelInfo;
import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
@@ -30,7 +31,7 @@ public abstract class IndexedInputGate extends InputGate implements Checkpointab
public abstract int getGateIndex();
@Override
- public void checkpointStarted(CheckpointBarrier barrier) {
+ public void checkpointStarted(CheckpointBarrier barrier) throws CheckpointException {
for (int index = 0, numChannels = getNumberOfInputChannels(); index < numChannels; index++) {
getChannel(index).checkpointStarted(barrier);
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java
index 02a0526..792392b9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.io.network.partition.consumer;
import org.apache.flink.metrics.Counter;
+import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.channel.InputChannelInfo;
import org.apache.flink.runtime.event.TaskEvent;
import org.apache.flink.runtime.execution.CancelTaskException;
@@ -173,7 +174,7 @@ public abstract class InputChannel {
/**
* Called by task thread when checkpointing is started (e.g., any input channel received barrier).
*/
- public void checkpointStarted(CheckpointBarrier barrier) {
+ public void checkpointStarted(CheckpointBarrier barrier) throws CheckpointException {
}
/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
index 85fbe4a..1992cdb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
@@ -20,6 +20,8 @@ package org.apache.flink.runtime.io.network.partition.consumer;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.metrics.Counter;
+import org.apache.flink.runtime.checkpoint.CheckpointException;
+import org.apache.flink.runtime.checkpoint.CheckpointFailureReason;
import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
import org.apache.flink.runtime.event.AbstractEvent;
import org.apache.flink.runtime.event.TaskEvent;
@@ -36,7 +38,6 @@ import org.apache.flink.runtime.io.network.buffer.BufferProvider;
import org.apache.flink.runtime.io.network.partition.PartitionNotFoundException;
import org.apache.flink.runtime.io.network.partition.PrioritizedDeque;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
-import org.apache.flink.util.Preconditions;
import org.apache.flink.shaded.guava18.com.google.common.collect.Iterators;
@@ -507,7 +508,7 @@ public class RemoteInputChannel extends InputChannel {
* Spills all queued buffers on checkpoint start. If barrier has already been received (and reordered), spill only
* the overtaken buffers.
*/
- public void checkpointStarted(CheckpointBarrier barrier) {
+ public void checkpointStarted(CheckpointBarrier barrier) throws CheckpointException {
synchronized (receivedBuffers) {
channelStatePersister.startPersisting(
barrier.getId(),
@@ -526,7 +527,7 @@ public class RemoteInputChannel extends InputChannel {
}
@VisibleForTesting
- List<Buffer> getInflightBuffers(long checkpointId) {
+ List<Buffer> getInflightBuffers(long checkpointId) throws CheckpointException {
synchronized (receivedBuffers) {
return getInflightBuffersUnsafe(checkpointId);
}
@@ -535,13 +536,16 @@ public class RemoteInputChannel extends InputChannel {
/**
* Returns a list of buffers, checking the first n non-priority buffers, and skipping all events.
*/
- private List<Buffer> getInflightBuffersUnsafe(long checkpointId) {
+ private List<Buffer> getInflightBuffersUnsafe(long checkpointId) throws CheckpointException {
assert Thread.holdsLock(receivedBuffers);
+ if (checkpointId < lastBarrierId) {
+ throw new CheckpointException(
+ String.format("Sequence number for checkpoint %d is not known (it was likely been overwritten by a newer checkpoint %d)", checkpointId, lastBarrierId),
+ CheckpointFailureReason.CHECKPOINT_SUBSUMED); // currently, at most one active unaligned checkpoint is possible
+ }
+
final List<Buffer> inflightBuffers = new ArrayList<>();
- Preconditions.checkState(
- checkpointId >= lastBarrierId,
- "Sequence number for checkpoint %s is not known (it was likely been overwritten by a newer checkpoint %s)", checkpointId, lastBarrierId);
Iterator<SequenceBuffer> iterator = receivedBuffers.iterator();
// skip all priority events (only buffers are stored anyways)
Iterators.advance(iterator, receivedBuffers.getNumPriorityElements());
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
index 2433451..c74cf4e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.io.network.partition.consumer;
import org.apache.flink.core.testutils.OneShotLatch;
+import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.execution.CancelTaskException;
import org.apache.flink.runtime.execution.ExecutionState;
@@ -1254,7 +1255,7 @@ public class RemoteInputChannelTest {
channel.checkError();
}
- private void assertInflightBufferSizes(RemoteInputChannel channel, Integer ...bufferSizes) {
+ private void assertInflightBufferSizes(RemoteInputChannel channel, Integer ...bufferSizes) throws CheckpointException {
assertEquals(Arrays.asList(bufferSizes), toBufferSizes(channel.getInflightBuffers(CHECKPOINT_ID)));
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AlternatingController.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AlternatingController.java
index 3f241c1..eb302ce 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AlternatingController.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AlternatingController.java
@@ -52,7 +52,7 @@ public class AlternatingController implements CheckpointBarrierBehaviourControll
@Override
public boolean preProcessFirstBarrier(
InputChannelInfo channelInfo,
- CheckpointBarrier barrier) throws IOException {
+ CheckpointBarrier barrier) throws IOException, CheckpointException {
activeController = chooseController(barrier);
return activeController.preProcessFirstBarrier(channelInfo, barrier);
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierBehaviourController.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierBehaviourController.java
index 81159e2..f38d745 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierBehaviourController.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointBarrierBehaviourController.java
@@ -41,7 +41,7 @@ public interface CheckpointBarrierBehaviourController {
* {@link #barrierReceived(InputChannelInfo, CheckpointBarrier)} for that given checkpoint.
* @return {@code true} if checkpoint should be triggered.
*/
- boolean preProcessFirstBarrier(InputChannelInfo channelInfo, CheckpointBarrier barrier) throws IOException;
+ boolean preProcessFirstBarrier(InputChannelInfo channelInfo, CheckpointBarrier barrier) throws IOException, CheckpointException;
/**
* Invoked once per checkpoint, after the last invocation of
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/SingleCheckpointBarrierHandler.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/SingleCheckpointBarrierHandler.java
index 54f31a1..ef8eb45 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/SingleCheckpointBarrierHandler.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/SingleCheckpointBarrierHandler.java
@@ -118,12 +118,17 @@ public class SingleCheckpointBarrierHandler extends CheckpointBarrierHandler {
currentCheckpointId = barrierId;
numBarriersReceived = 0;
allBarriersReceivedFuture = new CompletableFuture<>();
- if (controller.preProcessFirstBarrier(channelInfo, barrier)) {
- LOG.debug("{}: Triggering checkpoint {} on the first barrier at {}.",
- taskName,
- barrier.getId(),
- barrier.getTimestamp());
- notifyCheckpoint(barrier);
+ try {
+ if (controller.preProcessFirstBarrier(channelInfo, barrier)) {
+ LOG.debug("{}: Triggering checkpoint {} on the first barrier at {}.",
+ taskName,
+ barrier.getId(),
+ barrier.getTimestamp());
+ notifyCheckpoint(barrier);
+ }
+ } catch (CheckpointException e) {
+ abortInternal(barrier.getId(), e);
+ return;
}
}
@@ -158,14 +163,16 @@ public class SingleCheckpointBarrierHandler extends CheckpointBarrierHandler {
@Override
public void processCancellationBarrier(CancelCheckpointMarker cancelBarrier) throws IOException {
final long cancelledId = cancelBarrier.getCheckpointId();
- if (currentCheckpointId > cancelledId || (currentCheckpointId == cancelledId && numBarriersReceived == 0)) {
- return;
+ if (cancelledId > currentCheckpointId || (cancelledId == currentCheckpointId && numBarriersReceived > 0)) {
+ abortInternal(cancelledId, new CheckpointException(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER));
}
+ }
+
+ private void abortInternal(long cancelledId, CheckpointException exception) throws IOException {
// by setting the currentCheckpointId to this checkpoint while keeping the numBarriers
// at zero means that no checkpoint barrier can start a new alignment
- currentCheckpointId = cancelledId;
+ currentCheckpointId = Math.max(cancelledId, currentCheckpointId);
numBarriersReceived = 0;
- CheckpointException exception = new CheckpointException(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER);
controller.abortPendingCheckpoint(cancelledId, exception);
allBarriersReceivedFuture.completeExceptionally(exception);
notifyAbort(cancelledId, exception);
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/UnalignedController.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/UnalignedController.java
index 7cc109af..d53b4ec 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/UnalignedController.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/UnalignedController.java
@@ -48,7 +48,7 @@ public class UnalignedController implements CheckpointBarrierBehaviourController
}
@Override
- public boolean preProcessFirstBarrier(InputChannelInfo channelInfo, CheckpointBarrier barrier) throws IOException {
+ public boolean preProcessFirstBarrier(InputChannelInfo channelInfo, CheckpointBarrier barrier) throws IOException, CheckpointException {
checkpointCoordinator.initCheckpoint(barrier.getId(), barrier.getCheckpointOptions());
for (final CheckpointableInput input : inputs) {
input.checkpointStarted(barrier);