You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2016/07/25 12:53:03 UTC
flink git commit: [hotfix] Prevent CheckpointCommitter from failing
job
Repository: flink
Updated Branches:
refs/heads/master b71ac354d -> 2818ee009
[hotfix] Prevent CheckpointCommitter from failing job
Prevents the CheckpointCommitter from failing a job, if either
commitCheckpoint() or isCheckpointCommitter() failed. Instead, we will
try again on the next notify().
This closes #2287
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2818ee00
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2818ee00
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2818ee00
Branch: refs/heads/master
Commit: 2818ee00974da917785074812e6e994b98ff52f2
Parents: b71ac35
Author: zentol <ch...@apache.org>
Authored: Fri Jul 22 17:50:10 2016 +0200
Committer: zentol <ch...@apache.org>
Committed: Mon Jul 25 14:52:55 2016 +0200
----------------------------------------------------------------------
.../operators/GenericWriteAheadSink.java | 21 ++--
.../operators/GenericWriteAheadSinkTest.java | 120 +++++++++++++++++++
.../operators/WriteAheadSinkTestBase.java | 2 +-
3 files changed, 134 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/2818ee00/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
index 23cfc3a..afc28f4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
@@ -140,16 +140,21 @@ public abstract class GenericWriteAheadSink<IN> extends AbstractStreamOperator<I
Set<Long> checkpointsToRemove = new HashSet<>();
for (Long pastCheckpointId : pastCheckpointIds) {
if (pastCheckpointId <= checkpointId) {
- if (!committer.isCheckpointCommitted(pastCheckpointId)) {
- Tuple2<Long, StateHandle<DataInputView>> handle = state.pendingHandles.get(pastCheckpointId);
- DataInputView in = handle.f1.getState(getUserCodeClassloader());
- boolean success = sendValues(new ReusingMutableToRegularIteratorWrapper<>(new InputViewIterator<>(in, serializer), serializer), handle.f0);
- if (success) { //if the sending has failed we will retry on the next notify
- committer.commitCheckpoint(pastCheckpointId);
+ try {
+ if (!committer.isCheckpointCommitted(pastCheckpointId)) {
+ Tuple2<Long, StateHandle<DataInputView>> handle = state.pendingHandles.get(pastCheckpointId);
+ DataInputView in = handle.f1.getState(getUserCodeClassloader());
+ boolean success = sendValues(new ReusingMutableToRegularIteratorWrapper<>(new InputViewIterator<>(in, serializer), serializer), handle.f0);
+ if (success) { //if the sending has failed we will retry on the next notify
+ committer.commitCheckpoint(pastCheckpointId);
+ checkpointsToRemove.add(pastCheckpointId);
+ }
+ } else {
checkpointsToRemove.add(pastCheckpointId);
}
- } else {
- checkpointsToRemove.add(pastCheckpointId);
+ } catch (Exception e) {
+ LOG.error("Could not commit checkpoint.", e);
+ break; // we have to break here to prevent a new checkpoint from being committed before this one
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2818ee00/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
index 33896e8..2c406d9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
@@ -18,13 +18,17 @@
package org.apache.flink.streaming.runtime.operators;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.OneInputStreamTask;
import org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness;
import org.junit.Assert;
+import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
@@ -106,6 +110,64 @@ public class GenericWriteAheadSinkTest extends WriteAheadSinkTestBase<Tuple1<Int
Assert.assertTrue("The sink emitted to many values: " + (sink.values.size() - 40), sink.values.size() == 40);
}
+ @Test
+ /**
+ * Verifies that exceptions thrown by a committer do not fail a job and lead to an abort of notify()
+ * and later retry of the affected checkpoints.
+ */
+ public void testCommitterException() throws Exception {
+ OperatorExposingTask<Tuple1<Integer>> task = createTask();
+ TypeInformation<Tuple1<Integer>> info = createTypeInfo();
+ OneInputStreamTaskTestHarness<Tuple1<Integer>, Tuple1<Integer>> testHarness = new OneInputStreamTaskTestHarness<>(task, 1, 1, info, info);
+ StreamConfig streamConfig = testHarness.getStreamConfig();
+ streamConfig.setCheckpointingEnabled(true);
+ streamConfig.setStreamOperator(new ListSink2());
+
+ int elementCounter = 1;
+
+ testHarness.invoke();
+ testHarness.waitForTaskRunning();
+
+ for (int x = 0; x < 10; x++) {
+ testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
+ elementCounter++;
+ }
+ testHarness.waitForInputProcessing();
+
+ task.getOperator().snapshotOperatorState(0, 0);
+ task.notifyCheckpointComplete(0);
+
+ //isCommitted should have failed, thus sendValues() should never have been called
+ Assert.assertTrue(((ListSink2) task.getOperator()).values.size() == 0);
+
+ for (int x = 0; x < 10; x++) {
+ testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
+ elementCounter++;
+ }
+ testHarness.waitForInputProcessing();
+
+ task.getOperator().snapshotOperatorState(1, 0);
+ task.notifyCheckpointComplete(1);
+
+ //previous CP should be retried, but will fail the CP commit. Second CP should be skipped.
+ Assert.assertTrue(((ListSink2) task.getOperator()).values.size() == 10);
+
+ for (int x = 0; x < 10; x++) {
+ testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
+ elementCounter++;
+ }
+ testHarness.waitForInputProcessing();
+
+ task.getOperator().snapshotOperatorState(2, 0);
+ task.notifyCheckpointComplete(2);
+
+ //all CP's should be retried and succeed; since one CP was written twice we have 2 * 10 + 10 + 10 = 40 values
+ Assert.assertTrue(((ListSink2) task.getOperator()).values.size() == 40);
+
+ testHarness.endInput();
+ testHarness.waitForTaskCompletion();
+ }
+
/**
* Simple sink that stores all records in a public list.
*/
@@ -151,4 +213,62 @@ public class GenericWriteAheadSinkTest extends WriteAheadSinkTestBase<Tuple1<Int
return checkpoints.contains(checkpointID);
}
}
+
+ /**
+ * Simple sink that stores all records in a public list.
+ */
+ public static class ListSink2 extends GenericWriteAheadSink<Tuple1<Integer>> {
+ public List<Integer> values = new ArrayList<>();
+
+ public ListSink2() throws Exception {
+ super(new FailingCommitter(), TypeExtractor.getForObject(new Tuple1<>(1)).createSerializer(new ExecutionConfig()), "job");
+ }
+
+ @Override
+ protected boolean sendValues(Iterable<Tuple1<Integer>> values, long timestamp) throws Exception {
+ for (Tuple1<Integer> value : values) {
+ this.values.add(value.f0);
+ }
+ return true;
+ }
+ }
+
+ public static class FailingCommitter extends CheckpointCommitter {
+ private List<Long> checkpoints;
+ private boolean failIsCommitted = true;
+ private boolean failCommit = true;
+
+ @Override
+ public void open() throws Exception {
+ }
+
+ @Override
+ public void close() throws Exception {
+ }
+
+ @Override
+ public void createResource() throws Exception {
+ checkpoints = new ArrayList<>();
+ }
+
+ @Override
+ public void commitCheckpoint(long checkpointID) {
+ if (failCommit) {
+ failCommit = false;
+ throw new RuntimeException("Expected exception");
+ } else {
+ checkpoints.add(checkpointID);
+ }
+ }
+
+ @Override
+ public boolean isCheckpointCommitted(long checkpointID) {
+ if (failIsCommitted) {
+ failIsCommitted = false;
+ throw new RuntimeException("Expected exception");
+ } else {
+ return false;
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/2818ee00/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
index 1d706d1..1b0ca3a 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
@@ -219,7 +219,7 @@ public abstract class WriteAheadSinkTestBase<IN, S extends GenericWriteAheadSink
verifyResultsDataDiscardingUponRestore(testHarness, task, (S) task.getOperator());
}
- private StreamTaskState copyTaskState(StreamTaskState toCopy) throws IOException, ClassNotFoundException {
+ protected StreamTaskState copyTaskState(StreamTaskState toCopy) throws IOException, ClassNotFoundException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(baos);
oos.writeObject(toCopy);