You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2016/07/25 12:53:03 UTC

flink git commit: [hotfix] Prevent CheckpointCommitter from failing job

Repository: flink
Updated Branches:
  refs/heads/master b71ac354d -> 2818ee009


[hotfix] Prevent CheckpointCommitter from failing job

Prevents the CheckpointCommitter from failing a job, if either
commitCheckpoint() or isCheckpointCommitter() failed. Instead, we will
try again on the next notify().

This closes #2287


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2818ee00
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2818ee00
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2818ee00

Branch: refs/heads/master
Commit: 2818ee00974da917785074812e6e994b98ff52f2
Parents: b71ac35
Author: zentol <ch...@apache.org>
Authored: Fri Jul 22 17:50:10 2016 +0200
Committer: zentol <ch...@apache.org>
Committed: Mon Jul 25 14:52:55 2016 +0200

----------------------------------------------------------------------
 .../operators/GenericWriteAheadSink.java        |  21 ++--
 .../operators/GenericWriteAheadSinkTest.java    | 120 +++++++++++++++++++
 .../operators/WriteAheadSinkTestBase.java       |   2 +-
 3 files changed, 134 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/2818ee00/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
index 23cfc3a..afc28f4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSink.java
@@ -140,16 +140,21 @@ public abstract class GenericWriteAheadSink<IN> extends AbstractStreamOperator<I
 			Set<Long> checkpointsToRemove = new HashSet<>();
 			for (Long pastCheckpointId : pastCheckpointIds) {
 				if (pastCheckpointId <= checkpointId) {
-					if (!committer.isCheckpointCommitted(pastCheckpointId)) {
-						Tuple2<Long, StateHandle<DataInputView>> handle = state.pendingHandles.get(pastCheckpointId);
-						DataInputView in = handle.f1.getState(getUserCodeClassloader());
-						boolean success = sendValues(new ReusingMutableToRegularIteratorWrapper<>(new InputViewIterator<>(in, serializer), serializer), handle.f0);
-						if (success) { //if the sending has failed we will retry on the next notify
-							committer.commitCheckpoint(pastCheckpointId);
+					try {
+						if (!committer.isCheckpointCommitted(pastCheckpointId)) {
+							Tuple2<Long, StateHandle<DataInputView>> handle = state.pendingHandles.get(pastCheckpointId);
+							DataInputView in = handle.f1.getState(getUserCodeClassloader());
+							boolean success = sendValues(new ReusingMutableToRegularIteratorWrapper<>(new InputViewIterator<>(in, serializer), serializer), handle.f0);
+							if (success) { //if the sending has failed we will retry on the next notify
+								committer.commitCheckpoint(pastCheckpointId);
+								checkpointsToRemove.add(pastCheckpointId);
+							}
+						} else {
 							checkpointsToRemove.add(pastCheckpointId);
 						}
-					} else {
-						checkpointsToRemove.add(pastCheckpointId);
+					} catch (Exception e) {
+						LOG.error("Could not commit checkpoint.", e);
+						break; // we have to break here to prevent a new checkpoint from being committed before this one
 					}
 				}
 			}

http://git-wip-us.apache.org/repos/asf/flink/blob/2818ee00/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
index 33896e8..2c406d9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/GenericWriteAheadSinkTest.java
@@ -18,13 +18,17 @@
 package org.apache.flink.streaming.runtime.operators;
 
 import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.tuple.Tuple1;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.tasks.OneInputStreamTask;
 import org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness;
 import org.junit.Assert;
+import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.core.classloader.annotations.PrepareForTest;
@@ -106,6 +110,64 @@ public class GenericWriteAheadSinkTest extends WriteAheadSinkTestBase<Tuple1<Int
 		Assert.assertTrue("The sink emitted to many values: " + (sink.values.size() - 40), sink.values.size() == 40);
 	}
 
+	@Test
+	/**
+	 * Verifies that exceptions thrown by a committer do not fail a job and lead to an abort of notify()
+	 * and later retry of the affected checkpoints.
+	 */
+	public void testCommitterException() throws Exception {
+		OperatorExposingTask<Tuple1<Integer>> task = createTask();
+		TypeInformation<Tuple1<Integer>> info = createTypeInfo();
+		OneInputStreamTaskTestHarness<Tuple1<Integer>, Tuple1<Integer>> testHarness = new OneInputStreamTaskTestHarness<>(task, 1, 1, info, info);
+		StreamConfig streamConfig = testHarness.getStreamConfig();
+		streamConfig.setCheckpointingEnabled(true);
+		streamConfig.setStreamOperator(new ListSink2());
+
+		int elementCounter = 1;
+
+		testHarness.invoke();
+		testHarness.waitForTaskRunning();
+		
+		for (int x = 0; x < 10; x++) {
+			testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
+			elementCounter++;
+		}
+		testHarness.waitForInputProcessing();
+
+		task.getOperator().snapshotOperatorState(0, 0);
+		task.notifyCheckpointComplete(0);
+		
+		//isCommitted should have failed, thus sendValues() should never have been called
+		Assert.assertTrue(((ListSink2) task.getOperator()).values.size() == 0);
+
+		for (int x = 0; x < 10; x++) {
+			testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1)));
+			elementCounter++;
+		}
+		testHarness.waitForInputProcessing();
+
+		task.getOperator().snapshotOperatorState(1, 0);
+		task.notifyCheckpointComplete(1);
+
+		//previous CP should be retried, but will fail the CP commit. Second CP should be skipped.
+		Assert.assertTrue(((ListSink2) task.getOperator()).values.size() == 10);
+
+		for (int x = 0; x < 10; x++) {
+			testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2)));
+			elementCounter++;
+		}
+		testHarness.waitForInputProcessing();
+
+		task.getOperator().snapshotOperatorState(2, 0);
+		task.notifyCheckpointComplete(2);
+
+		//all CP's should be retried and succeed; since one CP was written twice we have 2 * 10 + 10 + 10 = 40 values
+		Assert.assertTrue(((ListSink2) task.getOperator()).values.size() == 40);
+
+		testHarness.endInput();
+		testHarness.waitForTaskCompletion();
+	}
+
 	/**
 	 * Simple sink that stores all records in a public list.
 	 */
@@ -151,4 +213,62 @@ public class GenericWriteAheadSinkTest extends WriteAheadSinkTestBase<Tuple1<Int
 			return checkpoints.contains(checkpointID);
 		}
 	}
+
+	/**
+	 * Simple sink that stores all records in a public list.
+	 */
+	public static class ListSink2 extends GenericWriteAheadSink<Tuple1<Integer>> {
+		public List<Integer> values = new ArrayList<>();
+
+		public ListSink2() throws Exception {
+			super(new FailingCommitter(), TypeExtractor.getForObject(new Tuple1<>(1)).createSerializer(new ExecutionConfig()), "job");
+		}
+
+		@Override
+		protected boolean sendValues(Iterable<Tuple1<Integer>> values, long timestamp) throws Exception {
+			for (Tuple1<Integer> value : values) {
+				this.values.add(value.f0);
+			}
+			return true;
+		}
+	}
+
+	public static class FailingCommitter extends CheckpointCommitter {
+		private List<Long> checkpoints;
+		private boolean failIsCommitted = true;
+		private boolean failCommit = true;
+
+		@Override
+		public void open() throws Exception {
+		}
+
+		@Override
+		public void close() throws Exception {
+		}
+
+		@Override
+		public void createResource() throws Exception {
+			checkpoints = new ArrayList<>();
+		}
+
+		@Override
+		public void commitCheckpoint(long checkpointID) {
+			if (failCommit) {
+				failCommit = false;
+				throw new RuntimeException("Expected exception");
+			} else {
+				checkpoints.add(checkpointID);
+			}
+		}
+
+		@Override
+		public boolean isCheckpointCommitted(long checkpointID) {
+			if (failIsCommitted) {
+				failIsCommitted = false;
+				throw new RuntimeException("Expected exception");
+			} else {
+				return false;
+			}
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/2818ee00/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
index 1d706d1..1b0ca3a 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/WriteAheadSinkTestBase.java
@@ -219,7 +219,7 @@ public abstract class WriteAheadSinkTestBase<IN, S extends GenericWriteAheadSink
 		verifyResultsDataDiscardingUponRestore(testHarness, task, (S) task.getOperator());
 	}
 
-	private StreamTaskState copyTaskState(StreamTaskState toCopy) throws IOException, ClassNotFoundException {
+	protected StreamTaskState copyTaskState(StreamTaskState toCopy) throws IOException, ClassNotFoundException {
 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
 		ObjectOutputStream oos = new ObjectOutputStream(baos);
 		oos.writeObject(toCopy);