You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ew...@apache.org on 2016/08/23 17:19:17 UTC
kafka git commit: KAFKA-2894: WorkerSinkTask should rewind offsets on
rebalance
Repository: kafka
Updated Branches:
refs/heads/trunk 313aad423 -> cec2769e2
KAFKA-2894: WorkerSinkTask should rewind offsets on rebalance
Author: Konstantine Karantasis <k....@gmail.com>
Reviewers: Ewen Cheslack-Postava <ew...@confluent.io>
Closes #1771 from kkonstantine/KAFKA-2894-rewind-offsets-on-rebalance
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/cec2769e
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/cec2769e
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/cec2769e
Branch: refs/heads/trunk
Commit: cec2769e2281ac731e8cfd12b5d8e34e0ebe38cd
Parents: 313aad4
Author: Konstantine Karantasis <k....@gmail.com>
Authored: Tue Aug 23 10:19:07 2016 -0700
Committer: Ewen Cheslack-Postava <me...@ewencp.org>
Committed: Tue Aug 23 10:19:07 2016 -0700
----------------------------------------------------------------------
.../kafka/connect/runtime/WorkerSinkTask.java | 9 +-
.../runtime/WorkerSinkTaskThreadedTest.java | 90 ++++++++++++++++++--
2 files changed, 88 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/cec2769e/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
----------------------------------------------------------------------
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index 2a16934..624b032 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -412,13 +412,16 @@ class WorkerSinkTask extends WorkerTask {
if (offsets.isEmpty()) {
return;
}
- for (TopicPartition tp: offsets.keySet()) {
- Long offset = offsets.get(tp);
+ for (Map.Entry<TopicPartition, Long> entry: offsets.entrySet()) {
+ TopicPartition tp = entry.getKey();
+ Long offset = entry.getValue();
if (offset != null) {
log.trace("Rewind {} to offset {}.", tp, offset);
consumer.seek(tp, offset);
lastCommittedOffsets.put(tp, new OffsetAndMetadata(offset));
currentOffsets.put(tp, new OffsetAndMetadata(offset));
+ } else {
+ log.warn("Cannot rewind {} to null offset.", tp);
}
}
context.clearOffsets();
@@ -462,6 +465,8 @@ class WorkerSinkTask extends WorkerTask {
if (rebalanceException == null || rebalanceException instanceof WakeupException) {
try {
openPartitions(partitions);
+ // Rewind should be applied only if openPartitions succeeds.
+ rewind();
} catch (RuntimeException e) {
// The consumer swallows exceptions raised in the rebalance listener, so we need to store
// exceptions and rethrow when poll() returns.
http://git-wip-us.apache.org/repos/asf/kafka/blob/cec2769e/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
----------------------------------------------------------------------
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
index 392985b..52a86ab 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
@@ -139,7 +139,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
expectPollInitialAssignment();
Capture<Collection<SinkRecord>> capturedRecords = expectPolls(1L);
- expectStopTask(10L);
+ expectStopTask();
PowerMock.replayAll();
@@ -182,7 +182,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
Capture<Collection<SinkRecord>> capturedRecords
= expectPolls(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_DEFAULT);
expectOffsetFlush(1L, null, null, 0, true);
- expectStopTask(2);
+ expectStopTask();
PowerMock.replayAll();
@@ -221,7 +221,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
PowerMock.expectLastCall();
consumer.seek(TOPIC_PARTITION3, FIRST_OFFSET);
PowerMock.expectLastCall();
- expectStopTask(2);
+ expectStopTask();
PowerMock.replayAll();
@@ -259,7 +259,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
PowerMock.expectLastCall();
consumer.seek(TOPIC_PARTITION3, FIRST_OFFSET);
PowerMock.expectLastCall();
- expectStopTask(2);
+ expectStopTask();
PowerMock.replayAll();
@@ -291,7 +291,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
Capture<Collection<SinkRecord>> capturedRecords
= expectPolls(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_DEFAULT);
expectOffsetFlush(1L, null, new Exception(), 0, true);
- expectStopTask(2);
+ expectStopTask();
PowerMock.replayAll();
@@ -323,7 +323,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
Capture<Collection<SinkRecord>> capturedRecords
= expectPolls(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_DEFAULT / 2);
expectOffsetFlush(2L, null, null, WorkerConfig.OFFSET_COMMIT_TIMEOUT_MS_DEFAULT, false);
- expectStopTask(4);
+ expectStopTask();
PowerMock.replayAll();
@@ -402,7 +402,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
consumer.resume(Arrays.asList(TOPIC_PARTITION, TOPIC_PARTITION2));
PowerMock.expectLastCall();
- expectStopTask(0);
+ expectStopTask();
PowerMock.replayAll();
@@ -446,7 +446,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
}
});
- expectStopTask(3);
+ expectStopTask();
PowerMock.replayAll();
workerTask.initialize(TASK_CONFIG);
@@ -460,6 +460,33 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
PowerMock.verifyAll();
}
+ @Test
+ public void testRewindOnRebalanceDuringPoll() throws Exception {
+ expectInitializeTask();
+ expectPollInitialAssignment();
+
+ expectRebalanceDuringPoll().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() throws Throwable {
+ Map<TopicPartition, Long> offsets = sinkTaskContext.getValue().offsets();
+ assertEquals(0, offsets.size());
+ return null;
+ }
+ });
+
+ expectStopTask();
+ PowerMock.replayAll();
+
+ workerTask.initialize(TASK_CONFIG);
+ workerTask.initializeAndStart();
+ workerTask.iteration();
+ workerTask.iteration();
+ workerTask.stop();
+ workerTask.close();
+
+ PowerMock.verifyAll();
+ }
+
private void expectInitializeTask() throws Exception {
PowerMock.expectPrivate(workerTask, "createConsumer").andReturn(consumer);
@@ -493,7 +520,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
EasyMock.expectLastCall();
}
- private void expectStopTask(final long expectedMessages) throws Exception {
+ private void expectStopTask() throws Exception {
sinkTask.stop();
PowerMock.expectLastCall();
@@ -561,6 +588,51 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
return EasyMock.expectLastCall();
}
+ @SuppressWarnings("unchecked")
+ private IExpectationSetters<Object> expectRebalanceDuringPoll() throws Exception {
+ final List<TopicPartition> partitions = Arrays.asList(TOPIC_PARTITION, TOPIC_PARTITION2, TOPIC_PARTITION3);
+
+ final long startOffset = 40L;
+ final Map<TopicPartition, Long> offsets = new HashMap<>();
+ offsets.put(TOPIC_PARTITION, startOffset);
+
+ EasyMock.expect(consumer.poll(EasyMock.anyLong())).andAnswer(
+ new IAnswer<ConsumerRecords<byte[], byte[]>>() {
+ @Override
+ public ConsumerRecords<byte[], byte[]> answer() throws Throwable {
+ // "Sleep" so time will progress
+ time.sleep(1L);
+
+ sinkTaskContext.getValue().offset(offsets);
+ rebalanceListener.getValue().onPartitionsAssigned(partitions);
+
+ ConsumerRecords<byte[], byte[]> records = new ConsumerRecords<>(
+ Collections.singletonMap(
+ new TopicPartition(TOPIC, PARTITION),
+ Arrays.asList(
+ new ConsumerRecord<>(TOPIC, PARTITION, FIRST_OFFSET + recordsReturned, TIMESTAMP, TIMESTAMP_TYPE, 0L, 0, 0, RAW_KEY, RAW_VALUE)
+ )));
+ recordsReturned++;
+ return records;
+ }
+ });
+
+ EasyMock.expect(consumer.position(TOPIC_PARTITION)).andReturn(FIRST_OFFSET);
+ EasyMock.expect(consumer.position(TOPIC_PARTITION2)).andReturn(FIRST_OFFSET);
+ EasyMock.expect(consumer.position(TOPIC_PARTITION3)).andReturn(FIRST_OFFSET);
+
+ sinkTask.open(partitions);
+ EasyMock.expectLastCall();
+
+ consumer.seek(TOPIC_PARTITION, startOffset);
+ EasyMock.expectLastCall();
+
+ EasyMock.expect(keyConverter.toConnectData(TOPIC, RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY));
+ EasyMock.expect(valueConverter.toConnectData(TOPIC, RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE));
+ sinkTask.put(EasyMock.anyObject(Collection.class));
+ return EasyMock.expectLastCall();
+ }
+
private Capture<OffsetCommitCallback> expectOffsetFlush(final long expectedMessages,
final RuntimeException flushError,
final Exception consumerCommitError,