You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ew...@apache.org on 2016/08/23 17:19:17 UTC

kafka git commit: KAFKA-2894: WorkerSinkTask should rewind offsets on rebalance

Repository: kafka
Updated Branches:
  refs/heads/trunk 313aad423 -> cec2769e2


KAFKA-2894: WorkerSinkTask should rewind offsets on rebalance

Author: Konstantine Karantasis <k....@gmail.com>

Reviewers: Ewen Cheslack-Postava <ew...@confluent.io>

Closes #1771 from kkonstantine/KAFKA-2894-rewind-offsets-on-rebalance


Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/cec2769e
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/cec2769e
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/cec2769e

Branch: refs/heads/trunk
Commit: cec2769e2281ac731e8cfd12b5d8e34e0ebe38cd
Parents: 313aad4
Author: Konstantine Karantasis <k....@gmail.com>
Authored: Tue Aug 23 10:19:07 2016 -0700
Committer: Ewen Cheslack-Postava <me...@ewencp.org>
Committed: Tue Aug 23 10:19:07 2016 -0700

----------------------------------------------------------------------
 .../kafka/connect/runtime/WorkerSinkTask.java   |  9 +-
 .../runtime/WorkerSinkTaskThreadedTest.java     | 90 ++++++++++++++++++--
 2 files changed, 88 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kafka/blob/cec2769e/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
----------------------------------------------------------------------
diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
index 2a16934..624b032 100644
--- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
+++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSinkTask.java
@@ -412,13 +412,16 @@ class WorkerSinkTask extends WorkerTask {
         if (offsets.isEmpty()) {
             return;
         }
-        for (TopicPartition tp: offsets.keySet()) {
-            Long offset = offsets.get(tp);
+        for (Map.Entry<TopicPartition, Long> entry: offsets.entrySet()) {
+            TopicPartition tp = entry.getKey();
+            Long offset = entry.getValue();
             if (offset != null) {
                 log.trace("Rewind {} to offset {}.", tp, offset);
                 consumer.seek(tp, offset);
                 lastCommittedOffsets.put(tp, new OffsetAndMetadata(offset));
                 currentOffsets.put(tp, new OffsetAndMetadata(offset));
+            } else {
+                log.warn("Cannot rewind {} to null offset.", tp);
             }
         }
         context.clearOffsets();
@@ -462,6 +465,8 @@ class WorkerSinkTask extends WorkerTask {
             if (rebalanceException == null || rebalanceException instanceof WakeupException) {
                 try {
                     openPartitions(partitions);
+                    // Rewind should be applied only if openPartitions succeeds.
+                    rewind();
                 } catch (RuntimeException e) {
                     // The consumer swallows exceptions raised in the rebalance listener, so we need to store
                     // exceptions and rethrow when poll() returns.

http://git-wip-us.apache.org/repos/asf/kafka/blob/cec2769e/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
----------------------------------------------------------------------
diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
index 392985b..52a86ab 100644
--- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
+++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/WorkerSinkTaskThreadedTest.java
@@ -139,7 +139,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         expectPollInitialAssignment();
 
         Capture<Collection<SinkRecord>> capturedRecords = expectPolls(1L);
-        expectStopTask(10L);
+        expectStopTask();
 
         PowerMock.replayAll();
 
@@ -182,7 +182,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         Capture<Collection<SinkRecord>> capturedRecords
                 = expectPolls(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_DEFAULT);
         expectOffsetFlush(1L, null, null, 0, true);
-        expectStopTask(2);
+        expectStopTask();
 
         PowerMock.replayAll();
 
@@ -221,7 +221,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         PowerMock.expectLastCall();
         consumer.seek(TOPIC_PARTITION3, FIRST_OFFSET);
         PowerMock.expectLastCall();
-        expectStopTask(2);
+        expectStopTask();
 
         PowerMock.replayAll();
 
@@ -259,7 +259,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         PowerMock.expectLastCall();
         consumer.seek(TOPIC_PARTITION3, FIRST_OFFSET);
         PowerMock.expectLastCall();
-        expectStopTask(2);
+        expectStopTask();
 
         PowerMock.replayAll();
 
@@ -291,7 +291,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         Capture<Collection<SinkRecord>> capturedRecords
                 = expectPolls(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_DEFAULT);
         expectOffsetFlush(1L, null, new Exception(), 0, true);
-        expectStopTask(2);
+        expectStopTask();
 
         PowerMock.replayAll();
 
@@ -323,7 +323,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         Capture<Collection<SinkRecord>> capturedRecords
                 = expectPolls(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_DEFAULT / 2);
         expectOffsetFlush(2L, null, null, WorkerConfig.OFFSET_COMMIT_TIMEOUT_MS_DEFAULT, false);
-        expectStopTask(4);
+        expectStopTask();
 
         PowerMock.replayAll();
 
@@ -402,7 +402,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         consumer.resume(Arrays.asList(TOPIC_PARTITION, TOPIC_PARTITION2));
         PowerMock.expectLastCall();
 
-        expectStopTask(0);
+        expectStopTask();
 
         PowerMock.replayAll();
 
@@ -446,7 +446,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
             }
         });
 
-        expectStopTask(3);
+        expectStopTask();
         PowerMock.replayAll();
 
         workerTask.initialize(TASK_CONFIG);
@@ -460,6 +460,33 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         PowerMock.verifyAll();
     }
 
+    @Test
+    public void testRewindOnRebalanceDuringPoll() throws Exception {
+        expectInitializeTask();
+        expectPollInitialAssignment();
+
+        expectRebalanceDuringPoll().andAnswer(new IAnswer<Object>() {
+            @Override
+            public Object answer() throws Throwable {
+                Map<TopicPartition, Long> offsets = sinkTaskContext.getValue().offsets();
+                assertEquals(0, offsets.size());
+                return null;
+            }
+        });
+
+        expectStopTask();
+        PowerMock.replayAll();
+
+        workerTask.initialize(TASK_CONFIG);
+        workerTask.initializeAndStart();
+        workerTask.iteration();
+        workerTask.iteration();
+        workerTask.stop();
+        workerTask.close();
+
+        PowerMock.verifyAll();
+    }
+
     private void expectInitializeTask() throws Exception {
         PowerMock.expectPrivate(workerTask, "createConsumer").andReturn(consumer);
 
@@ -493,7 +520,7 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         EasyMock.expectLastCall();
     }
 
-    private void expectStopTask(final long expectedMessages) throws Exception {
+    private void expectStopTask() throws Exception {
         sinkTask.stop();
         PowerMock.expectLastCall();
 
@@ -561,6 +588,51 @@ public class WorkerSinkTaskThreadedTest extends ThreadedTest {
         return EasyMock.expectLastCall();
     }
 
+    @SuppressWarnings("unchecked")
+    private IExpectationSetters<Object> expectRebalanceDuringPoll() throws Exception {
+        final List<TopicPartition> partitions = Arrays.asList(TOPIC_PARTITION, TOPIC_PARTITION2, TOPIC_PARTITION3);
+
+        final long startOffset = 40L;
+        final Map<TopicPartition, Long> offsets = new HashMap<>();
+        offsets.put(TOPIC_PARTITION, startOffset);
+
+        EasyMock.expect(consumer.poll(EasyMock.anyLong())).andAnswer(
+                new IAnswer<ConsumerRecords<byte[], byte[]>>() {
+                    @Override
+                    public ConsumerRecords<byte[], byte[]> answer() throws Throwable {
+                        // "Sleep" so time will progress
+                        time.sleep(1L);
+
+                        sinkTaskContext.getValue().offset(offsets);
+                        rebalanceListener.getValue().onPartitionsAssigned(partitions);
+
+                        ConsumerRecords<byte[], byte[]> records = new ConsumerRecords<>(
+                                Collections.singletonMap(
+                                        new TopicPartition(TOPIC, PARTITION),
+                                        Arrays.asList(
+                                                new ConsumerRecord<>(TOPIC, PARTITION, FIRST_OFFSET + recordsReturned, TIMESTAMP, TIMESTAMP_TYPE, 0L, 0, 0, RAW_KEY, RAW_VALUE)
+                                        )));
+                        recordsReturned++;
+                        return records;
+                    }
+                });
+
+        EasyMock.expect(consumer.position(TOPIC_PARTITION)).andReturn(FIRST_OFFSET);
+        EasyMock.expect(consumer.position(TOPIC_PARTITION2)).andReturn(FIRST_OFFSET);
+        EasyMock.expect(consumer.position(TOPIC_PARTITION3)).andReturn(FIRST_OFFSET);
+
+        sinkTask.open(partitions);
+        EasyMock.expectLastCall();
+
+        consumer.seek(TOPIC_PARTITION, startOffset);
+        EasyMock.expectLastCall();
+
+        EasyMock.expect(keyConverter.toConnectData(TOPIC, RAW_KEY)).andReturn(new SchemaAndValue(KEY_SCHEMA, KEY));
+        EasyMock.expect(valueConverter.toConnectData(TOPIC, RAW_VALUE)).andReturn(new SchemaAndValue(VALUE_SCHEMA, VALUE));
+        sinkTask.put(EasyMock.anyObject(Collection.class));
+        return EasyMock.expectLastCall();
+    }
+
     private Capture<OffsetCommitCallback> expectOffsetFlush(final long expectedMessages,
                                                             final RuntimeException flushError,
                                                             final Exception consumerCommitError,