You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/10/09 23:14:25 UTC
[3/4] storm git commit: Backport to 1.1.x
Backport to 1.1.x
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d8da7ca0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d8da7ca0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d8da7ca0
Branch: refs/heads/1.1.x-branch
Commit: d8da7ca08e6979df81544eb2add1adc7358bad30
Parents: 7d290af
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Wed Oct 4 21:19:16 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Wed Oct 4 21:19:16 2017 +0200
----------------------------------------------------------------------
.../org/apache/storm/kafka/spout/KafkaSpout.java | 3 ++-
.../storm/kafka/spout/KafkaSpoutRebalanceTest.java | 15 +++++++++------
.../storm/kafka/spout/SingleTopicKafkaSpoutTest.java | 6 ++++--
3 files changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d8da7ca0/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index e0fde9c..4d88205 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -133,6 +133,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
LOG.info("Partitions revoked. [consumer-group={}, consumer={}, topic-partitions={}]",
kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
+ previousAssignment = partitions;
if (!consumerAutoCommitMode && initialized) {
initialized = false;
commitOffsetsForAckedTuples();
@@ -150,7 +151,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
private void initialize(Collection<TopicPartition> partitions) {
if (!consumerAutoCommitMode) {
offsetManagers.keySet().retainAll(partitions); // remove from acked all partitions that are no longer assigned to this spout
- }
+ retryService.retainAll(partitions);
/*
* Emitted messages for partitions that are no longer assigned to this spout can't
http://git-wip-us.apache.org/repos/asf/storm/blob/d8da7ca0/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
index 79a1c9a..1c7f723 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
@@ -17,11 +17,13 @@ package org.apache.storm.kafka.spout;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.getKafkaSpoutConfigBuilder;
import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.Matchers.any;
import static org.hamcrest.Matchers.hasKey;
import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyCollection;
import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
@@ -32,10 +34,13 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
@@ -194,11 +199,7 @@ public class KafkaSpoutRebalanceTest {
*/
ArgumentCaptor<ConsumerRebalanceListener> rebalanceListenerCapture = ArgumentCaptor.forClass(ConsumerRebalanceListener.class);
- Subscription subscriptionMock = mock(Subscription.class);
- doNothing()
- .when(subscriptionMock)
- .subscribe(any(KafkaConsumer.class), rebalanceListenerCapture.capture(), any(TopologyContext.class));
- KafkaSpout<String, String> spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(subscriptionMock, -1)
+ KafkaSpout<String, String> spout = new KafkaSpout<>(getKafkaSpoutConfigBuilder(-1)
.setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST)
.build(), consumerFactoryMock);
String topic = SingleTopicKafkaSpoutConfiguration.TOPIC;
@@ -208,6 +209,8 @@ public class KafkaSpoutRebalanceTest {
//Setup spout with mock consumer so we can get at the rebalance listener
spout.open(conf, contextMock, collectorMock);
spout.activate();
+
+ verify(consumerMock).subscribe(anyCollection(), rebalanceListenerCapture.capture());
//Assign partitions to the spout
ConsumerRebalanceListener consumerRebalanceListener = rebalanceListenerCapture.getValue();
http://git-wip-us.apache.org/repos/asf/storm/blob/d8da7ca0/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
index 8ff6142..fff6902 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
@@ -36,7 +36,6 @@ import java.util.concurrent.TimeoutException;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.mockito.Matchers.anyCollection;
import static org.mockito.Matchers.anyList;
import static org.mockito.Matchers.anyListOf;
import static org.mockito.Matchers.anyObject;
@@ -53,12 +52,14 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.TopicPartition;
import org.apache.storm.kafka.spout.internal.KafkaConsumerFactory;
import org.apache.storm.kafka.spout.internal.KafkaConsumerFactoryDefault;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Time.SimulatedTime;
+import org.hamcrest.Matchers;
import org.junit.Before;
import org.mockito.Captor;
import org.mockito.MockitoAnnotations;
@@ -79,6 +80,7 @@ public class SingleTopicKafkaSpoutTest {
private KafkaConsumerFactory<String, String> consumerFactory;
private KafkaSpout<String, String> spout;
private final int maxPollRecords = 10;
+ private final int maxRetries = 3;
@Before
public void setUp() {
@@ -134,7 +136,7 @@ public class SingleTopicKafkaSpoutTest {
public void testSeekToCommittedOffsetIfConsumerPositionIsBehindWhenCommitting() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = maxPollRecords * 2;
- prepareSpout(messageCount);
+ initializeSpout(messageCount);
//Emit all messages and fail the first one while acking the rest
for (int i = 0; i < messageCount; i++) {