You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2018/07/09 21:48:05 UTC
[1/2] storm git commit: STORM-3082: Added support to handle absent
topics
Repository: storm
Updated Branches:
refs/heads/1.x-branch 27498b32c -> 4e1e62667
STORM-3082: Added support to handle absent topics
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e463cf02
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e463cf02
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e463cf02
Branch: refs/heads/1.x-branch
Commit: e463cf02df443ced705588bcfe2873bccc9ce8ce
Parents: 8a16fec
Author: Aniket Alhat <an...@rakuten.com>
Authored: Mon Jul 9 12:11:28 2018 +0900
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Jul 9 23:36:54 2018 +0200
----------------------------------------------------------------------
.../apache/storm/kafka/spout/NamedTopicFilter.java | 12 ++++++++++--
.../storm/kafka/spout/NamedTopicFilterTest.java | 15 +++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e463cf02/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
index 3d1ec1e..e6b79dc 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
@@ -26,12 +26,15 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Filter that returns all partitions for the specified topics.
*/
public class NamedTopicFilter implements TopicFilter {
+ private static final Logger LOG = LoggerFactory.getLogger(NamedTopicFilter.class);
private final Set<String> topics;
/**
@@ -54,8 +57,13 @@ public class NamedTopicFilter implements TopicFilter {
public List<TopicPartition> getFilteredTopicPartitions(KafkaConsumer<?, ?> consumer) {
List<TopicPartition> allPartitions = new ArrayList<>();
for (String topic : topics) {
- for (PartitionInfo partitionInfo: consumer.partitionsFor(topic)) {
- allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
+ List<PartitionInfo> partitionInfoList = consumer.partitionsFor(topic);
+ if(partitionInfoList != null) {
+ for (PartitionInfo partitionInfo : partitionInfoList) {
+ allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
+ }
+ } else {
+ LOG.warn("Topic {} not found, skipping addition of the topic", topic);
}
}
return allPartitions;
http://git-wip-us.apache.org/repos/asf/storm/blob/e463cf02/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
index fe3325c..c196a32 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
@@ -18,6 +18,7 @@ package org.apache.storm.kafka.spout;
import org.apache.storm.kafka.spout.NamedTopicFilter;
+import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
@@ -63,6 +64,20 @@ public class NamedTopicFilterTest {
}
+ @Test
+ public void testFilterOnAbsentTopic() {
+ String presentTopic = "present";
+ String absentTopic = "absent";
+
+ NamedTopicFilter filter = new NamedTopicFilter(presentTopic, absentTopic);
+ when(consumerMock.partitionsFor(presentTopic)).thenReturn(Collections.singletonList(createPartitionInfo(presentTopic, 2)));
+ when(consumerMock.partitionsFor(absentTopic)).thenReturn(null);
+
+ List<TopicPartition> presentPartitions = filter.getFilteredTopicPartitions(consumerMock);
+ assertThat("Expected filter to pass only topics which are present", presentPartitions,
+ contains(new TopicPartition(presentTopic, 2)));
+ }
+
private PartitionInfo createPartitionInfo(String topic, int partition) {
return new PartitionInfo(topic, partition, null, null, null);
}
[2/2] storm git commit: Merge branch 'STORM-3082-1.x-merge' into
asfgit-1.x-branch
Posted by sr...@apache.org.
Merge branch 'STORM-3082-1.x-merge' into asfgit-1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4e1e6266
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4e1e6266
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4e1e6266
Branch: refs/heads/1.x-branch
Commit: 4e1e62667d97fe684b1d73cd8a1c41d3175cf6bc
Parents: 27498b3 e463cf0
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Mon Jul 9 23:47:49 2018 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Jul 9 23:47:49 2018 +0200
----------------------------------------------------------------------
.../apache/storm/kafka/spout/NamedTopicFilter.java | 12 ++++++++++--
.../storm/kafka/spout/NamedTopicFilterTest.java | 15 +++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------