You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2018/07/09 21:48:05 UTC

[1/2] storm git commit: STORM-3082: Added support to handle absent topics

Repository: storm
Updated Branches:
  refs/heads/1.x-branch 27498b32c -> 4e1e62667


STORM-3082: Added support to handle absent topics


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e463cf02
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e463cf02
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e463cf02

Branch: refs/heads/1.x-branch
Commit: e463cf02df443ced705588bcfe2873bccc9ce8ce
Parents: 8a16fec
Author: Aniket Alhat <an...@rakuten.com>
Authored: Mon Jul 9 12:11:28 2018 +0900
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Jul 9 23:36:54 2018 +0200

----------------------------------------------------------------------
 .../apache/storm/kafka/spout/NamedTopicFilter.java   | 12 ++++++++++--
 .../storm/kafka/spout/NamedTopicFilterTest.java      | 15 +++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/e463cf02/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
index 3d1ec1e..e6b79dc 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/NamedTopicFilter.java
@@ -26,12 +26,15 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Filter that returns all partitions for the specified topics.
  */
 public class NamedTopicFilter implements TopicFilter {
 
+    private static final Logger LOG = LoggerFactory.getLogger(NamedTopicFilter.class);
     private final Set<String> topics;
     
     /**
@@ -54,8 +57,13 @@ public class NamedTopicFilter implements TopicFilter {
     public List<TopicPartition> getFilteredTopicPartitions(KafkaConsumer<?, ?> consumer) {
         List<TopicPartition> allPartitions = new ArrayList<>();
         for (String topic : topics) {
-            for (PartitionInfo partitionInfo: consumer.partitionsFor(topic)) {
-                allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
+            List<PartitionInfo> partitionInfoList = consumer.partitionsFor(topic);
+            if(partitionInfoList != null) {
+                for (PartitionInfo partitionInfo : partitionInfoList) {
+                    allPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
+                }
+            } else {
+                LOG.warn("Topic {} not found, skipping addition of the topic", topic);
             }
         }
         return allPartitions;

http://git-wip-us.apache.org/repos/asf/storm/blob/e463cf02/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
index fe3325c..c196a32 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/NamedTopicFilterTest.java
@@ -18,6 +18,7 @@ package org.apache.storm.kafka.spout;
 
 import org.apache.storm.kafka.spout.NamedTopicFilter;
 
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.junit.Assert.assertThat;
 import static org.mockito.Mockito.mock;
@@ -63,6 +64,20 @@ public class NamedTopicFilterTest {
             
     }
     
+    @Test
+    public void testFilterOnAbsentTopic() {
+        String presentTopic = "present";
+        String absentTopic = "absent";
+        
+        NamedTopicFilter filter = new NamedTopicFilter(presentTopic, absentTopic);
+        when(consumerMock.partitionsFor(presentTopic)).thenReturn(Collections.singletonList(createPartitionInfo(presentTopic, 2)));
+        when(consumerMock.partitionsFor(absentTopic)).thenReturn(null);
+        
+        List<TopicPartition> presentPartitions = filter.getFilteredTopicPartitions(consumerMock);
+        assertThat("Expected filter to pass only topics which are present", presentPartitions,
+            contains(new TopicPartition(presentTopic, 2)));
+    }
+    
     private PartitionInfo createPartitionInfo(String topic, int partition) {
         return new PartitionInfo(topic, partition, null, null, null);
     }


[2/2] storm git commit: Merge branch 'STORM-3082-1.x-merge' into asfgit-1.x-branch

Posted by sr...@apache.org.
Merge branch 'STORM-3082-1.x-merge' into asfgit-1.x-branch


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4e1e6266
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4e1e6266
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4e1e6266

Branch: refs/heads/1.x-branch
Commit: 4e1e62667d97fe684b1d73cd8a1c41d3175cf6bc
Parents: 27498b3 e463cf0
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Mon Jul 9 23:47:49 2018 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Jul 9 23:47:49 2018 +0200

----------------------------------------------------------------------
 .../apache/storm/kafka/spout/NamedTopicFilter.java   | 12 ++++++++++--
 .../storm/kafka/spout/NamedTopicFilterTest.java      | 15 +++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------