You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/17 06:30:29 UTC

[2/2] incubator-kylin git commit: Kafka consumer thread factory changed to daemon

Kafka consumer thread factory changed to daemon


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b980228b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b980228b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b980228b

Branch: refs/heads/0.8
Commit: b980228bf74b0c5fe6ee1526406ba61ea930d752
Parents: c698de9
Author: honma <ho...@ebay.com>
Authored: Wed Jun 17 11:16:37 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 17 11:16:37 2015 +0800

----------------------------------------------------------------------
 .../kylin/common/util/DaemonThreadFactory.java       | 15 +++++++++++++++
 .../kylin/job/streaming/StreamingBootstrap.java      | 11 ++++-------
 .../apache/kylin/streaming/ITKafkaConsumerTest.java  |  3 ++-
 3 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b980228b/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java b/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
new file mode 100644
index 0000000..bc4502c
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
@@ -0,0 +1,15 @@
+package org.apache.kylin.common.util;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+
+/**
+ */
+public class DaemonThreadFactory implements ThreadFactory {
+    @Override
+    public Thread newThread(Runnable r) {
+        Thread t = Executors.defaultThreadFactory().newThread(r);
+        t.setDaemon(true);
+        return t;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b980228b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index ae6b282..6e2c02b 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -43,6 +43,7 @@ import kafka.cluster.Broker;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.HBaseConnection;
+import org.apache.kylin.common.util.DaemonThreadFactory;
 import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.common.util.TimeUtil;
 import org.apache.kylin.cube.CubeInstance;
@@ -160,7 +161,7 @@ public class StreamingBootstrap {
             }
             logger.info("starting offset:" + streamingOffset + " cluster id:" + clusterID + " partitionId:" + partitionId + " transferredPartitionId:" + transferredPartitionId);
             KafkaConsumer consumer = new KafkaConsumer(clusterID, kafkaClusterConfig.getTopic(), partitionId, streamingOffset, kafkaClusterConfig.getBrokers(), kafkaClusterConfig);
-            Executors.newSingleThreadExecutor().submit(consumer);
+            Executors.newSingleThreadExecutor(new DaemonThreadFactory()).submit(consumer);
             result.add(consumer.getStreamQueue(0));
         }
         return result;
@@ -314,14 +315,10 @@ public class StreamingBootstrap {
 
         final IIDesc iiDesc = iiSegment.getIIDesc();
 
-        Executors.newSingleThreadExecutor().submit(consumer);
+        Executors.newSingleThreadExecutor(new DaemonThreadFactory()).submit(consumer);
         final ExecutorService streamingBuilderPool = Executors.newFixedThreadPool(parallelism);
         for (int i = startShard; i < endShard; ++i) {
-            final StreamBuilder task = StreamBuilder.newLimitedSizeStreamBuilder(streamingConfig.getName(),
-                    consumer.getStreamQueue(i % parallelism),
-                    new IIStreamConsumer(streamingConfig.getName(), iiSegment.getStorageLocationIdentifier(), iiDesc, i),
-                    0L,
-                    iiDesc.getSliceSize());
+            final StreamBuilder task = StreamBuilder.newLimitedSizeStreamBuilder(streamingConfig.getName(), consumer.getStreamQueue(i % parallelism), new IIStreamConsumer(streamingConfig.getName(), iiSegment.getStorageLocationIdentifier(), iiDesc, i), 0L, iiDesc.getSliceSize());
             task.setStreamParser(getStreamParser(streamingConfig, ii.getDescriptor().listAllColumns()));
             if (i == endShard - 1) {
                 streamingBuilderPool.submit(task).get();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b980228b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
index 4ed7094..ca6fe87 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
@@ -36,6 +36,7 @@ package org.apache.kylin.streaming;
 
 import com.google.common.collect.Lists;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.DaemonThreadFactory;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -87,7 +88,7 @@ public class ITKafkaConsumerTest extends KafkaBaseTest {
     @Ignore("since ci does not have the topic")
     public void test() throws InterruptedException {
         final TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig);
-        final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size());
+        final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size(), new DaemonThreadFactory());
         List<BlockingQueue<StreamMessage>> queues = Lists.newArrayList();
         for (Integer partitionId : kafkaTopicMeta.getPartitionIds()) {
             KafkaConsumer consumer = new KafkaConsumer(0, kafkaTopicMeta.getName(), partitionId, 0, kafkaClusterConfig.getBrokers(), kafkaClusterConfig);