You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/06/17 06:30:29 UTC
[2/2] incubator-kylin git commit: Kafka consumer thread factory
changed to daemon
Kafka consumer thread factory changed to daemon
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b980228b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b980228b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b980228b
Branch: refs/heads/0.8
Commit: b980228bf74b0c5fe6ee1526406ba61ea930d752
Parents: c698de9
Author: honma <ho...@ebay.com>
Authored: Wed Jun 17 11:16:37 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jun 17 11:16:37 2015 +0800
----------------------------------------------------------------------
.../kylin/common/util/DaemonThreadFactory.java | 15 +++++++++++++++
.../kylin/job/streaming/StreamingBootstrap.java | 11 ++++-------
.../apache/kylin/streaming/ITKafkaConsumerTest.java | 3 ++-
3 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b980228b/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java b/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
new file mode 100644
index 0000000..bc4502c
--- /dev/null
+++ b/common/src/main/java/org/apache/kylin/common/util/DaemonThreadFactory.java
@@ -0,0 +1,15 @@
+package org.apache.kylin.common.util;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+
+/**
+ */
+public class DaemonThreadFactory implements ThreadFactory {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = Executors.defaultThreadFactory().newThread(r);
+ t.setDaemon(true);
+ return t;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b980228b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index ae6b282..6e2c02b 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -43,6 +43,7 @@ import kafka.cluster.Broker;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.HBaseConnection;
+import org.apache.kylin.common.util.DaemonThreadFactory;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.TimeUtil;
import org.apache.kylin.cube.CubeInstance;
@@ -160,7 +161,7 @@ public class StreamingBootstrap {
}
logger.info("starting offset:" + streamingOffset + " cluster id:" + clusterID + " partitionId:" + partitionId + " transferredPartitionId:" + transferredPartitionId);
KafkaConsumer consumer = new KafkaConsumer(clusterID, kafkaClusterConfig.getTopic(), partitionId, streamingOffset, kafkaClusterConfig.getBrokers(), kafkaClusterConfig);
- Executors.newSingleThreadExecutor().submit(consumer);
+ Executors.newSingleThreadExecutor(new DaemonThreadFactory()).submit(consumer);
result.add(consumer.getStreamQueue(0));
}
return result;
@@ -314,14 +315,10 @@ public class StreamingBootstrap {
final IIDesc iiDesc = iiSegment.getIIDesc();
- Executors.newSingleThreadExecutor().submit(consumer);
+ Executors.newSingleThreadExecutor(new DaemonThreadFactory()).submit(consumer);
final ExecutorService streamingBuilderPool = Executors.newFixedThreadPool(parallelism);
for (int i = startShard; i < endShard; ++i) {
- final StreamBuilder task = StreamBuilder.newLimitedSizeStreamBuilder(streamingConfig.getName(),
- consumer.getStreamQueue(i % parallelism),
- new IIStreamConsumer(streamingConfig.getName(), iiSegment.getStorageLocationIdentifier(), iiDesc, i),
- 0L,
- iiDesc.getSliceSize());
+ final StreamBuilder task = StreamBuilder.newLimitedSizeStreamBuilder(streamingConfig.getName(), consumer.getStreamQueue(i % parallelism), new IIStreamConsumer(streamingConfig.getName(), iiSegment.getStorageLocationIdentifier(), iiDesc, i), 0L, iiDesc.getSliceSize());
task.setStreamParser(getStreamParser(streamingConfig, ii.getDescriptor().listAllColumns()));
if (i == endShard - 1) {
streamingBuilderPool.submit(task).get();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b980228b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
index 4ed7094..ca6fe87 100644
--- a/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
+++ b/streaming/src/test/java/org/apache/kylin/streaming/ITKafkaConsumerTest.java
@@ -36,6 +36,7 @@ package org.apache.kylin.streaming;
import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.DaemonThreadFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -87,7 +88,7 @@ public class ITKafkaConsumerTest extends KafkaBaseTest {
@Ignore("since ci does not have the topic")
public void test() throws InterruptedException {
final TopicMeta kafkaTopicMeta = KafkaRequester.getKafkaTopicMeta(kafkaClusterConfig);
- final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size());
+ final ExecutorService executorService = Executors.newFixedThreadPool(kafkaTopicMeta.getPartitionIds().size(), new DaemonThreadFactory());
List<BlockingQueue<StreamMessage>> queues = Lists.newArrayList();
for (Integer partitionId : kafkaTopicMeta.getPartitionIds()) {
KafkaConsumer consumer = new KafkaConsumer(0, kafkaTopicMeta.getName(), partitionId, 0, kafkaClusterConfig.getBrokers(), kafkaClusterConfig);