You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2017/04/06 09:12:35 UTC
hive git commit: HIVE-16371: Add bitmap selection strategy for druid
storage handler (Slim Bouguerra, reviewed by Jesus Camacho Rodriguez)
Repository: hive
Updated Branches:
refs/heads/master db24ce722 -> ecabdd3e7
HIVE-16371: Add bitmap selection strategy for druid storage handler (Slim Bouguerra, reviewed by Jesus Camacho Rodriguez)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ecabdd3e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ecabdd3e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ecabdd3e
Branch: refs/heads/master
Commit: ecabdd3e7eff59fb83b999e4f31533f91fd6391c
Parents: db24ce7
Author: Slim Bouguerra <sl...@gmail.com>
Authored: Thu Apr 6 11:01:09 2017 +0200
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Thu Apr 6 11:01:09 2017 +0200
----------------------------------------------------------------------
.../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 1 +
druid-handler/pom.xml | 1 +
.../apache/hadoop/hive/druid/io/DruidOutputFormat.java | 11 ++++++++++-
.../apache/hadoop/hive/ql/io/TestDruidRecordWriter.java | 8 ++++++--
4 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6de6ed6..7d4a6a0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2001,6 +2001,7 @@ public class HiveConf extends Configuration {
HIVE_DRUID_PASSIVE_WAIT_TIME("hive.druid.passiveWaitTimeMs", 30000,
"Wait time in ms default to 30 seconds."
),
+ HIVE_DRUID_BITMAP_FACTORY_TYPE("hive.druid.bitmap.type", "roaring", new PatternSet("roaring", "concise"), "Coding algorithm use to encode the bitmaps"),
// For HBase storage handler
HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true,
"Whether writes to HBase should be forced to the write-ahead log. \n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/pom.xml
----------------------------------------------------------------------
diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml
index ffc9d86..6afb5b8 100644
--- a/druid-handler/pom.xml
+++ b/druid-handler/pom.xml
@@ -329,6 +329,7 @@
<include>org.jdbi:*</include>
<include>net.jpountz.lz4:*</include>
<include>org.apache.commons:*</include>
+ <include>org.roaringbitmap:*</include>
</includes>
</artifactSet>
<filters>
http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
index 40a2022..4385dfe 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
@@ -33,6 +33,9 @@ import io.druid.granularity.QueryGranularity;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.DoubleSumAggregatorFactory;
import io.druid.query.aggregation.LongSumAggregatorFactory;
+import io.druid.segment.IndexSpec;
+import io.druid.segment.data.ConciseBitmapSerdeFactory;
+import io.druid.segment.data.RoaringBitmapSerdeFactory;
import io.druid.segment.indexing.DataSchema;
import io.druid.segment.indexing.RealtimeTuningConfig;
import io.druid.segment.indexing.granularity.GranularitySpec;
@@ -200,6 +203,12 @@ public class DruidOutputFormat<K, V> implements HiveOutputFormat<K, DruidWritabl
}
Integer maxRowInMemory = HiveConf.getIntVar(jc, HiveConf.ConfVars.HIVE_DRUID_MAX_ROW_IN_MEMORY);
+ IndexSpec indexSpec;
+ if ("concise".equals(HiveConf.getVar(jc, HiveConf.ConfVars.HIVE_DRUID_BITMAP_FACTORY_TYPE))) {
+ indexSpec = new IndexSpec(new ConciseBitmapSerdeFactory(), null, null, null);
+ } else {
+ indexSpec = new IndexSpec(new RoaringBitmapSerdeFactory(true), null, null, null);
+ }
RealtimeTuningConfig realtimeTuningConfig = new RealtimeTuningConfig(maxRowInMemory,
null,
null,
@@ -208,7 +217,7 @@ public class DruidOutputFormat<K, V> implements HiveOutputFormat<K, DruidWritabl
null,
null,
null,
- null,
+ indexSpec,
true,
0,
0,
http://git-wip-us.apache.org/repos/asf/hive/blob/ecabdd3e/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java b/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
index 9ec82c0..d9e01fe 100644
--- a/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
+++ b/druid-handler/src/test/org/apache/hadoop/hive/ql/io/TestDruidRecordWriter.java
@@ -37,8 +37,10 @@ import io.druid.granularity.QueryGranularities;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.LongSumAggregatorFactory;
import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
+import io.druid.segment.IndexSpec;
import io.druid.segment.QueryableIndex;
import io.druid.segment.QueryableIndexStorageAdapter;
+import io.druid.segment.data.RoaringBitmapSerdeFactory;
import io.druid.segment.indexing.DataSchema;
import io.druid.segment.indexing.RealtimeTuningConfig;
import io.druid.segment.indexing.granularity.UniformGranularitySpec;
@@ -139,8 +141,10 @@ public class TestDruidRecordWriter {
objectMapper
);
- RealtimeTuningConfig tuningConfig = RealtimeTuningConfig
- .makeDefaultTuningConfig(temporaryFolder.newFolder());
+ IndexSpec indexSpec = new IndexSpec(new RoaringBitmapSerdeFactory(true), null, null, null);
+ RealtimeTuningConfig tuningConfig = new RealtimeTuningConfig(null, null, null,
+ temporaryFolder.newFolder(), null, null, null, null, indexSpec, null, 0, 0, null, null
+ );
LocalFileSystem localFileSystem = FileSystem.getLocal(config);
DataSegmentPusher dataSegmentPusher = new LocalDataSegmentPusher(
new LocalDataSegmentPusherConfig() {