You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/10/10 02:56:59 UTC
hive git commit: HIVE-20700 : Add config to diable rollup for druid
tables (Nishant Bangarwa via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master c29038af9 -> 2ff9c5229
HIVE-20700 : Add config to diable rollup for druid tables (Nishant Bangarwa via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2ff9c522
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2ff9c522
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2ff9c522
Branch: refs/heads/master
Commit: 2ff9c5229d1470afab12258862f801cf14dc2a2d
Parents: c29038a
Author: Nishant <ni...@gmail.com>
Authored: Sat Oct 6 02:46:25 2018 +0530
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Oct 9 19:56:06 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/Constants.java | 1 +
.../org/apache/hadoop/hive/conf/HiveConf.java | 1 +
.../hive/druid/DruidStorageHandlerUtils.java | 4 ++++
.../clientpositive/druidmini_test_insert.q | 9 +++++++
.../druid/druidmini_test_insert.q.out | 25 ++++++++++++++++++++
5 files changed, 40 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/common/src/java/org/apache/hadoop/hive/conf/Constants.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
index 4badfa3..1190679 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
@@ -32,6 +32,7 @@ public class Constants {
"org.apache.hadoop.hive.druid.io.DruidOutputFormat";
public static final String DRUID_DATA_SOURCE = "druid.datasource";
public static final String DRUID_SEGMENT_GRANULARITY = "druid.segment.granularity";
+ public static final String DRUID_ROLLUP = "druid.rollup";
public static final String DRUID_QUERY_GRANULARITY = "druid.query.granularity";
public static final String DRUID_TARGET_SHARDS_PER_GRANULARITY =
"druid.segment.targetShardsPerGranularity";
http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d0adc35..cc6239c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2755,6 +2755,7 @@ public class HiveConf extends Configuration {
HIVE_DRUID_BASE_PERSIST_DIRECTORY("hive.druid.basePersistDirectory", "",
"Local temporary directory used to persist intermediate indexing state, will default to JVM system property java.io.tmpdir."
),
+ HIVE_DRUID_ROLLUP("hive.druid.rollup", true, "Whether to rollup druid rows or not."),
DRUID_SEGMENT_DIRECTORY("hive.druid.storage.storageDirectory", "/druid/segments"
, "druid deep storage location."),
DRUID_METADATA_BASE("hive.druid.metadata.base", "druid", "Default prefix for metadata tables"),
http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
index e67de89..c3e7e5d 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
@@ -827,12 +827,16 @@ public final class DruidStorageHandlerUtils {
tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) != null ?
tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) :
HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_DRUID_INDEXING_GRANULARITY);
+ final boolean rollup = tableProperties.getProperty(Constants.DRUID_ROLLUP) != null ?
+ Boolean.parseBoolean(tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY)):
+ HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_DRUID_ROLLUP);
return new UniformGranularitySpec(
Granularity.fromString(segmentGranularity),
Granularity.fromString(
tableProperties.getProperty(Constants.DRUID_QUERY_GRANULARITY) == null
? "NONE"
: tableProperties.getProperty(Constants.DRUID_QUERY_GRANULARITY)),
+ rollup,
null
);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/ql/src/test/queries/clientpositive/druidmini_test_insert.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_test_insert.q b/ql/src/test/queries/clientpositive/druidmini_test_insert.q
index e97fe5b..28fa6d7 100644
--- a/ql/src/test/queries/clientpositive/druidmini_test_insert.q
+++ b/ql/src/test/queries/clientpositive/druidmini_test_insert.q
@@ -141,6 +141,15 @@ EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-09 23:5
EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 00:00:00' as timestamp );
EXPLAIN select * from druid_test_table_n9 where `__time` = cast('2015-03-10 23:59:59' as timestamp );
+-- disable rollup and insert identical rows
+SET hive.druid.rollup=false;
+insert into druid_test_table_n9 values
+('2015-03-12 00:00:00', 'i4-1', 4),
+('2015-03-12 00:00:00', 'i4-1', 2),
+('2015-03-12 00:00:00', 'i4-1', 1);
+
+select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp );
+
DROP TABLE test_base_table;
DROP TABLE druid_test_table_n9;
http://git-wip-us.apache.org/repos/asf/hive/blob/2ff9c522/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
index e568b14..b74d33c 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
@@ -824,6 +824,31 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
ListSink
+PREHOOK: query: insert into druid_test_table_n9 values
+('2015-03-12 00:00:00', 'i4-1', 4),
+('2015-03-12 00:00:00', 'i4-1', 2),
+('2015-03-12 00:00:00', 'i4-1', 1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: druid_test_dst@druid_test_table_n9
+POSTHOOK: query: insert into druid_test_table_n9 values
+('2015-03-12 00:00:00', 'i4-1', 4),
+('2015-03-12 00:00:00', 'i4-1', 2),
+('2015-03-12 00:00:00', 'i4-1', 1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: druid_test_dst@druid_test_table_n9
+PREHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp )
+PREHOOK: type: QUERY
+PREHOOK: Input: druid_test_dst@druid_test_table_n9
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as timestamp )
+POSTHOOK: type: QUERY
+POSTHOOK: Input: druid_test_dst@druid_test_table_n9
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+2015-03-12 00:00:00.0 US/Pacific i4-1 4.0
+2015-03-12 00:00:00.0 US/Pacific i4-1 2.0
+2015-03-12 00:00:00.0 US/Pacific i4-1 1.0
PREHOOK: query: DROP TABLE test_base_table
PREHOOK: type: DROPTABLE
PREHOOK: Input: druid_test_dst@test_base_table