You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2017/10/16 23:48:45 UTC
[1/4] hive git commit: HIVE-17672: Upgrade Calcite version to 1.14
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 45b9b8d76 -> 9975131cc
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out b/ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out
index 02636eb..1c45853 100644
--- a/ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out
+++ b/ql/src/test/results/clientpositive/groupby_grouping_sets_limit.q.out
@@ -357,23 +357,22 @@ STAGE PLANS:
outputColumnNames: a
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: a (type: string), 0 (type: int)
+ keys: a (type: string)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Reduce Operator Tree:
Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: int)
+ keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
Limit
Number of rows: 10
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
@@ -427,39 +426,34 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: _col0 (type: double), 0 (type: int)
+ keys: _col0 (type: double)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: double), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: double), _col1 (type: int)
+ key expressions: _col0 (type: double)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: double)
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
- value expressions: _col2 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: double), KEY._col1 (type: int)
+ keys: KEY._col0 (type: double)
mode: mergepartial
- outputColumnNames: _col0, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
- Select Operator
- expressions: _col0 (type: double), _col2 (type: bigint)
- outputColumnNames: _col0, _col1
+ Limit
+ Number of rows: 10
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
+ File Output Operator
+ compressed: false
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
index f514448..43e5958 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
@@ -600,7 +600,7 @@ STAGE PLANS:
Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: int)
- outputColumnNames: _col4
+ outputColumnNames: _col2
Select Vectorization:
className: VectorSelectOperator
native: true
@@ -617,7 +617,7 @@ STAGE PLANS:
native: false
vectorProcessingMode: HASH
projectedOutputColumns: [0]
- keys: _col4 (type: int)
+ keys: _col2 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE
@@ -912,7 +912,7 @@ STAGE PLANS:
Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: int)
- outputColumnNames: _col4
+ outputColumnNames: _col2
Select Vectorization:
className: VectorSelectOperator
native: true
@@ -929,7 +929,7 @@ STAGE PLANS:
native: false
vectorProcessingMode: HASH
projectedOutputColumns: [0]
- keys: _col4 (type: int)
+ keys: _col2 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE
@@ -2084,7 +2084,7 @@ STAGE PLANS:
Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: int)
- outputColumnNames: _col4
+ outputColumnNames: _col2
Select Vectorization:
className: VectorSelectOperator
native: true
@@ -2101,7 +2101,7 @@ STAGE PLANS:
native: false
vectorProcessingMode: HASH
projectedOutputColumns: [0]
- keys: _col4 (type: int)
+ keys: _col2 (type: int)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out
index c759170..cc9a67d 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets1.q.out
@@ -534,14 +534,14 @@ STAGE PLANS:
outputColumnNames: a
Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: a (type: string), 0 (type: int)
+ keys: a (type: string)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -549,11 +549,10 @@ STAGE PLANS:
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: int)
+ keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
File Output Operator
compressed: false
Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE
@@ -612,16 +611,16 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: _col0 (type: double), 0 (type: int)
+ keys: _col0 (type: double)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: double), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: double), _col1 (type: int)
+ key expressions: _col0 (type: double)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: double)
Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint)
+ value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -629,22 +628,17 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: double), KEY._col1 (type: int)
+ keys: KEY._col0 (type: double)
mode: mergepartial
- outputColumnNames: _col0, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
- Select Operator
- expressions: _col0 (type: double), _col2 (type: bigint)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out
index 15190b8..b268de8 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out
@@ -482,14 +482,14 @@ STAGE PLANS:
outputColumnNames: a
Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: a (type: string), 0 (type: int)
+ keys: a (type: string)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized, llap
@@ -498,11 +498,10 @@ STAGE PLANS:
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: int)
+ keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 3 Data size: 552 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -577,17 +576,17 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: _col0 (type: double), 0 (type: int)
+ keys: _col0 (type: double)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: double), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: double), _col1 (type: int)
+ key expressions: _col0 (type: double)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: double)
Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
- value expressions: _col2 (type: bigint)
+ value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -595,21 +594,16 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: double), KEY._col1 (type: int)
+ keys: KEY._col0 (type: double)
mode: mergepartial
- outputColumnNames: _col0, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
- Select Operator
- expressions: _col0 (type: double), _col2 (type: bigint)
- outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions: _col0 (type: double)
+ sort order: +
Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: double)
- sort order: +
- Statistics: Num rows: 3 Data size: 1104 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col1 (type: bigint)
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col1 (type: bigint)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/spark/query36.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query36.q.out b/ql/src/test/results/clientpositive/perf/spark/query36.q.out
index c0d1cc7..d313337 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query36.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query36.q.out
@@ -212,43 +212,43 @@ STAGE PLANS:
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2)), _col2 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col5, _col6
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: (grouping(_col6, 1) + grouping(_col6, 0)) (type: int), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string), (_col4 / _col5) (type: decimal(37,20))
+ key expressions: (grouping(_col4, 1) + grouping(_col4, 0)) (type: int), CASE WHEN ((grouping(_col4, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END (type: string), (_col2 / _col3) (type: decimal(37,20))
sort order: +++
- Map-reduce partition columns: (grouping(_col6, 1) + grouping(_col6, 0)) (type: int), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string)
+ Map-reduce partition columns: (grouping(_col4, 1) + grouping(_col4, 0)) (type: int), CASE WHEN ((grouping(_col4, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END (type: string)
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: string), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col6 (type: int)
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: decimal(17,2)), _col3 (type: decimal(17,2)), _col4 (type: int)
Reducer 5
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: decimal(17,2)), VALUE._col3 (type: decimal(17,2)), VALUE._col4 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col5, _col6
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col4: decimal(17,2), _col5: decimal(17,2), _col6: int
+ output shape: _col0: string, _col1: string, _col2: decimal(17,2), _col3: decimal(17,2), _col4: int
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: (_col4 / _col5) ASC NULLS FIRST
- partition by: (grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END
+ order by: (_col2 / _col3) ASC NULLS FIRST
+ partition by: (grouping(_col4, 1) + grouping(_col4, 0)), CASE WHEN ((grouping(_col4, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: (_col4 / _col5)
+ arguments: (_col2 / _col3)
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: (_col4 / _col5) (type: decimal(37,20)), _col0 (type: string), _col1 (type: string), (grouping(_col6, 1) + grouping(_col6, 0)) (type: int), rank_window_0 (type: int), CASE WHEN (((grouping(_col6, 1) + grouping(_col6, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string)
+ expressions: (_col2 / _col3) (type: decimal(37,20)), _col0 (type: string), _col1 (type: string), (grouping(_col4, 1) + grouping(_col4, 0)) (type: int), rank_window_0 (type: int), CASE WHEN (((grouping(_col4, 1) + grouping(_col4, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/spark/query67.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query67.q.out b/ql/src/test/results/clientpositive/perf/spark/query67.q.out
index 9affd8f..25d37bd 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query67.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query67.q.out
@@ -242,10 +242,10 @@ STAGE PLANS:
pruneGroupingSetId: true
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col9 (type: decimal(28,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col16
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 3449926075 Data size: 304353479997 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col16 (type: decimal(28,2))
+ key expressions: _col0 (type: string), _col8 (type: decimal(28,2))
sort order: +-
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 3449926075 Data size: 304353479997 Basic stats: COMPLETE Column stats: NONE
@@ -255,24 +255,24 @@ STAGE PLANS:
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: string), KEY.reducesinkkey1 (type: decimal(28,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col16
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 3449926075 Data size: 304353479997 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: string, _col3: string, _col4: int, _col5: int, _col6: int, _col7: string, _col16: decimal(28,2)
+ output shape: _col0: string, _col1: string, _col2: string, _col3: string, _col4: int, _col5: int, _col6: int, _col7: string, _col8: decimal(28,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col16 DESC NULLS LAST
+ order by: _col8 DESC NULLS LAST
partition by: _col0
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: _col16
+ arguments: _col8
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
@@ -282,7 +282,7 @@ STAGE PLANS:
predicate: (rank_window_0 <= 100) (type: boolean)
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col16 (type: decimal(28,2)), rank_window_0 (type: int)
+ expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: decimal(28,2)), rank_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/spark/query70.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query70.q.out b/ql/src/test/results/clientpositive/perf/spark/query70.q.out
index 9f3b8a0..862bdb0 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query70.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query70.q.out
@@ -359,43 +359,43 @@ STAGE PLANS:
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col3 (type: decimal(17,2)), _col2 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string), _col4 (type: decimal(17,2))
+ key expressions: (grouping(_col3, 1) + grouping(_col3, 0)) (type: int), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END (type: string), _col2 (type: decimal(17,2))
sort order: ++-
- Map-reduce partition columns: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string)
+ Map-reduce partition columns: (grouping(_col3, 1) + grouping(_col3, 0)) (type: int), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END (type: string)
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: int)
+ value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: int)
Reducer 5
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: decimal(17,2)), VALUE._col2 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col4: decimal(17,2), _col5: int
+ output shape: _col0: string, _col1: string, _col2: decimal(17,2), _col3: int
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col4 DESC NULLS LAST
- partition by: (grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END
+ order by: _col2 DESC NULLS LAST
+ partition by: (grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: _col4
+ arguments: _col2
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col4 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), rank_window_0 (type: int), CASE WHEN (((grouping(_col5, 1) + grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string)
+ expressions: _col2 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col3, 1) + grouping(_col3, 0)) (type: int), rank_window_0 (type: int), CASE WHEN (((grouping(_col3, 1) + grouping(_col3, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/spark/query86.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query86.q.out b/ql/src/test/results/clientpositive/perf/spark/query86.q.out
index 3642e65..8e159fd 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query86.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query86.q.out
@@ -170,43 +170,43 @@ STAGE PLANS:
Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col3 (type: decimal(17,2)), _col2 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string), _col4 (type: decimal(17,2))
+ key expressions: (grouping(_col3, 1) + grouping(_col3, 0)) (type: int), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END (type: string), _col2 (type: decimal(17,2))
sort order: ++-
- Map-reduce partition columns: (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END (type: string)
+ Map-reduce partition columns: (grouping(_col3, 1) + grouping(_col3, 0)) (type: int), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END (type: string)
Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: int)
+ value expressions: _col0 (type: string), _col1 (type: string), _col3 (type: int)
Reducer 4
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: decimal(17,2)), VALUE._col2 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col4: decimal(17,2), _col5: int
+ output shape: _col0: string, _col1: string, _col2: decimal(17,2), _col3: int
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col4 DESC NULLS LAST
- partition by: (grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END
+ order by: _col2 DESC NULLS LAST
+ partition by: (grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: _col4
+ arguments: _col2
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col4 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col5, 1) + grouping(_col5, 0)) (type: int), rank_window_0 (type: int), CASE WHEN (((grouping(_col5, 1) + grouping(_col5, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string)
+ expressions: _col2 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col3, 1) + grouping(_col3, 0)) (type: int), rank_window_0 (type: int), CASE WHEN (((grouping(_col3, 1) + grouping(_col3, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/tez/query14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
index b2a45f1..77959cb 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
@@ -1,6 +1,6 @@
Warning: Shuffle Join MERGEJOIN[890][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product
-Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product
Warning: Shuffle Join MERGEJOIN[892][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product
+Warning: Shuffle Join MERGEJOIN[891][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product
PREHOOK: query: explain
with cross_items as
(select i_item_sk ss_item_sk
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/tez/query22.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query22.q.out b/ql/src/test/results/clientpositive/perf/tez/query22.q.out
index 995f61d..d13169c 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query22.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query22.q.out
@@ -54,13 +54,13 @@ Stage-0
limit:100
Stage-1
Reducer 6
- File Output Operator [FS_30]
- Limit [LIM_29] (rows=100 width=15)
+ File Output Operator [FS_29]
+ Limit [LIM_28] (rows=100 width=15)
Number of rows:100
- Select Operator [SEL_28] (rows=125060762 width=15)
+ Select Operator [SEL_27] (rows=125060762 width=15)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Reducer 5 [SIMPLE_EDGE]
- SHUFFLE [RS_27]
+ SHUFFLE [RS_26]
Select Operator [SEL_25] (rows=125060762 width=15)
Output:["_col0","_col1","_col2","_col3","_col4"]
Group By Operator [GBY_24] (rows=125060762 width=15)
@@ -70,42 +70,42 @@ Stage-0
PartitionCols:_col0, _col1, _col2, _col3, _col4
Group By Operator [GBY_22] (rows=250121525 width=15)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["avg(_col3)"],keys:_col8, _col9, _col10, _col11, 0
- Merge Join Operator [MERGEJOIN_46] (rows=50024305 width=15)
+ Merge Join Operator [MERGEJOIN_45] (rows=50024305 width=15)
Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col8","_col9","_col10","_col11"]
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
Select Operator [SEL_11] (rows=462000 width=1436)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_43] (rows=462000 width=1436)
+ Filter Operator [FIL_42] (rows=462000 width=1436)
predicate:i_item_sk is not null
TableScan [TS_9] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_class","i_category","i_product_name"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_45] (rows=45476640 width=15)
+ Merge Join Operator [MERGEJOIN_44] (rows=45476640 width=15)
Conds:RS_15._col2=RS_16._col0(Inner),Output:["_col1","_col3"]
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
Select Operator [SEL_8] (rows=27 width=1029)
Output:["_col0"]
- Filter Operator [FIL_42] (rows=27 width=1029)
+ Filter Operator [FIL_41] (rows=27 width=1029)
predicate:w_warehouse_sk is not null
TableScan [TS_6] (rows=27 width=1029)
default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk"]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_44] (rows=41342400 width=15)
+ Merge Join Operator [MERGEJOIN_43] (rows=41342400 width=15)
Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
Select Operator [SEL_2] (rows=37584000 width=15)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_40] (rows=37584000 width=15)
+ Filter Operator [FIL_39] (rows=37584000 width=15)
predicate:(inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null)
TableScan [TS_0] (rows=37584000 width=15)
default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"]
@@ -114,7 +114,7 @@ Stage-0
PartitionCols:_col0
Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_41] (rows=8116 width=1119)
+ Filter Operator [FIL_40] (rows=8116 width=1119)
predicate:(d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"]
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/tez/query36.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query36.q.out b/ql/src/test/results/clientpositive/perf/tez/query36.q.out
index 9a4349d..a30ac2e 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query36.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query36.q.out
@@ -81,14 +81,14 @@ Stage-0
Select Operator [SEL_30] (rows=1149975358 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
PTF Operator [PTF_29] (rows=1149975358 width=88)
- Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(_col4 / _col5) ASC NULLS FIRST","partition by:":"(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END"}]
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"(_col2 / _col3) ASC NULLS FIRST","partition by:":"(grouping(_col4, 1) + grouping(_col4, 0)), CASE WHEN ((grouping(_col4, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END"}]
Select Operator [SEL_28] (rows=1149975358 width=88)
- Output:["_col0","_col1","_col4","_col5","_col6"]
+ Output:["_col0","_col1","_col2","_col3","_col4"]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_27]
- PartitionCols:(grouping(_col6, 1) + grouping(_col6, 0)), CASE WHEN ((grouping(_col6, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END
+ PartitionCols:(grouping(_col4, 1) + grouping(_col4, 0)), CASE WHEN ((grouping(_col4, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END
Select Operator [SEL_26] (rows=1149975358 width=88)
- Output:["_col0","_col1","_col4","_col5","_col6"]
+ Output:["_col0","_col1","_col2","_col3","_col4"]
Group By Operator [GBY_25] (rows=1149975358 width=88)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Reducer 4 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/tez/query67.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query67.q.out b/ql/src/test/results/clientpositive/perf/tez/query67.q.out
index 803af6f..86c4d7c 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query67.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query67.q.out
@@ -111,14 +111,14 @@ Stage-0
Filter Operator [FIL_47] (rows=1149975358 width=88)
predicate:(rank_window_0 <= 100)
PTF Operator [PTF_29] (rows=3449926075 width=88)
- Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col16 DESC NULLS LAST","partition by:":"_col0"}]
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col8 DESC NULLS LAST","partition by:":"_col0"}]
Select Operator [SEL_28] (rows=3449926075 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col16"]
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
Select Operator [SEL_26] (rows=3449926075 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col16"]
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
Group By Operator [GBY_25] (rows=3449926075 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8
<-Reducer 4 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/tez/query70.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query70.q.out b/ql/src/test/results/clientpositive/perf/tez/query70.q.out
index 69f08c0..0f6891e 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query70.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query70.q.out
@@ -101,14 +101,14 @@ Stage-0
Select Operator [SEL_55] (rows=1149975358 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
PTF Operator [PTF_54] (rows=1149975358 width=88)
- Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END"}]
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col2 DESC NULLS LAST","partition by:":"(grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END"}]
Select Operator [SEL_53] (rows=1149975358 width=88)
- Output:["_col0","_col1","_col4","_col5"]
+ Output:["_col0","_col1","_col2","_col3"]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_52]
- PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END
+ PartitionCols:(grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END
Select Operator [SEL_51] (rows=1149975358 width=88)
- Output:["_col0","_col1","_col4","_col5"]
+ Output:["_col0","_col1","_col2","_col3"]
Group By Operator [GBY_50] (rows=1149975358 width=88)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Reducer 4 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/perf/tez/query86.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query86.q.out b/ql/src/test/results/clientpositive/perf/tez/query86.q.out
index c37b384..be7e858 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query86.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query86.q.out
@@ -72,14 +72,14 @@ Stage-0
Select Operator [SEL_24] (rows=261364852 width=135)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
PTF Operator [PTF_23] (rows=261364852 width=135)
- Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 DESC NULLS LAST","partition by:":"(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END"}]
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col2 DESC NULLS LAST","partition by:":"(grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END"}]
Select Operator [SEL_22] (rows=261364852 width=135)
- Output:["_col0","_col1","_col4","_col5"]
+ Output:["_col0","_col1","_col2","_col3"]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_21]
- PartitionCols:(grouping(_col5, 1) + grouping(_col5, 0)), CASE WHEN ((grouping(_col5, 0) = 0)) THEN (_col0) ELSE (CAST( null AS varchar(65535))) END
+ PartitionCols:(grouping(_col3, 1) + grouping(_col3, 0)), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (UDFToString(null)) END
Select Operator [SEL_20] (rows=261364852 width=135)
- Output:["_col0","_col1","_col4","_col5"]
+ Output:["_col0","_col1","_col2","_col3"]
Group By Operator [GBY_19] (rows=261364852 width=135)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Reducer 3 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/show_functions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out
index 31653a6..43e4a5d 100644
--- a/ql/src/test/results/clientpositive/show_functions.q.out
+++ b/ql/src/test/results/clientpositive/show_functions.q.out
@@ -250,6 +250,7 @@ substring_index
sum
tan
to_date
+to_epoch_milli
to_unix_timestamp
to_utc_timestamp
translate
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/timestamptz_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/timestamptz_3.q.out b/ql/src/test/results/clientpositive/timestamptz_3.q.out
new file mode 100644
index 0000000..196c584
--- /dev/null
+++ b/ql/src/test/results/clientpositive/timestamptz_3.q.out
@@ -0,0 +1,57 @@
+PREHOOK: query: drop table tstz1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tstz1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tstz1(t timestamp with local time zone)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tstz1
+POSTHOOK: query: create table tstz1(t timestamp with local time zone)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tstz1
+PREHOOK: query: insert overwrite table tstz1 select cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@tstz1
+POSTHOOK: query: insert overwrite table tstz1 select cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@tstz1
+POSTHOOK: Lineage: tstz1.t SIMPLE []
+PREHOOK: query: select cast(t as timestamp) from tstz1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast(t as timestamp) from tstz1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+2016-01-03 12:26:34
+PREHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+2016-01-03 12:26:34
+PREHOOK: query: select cast(t as timestamp) from tstz1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast(t as timestamp) from tstz1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+2016-01-03 20:26:34
+PREHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+POSTHOOK: query: select cast(to_epoch_milli(t) as timestamp) from tstz1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstz1
+#### A masked pattern was here ####
+2016-01-03 12:26:34
[4/4] hive git commit: HIVE-17672: Upgrade Calcite version to 1.14
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
HIVE-17672: Upgrade Calcite version to 1.14 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9975131c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9975131c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9975131c
Branch: refs/heads/master
Commit: 9975131cce0b6af71f3ae77e20de294adc1b3466
Parents: 45b9b8d
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Oct 3 16:08:46 2017 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon Oct 16 16:48:37 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/druid/io/DruidOutputFormat.java | 12 +-
.../hadoop/hive/druid/serde/DruidSerDe.java | 34 ++-
.../hive/druid/serde/DruidSerDeUtils.java | 2 +-
.../hadoop/hive/druid/serde/TestDruidSerDe.java | 87 ++++---
pom.xml | 2 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 2 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../metadata/HiveMaterializedViewsRegistry.java | 8 +-
...tedDynPartitionTimeGranularityOptimizer.java | 46 +++-
.../ql/optimizer/calcite/HiveCalciteUtil.java | 2 +-
.../calcite/HiveConfPlannerContext.java | 4 -
.../ql/optimizer/calcite/HiveRelFactories.java | 8 +-
.../hive/ql/optimizer/calcite/HiveType.java | 48 ----
.../optimizer/calcite/HiveTypeSystemImpl.java | 4 +
.../functions/HiveSqlCountAggFunction.java | 46 +++-
.../HiveSqlSumEmptyIsZeroAggFunction.java | 59 +++++
.../calcite/reloperators/HiveAggregate.java | 16 +-
.../calcite/reloperators/HiveGroupingID.java | 8 +-
.../rules/HiveAggregateJoinTransposeRule.java | 120 ++++-----
.../rules/HiveAggregateProjectMergeRule.java | 31 +--
.../calcite/rules/HiveExceptRewriteRule.java | 16 +-
.../rules/HiveExpandDistinctAggregatesRule.java | 4 +-
.../calcite/rules/HiveIntersectRewriteRule.java | 16 +-
.../calcite/rules/HiveRelColumnsAlignment.java | 3 +-
.../calcite/rules/HiveRelDecorrelator.java | 41 ++-
.../calcite/translator/ASTBuilder.java | 7 +
.../calcite/translator/ASTConverter.java | 41 ++-
.../calcite/translator/ExprNodeConverter.java | 35 ++-
.../calcite/translator/HiveGBOpConvUtil.java | 3 +-
.../calcite/translator/HiveOpConverter.java | 15 +-
.../calcite/translator/RexNodeConverter.java | 18 +-
.../translator/SqlFunctionConverter.java | 5 +-
.../calcite/translator/TypeConverter.java | 32 +--
.../hadoop/hive/ql/parse/CalcitePlanner.java | 35 +--
.../apache/hadoop/hive/ql/parse/HiveParser.g | 1 +
.../hadoop/hive/ql/parse/IdentifiersParser.g | 9 +
.../hive/ql/parse/TypeCheckProcFactory.java | 15 +-
.../apache/hadoop/hive/ql/udf/UDFDateFloor.java | 31 ++-
.../ql/udf/generic/GenericUDFEpochMilli.java | 62 +++++
.../ql/udf/TestUDFDateFormatGranularity.java | 78 ++++++
.../queries/clientpositive/druid_timeseries.q | 8 +-
ql/src/test/queries/clientpositive/druid_topn.q | 4 +-
.../test/queries/clientpositive/timestamptz_3.q | 15 ++
.../cbo_rp_annotate_stats_groupby.q.out | 256 +++++++------------
.../results/clientpositive/druid_basic1.q.out | 4 +-
.../results/clientpositive/druid_basic2.q.out | 106 ++++----
.../clientpositive/druid_intervals.q.out | 42 +--
.../clientpositive/druid_timeseries.q.out | 68 ++---
.../results/clientpositive/druid_topn.q.out | 56 ++--
.../clientpositive/groupby_grouping_sets1.q.out | 46 ++--
.../groupby_grouping_sets_limit.q.out | 50 ++--
.../llap/vector_groupby_grouping_id2.q.out | 12 +-
.../llap/vector_groupby_grouping_sets1.q.out | 46 ++--
.../vector_groupby_grouping_sets_limit.q.out | 44 ++--
.../clientpositive/perf/spark/query36.q.out | 20 +-
.../clientpositive/perf/spark/query67.q.out | 14 +-
.../clientpositive/perf/spark/query70.q.out | 20 +-
.../clientpositive/perf/spark/query86.q.out | 20 +-
.../clientpositive/perf/tez/query14.q.out | 2 +-
.../clientpositive/perf/tez/query22.q.out | 22 +-
.../clientpositive/perf/tez/query36.q.out | 8 +-
.../clientpositive/perf/tez/query67.q.out | 6 +-
.../clientpositive/perf/tez/query70.q.out | 8 +-
.../clientpositive/perf/tez/query86.q.out | 8 +-
.../results/clientpositive/show_functions.q.out | 1 +
.../results/clientpositive/timestamptz_3.q.out | 57 +++++
66 files changed, 1113 insertions(+), 838 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
index 664e02f..0888ff2 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
@@ -149,13 +149,21 @@ public class DruidOutputFormat<K, V> implements HiveOutputFormat<K, DruidWritabl
af = new DoubleSumAggregatorFactory(columnNames.get(i), columnNames.get(i));
break;
case TIMESTAMP:
+ // Granularity column
String tColumnName = columnNames.get(i);
- if (!tColumnName.equals(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN) && !tColumnName
- .equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME)) {
+ if (!tColumnName.equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME)) {
throw new IOException("Dimension " + tColumnName + " does not have STRING type: " +
primitiveCategory);
}
continue;
+ case TIMESTAMPLOCALTZ:
+ // Druid timestamp column
+ String tLocalTZColumnName = columnNames.get(i);
+ if (!tLocalTZColumnName.equals(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN)) {
+ throw new IOException("Dimension " + tLocalTZColumnName + " does not have STRING type: " +
+ primitiveCategory);
+ }
+ continue;
default:
// Dimension
String dColumnName = columnNames.get(i);
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
index a43f62e..4a7952e 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
@@ -19,7 +19,8 @@ package org.apache.hadoop.hive.druid.serde;
import java.io.IOException;
import java.io.InputStream;
-import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -31,6 +32,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.druid.DruidStorageHandler;
@@ -47,7 +49,7 @@ import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
@@ -64,9 +66,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspect
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
import org.apache.hadoop.io.BooleanWritable;
@@ -175,7 +179,7 @@ public class DruidSerDe extends AbstractSerDe {
if (columnInfo.getKey().equals(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN)) {
// Special handling for timestamp column
columnNames.add(columnInfo.getKey()); // field name
- PrimitiveTypeInfo type = TypeInfoFactory.timestampTypeInfo; // field type
+ PrimitiveTypeInfo type = TypeInfoFactory.timestampLocalTZTypeInfo; // field type
columnTypes.add(type);
inspectors
.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type));
@@ -300,7 +304,7 @@ public class DruidSerDe extends AbstractSerDe {
Map<String, PrimitiveTypeInfo> mapColumnNamesTypes) {
// Timestamp column
columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN);
- columnTypes.add(TypeInfoFactory.timestampTypeInfo);
+ columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo);
// Aggregator columns
for (AggregatorFactory af : query.getAggregatorSpecs()) {
columnNames.add(af.getName());
@@ -328,7 +332,7 @@ public class DruidSerDe extends AbstractSerDe {
Map<String, PrimitiveTypeInfo> mapColumnNamesTypes) {
// Timestamp column
columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN);
- columnTypes.add(TypeInfoFactory.timestampTypeInfo);
+ columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo);
// Dimension column
columnNames.add(query.getDimensionSpec().getOutputName());
columnTypes.add(TypeInfoFactory.stringTypeInfo);
@@ -360,7 +364,7 @@ public class DruidSerDe extends AbstractSerDe {
throws SerDeException {
// Timestamp column
columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN);
- columnTypes.add(TypeInfoFactory.timestampTypeInfo);
+ columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo);
// Dimension columns
for (DimensionSpec ds : query.getDimensions()) {
columnNames.add(ds.getOutputName());
@@ -402,7 +406,7 @@ public class DruidSerDe extends AbstractSerDe {
Map<String, PrimitiveTypeInfo> mapColumnNamesTypes) {
// Timestamp column
columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN);
- columnTypes.add(TypeInfoFactory.timestampTypeInfo);
+ columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo);
// Dimension columns
for (DimensionSpec ds : query.getDimensions()) {
columnNames.add(ds.getOutputName());
@@ -456,10 +460,9 @@ public class DruidSerDe extends AbstractSerDe {
}
final Object res;
switch (types[i].getPrimitiveCategory()) {
- case TIMESTAMP:
- res = ((TimestampObjectInspector) fields.get(i).getFieldObjectInspector())
- .getPrimitiveJavaObject(
- values.get(i)).getTime();
+ case TIMESTAMPLOCALTZ:
+ res = ((TimestampLocalTZObjectInspector) fields.get(i).getFieldObjectInspector())
+ .getPrimitiveJavaObject(values.get(i)).getZonedDateTime().toInstant().toEpochMilli();
break;
case BYTE:
res = ((ByteObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
@@ -529,8 +532,13 @@ public class DruidSerDe extends AbstractSerDe {
continue;
}
switch (types[i].getPrimitiveCategory()) {
- case TIMESTAMP:
- output.add(new TimestampWritable(new Timestamp((Long) value)));
+ case TIMESTAMPLOCALTZ:
+ output.add(
+ new TimestampLocalTZWritable(
+ new TimestampTZ(
+ ZonedDateTime.ofInstant(
+ Instant.ofEpochMilli((Long) value),
+ ((TimestampLocalTZTypeInfo) types[i]).timeZone()))));
break;
case BYTE:
output.add(new ByteWritable(((Number) value).byteValue()));
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java
index 86c325b..bf55308 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java
@@ -73,7 +73,7 @@ public final class DruidSerDeUtils {
TimeFormatExtractionFn tfe = (TimeFormatExtractionFn) eds.getExtractionFn();
if (tfe.getFormat() == null || tfe.getFormat().equals(ISO_TIME_FORMAT)) {
// Timestamp (null or default used by FLOOR)
- return TypeInfoFactory.timestampTypeInfo;
+ return TypeInfoFactory.timestampLocalTZTypeInfo;
} else {
// EXTRACT from timestamp
return TypeInfoFactory.intTypeInfo;
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java
----------------------------------------------------------------------
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java
index 655a96a..2d351e4 100644
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java
+++ b/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java
@@ -28,6 +28,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
@@ -47,6 +49,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
import org.apache.hadoop.hive.druid.QTestDruidSerDe;
@@ -62,6 +65,7 @@ import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -148,19 +152,19 @@ public class TestDruidSerDe {
// Timeseries query results as records
private static final Object[][] TIMESERIES_QUERY_RESULTS_RECORDS = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new LongWritable(0),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new LongWritable(0),
new FloatWritable(1.0F), new FloatWritable(2.2222F) },
- new Object[] { new TimestampWritable(new Timestamp(1325462400000L)), new LongWritable(2),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325462400000L).atZone(ZoneOffset.UTC))), new LongWritable(2),
new FloatWritable(3.32F), new FloatWritable(4F) }
};
// Timeseries query results as records (types defined by metastore)
private static final String TIMESERIES_COLUMN_NAMES = "__time,sample_name1,sample_name2,sample_divide";
- private static final String TIMESERIES_COLUMN_TYPES = "timestamp,smallint,double,float";
+ private static final String TIMESERIES_COLUMN_TYPES = "timestamp with local time zone,smallint,double,float";
private static final Object[][] TIMESERIES_QUERY_RESULTS_RECORDS_2 = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new ShortWritable((short) 0),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new ShortWritable((short) 0),
new DoubleWritable(1.0d), new FloatWritable(2.2222F) },
- new Object[] { new TimestampWritable(new Timestamp(1325462400000L)), new ShortWritable((short) 2),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325462400000L).atZone(ZoneOffset.UTC))), new ShortWritable((short) 2),
new DoubleWritable(3.32d), new FloatWritable(4F) }
};
@@ -263,40 +267,40 @@ public class TestDruidSerDe {
// TopN query results as records
private static final Object[][] TOPN_QUERY_RESULTS_RECORDS = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)), new Text("dim1_val"),
- new LongWritable(111), new FloatWritable(10669F),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
+ new Text("dim1_val"), new LongWritable(111), new FloatWritable(10669F),
new FloatWritable(96.11711711711712F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("another_dim1_val"), new LongWritable(88), new FloatWritable(28344F),
new FloatWritable(322.09090909090907F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val3"), new LongWritable(70), new FloatWritable(871F),
new FloatWritable(12.442857142857143F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val4"), new LongWritable(62), new FloatWritable(815F),
new FloatWritable(13.14516129032258F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val5"), new LongWritable(60), new FloatWritable(2787F),
new FloatWritable(46.45F) }
};
// TopN query results as records (types defined by metastore)
private static final String TOPN_COLUMN_NAMES = "__time,sample_dim,count,some_metric,sample_divide";
- private static final String TOPN_COLUMN_TYPES = "timestamp,string,bigint,double,float";
+ private static final String TOPN_COLUMN_TYPES = "timestamp with local time zone,string,bigint,double,float";
private static final Object[][] TOPN_QUERY_RESULTS_RECORDS_2 = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val"), new LongWritable(111), new DoubleWritable(10669d),
new FloatWritable(96.11711711711712F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("another_dim1_val"), new LongWritable(88), new DoubleWritable(28344d),
new FloatWritable(322.09090909090907F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val3"), new LongWritable(70), new DoubleWritable(871d),
new FloatWritable(12.442857142857143F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val4"), new LongWritable(62), new DoubleWritable(815d),
new FloatWritable(13.14516129032258F) },
- new Object[] { new TimestampWritable(new Timestamp(1377907200000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val5"), new LongWritable(60), new DoubleWritable(2787d),
new FloatWritable(46.45F) }
};
@@ -425,38 +429,39 @@ public class TestDruidSerDe {
// GroupBy query results as records
private static final Object[][] GROUP_BY_QUERY_EXTRACTION_RESULTS_RECORDS = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1325376000000L)),
- new TimestampWritable(new Timestamp(1325376000000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))),
+ new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))),
new LongWritable(200) },
- new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new TimestampWritable(new Timestamp(1325376012000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))),
+ new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))),
new LongWritable(400) }
};
private static final Object[][] GROUP_BY_QUERY_RESULTS_RECORDS = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new Text("India"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new Text("India"),
new Text("phone"), new LongWritable(88), new FloatWritable(29.91233453F),
new FloatWritable(60.32F) },
- new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new Text("Spain"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), new Text("Spain"),
new Text("pc"), new LongWritable(16), new FloatWritable(172.93494959F),
new FloatWritable(6.333333F) }
};
private static final Object[][] GB_MONTH_EXTRACTION_RESULTS_RECORDS = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1325376000000L)),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))),
new IntWritable(1),
new LongWritable(200) },
- new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new IntWritable(1),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), new IntWritable(1),
new LongWritable(400) }
};
// GroupBy query results as records (types defined by metastore)
private static final String GROUP_BY_COLUMN_NAMES = "__time,country,device,total_usage,data_transfer,avg_usage";
- private static final String GROUP_BY_COLUMN_TYPES = "timestamp,string,string,int,double,float";
+ private static final String GROUP_BY_COLUMN_TYPES = "timestamp with local time zone,string,string,int,double,float";
private static final Object[][] GROUP_BY_QUERY_RESULTS_RECORDS_2 = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1325376000000L)), new Text("India"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC))), new Text("India"),
new Text("phone"), new IntWritable(88), new DoubleWritable(29.91233453),
new FloatWritable(60.32F) },
- new Object[] { new TimestampWritable(new Timestamp(1325376012000L)), new Text("Spain"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1325376012000L).atZone(ZoneOffset.UTC))), new Text("Spain"),
new Text("pc"), new IntWritable(16), new DoubleWritable(172.93494959),
new FloatWritable(6.333333F) }
};
@@ -577,28 +582,28 @@ public class TestDruidSerDe {
// Select query results as records
private static final Object[][] SELECT_QUERY_RESULTS_RECORDS = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("1"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("1"),
new Text("article"), new Text("0"), new Text("0"),
new Text("11._korpus_(NOVJ)"), new Text("sl"), new Text("0"),
new Text("EmausBot"),
new FloatWritable(1.0F), new FloatWritable(39.0F), new FloatWritable(39.0F),
new FloatWritable(39.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("112_U.S._580"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new FloatWritable(1.0F), new FloatWritable(70.0F), new FloatWritable(70.0F),
new FloatWritable(70.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("113_U.S._243"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new FloatWritable(1.0F), new FloatWritable(77.0F), new FloatWritable(77.0F),
new FloatWritable(77.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("113_U.S._73"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new FloatWritable(1.0F), new FloatWritable(70.0F), new FloatWritable(70.0F),
new FloatWritable(70.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("113_U.S._756"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new FloatWritable(1.0F), new FloatWritable(68.0F), new FloatWritable(68.0F),
@@ -607,30 +612,30 @@ public class TestDruidSerDe {
// Select query results as records (types defined by metastore)
private static final String SELECT_COLUMN_NAMES = "__time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted";
- private static final String SELECT_COLUMN_TYPES = "timestamp,string,string,string,string,string,string,string,string,double,double,float,float,float";
+ private static final String SELECT_COLUMN_TYPES = "timestamp with local time zone,string,string,string,string,string,string,string,string,double,double,float,float,float";
private static final Object[][] SELECT_QUERY_RESULTS_RECORDS_2 = new Object[][] {
- new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("1"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("1"),
new Text("article"), new Text("0"), new Text("0"),
new Text("11._korpus_(NOVJ)"), new Text("sl"), new Text("0"),
new Text("EmausBot"),
new DoubleWritable(1.0d), new DoubleWritable(39.0d), new FloatWritable(39.0F),
new FloatWritable(39.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998400000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998400000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("112_U.S._580"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new DoubleWritable(1.0d), new DoubleWritable(70.0d), new FloatWritable(70.0F),
new FloatWritable(70.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("113_U.S._243"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new DoubleWritable(1.0d), new DoubleWritable(77.0d), new FloatWritable(77.0F),
new FloatWritable(77.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("113_U.S._73"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new DoubleWritable(1.0d), new DoubleWritable(70.0d), new FloatWritable(70.0F),
new FloatWritable(70.0F), new FloatWritable(0.0F) },
- new Object[] { new TimestampWritable(new Timestamp(1356998412000L)), new Text("0"),
+ new Object[] { new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1356998412000L).atZone(ZoneOffset.UTC))), new Text("0"),
new Text("article"), new Text("0"), new Text("0"),
new Text("113_U.S._756"), new Text("en"), new Text("1"), new Text("MZMcBride"),
new DoubleWritable(1.0d), new DoubleWritable(68.0d), new FloatWritable(68.0F),
@@ -827,9 +832,9 @@ public class TestDruidSerDe {
private static final String COLUMN_NAMES = "__time,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9";
- private static final String COLUMN_TYPES = "timestamp,string,char(6),varchar(8),double,float,decimal(38,18),bigint,int,smallint,tinyint";
+ private static final String COLUMN_TYPES = "timestamp with local time zone,string,char(6),varchar(8),double,float,decimal(38,18),bigint,int,smallint,tinyint";
private static final Object[] ROW_OBJECT = new Object[] {
- new TimestampWritable(new Timestamp(1377907200000L)),
+ new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val"),
new HiveCharWritable(new HiveChar("dim2_v", 6)),
new HiveVarcharWritable(new HiveVarchar("dim3_val", 8)),
@@ -933,7 +938,7 @@ public class TestDruidSerDe {
}
private static final Object[] ROW_OBJECT_2 = new Object[] {
- new TimestampWritable(new Timestamp(1377907200000L)),
+ new TimestampLocalTZWritable(new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
new Text("dim1_val"),
new HiveCharWritable(new HiveChar("dim2_v", 6)),
new HiveVarcharWritable(new HiveVarchar("dim3_val", 8)),
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 32b09ef..b521cc4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -119,7 +119,7 @@
<avatica.version>1.10.0</avatica.version>
<avro.version>1.7.7</avro.version>
<bonecp.version>0.8.0.RELEASE</bonecp.version>
- <calcite.version>1.13.0</calcite.version>
+ <calcite.version>1.14.0</calcite.version>
<datanucleus-api-jdo.version>4.2.4</datanucleus-api-jdo.version>
<datanucleus-core.version>4.1.17</datanucleus-core.version>
<datanucleus-rdbms.version>4.1.19</datanucleus-rdbms.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 1d1d2a3..2308bf4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -497,6 +497,8 @@ public final class FunctionRegistry {
system.registerGenericUDF("to_unix_timestamp", GenericUDFToUnixTimeStamp.class);
system.registerGenericUDF("internal_interval", GenericUDFInternalInterval.class);
+
+ system.registerGenericUDF("to_epoch_milli", GenericUDFEpochMilli.class);
// Generic UDTF's
system.registerGenericUDTF("explode", GenericUDTFExplode.class);
system.registerGenericUDTF("replicate_rows", GenericUDTFReplicateRows.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 68c9482..793ce9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1509,7 +1509,7 @@ public class Hive {
Map<String, RelOptMaterialization> qualifiedNameToView =
new HashMap<String, RelOptMaterialization>();
for (RelOptMaterialization materialization : cachedViews) {
- qualifiedNameToView.put(materialization.table.getQualifiedName().get(0), materialization);
+ qualifiedNameToView.put(materialization.qualifiedTableName.get(0), materialization);
}
for (String table : tables) {
// Compose qualified name
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 154ea68..ba46ff7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -188,7 +188,8 @@ public final class HiveMaterializedViewsRegistry {
" ignored; error parsing original query");
return null;
}
- RelOptMaterialization materialization = new RelOptMaterialization(tableRel, queryRel, null);
+ RelOptMaterialization materialization = new RelOptMaterialization(tableRel, queryRel,
+ null, tableRel.getTable().getQualifiedName());
cq.put(vk, materialization);
if (LOG.isDebugEnabled()) {
LOG.debug("Cached materialized view for rewriting: " + tableRel.getTable().getQualifiedName());
@@ -314,10 +315,13 @@ public final class HiveMaterializedViewsRegistry {
}
metrics.add(field.getName());
}
+ // TODO: Default interval will be an Interval once Calcite 1.15.0 is released.
+ // We will need to update the type of this list.
List<LocalInterval> intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false),
- dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals);
+ dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN,
+ intervals, null, null);
final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
optTable, viewTable.getTableName(), null, false, false);
tableRel = DruidQuery.create(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
index 4a248d6..8c4b690 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
@@ -62,6 +62,8 @@ import org.apache.hadoop.hive.ql.udf.UDFDateFloorSecond;
import org.apache.hadoop.hive.ql.udf.UDFDateFloorWeek;
import org.apache.hadoop.hive.ql.udf.UDFDateFloorYear;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEpochMilli;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -158,9 +160,7 @@ public class SortedDynPartitionTimeGranularityOptimizer extends Transform {
fsParent.getChildOperators().clear();
// Create SelectOp with granularity column
- Operator<? extends OperatorDesc> granularitySelOp = getGranularitySelOp(fsParent,
- segmentGranularity
- );
+ Operator<? extends OperatorDesc> granularitySelOp = getGranularitySelOp(fsParent, segmentGranularity);
// Create ReduceSinkOp operator
ArrayList<ColumnInfo> parentCols = Lists.newArrayList(granularitySelOp.getSchema().getSignature());
@@ -230,17 +230,17 @@ public class SortedDynPartitionTimeGranularityOptimizer extends Transform {
descs.add(columnDesc);
colNames.add(columnDesc.getExprString());
if (columnDesc.getTypeInfo().getCategory() == ObjectInspector.Category.PRIMITIVE
- && ((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory() == PrimitiveCategory.TIMESTAMP) {
+ && ((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory() == PrimitiveCategory.TIMESTAMPLOCALTZ) {
if (timestampPos != -1) {
- throw new SemanticException("Multiple columns with timestamp type on query result; "
- + "could not resolve which one is the timestamp column");
+ throw new SemanticException("Multiple columns with timestamp with local time-zone type on query result; "
+ + "could not resolve which one is the timestamp with local time-zone column");
}
timestampPos = i;
}
}
if (timestampPos == -1) {
- throw new SemanticException("No column with timestamp type on query result; "
- + "one column should be of timestamp type");
+ throw new SemanticException("No column with timestamp with local time-zone type on query result; "
+ + "one column should be of timestamp with local time-zone type");
}
RowSchema selRS = new RowSchema(fsParent.getSchema());
// Granularity (partition) column
@@ -279,11 +279,33 @@ public class SortedDynPartitionTimeGranularityOptimizer extends Transform {
default:
throw new SemanticException("Granularity for Druid segment not recognized");
}
+
+ // Timestamp column type in Druid is timestamp with local time-zone, as it represents
+ // a specific instant in time. Thus, we have this value and we need to extract the
+ // granularity to split the data when we are storing it in Druid. However, Druid stores
+ // the data in UTC. Thus, we need to apply the following logic on the data to extract
+ // the granularity correctly:
+ // 1) Read the timestamp with local time-zone value.
+ // 2) Extract UTC epoch (millis) from timestamp with local time-zone.
+ // 3) Cast the long to a timestamp.
+ // 4) Apply the granularity function on the timestamp value.
+ // That way, '2010-01-01 00:00:00 UTC' and '2009-12-31 16:00:00 PST' (same instant)
+ // will end up in the same Druid segment.
+
+ // #1 - Read the column value
ExprNodeDesc expr = new ExprNodeColumnDesc(parentCols.get(timestampPos));
- descs.add(new ExprNodeGenericFuncDesc(
- TypeInfoFactory.timestampTypeInfo,
- new GenericUDFBridge(udfName, false, udfClass.getName()),
- Lists.newArrayList(expr)));
+ // #2 - UTC epoch for instant
+ ExprNodeGenericFuncDesc f1 = new ExprNodeGenericFuncDesc(
+ TypeInfoFactory.longTypeInfo, new GenericUDFEpochMilli(), Lists.newArrayList(expr));
+ // #3 - Cast to timestamp
+ ExprNodeGenericFuncDesc f2 = new ExprNodeGenericFuncDesc(
+ TypeInfoFactory.timestampTypeInfo, new GenericUDFTimestamp(), Lists.newArrayList(f1));
+ // #4 - We apply the granularity function
+ ExprNodeGenericFuncDesc f3 = new ExprNodeGenericFuncDesc(
+ TypeInfoFactory.timestampTypeInfo,
+ new GenericUDFBridge(udfName, false, udfClass.getName()),
+ Lists.newArrayList(f2));
+ descs.add(f3);
colNames.add(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME);
// Add granularity to the row schema
ColumnInfo ci = new ColumnInfo(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, TypeInfoFactory.timestampTypeInfo,
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index f70aa75..692d7a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -964,7 +964,7 @@ public class HiveCalciteUtil {
aggArgRelDTBldr.build(), aggFnRetType);
List<Integer> argList = new ArrayList<Integer>();
argList.add(pos);
- return new AggregateCall(aggFunction, false, argList, aggFnRetType, null);
+ return AggregateCall.create(aggFunction, false, argList, -1, aggFnRetType, null);
}
public static HiveTableFunctionScan createUDTFForSetOp(RelOptCluster cluster, RelNode input)
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java
index 756b671..60c3bdd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveConfPlannerContext.java
@@ -17,10 +17,6 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite;
-import org.apache.calcite.rel.RelNode;
-
-import java.util.Set;
-
public class HiveConfPlannerContext{
private boolean isCorrelatedColumns;
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java
index 823b099..afebad3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java
@@ -44,10 +44,10 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
@@ -193,7 +193,11 @@ public class HiveRelFactories {
public RelNode createAggregate(RelNode child, boolean indicator,
ImmutableBitSet groupSet, ImmutableList<ImmutableBitSet> groupSets,
List<AggregateCall> aggCalls) {
- return new HiveAggregate(child.getCluster(), child.getTraitSet(), child, indicator,
+ if (indicator) {
+ throw new IllegalStateException("Hive does not support indicator columns but Calcite "
+ + "created an Aggregate operator containing them");
+ }
+ return new HiveAggregate(child.getCluster(), child.getTraitSet(), child,
groupSet, groupSets, aggCalls);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveType.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveType.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveType.java
deleted file mode 100644
index 31a628b..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveType.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.optimizer.calcite;
-
-import org.apache.calcite.sql.type.AbstractSqlType;
-import org.apache.calcite.sql.type.SqlTypeName;
-
-/**
- * Hive-specific type.
- *
- * TODO: Created to represent timestamp with time-zone type.
- * It can be removed once the type exists in Calcite.
- */
-public class HiveType extends AbstractSqlType {
- private final Class clazz;
-
- public HiveType(Class clazz) {
- super(SqlTypeName.NULL, true, null);
- this.clazz = clazz;
- computeDigest();
- }
-
- protected void generateTypeString(StringBuilder sb, boolean withDetail) {
- sb.append("HiveType(");
- sb.append(clazz);
- sb.append(")");
- }
-
- public Class getTypeClass() {
- return clazz;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java
index dad783d..bbc26bc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java
@@ -36,6 +36,7 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl {
private static final int DEFAULT_CHAR_PRECISION = 255;
private static final int MAX_BINARY_PRECISION = Integer.MAX_VALUE;
private static final int MAX_TIMESTAMP_PRECISION = 9;
+ private static final int MAX_TIMESTAMP_WITH_LOCAL_TIME_ZONE_PRECISION = 15; // Up to nanos
@Override
public int getMaxScale(SqlTypeName typeName) {
@@ -70,6 +71,7 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl {
case VARBINARY:
case TIME:
case TIMESTAMP:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
return getMaxPrecision(typeName);
case CHAR:
return DEFAULT_CHAR_PRECISION;
@@ -110,6 +112,8 @@ public class HiveTypeSystemImpl extends RelDataTypeSystemImpl {
case TIME:
case TIMESTAMP:
return MAX_TIMESTAMP_PRECISION;
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return MAX_TIMESTAMP_WITH_LOCAL_TIME_ZONE_PRECISION;
case INTERVAL_YEAR:
case INTERVAL_MONTH:
case INTERVAL_YEAR_MONTH:
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java
index 75b7ad2..661427e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlCountAggFunction.java
@@ -17,19 +17,29 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.functions;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlSplittableAggFunction;
import org.apache.calcite.sql.SqlSplittableAggFunction.CountSplitter;
+import org.apache.calcite.sql.SqlSplittableAggFunction.Registry;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.type.SqlOperandTypeChecker;
import org.apache.calcite.sql.type.SqlOperandTypeInference;
import org.apache.calcite.sql.type.SqlReturnTypeInference;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ImmutableIntList;
+import com.google.common.collect.ImmutableList;
+
public class HiveSqlCountAggFunction extends SqlAggFunction implements CanAggregateDistinct {
final boolean isDistinct;
@@ -65,15 +75,49 @@ public class HiveSqlCountAggFunction extends SqlAggFunction implements CanAggreg
return super.unwrap(clazz);
}
+ // We need to override these methods due to difference in nullability between Hive and
+ // Calcite for the return types of the aggregation (in particular, for COUNT and SUM0).
+ // TODO: We should close the semantics gaps between Hive and Calcite for nullability of
+ // aggregation calls return types. This might be useful to trigger some additional
+ // rewriting rules that would remove unnecessary predicates, etc.
class HiveCountSplitter extends CountSplitter {
@Override
public AggregateCall other(RelDataTypeFactory typeFactory, AggregateCall e) {
-
return AggregateCall.create(
new HiveSqlCountAggFunction(isDistinct, returnTypeInference, operandTypeInference, operandTypeChecker),
false, ImmutableIntList.of(), -1,
typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BIGINT), true), "count");
}
+
+ @Override
+ public AggregateCall topSplit(RexBuilder rexBuilder,
+ Registry<RexNode> extra, int offset, RelDataType inputRowType,
+ AggregateCall aggregateCall, int leftSubTotal, int rightSubTotal) {
+ final List<RexNode> merges = new ArrayList<>();
+ if (leftSubTotal >= 0) {
+ merges.add(
+ rexBuilder.makeInputRef(aggregateCall.type, leftSubTotal));
+ }
+ if (rightSubTotal >= 0) {
+ merges.add(
+ rexBuilder.makeInputRef(aggregateCall.type, rightSubTotal));
+ }
+ RexNode node;
+ switch (merges.size()) {
+ case 1:
+ node = merges.get(0);
+ break;
+ case 2:
+ node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, merges);
+ break;
+ default:
+ throw new AssertionError("unexpected count " + merges);
+ }
+ int ordinal = extra.register(node);
+ return AggregateCall.create(
+ new HiveSqlSumEmptyIsZeroAggFunction(isDistinct, returnTypeInference, operandTypeInference, operandTypeChecker),
+ false, ImmutableList.of(ordinal), -1, aggregateCall.type, aggregateCall.name);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java
new file mode 100644
index 0000000..31f6022
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/functions/HiveSqlSumEmptyIsZeroAggFunction.java
@@ -0,0 +1,59 @@
+package org.apache.hadoop.hive.ql.optimizer.calcite.functions;
+
+import java.util.List;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlSplittableAggFunction;
+import org.apache.calcite.sql.type.SqlOperandTypeChecker;
+import org.apache.calcite.sql.type.SqlOperandTypeInference;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ * <code>Sum0</code> is an aggregator which returns the sum of the values which
+ * go into it like <code>Sum</code>. It differs in that when no non null values
+ * are applied zero is returned instead of null. Can be used along with <code>
+ * Count</code> to implement <code>Sum</code>.
+ */
+public class HiveSqlSumEmptyIsZeroAggFunction extends SqlAggFunction {
+ //~ Constructors -----------------------------------------------------------
+
+ public HiveSqlSumEmptyIsZeroAggFunction(boolean isDistinct, SqlReturnTypeInference returnTypeInference,
+ SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker) {
+ super("$SUM0",
+ null,
+ SqlKind.SUM0,
+ returnTypeInference,
+ operandTypeInference,
+ operandTypeChecker,
+ SqlFunctionCategory.NUMERIC,
+ false,
+ false);
+ }
+
+ //~ Methods ----------------------------------------------------------------
+
+ public List<RelDataType> getParameterTypes(RelDataTypeFactory typeFactory) {
+ return ImmutableList.of(
+ typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.ANY), true));
+ }
+
+ public RelDataType getReturnType(RelDataTypeFactory typeFactory) {
+ return typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.ANY), true);
+ }
+
+ @Override public <T> T unwrap(Class<T> clazz) {
+ if (clazz == SqlSplittableAggFunction.class) {
+ return clazz.cast(SqlSplittableAggFunction.SumSplitter.INSTANCE);
+ }
+ return super.unwrap(clazz);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
index fa51b4e..1be15bc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveAggregate.java
@@ -34,8 +34,8 @@ import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttle;
+import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
import com.google.common.collect.Sets;
@@ -45,18 +45,22 @@ public class HiveAggregate extends Aggregate implements HiveRelNode {
public HiveAggregate(RelOptCluster cluster, RelTraitSet traitSet, RelNode child,
- boolean indicator, ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets,
+ ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets,
List<AggregateCall> aggCalls) {
- super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, indicator, groupSet,
- groupSets, aggCalls);
+ super(cluster, TraitsUtil.getDefaultTraitSet(cluster), child, false,
+ groupSet, groupSets, aggCalls);
}
@Override
public Aggregate copy(RelTraitSet traitSet, RelNode input,
boolean indicator, ImmutableBitSet groupSet,
List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls) {
- return new HiveAggregate(getCluster(), traitSet, input, indicator, groupSet,
- groupSets, aggCalls);
+ if (indicator) {
+ throw new IllegalStateException("Hive does not support indicator columns but tried "
+ + "to create an Aggregate operator containing them");
+ }
+ return new HiveAggregate(getCluster(), traitSet, input,
+ groupSet, groupSets, aggCalls);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
index 2fb9508..adcda26 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
@@ -17,17 +17,17 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
-import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlAbstractGroupFunction;
import org.apache.calcite.sql.type.InferTypes;
import org.apache.calcite.sql.type.OperandTypes;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-public class HiveGroupingID extends SqlAggFunction {
+public class HiveGroupingID extends SqlAbstractGroupFunction {
- public static final SqlAggFunction INSTANCE =
+ public static final HiveGroupingID INSTANCE =
new HiveGroupingID();
private HiveGroupingID() {
@@ -36,7 +36,7 @@ public class HiveGroupingID extends SqlAggFunction {
ReturnTypes.INTEGER,
InferTypes.BOOLEAN,
OperandTypes.NILADIC,
- SqlFunctionCategory.USER_DEFINED_FUNCTION);
+ SqlFunctionCategory.SYSTEM);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java
index 21b46cf..a09e7f6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateJoinTransposeRule.java
@@ -33,9 +33,9 @@ import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
-import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.rules.AggregateJoinTransposeRule;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
@@ -43,6 +43,8 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlSplittableAggFunction;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.Mappings;
@@ -64,29 +66,17 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
/** Extended instance of the rule that can push down aggregate functions. */
public static final HiveAggregateJoinTransposeRule INSTANCE =
- new HiveAggregateJoinTransposeRule(HiveAggregate.class, HiveRelFactories.HIVE_AGGREGATE_FACTORY,
- HiveJoin.class, HiveRelFactories.HIVE_JOIN_FACTORY, HiveRelFactories.HIVE_PROJECT_FACTORY,
- true);
-
- private final RelFactories.AggregateFactory aggregateFactory;
-
- private final RelFactories.JoinFactory joinFactory;
-
- private final RelFactories.ProjectFactory projectFactory;
+ new HiveAggregateJoinTransposeRule(HiveAggregate.class, HiveJoin.class,
+ HiveRelFactories.HIVE_BUILDER, true);
private final boolean allowFunctions;
/** Creates an AggregateJoinTransposeRule that may push down functions. */
private HiveAggregateJoinTransposeRule(Class<? extends Aggregate> aggregateClass,
- RelFactories.AggregateFactory aggregateFactory,
Class<? extends Join> joinClass,
- RelFactories.JoinFactory joinFactory,
- RelFactories.ProjectFactory projectFactory,
+ RelBuilderFactory relBuilderFactory,
boolean allowFunctions) {
- super(aggregateClass, aggregateFactory, joinClass, joinFactory, projectFactory, true);
- this.aggregateFactory = aggregateFactory;
- this.joinFactory = joinFactory;
- this.projectFactory = projectFactory;
+ super(aggregateClass, joinClass, relBuilderFactory, true);
this.allowFunctions = allowFunctions;
}
@@ -95,6 +85,7 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
final Aggregate aggregate = call.rel(0);
final Join join = call.rel(1);
final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
+ final RelBuilder relBuilder = call.builder();
// If any aggregate functions do not support splitting, bail out
// If any aggregate call has a filter, bail out
@@ -119,8 +110,8 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
}
// Do the columns used by the join appear in the output of the aggregate?
- final RelMetadataQuery mq = call.getMetadataQuery();
final ImmutableBitSet aggregateColumns = aggregate.getGroupSet();
+ final RelMetadataQuery mq = call.getMetadataQuery();
final ImmutableBitSet keyColumns = keyColumns(aggregateColumns,
mq.getPulledUpPredicates(join).pulledUpPredicates);
final ImmutableBitSet joinColumns =
@@ -171,8 +162,8 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
// any functions experiencing a cartesian product effect.
//
// But finding out whether the input is already unique requires a call
- // to areColumnsUnique that currently (until [CALCITE-794] "Detect
- // cycles when computing statistics" is fixed) places a heavy load on
+ // to areColumnsUnique that currently (until [CALCITE-1048] "Make
+ // metadata more robust" is fixed) places a heavy load on
// the metadata system.
//
// So we choose to imagine the the input is already unique, which is
@@ -213,8 +204,10 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
+ belowAggCallRegistry.register(call1));
}
}
- side.newInput = aggregateFactory.createAggregate(joinInput, false,
- belowAggregateKey, null, belowAggCalls);
+ side.newInput = relBuilder.push(joinInput)
+ .aggregate(relBuilder.groupKey(belowAggregateKey, null),
+ belowAggCalls)
+ .build();
}
offset += fieldCount;
belowOffset += side.newInput.getRowType().getFieldCount();
@@ -231,7 +224,6 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
// Update condition
final Mapping mapping = (Mapping) Mappings.target(
new Function<Integer, Integer>() {
- @Override
public Integer apply(Integer a0) {
return map.get(a0);
}
@@ -242,9 +234,9 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
RexUtil.apply(mapping, join.getCondition());
// Create new join
- RelNode newJoin = joinFactory.createJoin(sides.get(0).newInput,
- sides.get(1).newInput, newCondition, join.getJoinType(),
- join.getVariablesStopped(), join.isSemiJoinDone());
+ relBuilder.push(sides.get(0).newInput)
+ .push(sides.get(1).newInput)
+ .join(join.getJoinType(), newCondition);
// Aggregate above to sum up the sub-totals
final List<AggregateCall> newAggCalls = new ArrayList<>();
@@ -252,7 +244,8 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
aggregate.getGroupCount() + aggregate.getIndicatorCount();
final int newLeftWidth = sides.get(0).newInput.getRowType().getFieldCount();
final List<RexNode> projects =
- new ArrayList<>(rexBuilder.identityProjects(newJoin.getRowType()));
+ new ArrayList<>(
+ rexBuilder.identityProjects(relBuilder.peek().getRowType()));
for (Ord<AggregateCall> aggCall : Ord.zip(aggregate.getAggCallList())) {
final SqlAggFunction aggregation = aggCall.e.getAggregation();
final SqlSplittableAggFunction splitter =
@@ -262,47 +255,45 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
final Integer rightSubTotal = sides.get(1).split.get(aggCall.i);
newAggCalls.add(
splitter.topSplit(rexBuilder, registry(projects),
- groupIndicatorCount, newJoin.getRowType(), aggCall.e,
+ groupIndicatorCount, relBuilder.peek().getRowType(), aggCall.e,
leftSubTotal == null ? -1 : leftSubTotal,
rightSubTotal == null ? -1 : rightSubTotal + newLeftWidth));
}
- RelNode r = newJoin;
- b:
- if (allColumnsInAggregate && newAggCalls.isEmpty() &&
- RelOptUtil.areRowTypesEqual(r.getRowType(), aggregate.getRowType(), false)) {
- // no need to aggregate
- } else {
- r = RelOptUtil.createProject(r, projects, null, true,
- relBuilderFactory.create(aggregate.getCluster(), null));
- if (allColumnsInAggregate) {
- // let's see if we can convert
- List<RexNode> projects2 = new ArrayList<>();
- for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) {
- projects2.add(rexBuilder.makeInputRef(r, key));
- }
- for (AggregateCall newAggCall : newAggCalls) {
- final SqlSplittableAggFunction splitter =
- newAggCall.getAggregation()
- .unwrap(SqlSplittableAggFunction.class);
- if (splitter != null) {
- projects2.add(
- splitter.singleton(rexBuilder, r.getRowType(), newAggCall));
- }
- }
- if (projects2.size()
- == aggregate.getGroupSet().cardinality() + newAggCalls.size()) {
- // We successfully converted agg calls into projects.
- r = RelOptUtil.createProject(r, projects2, null, true,
- relBuilderFactory.create(aggregate.getCluster(), null));
- break b;
+
+ relBuilder.project(projects);
+
+ boolean aggConvertedToProjects = false;
+ if (allColumnsInAggregate) {
+ // let's see if we can convert aggregate into projects
+ List<RexNode> projects2 = new ArrayList<>();
+ for (int key : Mappings.apply(mapping, aggregate.getGroupSet())) {
+ projects2.add(relBuilder.field(key));
+ }
+ for (AggregateCall newAggCall : newAggCalls) {
+ final SqlSplittableAggFunction splitter =
+ newAggCall.getAggregation().unwrap(SqlSplittableAggFunction.class);
+ if (splitter != null) {
+ final RelDataType rowType = relBuilder.peek().getRowType();
+ projects2.add(splitter.singleton(rexBuilder, rowType, newAggCall));
}
}
- r = aggregateFactory.createAggregate(r, aggregate.indicator,
- Mappings.apply(mapping, aggregate.getGroupSet()),
- Mappings.apply2(mapping, aggregate.getGroupSets()), newAggCalls);
+ if (projects2.size()
+ == aggregate.getGroupSet().cardinality() + newAggCalls.size()) {
+ // We successfully converted agg calls into projects.
+ relBuilder.project(projects2);
+ aggConvertedToProjects = true;
+ }
+ }
+
+ if (!aggConvertedToProjects) {
+ relBuilder.aggregate(
+ relBuilder.groupKey(Mappings.apply(mapping, aggregate.getGroupSet()),
+ Mappings.apply2(mapping, aggregate.getGroupSets())),
+ newAggCalls);
}
// Make a cost based decision to pick cheaper plan
+ RelNode r = relBuilder.build();
RelOptCost afterCost = mq.getCumulativeCost(r);
RelOptCost beforeCost = mq.getCumulativeCost(aggregate);
if (afterCost.isLt(beforeCost)) {
@@ -316,8 +307,8 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
private static ImmutableBitSet keyColumns(ImmutableBitSet aggregateColumns,
ImmutableList<RexNode> predicates) {
SortedMap<Integer, BitSet> equivalence = new TreeMap<>();
- for (RexNode pred : predicates) {
- populateEquivalences(equivalence, pred);
+ for (RexNode predicate : predicates) {
+ populateEquivalences(equivalence, predicate);
}
ImmutableBitSet keyColumns = aggregateColumns;
for (Integer aggregateColumn : aggregateColumns) {
@@ -358,10 +349,9 @@ public class HiveAggregateJoinTransposeRule extends AggregateJoinTransposeRule {
/** Creates a {@link org.apache.calcite.sql.SqlSplittableAggFunction.Registry}
* that is a view of a list. */
- private static <E> SqlSplittableAggFunction.Registry<E>
- registry(final List<E> list) {
+ private static <E> SqlSplittableAggFunction.Registry<E> registry(
+ final List<E> list) {
return new SqlSplittableAggFunction.Registry<E>() {
- @Override
public int register(E e) {
int i = list.indexOf(e);
if (i < 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
index 7c3b4b0..b5f1775 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
@@ -20,16 +20,17 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Aggregate.Group;
import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
@@ -48,15 +49,13 @@ import com.google.common.collect.Lists;
* <p>In some cases, this rule has the effect of trimming: the aggregate will
* use fewer columns than the project did.
*/
-public class HiveAggregateProjectMergeRule extends RelOptRule {
+public class HiveAggregateProjectMergeRule extends AggregateProjectMergeRule {
public static final HiveAggregateProjectMergeRule INSTANCE =
new HiveAggregateProjectMergeRule();
/** Private constructor. */
private HiveAggregateProjectMergeRule() {
- super(
- operand(HiveAggregate.class,
- operand(HiveProject.class, any())));
+ super(HiveAggregate.class, HiveProject.class, HiveRelFactories.HIVE_BUILDER);
}
@Override
@@ -72,17 +71,16 @@ public class HiveAggregateProjectMergeRule extends RelOptRule {
return super.matches(call);
}
- @Override
public void onMatch(RelOptRuleCall call) {
final HiveAggregate aggregate = call.rel(0);
final HiveProject project = call.rel(1);
- RelNode x = apply(aggregate, project);
+ RelNode x = apply(call, aggregate, project);
if (x != null) {
call.transformTo(x);
}
}
- public static RelNode apply(HiveAggregate aggregate,
+ public static RelNode apply(RelOptRuleCall call, HiveAggregate aggregate,
HiveProject project) {
final List<Integer> newKeys = Lists.newArrayList();
final Map<Integer, Integer> map = new HashMap<>();
@@ -100,7 +98,7 @@ public class HiveAggregateProjectMergeRule extends RelOptRule {
final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map);
ImmutableList<ImmutableBitSet> newGroupingSets = null;
- if (aggregate.indicator) {
+ if (aggregate.getGroupType() != Group.SIMPLE) {
newGroupingSets =
ImmutableBitSet.ORDERING.immutableSortedCopy(
ImmutableBitSet.permute(aggregate.getGroupSets(), map));
@@ -139,7 +137,8 @@ public class HiveAggregateProjectMergeRule extends RelOptRule {
// Add a project if the group set is not in the same order or
// contains duplicates.
- RelNode rel = newAggregate;
+ final RelBuilder relBuilder = call.builder();
+ relBuilder.push(newAggregate);
if (!newKeys.equals(newGroupSet.asList())) {
final List<Integer> posList = Lists.newArrayList();
for (int newKey : newKeys) {
@@ -155,14 +154,12 @@ public class HiveAggregateProjectMergeRule extends RelOptRule {
i < newAggregate.getRowType().getFieldCount(); i++) {
posList.add(i);
}
- rel = HiveRelOptUtil.createProject(
- HiveRelFactories.HIVE_BUILDER.create(aggregate.getCluster(), null),
- rel, posList);
-
+ relBuilder.project(relBuilder.fields(posList));
}
- return rel;
+ return relBuilder.build();
}
+
}
-// End AggregateProjectMergeRule.java
+// End HiveAggregateProjectMergeRule.java
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java
index b63ea02..5b72dbd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExceptRewriteRule.java
@@ -27,7 +27,6 @@ import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
@@ -35,17 +34,13 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlOperator;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExcept;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIntersect;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
@@ -53,12 +48,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.calcite.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,7 +159,7 @@ public class HiveExceptRewriteRule extends RelOptRule {
final ImmutableBitSet groupSet = ImmutableBitSet.of(groupSetPositions);
HiveRelNode aggregateRel = new HiveAggregate(cluster,
- cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, false, groupSet, null,
+ cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, groupSet, null,
aggregateCalls);
// the schema after GB is like this
@@ -270,7 +260,7 @@ public class HiveExceptRewriteRule extends RelOptRule {
TypeInfoFactory.longTypeInfo, input.getRowType().getFieldList().size(), aggFnRetType);
aggregateCalls.add(aggregateCall);
return new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel,
- false, groupSet, null, aggregateCalls);
+ groupSet, null, aggregateCalls);
}
private RexNode multiply(RexNode r1, RexNode r2, RelOptCluster cluster, RexBuilder rexBuilder)
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
index 81bb730..89c5c23 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
@@ -268,7 +268,7 @@ public final class HiveExpandDistinctAggregatesRule extends RelOptRule {
aggregateCalls.add(aggregateCall);
}
Aggregate aggregate = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel,
- false, ImmutableBitSet.of(), null, aggregateCalls);
+ ImmutableBitSet.of(), null, aggregateCalls);
// create the project after GB. For those repeated values, e.g., select
// count(distinct x, y), count(distinct y, x), we find the correct mapping.
@@ -330,7 +330,7 @@ public final class HiveExpandDistinctAggregatesRule extends RelOptRule {
.createSqlType(SqlTypeName.INTEGER), HiveGroupingID.INSTANCE.getName());
aggregateCalls.add(aggCall);
return new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
- aggregate.getInput(), true, groupSet, origGroupSets, aggregateCalls);
+ aggregate.getInput(), groupSet, origGroupSets, aggregateCalls);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java
index 5b0a7d7..a5d950a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveIntersectRewriteRule.java
@@ -27,18 +27,14 @@ import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
-import org.apache.calcite.rel.core.Intersect;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
@@ -52,16 +48,10 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.calcite.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.esotericsoftware.minlog.Log;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
@@ -140,7 +130,7 @@ public class HiveIntersectRewriteRule extends RelOptRule {
aggregateCalls.add(aggregateCall);
HiveRelNode aggregateRel = new HiveAggregate(cluster,
- cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, false, groupSet, null,
+ cluster.traitSetOf(HiveRelNode.CONVENTION), gbInputRel, groupSet, null,
aggregateCalls);
bldr.add(aggregateRel);
}
@@ -173,7 +163,7 @@ public class HiveIntersectRewriteRule extends RelOptRule {
final ImmutableBitSet groupSet = ImmutableBitSet.of(groupSetPositions);
HiveRelNode aggregateRel = new HiveAggregate(cluster,
- cluster.traitSetOf(HiveRelNode.CONVENTION), union, false, groupSet, null, aggregateCalls);
+ cluster.traitSetOf(HiveRelNode.CONVENTION), union, groupSet, null, aggregateCalls);
// add a filter count(c) = #branches
int countInd = cInd;
[2/4] hive git commit: HIVE-17672: Upgrade Calcite version to 1.14
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/druid_basic2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_basic2.q.out b/ql/src/test/results/clientpositive/druid_basic2.q.out
index 5d6862c..753ccb4 100644
--- a/ql/src/test/results/clientpositive/druid_basic2.q.out
+++ b/ql/src/test/results/clientpositive/druid_basic2.q.out
@@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@druid_table_1
# col_name data_type comment
-__time timestamp from deserializer
+__time timestamp with local time zone from deserializer
robot string from deserializer
namespace string from deserializer
anonymous string from deserializer
@@ -271,7 +271,7 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot","language"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
@@ -280,7 +280,7 @@ STAGE PLANS:
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -297,7 +297,7 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["language"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
@@ -306,7 +306,7 @@ STAGE PLANS:
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -439,7 +439,7 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"filter":{"type":"selector","dimension":"language","value":"en"},"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
@@ -448,7 +448,7 @@ STAGE PLANS:
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -465,7 +465,7 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"filter":{"type":"selector","dimension":"language","value":"en"},"dimensions":["robot"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
@@ -474,7 +474,7 @@ STAGE PLANS:
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -553,12 +553,12 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
GatherStats: false
Select Operator
- expressions: robot (type: string), floor_day (type: timestamp), $f3 (type: float), $f4 (type: float), UDFToInteger(robot) (type: int)
+ expressions: robot (type: string), floor_day (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float), UDFToInteger(robot) (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col5
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
@@ -569,7 +569,7 @@ STAGE PLANS:
tag: -1
TopN: 10
TopN Hash Memory Usage: 0.1
- value expressions: _col0 (type: string), _col1 (type: timestamp), _col3 (type: float)
+ value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: float)
auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
@@ -586,16 +586,16 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
#### A masked pattern was here ####
name default.druid_table_1
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -612,16 +612,16 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"language"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"delta"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
#### A masked pattern was here ####
name default.druid_table_1
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -635,7 +635,7 @@ STAGE PLANS:
Needs Tagging: false
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: float)
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp with local time zone), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Limit
@@ -653,7 +653,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
properties:
columns _col0,_col1,_col2,_col3
- columns.types string:timestamp:float:float
+ columns.types string:timestamp with local time zone:float:float
escape.delim \
hive.serialization.extend.additional.nesting.levels true
serialization.escape.crlf true
@@ -737,32 +737,32 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: druid_table_1
- filterExpr: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean)
+ filterExpr: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean)
properties:
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean)
+ predicate: floor_day(__time) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_day(__time) (type: timestamp)
+ expressions: robot (type: string), floor_day(__time) (type: timestamp with local time zone)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- keys: _col0 (type: string), _col1 (type: timestamp)
+ keys: _col0 (type: string), _col1 (type: timestamp with local time zone)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: timestamp)
+ key expressions: _col0 (type: string), _col1 (type: timestamp with local time zone)
sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp)
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp with local time zone)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
TopN Hash Memory Usage: 0.1
Reduce Operator Tree:
Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: timestamp)
+ keys: KEY._col0 (type: string), KEY._col1 (type: timestamp with local time zone)
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -782,10 +782,10 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
TopN Hash Memory Usage: 0.1
- value expressions: _col1 (type: timestamp)
+ value expressions: _col1 (type: timestamp with local time zone)
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp)
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp with local time zone)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Limit
@@ -831,16 +831,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: druid_table_1
- filterExpr: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean)
+ filterExpr: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean)
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default"},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 AND 1999-11-10 00:00:00.0 (type: boolean)
+ predicate: floor_day(extract) BETWEEN 1999-11-01 00:00:00.0 US/Pacific AND 1999-11-10 00:00:00.0 US/Pacific (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), extract (type: timestamp)
+ expressions: robot (type: string), extract (type: timestamp with local time zone)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
@@ -848,10 +848,10 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
TopN Hash Memory Usage: 0.1
- value expressions: _col1 (type: timestamp)
+ value expressions: _col1 (type: timestamp with local time zone)
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp)
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: timestamp with local time zone)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Limit
@@ -898,11 +898,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"}]},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1999-11-01T00:00:00.000/1999-11-10T00:00:00.001"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_day","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"day","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"}]},"aggregations":[{"type":"longSum","name":"dummy_agg","fieldName":"dummy_agg"}],"intervals":["1999-11-01T08:00:00.000/1999-11-10T08:00:00.001"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_day (type: timestamp)
+ expressions: robot (type: string), floor_day (type: timestamp with local time zone)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -935,20 +935,20 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
GatherStats: false
Select Operator
- expressions: __time (type: timestamp), robot (type: string), language (type: string), added (type: float), delta (type: float)
+ expressions: __time (type: timestamp with local time zone), robot (type: string), language (type: string), added (type: float), delta (type: float)
outputColumnNames: __time, robot, language, added, delta
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: max(added), sum(delta)
- keys: robot (type: string), language (type: string), floor_day(__time) (type: timestamp)
+ keys: robot (type: string), language (type: string), floor_day(__time) (type: timestamp with local time zone)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp)
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp with local time zone)
null sort order: aaa
sort order: +++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp)
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: timestamp with local time zone)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
tag: -1
value expressions: _col3 (type: float), _col4 (type: double)
@@ -968,14 +968,14 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
#### A masked pattern was here ####
name default.druid_table_1
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -992,14 +992,14 @@ STAGE PLANS:
column.name.delimiter ,
columns __time,robot,namespace,anonymous,unpatrolled,page,language,newpage,user,count,added,delta,variation,deleted
columns.comments 'from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer','from deserializer'
- columns.types timestamp:string:string:string:string:string:string:string:string:float:float:float:float:float
+ columns.types timestamp with local time zone:string:string:string:string:string:string:string:string:float:float:float:float:float
druid.datasource wikipedia
#### A masked pattern was here ####
name default.druid_table_1
numFiles 0
numRows 0
rawDataSize 0
- serialization.ddl struct druid_table_1 { timestamp __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
+ serialization.ddl struct druid_table_1 { timestamp with local time zone __time, string robot, string namespace, string anonymous, string unpatrolled, string page, string language, string newpage, string user, float count, float added, float delta, float variation, float deleted}
serialization.format 1
serialization.lib org.apache.hadoop.hive.druid.QTestDruidSerDe
storage_handler org.apache.hadoop.hive.druid.QTestDruidStorageHandler
@@ -1014,12 +1014,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), sum(VALUE._col1)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: timestamp)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: timestamp with local time zone)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col2 (type: timestamp), _col3 (type: float), _col4 (type: double)
+ expressions: _col0 (type: string), _col2 (type: timestamp with local time zone), _col3 (type: float), _col4 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
@@ -1033,7 +1033,7 @@ STAGE PLANS:
properties:
column.name.delimiter ,
columns _col0,_col1,_col2,_col3
- columns.types string,timestamp,float,double
+ columns.types string,timestamp with local time zone,float,double
escape.delim \
serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
@@ -1054,7 +1054,7 @@ STAGE PLANS:
tag: -1
TopN: 10
TopN Hash Memory Usage: 0.1
- value expressions: _col0 (type: string), _col1 (type: timestamp), _col3 (type: double)
+ value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: double)
auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
@@ -1067,7 +1067,7 @@ STAGE PLANS:
properties:
column.name.delimiter ,
columns _col0,_col1,_col2,_col3
- columns.types string,timestamp,float,double
+ columns.types string,timestamp with local time zone,float,double
escape.delim \
serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
@@ -1077,7 +1077,7 @@ STAGE PLANS:
properties:
column.name.delimiter ,
columns _col0,_col1,_col2,_col3
- columns.types string,timestamp,float,double
+ columns.types string,timestamp with local time zone,float,double
escape.delim \
serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
@@ -1086,7 +1086,7 @@ STAGE PLANS:
Needs Tagging: false
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: double)
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp with local time zone), KEY.reducesinkkey1 (type: float), VALUE._col2 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Limit
@@ -1104,7 +1104,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
properties:
columns _col0,_col1,_col2,_col3
- columns.types string:timestamp:float:double
+ columns.types string:timestamp with local time zone:float:double
escape.delim \
hive.serialization.extend.additional.nesting.levels true
serialization.escape.crlf true
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/druid_intervals.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_intervals.q.out b/ql/src/test/results/clientpositive/druid_intervals.q.out
index 63f28d5..3cd28b5 100644
--- a/ql/src/test/results/clientpositive/druid_intervals.q.out
+++ b/ql/src/test/results/clientpositive/druid_intervals.q.out
@@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@druid_table_1
# col_name data_type comment
-__time timestamp from deserializer
+__time timestamp with local time zone from deserializer
robot string from deserializer
namespace string from deserializer
anonymous string from deserializer
@@ -82,7 +82,7 @@ STAGE PLANS:
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -108,11 +108,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/2012-03-01T00:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/2012-03-01T08:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -138,11 +138,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2012-03-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2012-03-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -170,11 +170,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2011-01-01T00:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2011-01-01T08:00:00.000"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -200,11 +200,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2011-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2011-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -232,11 +232,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2011-01-01T00:00:00.001","2012-01-01T00:00:00.000/2013-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2011-01-01T08:00:00.001","2012-01-01T08:00:00.000/2013-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -264,11 +264,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2012-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2012-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -294,11 +294,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2010-01-01T00:00:00.001","2011-01-01T00:00:00.000/2011-01-01T00:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2010-01-01T08:00:00.001","2011-01-01T08:00:00.000/2011-01-01T08:00:00.001"],"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp)
+ expressions: __time (type: timestamp with local time zone)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -324,11 +324,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T00:00:00.000/2010-01-01T00:00:00.001","2011-01-01T00:00:00.000/2011-01-01T00:00:00.001"],"filter":{"type":"selector","dimension":"robot","value":"user1"},"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
+ druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["2010-01-01T08:00:00.000/2010-01-01T08:00:00.001","2011-01-01T08:00:00.000/2011-01-01T08:00:00.001"],"filter":{"type":"selector","dimension":"robot","value":"user1"},"dimensions":[],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), 'user1' (type: string)
+ expressions: __time (type: timestamp with local time zone), 'user1' (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -353,16 +353,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: druid_table_1
- filterExpr: ((__time) IN (2010-01-01 00:00:00.0, 2011-01-01 00:00:00.0) or (robot = 'user1')) (type: boolean)
+ filterExpr: ((__time) IN (2010-01-01 00:00:00.0 US/Pacific, 2011-01-01 00:00:00.0 US/Pacific) or (robot = 'user1')) (type: boolean)
properties:
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot"],"metrics":[],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: ((__time) IN (2010-01-01 00:00:00.0, 2011-01-01 00:00:00.0) or (robot = 'user1')) (type: boolean)
+ predicate: ((__time) IN (2010-01-01 00:00:00.0 US/Pacific, 2011-01-01 00:00:00.0 US/Pacific) or (robot = 'user1')) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), robot (type: string)
+ expressions: __time (type: timestamp with local time zone), robot (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/druid_timeseries.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_timeseries.q.out b/ql/src/test/results/clientpositive/druid_timeseries.q.out
index 04db1e0..330c068 100644
--- a/ql/src/test/results/clientpositive/druid_timeseries.q.out
+++ b/ql/src/test/results/clientpositive/druid_timeseries.q.out
@@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@druid_table_1
# col_name data_type comment
-__time timestamp from deserializer
+__time timestamp with local time zone from deserializer
robot string from deserializer
namespace string from deserializer
anonymous string from deserializer
@@ -108,11 +108,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}}],"limitSpec":{"type":"default"},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: extract (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: extract (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -142,7 +142,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -172,7 +172,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -202,7 +202,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -232,7 +232,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -262,7 +262,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -292,7 +292,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -322,7 +322,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -352,7 +352,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -384,7 +384,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -393,16 +393,16 @@ PREHOOK: query: EXPLAIN
SELECT floor_hour(`__time`), max(added), sum(variation)
FROM druid_table_1
WHERE floor_hour(`__time`)
- BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+ BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
GROUP BY floor_hour(`__time`)
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
SELECT floor_hour(`__time`), max(added), sum(variation)
FROM druid_table_1
WHERE floor_hour(`__time`)
- BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+ BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
GROUP BY floor_hour(`__time`)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -415,34 +415,34 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: druid_table_1
- filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean)
+ filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean)
properties:
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":[],"metrics":["added","variation"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean)
+ predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: floor_hour(__time) (type: timestamp), added (type: float), variation (type: float)
+ expressions: floor_hour(__time) (type: timestamp with local time zone), added (type: float), variation (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: max(_col1), sum(_col2)
- keys: _col0 (type: timestamp)
+ keys: _col0 (type: timestamp with local time zone)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: timestamp)
+ key expressions: _col0 (type: timestamp with local time zone)
sort order: +
- Map-reduce partition columns: _col0 (type: timestamp)
+ Map-reduce partition columns: _col0 (type: timestamp with local time zone)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: float), _col2 (type: double)
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), sum(VALUE._col1)
- keys: KEY._col0 (type: timestamp)
+ keys: KEY._col0 (type: timestamp with local time zone)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -468,8 +468,8 @@ FROM
FROM druid_table_1
GROUP BY floor_hour(`__time`)
) subq
-WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
SELECT subq.h, subq.m, subq.s
@@ -479,8 +479,8 @@ FROM
FROM druid_table_1
GROUP BY floor_hour(`__time`)
) subq
-WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -492,34 +492,34 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: druid_table_1
- filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean)
+ filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean)
properties:
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":[],"metrics":["added","variation"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean)
+ predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: floor_hour(__time) (type: timestamp), added (type: float), variation (type: float)
+ expressions: floor_hour(__time) (type: timestamp with local time zone), added (type: float), variation (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: max(_col1), sum(_col2)
- keys: _col0 (type: timestamp)
+ keys: _col0 (type: timestamp with local time zone)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: timestamp)
+ key expressions: _col0 (type: timestamp with local time zone)
sort order: +
- Map-reduce partition columns: _col0 (type: timestamp)
+ Map-reduce partition columns: _col0 (type: timestamp with local time zone)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: float), _col2 (type: double)
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), sum(VALUE._col1)
- keys: KEY._col0 (type: timestamp)
+ keys: KEY._col0 (type: timestamp with local time zone)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/druid_topn.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_topn.q.out b/ql/src/test/results/clientpositive/druid_topn.q.out
index 5c31f85..3e2b477 100644
--- a/ql/src/test/results/clientpositive/druid_topn.q.out
+++ b/ql/src/test/results/clientpositive/druid_topn.q.out
@@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@druid_table_1
# col_name data_type comment
-__time timestamp from deserializer
+__time timestamp with local time zone from deserializer
robot string from deserializer
namespace string from deserializer
anonymous string from deserializer
@@ -118,11 +118,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"UTC"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), extract (type: timestamp), $f2 (type: float), $f3 (type: float)
+ expressions: robot (type: string), extract (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -152,11 +152,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"year","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"year","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_year (type: timestamp), $f2 (type: float), $f3 (type: float)
+ expressions: robot (type: string), floor_year (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -186,11 +186,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_month (type: timestamp), $f2 (type: float), $f3 (type: float)
+ expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -220,11 +220,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f4","direction":"descending","dimensionOrder":"numeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f4","direction":"descending","dimensionOrder":"numeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_month (type: timestamp), $f3 (type: float), $f4 (type: float)
+ expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -254,11 +254,11 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"UTC","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_month (type: timestamp), $f3 (type: float), $f4 (type: float)
+ expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -294,7 +294,7 @@ STAGE PLANS:
druid.query.type timeseries
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: __time (type: timestamp), $f1_0 (type: float), $f2 (type: float)
+ expressions: __time (type: timestamp with local time zone), $f1_0 (type: float), $f2 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
@@ -302,17 +302,17 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
TopN Hash Memory Usage: 0.1
- value expressions: _col0 (type: timestamp), _col1 (type: float)
+ value expressions: _col0 (type: timestamp with local time zone), _col1 (type: float)
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: timestamp), VALUE._col1 (type: float), KEY.reducesinkkey0 (type: float)
+ expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: float), KEY.reducesinkkey0 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: '1' (type: string), _col0 (type: timestamp), _col1 (type: float), _col2 (type: float)
+ expressions: '1' (type: string), _col0 (type: timestamp with local time zone), _col1 (type: float), _col2 (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
@@ -333,8 +333,8 @@ PREHOOK: query: EXPLAIN
SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation)
FROM druid_table_1
WHERE floor_hour(`__time`)
- BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+ BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
GROUP BY robot, floor_hour(`__time`)
ORDER BY m
LIMIT 100
@@ -343,8 +343,8 @@ POSTHOOK: query: EXPLAIN
SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation)
FROM druid_table_1
WHERE floor_hour(`__time`)
- BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+ BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
GROUP BY robot, floor_hour(`__time`)
ORDER BY m
LIMIT 100
@@ -360,34 +360,34 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: druid_table_1
- filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean)
+ filterExpr: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean)
properties:
druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000/3000-01-01T00:00:00.000"],"dimensions":["robot"],"metrics":["added","variation"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
druid.query.type select
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 AND 2014-01-01 00:00:00.0 (type: boolean)
+ predicate: floor_hour(__time) BETWEEN 2010-01-01 00:00:00.0 US/Pacific AND 2014-01-01 00:00:00.0 US/Pacific (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_hour(__time) (type: timestamp), added (type: float), variation (type: float)
+ expressions: robot (type: string), floor_hour(__time) (type: timestamp with local time zone), added (type: float), variation (type: float)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: max(_col2), sum(_col3)
- keys: _col0 (type: string), _col1 (type: timestamp)
+ keys: _col0 (type: string), _col1 (type: timestamp with local time zone)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: timestamp)
+ key expressions: _col0 (type: string), _col1 (type: timestamp with local time zone)
sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp)
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp with local time zone)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
value expressions: _col2 (type: float), _col3 (type: double)
Reduce Operator Tree:
Group By Operator
aggregations: max(VALUE._col0), sum(VALUE._col1)
- keys: KEY._col0 (type: string), KEY._col1 (type: timestamp)
+ keys: KEY._col0 (type: string), KEY._col1 (type: timestamp with local time zone)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
@@ -407,10 +407,10 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
TopN Hash Memory Usage: 0.1
- value expressions: _col0 (type: string), _col1 (type: timestamp), _col3 (type: double)
+ value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: double)
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp), KEY.reducesinkkey0 (type: float), VALUE._col2 (type: double)
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp with local time zone), KEY.reducesinkkey0 (type: float), VALUE._col2 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Limit
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out b/ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out
index 13eac20..fcf203e 100644
--- a/ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out
+++ b/ql/src/test/results/clientpositive/groupby_grouping_sets1.q.out
@@ -464,22 +464,21 @@ STAGE PLANS:
outputColumnNames: a
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: a (type: string), 0 (type: int)
+ keys: a (type: string)
mode: hash
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Reduce Operator Tree:
Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: int)
+ keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
File Output Operator
compressed: false
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
@@ -532,35 +531,30 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: _col0 (type: double), 0 (type: int)
+ keys: _col0 (type: double)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: double), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: double), _col1 (type: int)
+ key expressions: _col0 (type: double)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: double)
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint)
+ value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: double), KEY._col1 (type: int)
+ keys: KEY._col0 (type: double)
mode: mergepartial
- outputColumnNames: _col0, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- pruneGroupingSetId: true
- Select Operator
- expressions: _col0 (type: double), _col2 (type: bigint)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 36 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
[3/4] hive git commit: HIVE-17672: Upgrade Calcite version to 1.14
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java
index f35bf2f..4a35234 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelColumnsAlignment.java
@@ -35,6 +35,7 @@ import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.SetOp;
import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.Aggregate.Group;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
@@ -96,7 +97,7 @@ public class HiveRelColumnsAlignment implements ReflectiveVisitor {
// sort them so they respect it
LinkedHashSet<Integer> aggregateColumnsOrder = new LinkedHashSet<>();
ImmutableList.Builder<RelFieldCollation> propagateCollations = ImmutableList.builder();
- if (!rel.indicator && !collations.isEmpty()) {
+ if (rel.getGroupType() == Group.SIMPLE && !collations.isEmpty()) {
for (RelFieldCollation c : collations) {
if (c.getFieldIndex() < rel.getGroupCount()) {
// Group column found
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
index b922faa..62125f0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveRelDecorrelator.java
@@ -17,6 +17,23 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import javax.annotation.Nonnull;
+
import org.apache.calcite.linq4j.Ord;
import org.apache.calcite.linq4j.function.Function2;
import org.apache.calcite.plan.Context;
@@ -48,7 +65,6 @@ import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.logical.LogicalProject;
import org.apache.calcite.rel.metadata.RelMdUtil;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.rules.FilterCorrelateRule;
import org.apache.calcite.rel.rules.FilterJoinRule;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
import org.apache.calcite.rel.type.RelDataType;
@@ -83,12 +99,13 @@ import org.apache.calcite.util.ReflectiveVisitor;
import org.apache.calcite.util.Stacks;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.mapping.Mappings;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttleImpl;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
-import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,24 +121,6 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.collect.SortedSetMultimap;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelShuttleImpl;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Objects;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import javax.annotation.Nonnull;
/**
* NOTE: this whole logic is replicated from Calcite's RelDecorrelator
@@ -824,7 +823,7 @@ public class HiveRelDecorrelator implements ReflectiveVisitor {
}
relBuilder.push(
- new HiveAggregate(rel.getCluster(), rel.getTraitSet(), newProject, false, newGroupSet, null, newAggCalls) );
+ new HiveAggregate(rel.getCluster(), rel.getTraitSet(), newProject, newGroupSet, null, newAggCalls) );
if (!omittedConstants.isEmpty()) {
final List<RexNode> postProjects = new ArrayList<>(relBuilder.fields());
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
index eab8f6f..0525d04 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
@@ -182,6 +182,7 @@ public class ASTBuilder {
case DATE:
case TIME:
case TIMESTAMP:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
case INTERVAL_DAY:
case INTERVAL_DAY_HOUR:
case INTERVAL_DAY_MINUTE:
@@ -280,6 +281,12 @@ public class ASTBuilder {
val = "'" + literal.getValueAs(TimestampString.class).toString() + "'";
type = HiveParser.TOK_TIMESTAMPLITERAL;
break;
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ // Calcite stores timestamp with local time-zone in UTC internally, thus
+ // when we bring it back, we need to add the UTC suffix.
+ val = "'" + literal.getValueAs(TimestampString.class).toString() + " UTC'";
+ type = HiveParser.TOK_TIMESTAMPLOCALTZLITERAL;
+ break;
case INTERVAL_YEAR:
case INTERVAL_MONTH:
case INTERVAL_YEAR_MONTH: {
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
index d6b978f..f01cf21 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
@@ -63,8 +63,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExtractDate;
-import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan;
@@ -134,22 +132,23 @@ public class ASTConverter {
if (groupBy != null) {
ASTBuilder b;
boolean groupingSetsExpression = false;
- if (groupBy.indicator) {
- Group aggregateType = Aggregate.Group.induce(groupBy.getGroupSet(),
- groupBy.getGroupSets());
- if (aggregateType == Group.ROLLUP) {
+ Group aggregateType = groupBy.getGroupType();
+ switch (aggregateType) {
+ case SIMPLE:
+ b = ASTBuilder.construct(HiveParser.TOK_GROUPBY, "TOK_GROUPBY");
+ break;
+ case ROLLUP:
b = ASTBuilder.construct(HiveParser.TOK_ROLLUP_GROUPBY, "TOK_ROLLUP_GROUPBY");
- }
- else if (aggregateType == Group.CUBE) {
+ break;
+ case CUBE:
b = ASTBuilder.construct(HiveParser.TOK_CUBE_GROUPBY, "TOK_CUBE_GROUPBY");
- }
- else {
+ break;
+ case OTHER:
b = ASTBuilder.construct(HiveParser.TOK_GROUPING_SETS, "TOK_GROUPING_SETS");
groupingSetsExpression = true;
- }
- }
- else {
- b = ASTBuilder.construct(HiveParser.TOK_GROUPBY, "TOK_GROUPBY");
+ break;
+ default:
+ throw new CalciteSemanticException("Group type not recognized");
}
HiveAggregate hiveAgg = (HiveAggregate) groupBy;
@@ -201,14 +200,15 @@ public class ASTConverter {
ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT");
if (select instanceof Project) {
- if (select.getChildExps().isEmpty()) {
+ List<RexNode> childExps = ((Project) select).getChildExps();
+ if (childExps.isEmpty()) {
RexLiteral r = select.getCluster().getRexBuilder().makeExactLiteral(new BigDecimal(1));
ASTNode selectExpr = ASTBuilder.selectExpr(ASTBuilder.literal(r), "1");
b.add(selectExpr);
} else {
int i = 0;
- for (RexNode r : select.getChildExps()) {
+ for (RexNode r : childExps) {
ASTNode expr = r.accept(new RexVisitor(schema, r instanceof RexLiteral,
select.getCluster().getRexBuilder()));
String alias = select.getRowType().getFieldNames().get(i++);
@@ -770,15 +770,6 @@ public class ASTConverter {
ColumnInfo cI = src.get(i);
add(cI);
}
- // If we are using grouping sets, we add the
- // fields again, these correspond to the boolean
- // grouping in Calcite. They are not used by Hive.
- if(gBy.indicator) {
- for (int i : gBy.getGroupSet()) {
- ColumnInfo cI = src.get(i);
- add(cI);
- }
- }
List<AggregateCall> aggs = gBy.getAggCallList();
for (AggregateCall agg : aggs) {
if (agg.getAggregation() == HiveGroupingID.INSTANCE) {
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
index 3dcceab..3b42741 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
@@ -48,10 +48,11 @@ import org.apache.calcite.util.TimestampString;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
-import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveType;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -244,6 +245,15 @@ public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
case TIME:
case TIMESTAMP:
return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, null);
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ HiveConf conf;
+ try {
+ conf = Hive.get().getConf();
+ } catch (HiveException e) {
+ throw new RuntimeException(e);
+ }
+ return new ExprNodeConstantDesc(
+ TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()), null);
case BINARY:
return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, null);
case DECIMAL:
@@ -267,12 +277,7 @@ public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
case INTERVAL_MINUTE_SECOND:
case INTERVAL_SECOND:
return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, null);
- case NULL:
- case OTHER:
default:
- if (lType instanceof HiveType && ((HiveType) lType).getTypeClass() == TimestampTZ.class) {
- return new ExprNodeConstantDesc(TypeInfoFactory.timestampLocalTZTypeInfo, null);
- }
return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, null);
}
} else {
@@ -308,6 +313,17 @@ public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
case TIMESTAMP:
return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo,
Timestamp.valueOf(literal.getValueAs(TimestampString.class).toString()));
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ HiveConf conf;
+ try {
+ conf = Hive.get().getConf();
+ } catch (HiveException e) {
+ throw new RuntimeException(e);
+ }
+ // Calcite stores timestamp with local time-zone in UTC internally, thus
+ // when we bring it back, we need to add the UTC suffix.
+ return new ExprNodeConstantDesc(TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()),
+ literal.getValueAs(TimestampString.class).toString() + " UTC");
case BINARY:
return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, literal.getValue3());
case DECIMAL:
@@ -340,12 +356,7 @@ public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo,
new HiveIntervalDayTime(secsBd));
}
- case NULL:
- case OTHER:
default:
- if (lType instanceof HiveType && ((HiveType) lType).getTypeClass() == TimestampTZ.class) {
- return new ExprNodeConstantDesc(TypeInfoFactory.timestampLocalTZTypeInfo, literal.getValue3());
- }
return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, literal.getValue3());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java
index 0f6c5b5..6f4188c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveGBOpConvUtil.java
@@ -29,6 +29,7 @@ import java.util.TreeMap;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Aggregate.Group;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -173,7 +174,7 @@ public class HiveGBOpConvUtil {
}
// 2. Collect Grouping Set info
- if (aggRel.indicator) {
+ if (aggRel.getGroupType() != Group.SIMPLE) {
// 2.1 Translate Grouping set col bitset
ImmutableList<ImmutableBitSet> lstGrpSet = aggRel.getGroupSets();
int bitmap = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index 471675b..1885869 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
-import org.apache.hadoop.hive.ql.parse.*;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -40,7 +38,6 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
@@ -62,7 +59,6 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
@@ -74,8 +70,19 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
+import org.apache.hadoop.hive.ql.parse.JoinCond;
+import org.apache.hadoop.hive.ql.parse.JoinType;
+import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
+import org.apache.hadoop.hive.ql.parse.PTFTranslator;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.UnparseTranslator;
+import org.apache.hadoop.hive.ql.parse.WindowingComponentizer;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
index f8a959d..9d9bf89 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Timestamp;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
@@ -58,6 +59,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
@@ -436,7 +438,7 @@ public class RexNodeConverter {
// Calcite always needs the else clause to be defined explicitly
if (newChildRexNodeLst.size() % 2 == 0) {
newChildRexNodeLst.add(cluster.getRexBuilder().makeNullLiteral(
- newChildRexNodeLst.get(newChildRexNodeLst.size()-1).getType().getSqlTypeName()));
+ newChildRexNodeLst.get(newChildRexNodeLst.size()-1).getType()));
}
return newChildRexNodeLst;
}
@@ -687,6 +689,20 @@ public class RexNodeConverter {
rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
false);
break;
+ case TIMESTAMPLOCALTZ:
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = ((TimestampTZ)value).getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ calciteLiteral = rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ break;
case INTERVAL_YEAR_MONTH:
// Calcite year-month literal value is months as BigDecimal
BigDecimal totalMonths = BigDecimal.valueOf(((HiveIntervalYearMonth) value).getTotalMonths());
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
index 31a088b..7208f01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
@@ -181,7 +182,7 @@ public class SqlFunctionConverter {
castUDF = FunctionRegistry.getFunctionInfo("double");
} else if (castType.equals(TypeInfoFactory.timestampTypeInfo)) {
castUDF = FunctionRegistry.getFunctionInfo("timestamp");
- } else if (castType.equals(TypeInfoFactory.timestampLocalTZTypeInfo)) {
+ } else if (castType instanceof TimestampLocalTZTypeInfo) {
castUDF = handleCastForParameterizedType(castType,
FunctionRegistry.getFunctionInfo(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME));
} else if (castType.equals(TypeInfoFactory.dateTypeInfo)) {
@@ -335,7 +336,7 @@ public class SqlFunctionConverter {
registerFunction("-", SqlStdOperatorTable.MINUS, hToken(HiveParser.MINUS, "-"));
registerFunction("*", SqlStdOperatorTable.MULTIPLY, hToken(HiveParser.STAR, "*"));
registerFunction("/", SqlStdOperatorTable.DIVIDE, hToken(HiveParser.DIVIDE, "/"));
- registerFunction("%", SqlStdOperatorTable.MOD, hToken(HiveParser.Identifier, "%"));
+ registerFunction("%", SqlStdOperatorTable.MOD, hToken(HiveParser.MOD, "%"));
registerFunction("and", SqlStdOperatorTable.AND, hToken(HiveParser.KW_AND, "and"));
registerFunction("or", SqlStdOperatorTable.OR, hToken(HiveParser.KW_OR, "or"));
registerFunction("=", SqlStdOperatorTable.EQUALS, hToken(HiveParser.EQUAL, "="));
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
index 34886f3..e2c3b9a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
@@ -36,12 +36,13 @@ import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ConversionUtil;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveType;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter.HiveToken;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.RowResolver;
@@ -65,7 +66,6 @@ public class TypeConverter {
private static final Map<String, HiveToken> calciteToHiveTypeNameMap;
- // TODO: Handling of char[], varchar[], string...
static {
Builder<String, HiveToken> b = ImmutableMap.<String, HiveToken> builder();
b.put(SqlTypeName.BOOLEAN.getName(), new HiveToken(HiveParser.TOK_BOOLEAN, "TOK_BOOLEAN"));
@@ -203,7 +203,7 @@ public class TypeConverter {
convertedType = dtFactory.createSqlType(SqlTypeName.TIMESTAMP);
break;
case TIMESTAMPLOCALTZ:
- convertedType = new HiveType(TimestampTZ.class);
+ convertedType = dtFactory.createSqlType(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE);
break;
case INTERVAL_YEAR_MONTH:
convertedType = dtFactory.createSqlIntervalType(
@@ -330,6 +330,14 @@ public class TypeConverter {
return TypeInfoFactory.dateTypeInfo;
case TIMESTAMP:
return TypeInfoFactory.timestampTypeInfo;
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ HiveConf conf;
+ try {
+ conf = Hive.get().getConf();
+ } catch (HiveException e) {
+ throw new RuntimeException(e);
+ }
+ return TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone());
case INTERVAL_YEAR:
case INTERVAL_MONTH:
case INTERVAL_YEAR_MONTH:
@@ -361,13 +369,7 @@ public class TypeConverter {
return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
else
return TypeInfoFactory.getCharTypeInfo(charLength);
- case NULL:
- case OTHER:
default:
- if (rType instanceof HiveType && ((HiveType) rType).getTypeClass() == TimestampTZ.class) {
- // TODO: This block should be removed when we upgrade Calcite to use local time-zone
- return TypeInfoFactory.timestampLocalTZTypeInfo;
- }
return TypeInfoFactory.voidTypeInfo;
}
@@ -396,12 +398,10 @@ public class TypeConverter {
.getPrecision()), String.valueOf(calciteType.getScale()));
}
break;
- case NULL:
- if (calciteType instanceof HiveType && ((HiveType) calciteType).getTypeClass() == TimestampTZ.class) {
- ht = new HiveToken(HiveParser.TOK_TIMESTAMPLOCALTZ, "TOK_TIMESTAMPLOCALTZ");
- break;
- }
- // fall-through
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE: {
+ ht = new HiveToken(HiveParser.TOK_TIMESTAMPLOCALTZ, "TOK_TIMESTAMPLOCALTZ");
+ }
+ break;
default:
ht = calciteToHiveTypeNameMap.get(calciteType.getSqlTypeName().getName());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 6555269..ae27d60 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -231,8 +231,6 @@ import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PartitionSpec;
import org.apache.hadoop.hive.ql.parse.QBExpr.Opcode;
-import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContext;
-import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContextFactory;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowExpressionSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
@@ -1357,6 +1355,9 @@ public class CalcitePlanner extends SemanticAnalyzer {
HiveRulesRegistry registry = new HiveRulesRegistry();
Properties calciteConfigProperties = new Properties();
calciteConfigProperties.setProperty(
+ CalciteConnectionProperty.TIME_ZONE.camelName(),
+ conf.getLocalTimeZone().getId());
+ calciteConfigProperties.setProperty(
CalciteConnectionProperty.MATERIALIZATIONS_ENABLED.camelName(),
Boolean.FALSE.toString());
CalciteConnectionConfig calciteConfig = new CalciteConnectionConfigImpl(calciteConfigProperties);
@@ -1500,7 +1501,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
} else {
newViewScan = copyNodeScan(viewScan);
}
- return new RelOptMaterialization(newViewScan, materialization.queryRel, null);
+ return new RelOptMaterialization(newViewScan, materialization.queryRel, null,
+ materialization.qualifiedTableName);
}
private RelNode copyNodeScan(RelNode scan) {
@@ -2399,10 +2401,13 @@ public class CalcitePlanner extends SemanticAnalyzer {
}
metrics.add(field.getName());
}
+ // TODO: Default interval will be an Interval once Calcite 1.15.0 is released.
+ // We will need to update the type of this list.
List<LocalInterval> intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false),
- dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, intervals);
+ dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN,
+ intervals, null, null);
final TableScan scan = new HiveTableScan(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
optTable, null == tableAlias ? tabMetaData.getTableName() : tableAlias,
getAliasId(tableAlias, qb), HiveConf.getBoolVar(conf,
@@ -2821,8 +2826,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
}
if (hasGroupSets) {
// Create GroupingID column
- AggregateCall aggCall = new AggregateCall(HiveGroupingID.INSTANCE,
- false, new ImmutableList.Builder<Integer>().build(),
+ AggregateCall aggCall = AggregateCall.create(HiveGroupingID.INSTANCE,
+ false, new ImmutableList.Builder<Integer>().build(), -1,
this.cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
HiveGroupingID.INSTANCE.getName());
aggregateCalls.add(aggCall);
@@ -2836,8 +2841,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
RelNode gbInputRel = HiveProject.create(srcRel, gbChildProjLst, null);
HiveRelNode aggregateRel = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
- gbInputRel, (transformedGroupSets!=null ? true:false), groupSet,
- transformedGroupSets, aggregateCalls);
+ gbInputRel, groupSet, transformedGroupSets, aggregateCalls);
return aggregateRel;
}
@@ -3095,19 +3099,6 @@ public class CalcitePlanner extends SemanticAnalyzer {
} else if (qbp.getDestGroupingSets().contains(detsClauseName)) {
groupingSets = getGroupingSets(grpByAstExprs, qbp, detsClauseName);
}
-
- final int limit = groupingColsSize * 2;
- while (groupingColsSize < limit) {
- String field = getColumnInternalName(groupingColsSize);
- outputColumnNames.add(field);
- groupByOutputRowResolver.put(null, field,
- new ColumnInfo(
- field,
- TypeInfoFactory.booleanTypeInfo,
- null,
- false));
- groupingColsSize++;
- }
}
// 6. Construct aggregation function Info
@@ -4061,7 +4052,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
if (selForWindow != null && selExprList.getToken().getType() == HiveParser.TOK_SELECTDI) {
ImmutableBitSet groupSet = ImmutableBitSet.range(outputRel.getRowType().getFieldList().size());
outputRel = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
- outputRel, false, groupSet, null, new ArrayList<AggregateCall>());
+ outputRel, groupSet, null, new ArrayList<AggregateCall>());
RowResolver groupByOutputRowResolver = new RowResolver();
for (int i = 0; i < out_rwsch.getColumnInfos().size(); i++) {
ColumnInfo colInfo = out_rwsch.getColumnInfos().get(i);
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 020a300..c2d8904 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -127,6 +127,7 @@ TOK_DATETIME;
TOK_TIMESTAMP;
TOK_TIMESTAMPLITERAL;
TOK_TIMESTAMPLOCALTZ;
+TOK_TIMESTAMPLOCALTZLITERAL;
TOK_INTERVAL_YEAR_MONTH;
TOK_INTERVAL_YEAR_MONTH_LITERAL;
TOK_INTERVAL_DAY_TIME;
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index 0591fbf..0a7b7b2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -324,6 +324,7 @@ constant
| Number
| dateLiteral
| timestampLiteral
+ | timestampLocalTZLiteral
| StringLiteral
| stringLiteralSequence
| IntegralLiteral
@@ -367,6 +368,14 @@ timestampLiteral
KW_CURRENT_TIMESTAMP -> ^(TOK_FUNCTION KW_CURRENT_TIMESTAMP)
;
+timestampLocalTZLiteral
+ :
+ KW_TIMESTAMPLOCALTZ StringLiteral ->
+ {
+ adaptor.create(TOK_TIMESTAMPLOCALTZLITERAL, $StringLiteral.text)
+ }
+ ;
+
intervalValue
:
StringLiteral | Number
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index 391ee08..4ddf1d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.parse;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
+import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
@@ -207,7 +209,8 @@ public class TypeCheckProcFactory {
opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
+ HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor());
opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%|"
- + HiveParser.TOK_TIMESTAMPLITERAL + "%"), tf.getDateTimeExprProcessor());
+ + HiveParser.TOK_TIMESTAMPLITERAL + "%|"
+ + HiveParser.TOK_TIMESTAMPLOCALTZLITERAL + "%"), tf.getDateTimeExprProcessor());
opRules.put(new RuleRegExp("R6", HiveParser.TOK_INTERVAL_YEAR_MONTH_LITERAL + "%|"
+ HiveParser.TOK_INTERVAL_DAY_TIME_LITERAL + "%|"
+ HiveParser.TOK_INTERVAL_YEAR_LITERAL + "%|"
@@ -510,6 +513,16 @@ public class TypeCheckProcFactory {
return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo,
Timestamp.valueOf(timeString));
}
+ if (expr.getType() == HiveParser.TOK_TIMESTAMPLOCALTZLITERAL) {
+ HiveConf conf;
+ try {
+ conf = Hive.get().getConf();
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+ return new ExprNodeConstantDesc(TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()),
+ TimestampTZUtil.parse(timeString));
+ }
throw new IllegalArgumentException("Invalid time literal type " + expr.getType());
} catch (Exception err) {
throw new SemanticException(
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java
index 126ed44..f623636 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateFloor.java
@@ -17,11 +17,17 @@
*/
package org.apache.hadoop.hive.ql.udf;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.joda.time.Chronology;
import org.joda.time.DateTime;
@@ -40,11 +46,13 @@ import com.google.common.collect.ImmutableMap;
public abstract class UDFDateFloor extends UDF {
private final QueryGranularity granularity;
- private final TimestampWritable result;
+ private final TimestampWritable resultTS;
+ private final TimestampLocalTZWritable resultTSLTZ;
public UDFDateFloor(String granularity) {
this.granularity = QueryGranularity.fromString(granularity);
- this.result = new TimestampWritable();
+ this.resultTS = new TimestampWritable();
+ this.resultTSLTZ = new TimestampLocalTZWritable();
}
public TimestampWritable evaluate(TimestampWritable t) {
@@ -57,8 +65,23 @@ public abstract class UDFDateFloor extends UDF {
final long newTimestampUTC = granularity.truncate(originalTimestampUTC); // utc
final long newTimestamp = new DateTime(newTimestampUTC, DateTimeZone.UTC)
.withZoneRetainFields(DateTimeZone.getDefault()).getMillis(); // utc -> default
- result.setTime(newTimestamp);
- return result;
+ resultTS.setTime(newTimestamp);
+ return resultTS;
+ }
+
+ public TimestampLocalTZWritable evaluate(TimestampLocalTZWritable t) {
+ if (t == null) {
+ return null;
+ }
+ final ZonedDateTime localZDT = t.getTimestampTZ().getZonedDateTime(); // default
+ final long originalTimestampUTC = localZDT.withZoneSameLocal(ZoneOffset.UTC)
+ .toInstant().toEpochMilli(); // default -> utc
+ final long newTimestampUTC = granularity.truncate(originalTimestampUTC); // utc
+ final ZonedDateTime newLocalZDT = ZonedDateTime.of(
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(newTimestampUTC), ZoneOffset.UTC),
+ localZDT.getZone()); // utc -> default
+ resultTSLTZ.set(new TimestampTZ(newLocalZDT));
+ return resultTSLTZ;
}
/*
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEpochMilli.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEpochMilli.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEpochMilli.java
new file mode 100644
index 0000000..1fe8ad4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEpochMilli.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * GenericUDFEpochMilli.
+ */
+public class GenericUDFEpochMilli extends GenericUDF {
+
+ private transient final LongWritable result = new LongWritable();
+ private transient TimestampLocalTZObjectInspector oi;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ if (arguments.length != 1) {
+ throw new UDFArgumentLengthException(
+ "The operator GenericUDFEpochMilli only accepts 1 argument.");
+ }
+ oi = (TimestampLocalTZObjectInspector) arguments[0];
+ return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ Object a0 = arguments[0].get();
+ if (a0 == null) {
+ return null;
+ }
+ result.set(oi.getPrimitiveJavaObject(a0).getZonedDateTime().toInstant().toEpochMilli());
+ return result;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ assert (children.length == 1);
+ return "GenericUDFEpochMilli(" + children[0] + ")";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java
index 03b2d42..ccccd32 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFDateFormatGranularity.java
@@ -18,7 +18,12 @@
package org.apache.hadoop.hive.ql.udf;
import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.junit.Test;
@@ -84,4 +89,77 @@ public class TestUDFDateFormatGranularity extends TestCase {
assertEquals(494243222000L, i8.getTimestamp().getTime());
}
+ @Test
+ public void testTimestampWithLocalTZGranularity() throws Exception {
+ // Running example
+ // Friday 30th August 1985 02:47:02 AM
+ final TimestampLocalTZWritable t = new TimestampLocalTZWritable(
+ new TimestampTZ(Instant.ofEpochMilli(494243222000L).atZone(ZoneId.of("America/Los_Angeles"))));
+ UDFDateFloor g;
+
+ // Year granularity
+ // Tuesday 1st January 1985 12:00:00 AM
+ g = new UDFDateFloorYear();
+ TimestampLocalTZWritable i1 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(473414400000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i1.getTimestampTZ());
+
+ // Quarter granularity
+ // Monday 1st July 1985 12:00:00 AM
+ g = new UDFDateFloorQuarter();
+ TimestampLocalTZWritable i2 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(489049200000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i2.getTimestampTZ());
+
+ // Month granularity
+ // Thursday 1st August 1985 12:00:00 AM
+ g = new UDFDateFloorMonth();
+ TimestampLocalTZWritable i3 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(491727600000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i3.getTimestampTZ());
+
+ // Week granularity
+ // Monday 26th August 1985 12:00:00 AM
+ g = new UDFDateFloorWeek();
+ TimestampLocalTZWritable i4 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(493887600000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i4.getTimestampTZ());
+
+ // Day granularity
+ // Friday 30th August 1985 12:00:00 AM
+ g = new UDFDateFloorDay();
+ TimestampLocalTZWritable i5 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(494233200000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i5.getTimestampTZ());
+
+ // Hour granularity
+ // Friday 30th August 1985 02:00:00 AM
+ g = new UDFDateFloorHour();
+ TimestampLocalTZWritable i6 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(494240400000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i6.getTimestampTZ());
+
+ // Minute granularity
+ // Friday 30th August 1985 02:47:00 AM
+ g = new UDFDateFloorMinute();
+ TimestampLocalTZWritable i7 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(494243220000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i7.getTimestampTZ());
+
+ // Second granularity
+ // Friday 30th August 1985 02:47:02 AM
+ g = new UDFDateFloorSecond();
+ TimestampLocalTZWritable i8 = g.evaluate(t);
+ assertEquals(
+ new TimestampTZ(Instant.ofEpochMilli(494243222000L).atZone(ZoneId.of("America/Los_Angeles"))),
+ i8.getTimestampTZ());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/queries/clientpositive/druid_timeseries.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druid_timeseries.q b/ql/src/test/queries/clientpositive/druid_timeseries.q
index c0ad60f..f784f26 100644
--- a/ql/src/test/queries/clientpositive/druid_timeseries.q
+++ b/ql/src/test/queries/clientpositive/druid_timeseries.q
@@ -77,8 +77,8 @@ EXPLAIN
SELECT floor_hour(`__time`), max(added), sum(variation)
FROM druid_table_1
WHERE floor_hour(`__time`)
- BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+ BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
GROUP BY floor_hour(`__time`);
-- WITH FILTER ON TIME
@@ -90,5 +90,5 @@ FROM
FROM druid_table_1
GROUP BY floor_hour(`__time`)
) subq
-WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP);
+WHERE subq.h BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE);
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/queries/clientpositive/druid_topn.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druid_topn.q b/ql/src/test/queries/clientpositive/druid_topn.q
index b121b7e..24d5051 100644
--- a/ql/src/test/queries/clientpositive/druid_topn.q
+++ b/ql/src/test/queries/clientpositive/druid_topn.q
@@ -68,8 +68,8 @@ EXPLAIN
SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation)
FROM druid_table_1
WHERE floor_hour(`__time`)
- BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP)
- AND CAST('2014-01-01 00:00:00' AS TIMESTAMP)
+ BETWEEN CAST('2010-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
+ AND CAST('2014-01-01 00:00:00' AS TIMESTAMP WITH LOCAL TIME ZONE)
GROUP BY robot, floor_hour(`__time`)
ORDER BY m
LIMIT 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/queries/clientpositive/timestamptz_3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/timestamptz_3.q b/ql/src/test/queries/clientpositive/timestamptz_3.q
new file mode 100644
index 0000000..7c55a50
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/timestamptz_3.q
@@ -0,0 +1,15 @@
+set hive.fetch.task.conversion=more;
+
+drop table tstz1;
+
+create table tstz1(t timestamp with local time zone);
+
+insert overwrite table tstz1 select cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone);
+
+select cast(t as timestamp) from tstz1;
+select cast(to_epoch_milli(t) as timestamp) from tstz1;
+
+set time zone UTC;
+
+select cast(t as timestamp) from tstz1;
+select cast(to_epoch_milli(t) as timestamp) from tstz1;
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out b/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
index a603cc6..a4c9a6a 100644
--- a/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
+++ b/ql/src/test/results/clientpositive/cbo_rp_annotate_stats_groupby.q.out
@@ -315,17 +315,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -368,17 +364,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -421,17 +413,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -474,17 +462,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 8 Data size: 720 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -527,17 +511,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -580,17 +560,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 24 Data size: 2160 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -633,17 +609,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 32 Data size: 2880 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -735,17 +707,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 16 Data size: 1440 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -837,17 +805,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -890,17 +854,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -943,17 +903,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -996,17 +952,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 4 Data size: 398 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 4 Data size: 398 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 4 Data size: 398 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -1049,17 +1001,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -1102,17 +1050,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 12 Data size: 1194 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -1155,17 +1099,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -1257,17 +1197,13 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: state, locid
Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ File Output Operator
+ compressed: false
Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/9975131c/ql/src/test/results/clientpositive/druid_basic1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_basic1.q.out b/ql/src/test/results/clientpositive/druid_basic1.q.out
index 91f79bb..25ec0ca 100644
--- a/ql/src/test/results/clientpositive/druid_basic1.q.out
+++ b/ql/src/test/results/clientpositive/druid_basic1.q.out
@@ -17,7 +17,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_1
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@druid_table_1
# col_name data_type comment
-__time timestamp from deserializer
+__time timestamp with local time zone from deserializer
robot string from deserializer
namespace string from deserializer
anonymous string from deserializer
@@ -78,7 +78,7 @@ POSTHOOK: query: DESCRIBE FORMATTED druid_table_2
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@druid_table_2
# col_name data_type comment
-__time timestamp from deserializer
+__time timestamp with local time zone from deserializer
robot string from deserializer
namespace string from deserializer
anonymous string from deserializer