You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/03/24 17:32:40 UTC
[3/9] hive git commit: HIVE-18780 : Improve schema discovery For
Druid Storage Handler (Slim Bouguerra via Ashutosh Chauhan) HIVE-18993 : Use
Druid Expressions HIVE-14518 : Support 'having' translation for Druid GroupBy
queries HIVE-18957 : Upgrade Calci
http://git-wip-us.apache.org/repos/asf/hive/blob/696affa2/ql/src/test/results/clientpositive/druid_topn.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid_topn.q.out b/ql/src/test/results/clientpositive/druid_topn.q.out
index 7a4c1f9..f5400a0 100644
--- a/ql/src/test/results/clientpositive/druid_topn.q.out
+++ b/ql/src/test/results/clientpositive/druid_topn.q.out
@@ -84,11 +84,13 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f1","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+ druid.fieldNames robot,$f1,$f2
+ druid.fieldTypes string,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f1","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f1","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), $f1 (type: float), $f2 (type: float)
+ expressions: robot (type: string), $f1 (type: float), $f2 (type: double)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -118,11 +120,13 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"robot"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+ druid.fieldNames extract,robot,$f2,$f3
+ druid.fieldTypes timestamp with local time zone,string,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"extract","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","timeZone":"US/Pacific"}},{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), extract (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float)
+ expressions: robot (type: string), extract (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -152,11 +156,13 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"year","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+ druid.fieldNames robot,floor_year,$f2,$f3
+ druid.fieldTypes string,timestamp with local time zone,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"timeZone":"US/Pacific","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_year (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float)
+ expressions: robot (type: string), floor_year (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -186,11 +192,13 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+ druid.fieldNames robot,floor_month,$f2,$f3
+ druid.fieldTypes string,timestamp with local time zone,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1M","timeZone":"US/Pacific"},"timeZone":"US/Pacific","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f3","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f2 (type: float), $f3 (type: float)
+ expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -220,11 +228,13 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f4","direction":"descending","dimensionOrder":"numeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+ druid.fieldNames robot,namespace,floor_month,$f3,$f4
+ druid.fieldTypes string,string,timestamp with local time zone,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"default","dimension":"namespace","outputName":"namespace","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1M","timeZone":"US/Pacific"},"timeZone":"US/Pacific","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f4","direction":"descending","dimensionOrder":"numeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float)
+ expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -254,11 +264,13 @@ STAGE PLANS:
TableScan
alias: druid_table_1
properties:
- druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot"},{"type":"default","dimension":"namespace"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":"month","timeZone":"US/Pacific","locale":"en-US"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"alphanumeric"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+ druid.fieldNames robot,namespace,floor_month,$f3,$f4
+ druid.fieldTypes string,string,timestamp with local time zone,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"default","dimension":"namespace","outputName":"namespace","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_month","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1M","timeZone":"US/Pacific"},"timeZone":"US/Pacific","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"robot","direction":"ascending","dimensionOrder":"lexicographic"},{"dimension":"$f3","direction":"descending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f3","fieldName":"added"},{"type":"doubleSum","name":"$f4","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
druid.query.type groupBy
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: float)
+ expressions: robot (type: string), floor_month (type: timestamp with local time zone), $f3 (type: float), $f4 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
ListSink
@@ -280,54 +292,26 @@ ORDER BY s
LIMIT 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: druid_table_1
- properties:
- druid.query.json {"queryType":"timeseries","dataSource":"wikipedia","descending":false,"granularity":"year","filter":{"type":"selector","dimension":"robot","value":"1"},"aggregations":[{"type":"doubleMax","name":"$f1_0","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"context":{"skipEmptyBuckets":true}}
- druid.query.type timeseries
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Select Operator
- expressions: __time (type: timestamp with local time zone), $f1_0 (type: float), $f2 (type: float)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: float)
- sort order: +
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col0 (type: timestamp with local time zone), _col1 (type: float)
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: timestamp with local time zone), VALUE._col1 (type: float), KEY.reducesinkkey0 (type: float)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Select Operator
- expressions: '1' (type: string), _col0 (type: timestamp with local time zone), _col1 (type: float), _col2 (type: float)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
limit: -1
Processor Tree:
- ListSink
+ TableScan
+ alias: druid_table_1
+ properties:
+ druid.fieldNames floor_year,$f1_0,$f2
+ druid.fieldTypes timestamp with local time zone,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"extraction","dimension":"__time","outputName":"floor_year","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"P1Y","timeZone":"US/Pacific"},"timeZone":"US/Pacific","locale":"und"}}],"limitSpec":{"type":"default","limit":10,"columns":[{"dimension":"$f2","direction":"ascending","dimensionOrder":"numeric"}]},"filter":{"type":"selector","dimension":"robot","value":"1"},"aggregations":[{"type":"doubleMax","name":"$f1_0","fieldName":"added"},{"type":"doubleSum","name":"$f2","fieldName":"variation"}],"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"]}
+ druid.query.type groupBy
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: '1' (type: string), floor_year (type: timestamp with local time zone), $f1_0 (type: float), $f2 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ ListSink
PREHOOK: query: EXPLAIN
SELECT robot, floor_hour(`__time`), max(added) as m, sum(variation)
@@ -350,83 +334,24 @@ ORDER BY m
LIMIT 100
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: druid_table_1
- filterExpr: floor_hour(__time) BETWEEN TIMESTAMPLOCALTZ'2010-01-01 00:00:00.0 US/Pacific' AND TIMESTAMPLOCALTZ'2014-01-01 00:00:00.0 US/Pacific' (type: boolean)
- properties:
- druid.query.json {"queryType":"select","dataSource":"wikipedia","descending":false,"intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"dimensions":["robot"],"metrics":["added","variation"],"granularity":"all","pagingSpec":{"threshold":16384,"fromNext":true},"context":{"druid.query.fetch":false}}
- druid.query.type select
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Filter Operator
- predicate: floor_hour(__time) BETWEEN TIMESTAMPLOCALTZ'2010-01-01 00:00:00.0 US/Pacific' AND TIMESTAMPLOCALTZ'2014-01-01 00:00:00.0 US/Pacific' (type: boolean)
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Select Operator
- expressions: robot (type: string), floor_hour(__time) (type: timestamp with local time zone), added (type: float), variation (type: float)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Group By Operator
- aggregations: max(_col2), sum(_col3)
- keys: _col0 (type: string), _col1 (type: timestamp with local time zone)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: timestamp with local time zone)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: timestamp with local time zone)
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col2 (type: float), _col3 (type: double)
- Reduce Operator Tree:
- Group By Operator
- aggregations: max(VALUE._col0), sum(VALUE._col1)
- keys: KEY._col0 (type: string), KEY._col1 (type: timestamp with local time zone)
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- Reduce Output Operator
- key expressions: _col2 (type: float)
- sort order: +
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col0 (type: string), _col1 (type: timestamp with local time zone), _col3 (type: double)
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: string), VALUE._col1 (type: timestamp with local time zone), KEY.reducesinkkey0 (type: float), VALUE._col2 (type: double)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Limit
- Number of rows: 100
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
- limit: 100
+ limit: -1
Processor Tree:
- ListSink
+ TableScan
+ alias: druid_table_1
+ properties:
+ druid.fieldNames robot,floor_hour,$f2,$f3
+ druid.fieldTypes string,timestamp with local time zone,float,double
+ druid.query.json {"queryType":"groupBy","dataSource":"wikipedia","granularity":"all","dimensions":[{"type":"default","dimension":"robot","outputName":"robot","outputType":"STRING"},{"type":"extraction","dimension":"__time","outputName":"floor_hour","extractionFn":{"type":"timeFormat","format":"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'","granularity":{"type":"period","period":"PT1H","timeZone":"US/Pacific"},"timeZone":"US/Pacific","locale":"und"}}],"limitSpec":{"type":"default","limit":100,"columns":[{"dimension":"$f2","direction":"ascending","dimensionOrder":"numeric"}]},"aggregations":[{"type":"doubleMax","name":"$f2","fieldName":"added"},{"type":"doubleSum","name":"$f3","fieldName":"variation"}],"intervals":["2010-01-01T08:00:00.000Z/2014-01-01T08:00:00.001Z"]}
+ druid.query.type groupBy
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: robot (type: string), floor_hour (type: timestamp with local time zone), $f2 (type: float), $f3 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ ListSink
http://git-wip-us.apache.org/repos/asf/hive/blob/696affa2/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
index 86e3c56..617ae3b 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
@@ -278,7 +278,7 @@ POSTHOOK: query: alter materialized view cmv_mat_view2 enable rewrite
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
POSTHOOK: Input: default@cmv_mat_view2
POSTHOOK: Output: default@cmv_mat_view2
-Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[8][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: explain
select * from (
(select a, c from cmv_basetable where a = 3) table1
@@ -309,15 +309,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: default.cmv_mat_view2
- Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c (type: decimal(10,2))
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(10,2))
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ value expressions: a (type: int), c (type: decimal(10,2))
Execution mode: llap
LLAP IO: all inputs
Map 3
@@ -325,17 +321,10 @@ STAGE PLANS:
TableScan
alias: cmv_basetable
Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: c (type: decimal(10,2))
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: decimal(10,2))
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -347,19 +336,22 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2))
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ outputColumnNames: _col0, _col1, _col5, _col7, _col8
+ Statistics: Num rows: 10 Data size: 2370 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col5 = 3) and (_col8 = 3)) (type: boolean)
+ Statistics: Num rows: 2 Data size: 474 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col0 (type: int), _col7 (type: decimal(10,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 2 Data size: 474 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 474 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -367,7 +359,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[8][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select * from (
(select a, c from cmv_basetable where a = 3) table1
join
http://git-wip-us.apache.org/repos/asf/hive/blob/696affa2/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
index 86e3c56..617ae3b 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
@@ -278,7 +278,7 @@ POSTHOOK: query: alter materialized view cmv_mat_view2 enable rewrite
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
POSTHOOK: Input: default@cmv_mat_view2
POSTHOOK: Output: default@cmv_mat_view2
-Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[8][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: explain
select * from (
(select a, c from cmv_basetable where a = 3) table1
@@ -309,15 +309,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: default.cmv_mat_view2
- Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c (type: decimal(10,2))
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(10,2))
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE
+ value expressions: a (type: int), c (type: decimal(10,2))
Execution mode: llap
LLAP IO: all inputs
Map 3
@@ -325,17 +321,10 @@ STAGE PLANS:
TableScan
alias: cmv_basetable
Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: c (type: decimal(10,2))
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: decimal(10,2))
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 5 Data size: 600 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
Execution mode: llap
LLAP IO: may be used (ACID table)
Reducer 2
@@ -347,19 +336,22 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2))
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 450 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ outputColumnNames: _col0, _col1, _col5, _col7, _col8
+ Statistics: Num rows: 10 Data size: 2370 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col5 = 3) and (_col8 = 3)) (type: boolean)
+ Statistics: Num rows: 2 Data size: 474 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col0 (type: int), _col7 (type: decimal(10,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 2 Data size: 474 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 474 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -367,7 +359,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[8][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Reducer 2' is a cross product
PREHOOK: query: select * from (
(select a, c from cmv_basetable where a = 3) table1
join
http://git-wip-us.apache.org/repos/asf/hive/blob/696affa2/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
index c93bf36..3a5c272 100644
--- a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
@@ -1645,8 +1645,8 @@ STAGE PLANS:
selectExpressions: LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 15:int, LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 16:bigint, LongColUnaryMinus(col 3:bigint) -> 17:bigint, DoubleColUnaryMinus(col 4:float) -> 14:float, LongColAddLongColumn(col 18:bigint, col 3:bigint)(children: LongColSubtractLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint) -> 18:bigint) -> 19:bigint, DoubleColDivideDoubleColumn(col 5:double, col 5:double) -> 20:double, DoubleColUnaryMinus(col 5:double) -> 21:double, LongColMultiplyLongColumn(col 18:bigint, col 22:bigint)(children: col 18:int, LongColUnaryMinus(col 3:bigint) -> 22:bigint) -> 23:bigint, DoubleColAddDoubleColumn(col 24:double, col 25:double)(children: DoubleColUnaryMinus(col 5:double) -> 24:double, CastLongToDouble(col 3:bigint) -> 25:double) -> 26:double, DecimalScalarDivideDecimalColumn(val -1.389, col 27:decimal(3,0))(children: CastLongToDecima
l(col 0:tinyint) -> 27:decimal(3,0)) -> 28:decimal(8,7), DoubleColModuloDoubleColumn(col 24:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 24:double) -> 25:double, LongColUnaryMinus(col 1:smallint) -> 18:smallint, LongColAddLongColumn(col 1:int, col 22:int)(children: col 1:smallint, LongColAddLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 22:int) -> 29:int
Statistics: Num rows: 10922 Data size: 3594034 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
- key expressions: _col8 (type: boolean), _col1 (type: string), _col3 (type: timestamp), _col5 (type: float), _col6 (type: bigint), _col1 (type: string), _col4 (type: double), _col0 (type: int), _col7 (type: smallint), _col4 (type: double), _col9 (type: int), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: float), _col13 (type: bigint), _col14 (type: double), _col15 (type: double), _col16 (type: bigint), _col17 (type: double), _col18 (type: decimal(8,7)), _col19 (type: double), _col20 (type: smallint), _col21 (type: int)
- sort order: +++++++++++++++++++++++
+ key expressions: _col8 (type: boolean), _col1 (type: string), _col3 (type: timestamp), _col5 (type: float), _col6 (type: bigint), _col4 (type: double), _col0 (type: int), _col7 (type: smallint), _col9 (type: int), _col10 (type: bigint), _col11 (type: bigint), _col12 (type: float), _col13 (type: bigint), _col14 (type: double), _col15 (type: double), _col16 (type: bigint), _col17 (type: double), _col18 (type: decimal(8,7)), _col19 (type: double), _col20 (type: smallint), _col21 (type: int)
+ sort order: +++++++++++++++++++++
Reduce Sink Vectorization:
className: VectorReduceSinkObjectHashOperator
native: true
@@ -1675,12 +1675,12 @@ STAGE PLANS:
vectorized: true
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey7 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: boolean), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey6 (type: double), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: bigint), KEY.reducesinkkey8 (type: smallint), KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey10 (type: int), KEY.reducesinkkey11 (type: bigint), KEY.reducesinkkey12 (type: bigint), KEY.reducesinkkey13 (type: float), KEY.reducesinkkey14 (type: bigint), KEY.reducesinkkey15 (type: double), KEY.reducesinkkey16 (type: double), KEY.reducesinkkey17 (type: bigint), KEY.reducesinkkey18 (type: double), KEY.reducesinkkey19 (type: decimal(8,7)), KEY.reducesinkkey20 (type: double), KEY.reducesinkkey21 (type: smallint), KEY.reducesinkkey22 (type: int)
+ expressions: KEY.reducesinkkey6 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: boolean), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey5 (type: double), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: bigint), KEY.reducesinkkey7 (type: smallint), KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey8 (type: int), KEY.reducesinkkey9 (type: bigint), KEY.reducesinkkey10 (type: bigint), KEY.reducesinkkey11 (type: float), KEY.reducesinkkey12 (type: bigint), KEY.reducesinkkey13 (type: double), KEY.reducesinkkey14 (type: double), KEY.reducesinkkey15 (type: bigint), KEY.reducesinkkey16 (type: double), KEY.reducesinkkey17 (type: decimal(8,7)), KEY.reducesinkkey18 (type: double), KEY.reducesinkkey19 (type: smallint), KEY.reducesinkkey20 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
Select Vectorization:
className: VectorSelectOperator
native: true
- projectedOutputColumnNums: [7, 1, 23, 2, 6, 3, 4, 8, 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]
+ projectedOutputColumnNums: [6, 1, 21, 2, 5, 3, 4, 7, 0, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
Statistics: Num rows: 10922 Data size: 3594034 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 75
http://git-wip-us.apache.org/repos/asf/hive/blob/696affa2/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out
index e6e559f..4da3d09 100644
--- a/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/materialized_view_create_rewrite.q.out
@@ -266,7 +266,7 @@ POSTHOOK: query: alter materialized view cmv_mat_view2 enable rewrite
POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
POSTHOOK: Input: default@cmv_mat_view2
POSTHOOK: Output: default@cmv_mat_view2
-Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[4][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: explain
select * from (
(select a, c from cmv_basetable where a = 3) table1
@@ -292,28 +292,17 @@ STAGE PLANS:
TableScan
alias: default.cmv_mat_view2
Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c (type: decimal(10,2))
- outputColumnNames: _col0
+ Reduce Output Operator
+ sort order:
Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 2 Data size: 232 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(10,2))
+ value expressions: a (type: int), c (type: decimal(10,2))
TableScan
alias: cmv_basetable
Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((a = 3) and (d = 3)) (type: boolean)
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c (type: decimal(10,2))
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 241 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(10,2))
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 5 Data size: 1205 Basic stats: COMPLETE Column stats: NONE
+ value expressions: a (type: int), c (type: decimal(10,2)), d (type: int)
Reduce Operator Tree:
Join Operator
condition map:
@@ -321,19 +310,22 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: 3 (type: int), _col0 (type: decimal(10,2)), 3 (type: int), _col1 (type: decimal(10,2))
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col0, _col1, _col5, _col7, _col8
+ Statistics: Num rows: 10 Data size: 3580 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col5 = 3) and (_col8 = 3)) (type: boolean)
Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: decimal(10,2)), _col0 (type: int), _col7 (type: decimal(10,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 716 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -341,7 +333,7 @@ STAGE PLANS:
Processor Tree:
ListSink
-Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[4][tables = [default.cmv_mat_view2, cmv_basetable]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: select * from (
(select a, c from cmv_basetable where a = 3) table1
join
http://git-wip-us.apache.org/repos/asf/hive/blob/696affa2/ql/src/test/results/clientpositive/perf/spark/query23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query23.q.out b/ql/src/test/results/clientpositive/perf/spark/query23.q.out
index 746e44d..44e72cf 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query23.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query23.q.out
@@ -1,5 +1,5 @@
-Warning: Map Join MAPJOIN[275][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[276][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[285][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[286][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
PREHOOK: query: explain
with frequent_ss_items as
(select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt
@@ -131,15 +131,15 @@ STAGE PLANS:
predicate: (ss_customer_sk is not null and ss_sold_date_sk is not null) (type: boolean)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_sold_date_sk (type: int), ss_customer_sk (type: int)
- outputColumnNames: _col0, _col1
+ expressions: ss_sold_date_sk (type: int), ss_customer_sk (type: int), ss_quantity (type: int), ss_sales_price (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
Map 18
Map Operator Tree:
TableScan
@@ -234,13 +234,14 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col1, _col2, _col3
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: decimal(7,2))
Reducer 15
Reduce Operator Tree:
Join Operator
@@ -249,39 +250,44 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col6
+ outputColumnNames: _col2, _col3, _col6
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int)
- outputColumnNames: _col0
+ expressions: _col6 (type: int), (CAST( _col2 AS decimal(10,0)) * _col3) (type: decimal(18,2))
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
+ aggregations: sum(_col1)
keys: _col0 (type: int)
mode: hash
- outputColumnNames: _col0
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(28,2))
Reducer 16
Reduce Operator Tree:
Group By Operator
+ aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: int)
mode: mergepartial
- outputColumnNames: _col0
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count()
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
Reducer 17
Local Work:
@@ -291,17 +297,24 @@ STAGE PLANS:
aggregations: count(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (sq_count_check(_col0) <= 1) (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0
- 1
- 2
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: complete
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (sq_count_check(_col0) <= 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0
+ 1
+ 2
Reducer 21
Reduce Operator Tree:
Join Operator
@@ -402,15 +415,15 @@ STAGE PLANS:
predicate: (ss_customer_sk is not null and ss_sold_date_sk is not null) (type: boolean)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_sold_date_sk (type: int), ss_customer_sk (type: int)
- outputColumnNames: _col0, _col1
+ expressions: ss_sold_date_sk (type: int), ss_customer_sk (type: int), ss_quantity (type: int), ss_sales_price (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
Map 47
Map Operator Tree:
TableScan
@@ -505,13 +518,14 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col1, _col2, _col3
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: decimal(7,2))
Reducer 44
Reduce Operator Tree:
Join Operator
@@ -520,39 +534,44 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col6
+ outputColumnNames: _col2, _col3, _col6
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int)
- outputColumnNames: _col0
+ expressions: _col6 (type: int), (CAST( _col2 AS decimal(10,0)) * _col3) (type: decimal(18,2))
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
+ aggregations: sum(_col1)
keys: _col0 (type: int)
mode: hash
- outputColumnNames: _col0
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(28,2))
Reducer 45
Reduce Operator Tree:
Group By Operator
+ aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: int)
mode: mergepartial
- outputColumnNames: _col0
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Select Operator
+ expressions: _col0 (type: int)
+ outputColumnNames: _col0
Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: count()
+ aggregations: count(_col0)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
Reducer 46
Local Work:
@@ -562,17 +581,24 @@ STAGE PLANS:
aggregations: count(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (sq_count_check(_col0) <= 1) (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0
- 1
- 2
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: complete
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (sq_count_check(_col0) <= 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0
+ 1
+ 2
Reducer 50
Reduce Operator Tree:
Join Operator
@@ -661,10 +687,10 @@ STAGE PLANS:
Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 376), Reducer 2 (PARTITION-LEVEL SORT, 376)
Reducer 32 <- Map 31 (PARTITION-LEVEL SORT, 154), Map 35 (PARTITION-LEVEL SORT, 154)
Reducer 33 <- Reducer 10 (PARTITION-LEVEL SORT, 209), Reducer 32 (PARTITION-LEVEL SORT, 209)
- Reducer 34 <- Reducer 33 (PARTITION-LEVEL SORT, 451), Reducer 58 (PARTITION-LEVEL SORT, 451)
+ Reducer 34 <- Reducer 33 (PARTITION-LEVEL SORT, 543), Reducer 58 (PARTITION-LEVEL SORT, 543)
Reducer 37 <- Map 11 (PARTITION-LEVEL SORT, 398), Map 36 (PARTITION-LEVEL SORT, 398)
Reducer 38 <- Map 12 (PARTITION-LEVEL SORT, 442), Reducer 37 (PARTITION-LEVEL SORT, 442)
- Reducer 4 <- Reducer 29 (PARTITION-LEVEL SORT, 634), Reducer 3 (PARTITION-LEVEL SORT, 634)
+ Reducer 4 <- Reducer 29 (PARTITION-LEVEL SORT, 727), Reducer 3 (PARTITION-LEVEL SORT, 727)
Reducer 5 <- Reducer 34 (GROUP, 1), Reducer 4 (GROUP, 1)
Reducer 58 <- Reducer 28 (GROUP, 437)
#### A masked pattern was here ####
@@ -904,19 +930,19 @@ STAGE PLANS:
input vertices:
0 Reducer 17
1 Reducer 24
- Statistics: Num rows: 316797606 Data size: 101761818952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316797606 Data size: 137243150824 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col3 > (0.95 * _col1)) (type: boolean)
- Statistics: Num rows: 105599202 Data size: 33920606317 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 105599202 Data size: 45747716941 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 105599202 Data size: 33920606317 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 105599202 Data size: 45747716941 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 105599202 Data size: 33920606317 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 105599202 Data size: 45747716941 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Join Operator
@@ -1088,19 +1114,19 @@ STAGE PLANS:
input vertices:
0 Reducer 46
1 Reducer 53
- Statistics: Num rows: 316797606 Data size: 101761818952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316797606 Data size: 137243150824 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col3 > (0.95 * _col1)) (type: boolean)
- Statistics: Num rows: 105599202 Data size: 33920606317 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 105599202 Data size: 45747716941 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 105599202 Data size: 33920606317 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 105599202 Data size: 45747716941 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 105599202 Data size: 33920606317 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 105599202 Data size: 45747716941 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator