You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/11/11 02:45:48 UTC
[1/4] hive git commit: HIVE-20826 : Enhance HiveSemiJoin rule to
convert join + group by on left side to Left Semi Join (Vineet Garg via
Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master a1cb7e4f7 -> c049caceb
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/spark/semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out
index a787bce..38db76d 100644
--- a/ql/src/test/results/clientpositive/spark/semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out
@@ -2921,3 +2921,55 @@ POSTHOOK: query: select key, value from src outr left semi join
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
+PREHOOK: query: explain cbo select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(p_partkey=[$0])
+ HiveSemiJoin(condition=[=($1, $3)], joinType=[inner])
+ HiveProject(p_partkey=[$0], p_name=[$1])
+ HiveFilter(condition=[IS NOT NULL($1)])
+ HiveTableScan(table=[[default, part]], table:alias=[pp])
+ HiveProject(p_partkey=[$0], p_name=[$1], p_mfgr=[$2], p_brand=[$3], p_type=[$4], p_size=[$5], p_container=[$6], p_retailprice=[$7], p_comment=[$8], BLOCK__OFFSET__INSIDE__FILE=[$9], INPUT__FILE__NAME=[$10], ROW__ID=[$11])
+ HiveFilter(condition=[IS NOT NULL($1)])
+ HiveTableScan(table=[[default, part]], table:alias=[part])
+
+PREHOOK: query: select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+105685
+110592
+112398
+121152
+121152
+132666
+144293
+146985
+15103
+155733
+17273
+17927
+191709
+192697
+195606
+33357
+40982
+42669
+45261
+48427
+49671
+65667
+78486
+85768
+86428
+90681
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
index 0bdc44b..5c9d7fa 100644
--- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
@@ -2210,42 +2210,36 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 2 <- Map 1 (GROUP)
-Reducer 3 <- Map 4 (PARTITION-LEVEL SORT), Reducer 2 (PARTITION-LEVEL SORT)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT), Map 3 (PARTITION-LEVEL SORT)
Stage-0
Fetch Operator
limit:-1
Stage-1
- Reducer 3
- File Output Operator [FS_14]
- Select Operator [SEL_13] (rows=131 width=178)
- Output:["_col0","_col1"]
- Join Operator [JOIN_12] (rows=131 width=178)
- Output:["_col1","_col2"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"}
- <-Map 4 [PARTITION-LEVEL SORT]
- PARTITION-LEVEL SORT [RS_11]
- PartitionCols:_col0
- Select Operator [SEL_9] (rows=166 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_16] (rows=166 width=178)
+ Reducer 2
+ File Output Operator [FS_12]
+ Join Operator [JOIN_10] (rows=131 width=178)
+ Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"}
+ <-Map 1 [PARTITION-LEVEL SORT]
+ PARTITION-LEVEL SORT [RS_8]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=166 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_13] (rows=166 width=178)
+ predicate:(key > '9')
+ TableScan [TS_0] (rows=500 width=178)
+ default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Map 3 [PARTITION-LEVEL SORT]
+ PARTITION-LEVEL SORT [RS_9]
+ PartitionCols:_col0
+ Group By Operator [GBY_7] (rows=83 width=87)
+ Output:["_col0"],keys:_col0
+ Select Operator [SEL_5] (rows=166 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_14] (rows=166 width=87)
predicate:(key > '9')
- TableScan [TS_7] (rows=500 width=178)
- default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 2 [PARTITION-LEVEL SORT]
- PARTITION-LEVEL SORT [RS_10]
- PartitionCols:_col0
- Group By Operator [GBY_5] (rows=83 width=87)
- Output:["_col0"],keys:KEY._col0
- <-Map 1 [GROUP]
- GROUP [RS_4]
- PartitionCols:_col0
- Group By Operator [GBY_3] (rows=83 width=87)
- Output:["_col0"],keys:key
- Filter Operator [FIL_15] (rows=166 width=87)
- predicate:(key > '9')
- TableScan [TS_0] (rows=500 width=87)
- default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ TableScan [TS_3] (rows=500 width=87)
+ default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/spark/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
index 7063a79..92cdcd7 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out
@@ -20,84 +20,70 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: s1
+ alias: src
filterExpr: (key > '9') (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key > '9') (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Execution mode: vectorized
- Map 4
+ Map 3
Map Operator Tree:
TableScan
- alias: src
+ alias: s1
filterExpr: (key > '9') (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key > '9') (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: string), _col2 (type: string)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -1075,9 +1061,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
Reducer 5 <- Map 4 (GROUP, 2)
- Reducer 7 <- Map 6 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1131,10 +1116,10 @@ STAGE PLANS:
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_orderkey (type: int)
- outputColumnNames: l_orderkey
+ outputColumnNames: _col0
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: l_orderkey (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
@@ -1164,7 +1149,7 @@ STAGE PLANS:
Reduce Operator Tree:
Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
@@ -1194,19 +1179,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Reducer 7
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/spark/subquery_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/subquery_views.q.out b/ql/src/test/results/clientpositive/spark/subquery_views.q.out
index 1540089..c221392 100644
--- a/ql/src/test/results/clientpositive/spark/subquery_views.q.out
+++ b/ql/src/test/results/clientpositive/spark/subquery_views.q.out
@@ -107,18 +107,17 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 11 <- Map 10 (GROUP, 2)
- Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 2), Reducer 15 (PARTITION-LEVEL SORT, 2), Reducer 18 (PARTITION-LEVEL SORT, 2)
- Reducer 15 <- Map 14 (GROUP, 2)
- Reducer 17 <- Map 16 (GROUP, 2)
- Reducer 18 <- Reducer 17 (PARTITION-LEVEL SORT, 2), Reducer 20 (PARTITION-LEVEL SORT, 2)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 6 (PARTITION-LEVEL SORT, 2), Reducer 9 (PARTITION-LEVEL SORT, 2)
- Reducer 20 <- Map 10 (GROUP, 2)
- Reducer 3 <- Reducer 2 (GROUP, 2)
- Reducer 4 <- Reducer 13 (PARTITION-LEVEL SORT, 2), Reducer 3 (PARTITION-LEVEL SORT, 2)
- Reducer 6 <- Map 14 (GROUP, 2)
- Reducer 8 <- Map 16 (GROUP, 2)
- Reducer 9 <- Reducer 11 (PARTITION-LEVEL SORT, 2), Reducer 8 (PARTITION-LEVEL SORT, 2)
+ Reducer 10 <- Map 18 (GROUP, 2)
+ Reducer 12 <- Map 11 (PARTITION-LEVEL SORT, 2), Reducer 14 (PARTITION-LEVEL SORT, 2), Reducer 17 (PARTITION-LEVEL SORT, 2)
+ Reducer 14 <- Map 13 (GROUP, 2)
+ Reducer 16 <- Map 15 (GROUP, 2)
+ Reducer 17 <- Reducer 16 (PARTITION-LEVEL SORT, 2), Reducer 19 (PARTITION-LEVEL SORT, 2)
+ Reducer 19 <- Map 18 (GROUP, 2)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 5 (PARTITION-LEVEL SORT, 2), Reducer 8 (PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Reducer 12 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2)
+ Reducer 5 <- Map 13 (GROUP, 2)
+ Reducer 7 <- Map 15 (GROUP, 2)
+ Reducer 8 <- Reducer 10 (PARTITION-LEVEL SORT, 2), Reducer 7 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -142,29 +141,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
- Map 10
- Map Operator Tree:
- TableScan
- alias: b
- filterExpr: (key < '11') (type: boolean)
- properties:
- insideView TRUE
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (key < '11') (type: boolean)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Execution mode: vectorized
- Map 12
+ Map 11
Map Operator Tree:
TableScan
alias: b
@@ -185,7 +162,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
- Map 14
+ Map 13
Map Operator Tree:
TableScan
alias: a
@@ -207,7 +184,7 @@ STAGE PLANS:
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint), _col3 (type: bigint)
Execution mode: vectorized
- Map 16
+ Map 15
Map Operator Tree:
TableScan
alias: a
@@ -227,7 +204,29 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
- Reducer 11
+ Map 18
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: (key < '11') (type: boolean)
+ properties:
+ insideView TRUE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key < '11') (type: boolean)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 10
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
@@ -240,7 +239,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Reducer 13
+ Reducer 12
Reduce Operator Tree:
Join Operator
condition map:
@@ -250,22 +249,26 @@ STAGE PLANS:
0 _col0 (type: string), _col1 (type: string)
1 _col0 (type: string), _col1 (type: string)
2 _col3 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1, _col4, _col5, _col8
+ outputColumnNames: _col0, _col4, _col5, _col8
Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
+ predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Reducer 15
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
+ Reducer 14
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
@@ -280,7 +283,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint), _col3 (type: bigint)
- Reducer 17
+ Reducer 16
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
@@ -298,7 +301,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: boolean)
- Reducer 18
+ Reducer 17
Reduce Operator Tree:
Join Operator
condition map:
@@ -314,6 +317,19 @@ STAGE PLANS:
Map-reduce partition columns: _col3 (type: string), _col1 (type: string)
Statistics: Num rows: 91 Data size: 969 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: boolean)
+ Reducer 19
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -324,73 +340,39 @@ STAGE PLANS:
0 _col0 (type: string), _col1 (type: string)
1 _col0 (type: string), _col1 (type: string)
2 _col3 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col4, _col5, _col8
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8
Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
+ predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 182 Data size: 1933 Basic stats: COMPLETE Column stats: NONE
- Reducer 20
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 3
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 91 Data size: 966 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 91 Data size: 966 Basic stats: COMPLETE Column stats: NONE
- Reducer 4
Reduce Operator Tree:
Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 200 Data size: 2126 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: string), _col2 (type: string)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 200 Data size: 2126 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 200 Data size: 2126 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 6
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
@@ -405,7 +387,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint), _col3 (type: bigint)
- Reducer 8
+ Reducer 7
Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
@@ -423,7 +405,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 27 Data size: 286 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: boolean)
- Reducer 9
+ Reducer 8
Reduce Operator Tree:
Join Operator
condition map:
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
index 6634a04..e9095ef 100644
--- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out
@@ -28,7 +28,6 @@ STAGE PLANS:
Spark
Edges:
Reducer 3 <- Map 2 (GROUP, 2)
- Reducer 5 <- Map 4 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
Map 2
@@ -94,7 +93,7 @@ STAGE PLANS:
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_orderkey (type: int)
- outputColumnNames: l_orderkey
+ outputColumnNames: _col0
Select Vectorization:
className: VectorSelectOperator
native: true
@@ -108,19 +107,17 @@ STAGE PLANS:
native: false
vectorProcessingMode: HASH
projectedOutputColumnNums: []
- keys: l_orderkey (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkLongOperator
+ Spark HashTable Sink Operator
+ Spark Hash Table Sink Vectorization:
+ className: VectorSparkHashTableSinkOperator
native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -131,6 +128,8 @@ STAGE PLANS:
allNative: false
usesVectorUDFAdaptor: false
vectorized: true
+ Local Work:
+ Map Reduce Local Work
Reducer 3
Execution mode: vectorized
Local Work:
@@ -161,36 +160,6 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- Reducer 5
- Execution mode: vectorized
- Local Work:
- Map Reduce Local Work
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- Reduce Operator Tree:
- Group By Operator
- Group By Vectorization:
- className: VectorGroupByOperator
- groupByMode: MERGEPARTIAL
- keyExpressions: col 0:int
- native: false
- vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumnNums: []
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- Spark Hash Table Sink Vectorization:
- className: VectorSparkHashTableSinkOperator
- native: true
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
Stage: Stage-1
Spark
@@ -236,18 +205,18 @@ STAGE PLANS:
Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
Map Join Vectorization:
- className: VectorMapJoinInnerBigOnlyLongOperator
+ className: VectorMapJoinLeftSemiLongOperator
native: true
nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true
hashTableImplementationType: OPTIMIZED
outputColumnNames: _col1, _col2
input vertices:
- 1 Reducer 5
+ 1 Map 4
Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: int), _col2 (type: int)
[3/4] hive git commit: HIVE-20826 : Enhance HiveSemiJoin rule to
convert join + group by on left side to Left Semi Join (Vineet Garg via
Ashutosh Chauhan)
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out b/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out
index 9bb4f2e..43a4fab 100644
--- a/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/cbo_query14.q.out
@@ -1,9 +1,9 @@
-Warning: Shuffle Join MERGEJOIN[1446][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
-Warning: Shuffle Join MERGEJOIN[1458][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product
-Warning: Shuffle Join MERGEJOIN[1448][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 13' is a cross product
-Warning: Shuffle Join MERGEJOIN[1471][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 14' is a cross product
-Warning: Shuffle Join MERGEJOIN[1450][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 18' is a cross product
-Warning: Shuffle Join MERGEJOIN[1484][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 19' is a cross product
+Warning: Shuffle Join MERGEJOIN[1440][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
+Warning: Shuffle Join MERGEJOIN[1452][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product
+Warning: Shuffle Join MERGEJOIN[1442][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 13' is a cross product
+Warning: Shuffle Join MERGEJOIN[1465][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 14' is a cross product
+Warning: Shuffle Join MERGEJOIN[1444][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 18' is a cross product
+Warning: Shuffle Join MERGEJOIN[1478][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 19' is a cross product
PREHOOK: query: explain cbo
with cross_items as
(select i_item_sk ss_item_sk
@@ -230,60 +230,10 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[
HiveUnion(all=[true])
HiveProject(channel=[_UTF-16LE'store'], i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], sales=[$3], number_sales=[$4])
HiveJoin(condition=[>($3, $6)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3], $f4=[$4])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3], $f4=[$4])
HiveAggregate(group=[{0, 1, 2}], agg#0=[sum($3)], agg#1=[count()])
- HiveProject($f0=[$2], $f1=[$3], $f2=[$4], $f3=[*(CAST($7):DECIMAL(10, 0), $8)])
- HiveJoin(condition=[=($6, $0)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(i_item_sk=[$0])
- HiveAggregate(group=[{0}])
- HiveJoin(condition=[AND(AND(=($1, $4), =($2, $5)), =($3, $6))], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11), IS NOT NULL($0))])
- HiveTableScan(table=[[default, item]], table:alias=[item])
- HiveProject($f0=[$0], $f1=[$1], $f2=[$2])
- HiveFilter(condition=[=($3, 3)])
- HiveAggregate(group=[{0, 1, 2}], agg#0=[count($3)])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveUnion(all=[true])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(ss_sold_date_sk=[$0], ss_item_sk=[$2])
- HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($0))])
- HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d1])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[iss])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(cs_sold_date_sk=[$0], cs_item_sk=[$15])
- HiveFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($0))])
- HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d2])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[ics])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(ws_sold_date_sk=[$0], ws_item_sk=[$3])
- HiveFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($0))])
- HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d3])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[iws])
+ HiveProject(i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f3=[*(CAST($6):DECIMAL(10, 0), $7)])
+ HiveSemiJoin(condition=[=($5, $11)], joinType=[inner])
HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
HiveFilter(condition=[IS NOT NULL($0)])
@@ -295,6 +245,55 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[
HiveProject(d_date_sk=[$0], d_year=[CAST(2000):INTEGER], d_moy=[CAST(11):INTEGER])
HiveFilter(condition=[AND(=($6, 2000), =($8, 11), IS NOT NULL($0))])
HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f0=[$4], $f1=[$5], $f2=[$6])
+ HiveJoin(condition=[AND(AND(=($1, $4), =($2, $5)), =($3, $6))], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, item]], table:alias=[item])
+ HiveProject($f0=[$0], $f1=[$1], $f2=[$2])
+ HiveFilter(condition=[=($3, 3)])
+ HiveAggregate(group=[{0, 1, 2}], agg#0=[count($3)])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveUnion(all=[true])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(ss_sold_date_sk=[$0], ss_item_sk=[$2])
+ HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d1])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[iss])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(cs_sold_date_sk=[$0], cs_item_sk=[$15])
+ HiveFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d2])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[ics])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(ws_sold_date_sk=[$0], ws_item_sk=[$3])
+ HiveFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d3])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[iws])
HiveJoin(condition=[true], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(cnt=[$0])
HiveFilter(condition=[<=(sq_count_check($0), 1)])
@@ -359,60 +358,10 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[
HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim])
HiveProject(channel=[_UTF-16LE'catalog'], i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], sales=[$3], number_sales=[$4])
HiveJoin(condition=[>($3, $6)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3], $f4=[$4])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3], $f4=[$4])
HiveAggregate(group=[{0, 1, 2}], agg#0=[sum($3)], agg#1=[count()])
- HiveProject($f0=[$2], $f1=[$3], $f2=[$4], $f3=[*(CAST($7):DECIMAL(10, 0), $8)])
- HiveJoin(condition=[=($6, $0)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(i_item_sk=[$0])
- HiveAggregate(group=[{0}])
- HiveJoin(condition=[AND(AND(=($1, $4), =($2, $5)), =($3, $6))], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11), IS NOT NULL($0))])
- HiveTableScan(table=[[default, item]], table:alias=[item])
- HiveProject($f0=[$0], $f1=[$1], $f2=[$2])
- HiveFilter(condition=[=($3, 3)])
- HiveAggregate(group=[{0, 1, 2}], agg#0=[count($3)])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveUnion(all=[true])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(ss_sold_date_sk=[$0], ss_item_sk=[$2])
- HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($0))])
- HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d1])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[iss])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(cs_sold_date_sk=[$0], cs_item_sk=[$15])
- HiveFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($0))])
- HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d2])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[ics])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(ws_sold_date_sk=[$0], ws_item_sk=[$3])
- HiveFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($0))])
- HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d3])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[iws])
+ HiveProject(i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f3=[*(CAST($6):DECIMAL(10, 0), $7)])
+ HiveSemiJoin(condition=[=($5, $11)], joinType=[inner])
HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
HiveFilter(condition=[IS NOT NULL($0)])
@@ -424,6 +373,55 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[
HiveProject(d_date_sk=[$0], d_year=[CAST(2000):INTEGER], d_moy=[CAST(11):INTEGER])
HiveFilter(condition=[AND(=($6, 2000), =($8, 11), IS NOT NULL($0))])
HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f0=[$4], $f1=[$5], $f2=[$6])
+ HiveJoin(condition=[AND(AND(=($1, $4), =($2, $5)), =($3, $6))], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, item]], table:alias=[item])
+ HiveProject($f0=[$0], $f1=[$1], $f2=[$2])
+ HiveFilter(condition=[=($3, 3)])
+ HiveAggregate(group=[{0, 1, 2}], agg#0=[count($3)])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveUnion(all=[true])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(ss_sold_date_sk=[$0], ss_item_sk=[$2])
+ HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d1])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[iss])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(cs_sold_date_sk=[$0], cs_item_sk=[$15])
+ HiveFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d2])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[ics])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(ws_sold_date_sk=[$0], ws_item_sk=[$3])
+ HiveFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d3])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[iws])
HiveJoin(condition=[true], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(cnt=[$0])
HiveFilter(condition=[<=(sq_count_check($0), 1)])
@@ -488,60 +486,10 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[
HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim])
HiveProject(channel=[_UTF-16LE'web'], i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], sales=[$3], number_sales=[$4])
HiveJoin(condition=[>($3, $6)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3], $f4=[$4])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3], $f4=[$4])
HiveAggregate(group=[{0, 1, 2}], agg#0=[sum($3)], agg#1=[count()])
- HiveProject($f0=[$2], $f1=[$3], $f2=[$4], $f3=[*(CAST($7):DECIMAL(10, 0), $8)])
- HiveJoin(condition=[=($6, $0)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(i_item_sk=[$0])
- HiveAggregate(group=[{0}])
- HiveJoin(condition=[AND(AND(=($1, $4), =($2, $5)), =($3, $6))], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11), IS NOT NULL($0))])
- HiveTableScan(table=[[default, item]], table:alias=[item])
- HiveProject($f0=[$0], $f1=[$1], $f2=[$2])
- HiveFilter(condition=[=($3, 3)])
- HiveAggregate(group=[{0, 1, 2}], agg#0=[count($3)])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveUnion(all=[true])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(ss_sold_date_sk=[$0], ss_item_sk=[$2])
- HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($0))])
- HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d1])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[iss])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(cs_sold_date_sk=[$0], cs_item_sk=[$15])
- HiveFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($0))])
- HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d2])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[ics])
- HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
- HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
- HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(ws_sold_date_sk=[$0], ws_item_sk=[$3])
- HiveFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($0))])
- HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales])
- HiveProject(d_date_sk=[$0], d_year=[$6])
- HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
- HiveTableScan(table=[[default, date_dim]], table:alias=[d3])
- HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
- HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
- HiveTableScan(table=[[default, item]], table:alias=[iws])
+ HiveProject(i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f3=[*(CAST($6):DECIMAL(10, 0), $7)])
+ HiveSemiJoin(condition=[=($5, $11)], joinType=[inner])
HiveJoin(condition=[=($5, $0)], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
HiveFilter(condition=[IS NOT NULL($0)])
@@ -553,6 +501,55 @@ HiveSortLimit(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC], dir1=[
HiveProject(d_date_sk=[$0], d_year=[CAST(2000):INTEGER], d_moy=[CAST(11):INTEGER])
HiveFilter(condition=[AND(=($6, 2000), =($8, 11), IS NOT NULL($0))])
HiveTableScan(table=[[default, date_dim]], table:alias=[date_dim])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$1], i_class_id=[$2], i_category_id=[$3], $f0=[$4], $f1=[$5], $f2=[$6])
+ HiveJoin(condition=[AND(AND(=($1, $4), =($2, $5)), =($3, $6))], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, item]], table:alias=[item])
+ HiveProject($f0=[$0], $f1=[$1], $f2=[$2])
+ HiveFilter(condition=[=($3, 3)])
+ HiveAggregate(group=[{0, 1, 2}], agg#0=[count($3)])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveUnion(all=[true])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(ss_sold_date_sk=[$0], ss_item_sk=[$2])
+ HiveFilter(condition=[AND(IS NOT NULL($2), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d1])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[iss])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(cs_sold_date_sk=[$0], cs_item_sk=[$15])
+ HiveFilter(condition=[AND(IS NOT NULL($15), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, catalog_sales]], table:alias=[catalog_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d2])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[ics])
+ HiveProject(i_brand_id=[$0], i_class_id=[$1], i_category_id=[$2], $f3=[$3])
+ HiveAggregate(group=[{5, 6, 7}], agg#0=[count()])
+ HiveJoin(condition=[=($1, $4)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
+ HiveProject(ws_sold_date_sk=[$0], ws_item_sk=[$3])
+ HiveFilter(condition=[AND(IS NOT NULL($3), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, web_sales]], table:alias=[web_sales])
+ HiveProject(d_date_sk=[$0], d_year=[$6])
+ HiveFilter(condition=[AND(BETWEEN(false, $6, 1999, 2001), IS NOT NULL($0))])
+ HiveTableScan(table=[[default, date_dim]], table:alias=[d3])
+ HiveProject(i_item_sk=[$0], i_brand_id=[$7], i_class_id=[$9], i_category_id=[$11])
+ HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7), IS NOT NULL($9), IS NOT NULL($11))])
+ HiveTableScan(table=[[default, item]], table:alias=[iws])
HiveJoin(condition=[true], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(cnt=[$0])
HiveFilter(condition=[<=(sq_count_check($0), 1)])
[2/4] hive git commit: HIVE-20826 : Enhance HiveSemiJoin rule to
convert join + group by on left side to Left Semi Join (Vineet Garg via
Ashutosh Chauhan)
Posted by ha...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/perf/tez/query14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
index c078c27..2c28023 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
@@ -1,9 +1,9 @@
-Warning: Shuffle Join MERGEJOIN[1446][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
-Warning: Shuffle Join MERGEJOIN[1458][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product
-Warning: Shuffle Join MERGEJOIN[1448][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 13' is a cross product
-Warning: Shuffle Join MERGEJOIN[1471][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 14' is a cross product
-Warning: Shuffle Join MERGEJOIN[1450][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 18' is a cross product
-Warning: Shuffle Join MERGEJOIN[1484][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 19' is a cross product
+Warning: Shuffle Join MERGEJOIN[1440][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
+Warning: Shuffle Join MERGEJOIN[1452][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product
+Warning: Shuffle Join MERGEJOIN[1442][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 13' is a cross product
+Warning: Shuffle Join MERGEJOIN[1465][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 14' is a cross product
+Warning: Shuffle Join MERGEJOIN[1444][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 18' is a cross product
+Warning: Shuffle Join MERGEJOIN[1478][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 19' is a cross product
PREHOOK: query: explain
with cross_items as
(select i_item_sk ss_item_sk
@@ -225,35 +225,31 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
Plan optimized by CBO.
Vertex dependency in root stage
-Map 1 <- Reducer 108 (BROADCAST_EDGE)
-Map 100 <- Reducer 103 (BROADCAST_EDGE)
-Map 110 <- Reducer 105 (BROADCAST_EDGE)
-Map 111 <- Reducer 107 (BROADCAST_EDGE)
-Map 112 <- Reducer 63 (BROADCAST_EDGE), Reducer 83 (BROADCAST_EDGE)
-Map 113 <- Reducer 68 (BROADCAST_EDGE), Reducer 93 (BROADCAST_EDGE)
+Map 1 <- Reducer 102 (BROADCAST_EDGE)
+Map 104 <- Reducer 99 (BROADCAST_EDGE)
+Map 105 <- Reducer 101 (BROADCAST_EDGE)
+Map 106 <- Reducer 63 (BROADCAST_EDGE), Reducer 80 (BROADCAST_EDGE)
+Map 107 <- Reducer 68 (BROADCAST_EDGE), Reducer 85 (BROADCAST_EDGE)
Map 20 <- Reducer 25 (BROADCAST_EDGE)
Map 36 <- Reducer 41 (BROADCAST_EDGE)
-Map 46 <- Reducer 109 (BROADCAST_EDGE)
+Map 46 <- Reducer 103 (BROADCAST_EDGE)
Map 50 <- Reducer 29 (BROADCAST_EDGE)
Map 51 <- Reducer 43 (BROADCAST_EDGE)
-Map 52 <- Reducer 58 (BROADCAST_EDGE), Reducer 72 (BROADCAST_EDGE)
-Reducer 10 <- Map 1 (SIMPLE_EDGE), Map 102 (SIMPLE_EDGE), Union 11 (CONTAINS)
-Reducer 101 <- Map 100 (SIMPLE_EDGE), Map 102 (SIMPLE_EDGE)
-Reducer 103 <- Map 102 (CUSTOM_SIMPLE_EDGE)
-Reducer 104 <- Map 102 (SIMPLE_EDGE), Map 110 (SIMPLE_EDGE)
-Reducer 105 <- Map 102 (CUSTOM_SIMPLE_EDGE)
-Reducer 106 <- Map 102 (SIMPLE_EDGE), Map 111 (SIMPLE_EDGE)
-Reducer 107 <- Map 102 (CUSTOM_SIMPLE_EDGE)
-Reducer 108 <- Map 102 (CUSTOM_SIMPLE_EDGE)
-Reducer 109 <- Map 102 (CUSTOM_SIMPLE_EDGE)
+Map 52 <- Reducer 58 (BROADCAST_EDGE), Reducer 71 (BROADCAST_EDGE)
+Map 94 <- Reducer 97 (BROADCAST_EDGE)
+Reducer 10 <- Map 1 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE), Union 11 (CONTAINS)
+Reducer 100 <- Map 105 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE)
+Reducer 101 <- Map 96 (CUSTOM_SIMPLE_EDGE)
+Reducer 102 <- Map 96 (CUSTOM_SIMPLE_EDGE)
+Reducer 103 <- Map 96 (CUSTOM_SIMPLE_EDGE)
Reducer 12 <- Union 11 (CUSTOM_SIMPLE_EDGE)
Reducer 13 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 32 (CUSTOM_SIMPLE_EDGE)
Reducer 14 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 62 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS)
-Reducer 15 <- Map 1 (SIMPLE_EDGE), Map 102 (SIMPLE_EDGE), Union 16 (CONTAINS)
+Reducer 15 <- Map 1 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE), Union 16 (CONTAINS)
Reducer 17 <- Union 16 (CUSTOM_SIMPLE_EDGE)
Reducer 18 <- Reducer 17 (CUSTOM_SIMPLE_EDGE), Reducer 35 (CUSTOM_SIMPLE_EDGE)
Reducer 19 <- Reducer 18 (CUSTOM_SIMPLE_EDGE), Reducer 67 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 102 (SIMPLE_EDGE), Union 3 (CONTAINS)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE), Union 3 (CONTAINS)
Reducer 21 <- Map 20 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE), Union 3 (CONTAINS)
Reducer 22 <- Map 20 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE), Union 11 (CONTAINS)
Reducer 23 <- Map 20 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE), Union 16 (CONTAINS)
@@ -274,1157 +270,1107 @@ Reducer 42 <- Map 40 (SIMPLE_EDGE), Map 51 (SIMPLE_EDGE), Union 27 (CONTAINS)
Reducer 43 <- Map 40 (CUSTOM_SIMPLE_EDGE)
Reducer 44 <- Map 40 (SIMPLE_EDGE), Map 51 (SIMPLE_EDGE), Union 31 (CONTAINS)
Reducer 45 <- Map 40 (SIMPLE_EDGE), Map 51 (SIMPLE_EDGE), Union 34 (CONTAINS)
-Reducer 47 <- Map 102 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE), Union 27 (CONTAINS)
-Reducer 48 <- Map 102 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE), Union 31 (CONTAINS)
-Reducer 49 <- Map 102 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE), Union 34 (CONTAINS)
+Reducer 47 <- Map 46 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE), Union 27 (CONTAINS)
+Reducer 48 <- Map 46 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE), Union 31 (CONTAINS)
+Reducer 49 <- Map 46 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE), Union 34 (CONTAINS)
Reducer 5 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE)
Reducer 53 <- Map 52 (SIMPLE_EDGE), Map 57 (SIMPLE_EDGE)
Reducer 54 <- Map 69 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE)
-Reducer 55 <- Reducer 54 (ONE_TO_ONE_EDGE), Reducer 71 (ONE_TO_ONE_EDGE)
+Reducer 55 <- Reducer 54 (ONE_TO_ONE_EDGE), Reducer 70 (SIMPLE_EDGE)
Reducer 56 <- Reducer 55 (SIMPLE_EDGE)
Reducer 58 <- Map 57 (CUSTOM_SIMPLE_EDGE)
-Reducer 59 <- Map 112 (SIMPLE_EDGE), Map 57 (SIMPLE_EDGE)
+Reducer 59 <- Map 106 (SIMPLE_EDGE), Map 57 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE), Reducer 56 (CUSTOM_SIMPLE_EDGE), Union 7 (CONTAINS)
Reducer 60 <- Map 69 (SIMPLE_EDGE), Reducer 59 (SIMPLE_EDGE)
-Reducer 61 <- Reducer 60 (ONE_TO_ONE_EDGE), Reducer 82 (ONE_TO_ONE_EDGE)
+Reducer 61 <- Reducer 60 (ONE_TO_ONE_EDGE), Reducer 79 (SIMPLE_EDGE)
Reducer 62 <- Reducer 61 (SIMPLE_EDGE)
Reducer 63 <- Map 57 (CUSTOM_SIMPLE_EDGE)
-Reducer 64 <- Map 113 (SIMPLE_EDGE), Map 57 (SIMPLE_EDGE)
+Reducer 64 <- Map 107 (SIMPLE_EDGE), Map 57 (SIMPLE_EDGE)
Reducer 65 <- Map 69 (SIMPLE_EDGE), Reducer 64 (SIMPLE_EDGE)
-Reducer 66 <- Reducer 65 (ONE_TO_ONE_EDGE), Reducer 92 (ONE_TO_ONE_EDGE)
+Reducer 66 <- Reducer 65 (ONE_TO_ONE_EDGE), Reducer 84 (SIMPLE_EDGE)
Reducer 67 <- Reducer 66 (SIMPLE_EDGE)
Reducer 68 <- Map 57 (CUSTOM_SIMPLE_EDGE)
-Reducer 70 <- Map 69 (SIMPLE_EDGE), Reducer 76 (ONE_TO_ONE_EDGE)
-Reducer 71 <- Reducer 70 (SIMPLE_EDGE)
-Reducer 72 <- Reducer 71 (CUSTOM_SIMPLE_EDGE)
-Reducer 73 <- Map 69 (SIMPLE_EDGE), Reducer 101 (SIMPLE_EDGE)
-Reducer 74 <- Reducer 73 (SIMPLE_EDGE), Union 75 (CONTAINS)
-Reducer 76 <- Union 75 (SIMPLE_EDGE)
-Reducer 77 <- Map 69 (SIMPLE_EDGE), Reducer 104 (SIMPLE_EDGE)
-Reducer 78 <- Reducer 77 (SIMPLE_EDGE), Union 75 (CONTAINS)
-Reducer 79 <- Map 69 (SIMPLE_EDGE), Reducer 106 (SIMPLE_EDGE)
+Reducer 70 <- Map 69 (SIMPLE_EDGE), Reducer 75 (ONE_TO_ONE_EDGE)
+Reducer 71 <- Reducer 70 (CUSTOM_SIMPLE_EDGE)
+Reducer 72 <- Map 69 (SIMPLE_EDGE), Reducer 95 (SIMPLE_EDGE)
+Reducer 73 <- Reducer 72 (SIMPLE_EDGE), Union 74 (CONTAINS)
+Reducer 75 <- Union 74 (SIMPLE_EDGE)
+Reducer 76 <- Reducer 72 (SIMPLE_EDGE), Union 77 (CONTAINS)
+Reducer 78 <- Union 77 (SIMPLE_EDGE)
+Reducer 79 <- Map 69 (SIMPLE_EDGE), Reducer 78 (ONE_TO_ONE_EDGE)
Reducer 8 <- Union 7 (SIMPLE_EDGE)
-Reducer 80 <- Reducer 79 (SIMPLE_EDGE), Union 75 (CONTAINS)
-Reducer 81 <- Map 69 (SIMPLE_EDGE), Reducer 87 (ONE_TO_ONE_EDGE)
-Reducer 82 <- Reducer 81 (SIMPLE_EDGE)
-Reducer 83 <- Reducer 82 (CUSTOM_SIMPLE_EDGE)
-Reducer 84 <- Map 69 (SIMPLE_EDGE), Reducer 101 (SIMPLE_EDGE)
-Reducer 85 <- Reducer 84 (SIMPLE_EDGE), Union 86 (CONTAINS)
-Reducer 87 <- Union 86 (SIMPLE_EDGE)
-Reducer 88 <- Reducer 84 (SIMPLE_EDGE), Union 89 (CONTAINS)
+Reducer 80 <- Reducer 79 (CUSTOM_SIMPLE_EDGE)
+Reducer 81 <- Reducer 72 (SIMPLE_EDGE), Union 82 (CONTAINS)
+Reducer 83 <- Union 82 (SIMPLE_EDGE)
+Reducer 84 <- Map 69 (SIMPLE_EDGE), Reducer 83 (ONE_TO_ONE_EDGE)
+Reducer 85 <- Reducer 84 (CUSTOM_SIMPLE_EDGE)
+Reducer 86 <- Map 69 (SIMPLE_EDGE), Reducer 98 (SIMPLE_EDGE)
+Reducer 87 <- Reducer 86 (SIMPLE_EDGE), Union 74 (CONTAINS)
+Reducer 88 <- Reducer 86 (SIMPLE_EDGE), Union 77 (CONTAINS)
+Reducer 89 <- Reducer 86 (SIMPLE_EDGE), Union 82 (CONTAINS)
Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
-Reducer 90 <- Union 89 (SIMPLE_EDGE)
-Reducer 91 <- Map 69 (SIMPLE_EDGE), Reducer 90 (ONE_TO_ONE_EDGE)
-Reducer 92 <- Reducer 91 (SIMPLE_EDGE)
-Reducer 93 <- Reducer 92 (CUSTOM_SIMPLE_EDGE)
-Reducer 94 <- Map 69 (SIMPLE_EDGE), Reducer 104 (SIMPLE_EDGE)
-Reducer 95 <- Reducer 94 (SIMPLE_EDGE), Union 86 (CONTAINS)
-Reducer 96 <- Reducer 94 (SIMPLE_EDGE), Union 89 (CONTAINS)
-Reducer 97 <- Map 69 (SIMPLE_EDGE), Reducer 106 (SIMPLE_EDGE)
-Reducer 98 <- Reducer 97 (SIMPLE_EDGE), Union 86 (CONTAINS)
-Reducer 99 <- Reducer 97 (SIMPLE_EDGE), Union 89 (CONTAINS)
+Reducer 90 <- Map 69 (SIMPLE_EDGE), Reducer 100 (SIMPLE_EDGE)
+Reducer 91 <- Reducer 90 (SIMPLE_EDGE), Union 74 (CONTAINS)
+Reducer 92 <- Reducer 90 (SIMPLE_EDGE), Union 77 (CONTAINS)
+Reducer 93 <- Reducer 90 (SIMPLE_EDGE), Union 82 (CONTAINS)
+Reducer 95 <- Map 94 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE)
+Reducer 97 <- Map 96 (CUSTOM_SIMPLE_EDGE)
+Reducer 98 <- Map 104 (SIMPLE_EDGE), Map 96 (SIMPLE_EDGE)
+Reducer 99 <- Map 96 (CUSTOM_SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:100
Stage-1
Reducer 9 vectorized
- File Output Operator [FS_1726]
- Limit [LIM_1725] (rows=7 width=192)
+ File Output Operator [FS_1712]
+ Limit [LIM_1711] (rows=7 width=192)
Number of rows:100
- Select Operator [SEL_1724] (rows=7 width=192)
+ Select Operator [SEL_1710] (rows=7 width=192)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
<-Reducer 8 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1723]
- Select Operator [SEL_1722] (rows=7 width=192)
+ SHUFFLE [RS_1709]
+ Select Operator [SEL_1708] (rows=7 width=192)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Group By Operator [GBY_1721] (rows=7 width=200)
+ Group By Operator [GBY_1707] (rows=7 width=200)
Output:["_col0","_col1","_col2","_col3","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
<-Union 7 [SIMPLE_EDGE]
<-Reducer 14 [CONTAINS]
- Reduce Output Operator [RS_1477]
+ Reduce Output Operator [RS_1471]
PartitionCols:_col0, _col1, _col2, _col3, _col4
- Group By Operator [GBY_1476] (rows=7 width=200)
+ Group By Operator [GBY_1470] (rows=7 width=200)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0L
- Top N Key Operator [TNK_1475] (rows=3 width=221)
+ Top N Key Operator [TNK_1469] (rows=3 width=221)
keys:_col0, _col1, _col2, _col3, 0L,sort order:+++++,top n:100
- Select Operator [SEL_1473] (rows=1 width=223)
+ Select Operator [SEL_1467] (rows=1 width=223)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_1472] (rows=1 width=244)
+ Filter Operator [FIL_1466] (rows=1 width=244)
predicate:(_col5 > _col1)
- Merge Join Operator [MERGEJOIN_1471] (rows=1 width=244)
+ Merge Join Operator [MERGEJOIN_1465] (rows=1 width=244)
Conds:(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
<-Reducer 13 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_381]
- Merge Join Operator [MERGEJOIN_1448] (rows=1 width=112)
+ PARTITION_ONLY_SHUFFLE [RS_377]
+ Merge Join Operator [MERGEJOIN_1442] (rows=1 width=112)
Conds:(Inner),Output:["_col1"]
<-Reducer 12 [CUSTOM_SIMPLE_EDGE] vectorized
- PARTITION_ONLY_SHUFFLE [RS_1732]
- Select Operator [SEL_1731] (rows=1 width=8)
- Filter Operator [FIL_1730] (rows=1 width=8)
+ PARTITION_ONLY_SHUFFLE [RS_1718]
+ Select Operator [SEL_1717] (rows=1 width=8)
+ Filter Operator [FIL_1716] (rows=1 width=8)
predicate:(sq_count_check(_col0) <= 1)
- Group By Operator [GBY_1729] (rows=1 width=8)
+ Group By Operator [GBY_1715] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_1728] (rows=1 width=8)
- Group By Operator [GBY_1727] (rows=1 width=8)
+ Select Operator [SEL_1714] (rows=1 width=8)
+ Group By Operator [GBY_1713] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Union 11 [CUSTOM_SIMPLE_EDGE]
<-Reducer 10 [CONTAINS]
- Reduce Output Operator [RS_1470]
- Group By Operator [GBY_1469] (rows=1 width=8)
+ Reduce Output Operator [RS_1464]
+ Group By Operator [GBY_1463] (rows=1 width=8)
Output:["_col0"],aggregations:["count(_col0)"]
- Select Operator [SEL_1468] (rows=26270325 width=1)
+ Select Operator [SEL_1462] (rows=26270325 width=1)
Output:["_col0"]
- Select Operator [SEL_1466] (rows=14736682 width=0)
+ Select Operator [SEL_1460] (rows=14736682 width=0)
Output:["_col0"]
- Merge Join Operator [MERGEJOIN_1465] (rows=14736682 width=0)
- Conds:RS_1648._col0=RS_1629._col0(Inner),Output:["_col1"]
- <-Map 102 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1629]
+ Merge Join Operator [MERGEJOIN_1459] (rows=14736682 width=0)
+ Conds:RS_1642._col0=RS_1623._col0(Inner),Output:["_col1"]
+ <-Map 96 [SIMPLE_EDGE] vectorized
+ PARTITION_ONLY_SHUFFLE [RS_1623]
PartitionCols:_col0
- Select Operator [SEL_1618] (rows=1957 width=8)
+ Select Operator [SEL_1612] (rows=1957 width=8)
Output:["_col0"]
- Filter Operator [FIL_1617] (rows=1957 width=8)
+ Filter Operator [FIL_1611] (rows=1957 width=8)
predicate:(d_date_sk is not null and d_year BETWEEN 1999 AND 2001)
TableScan [TS_97] (rows=73049 width=8)
default@date_dim,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_year"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1648]
+ SHUFFLE [RS_1642]
PartitionCols:_col0
- Select Operator [SEL_1646] (rows=550076554 width=7)
+ Select Operator [SEL_1640] (rows=550076554 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_1645] (rows=550076554 width=7)
+ Filter Operator [FIL_1639] (rows=550076554 width=7)
predicate:((ss_sold_date_sk BETWEEN DynamicValue(RS_7_date_dim_d_date_sk_min) AND DynamicValue(RS_7_date_dim_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_7_date_dim_d_date_sk_bloom_filter))) and ss_sold_date_sk is not null)
TableScan [TS_0] (rows=575995635 width=7)
default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_quantity"]
- <-Reducer 108 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1644]
- Group By Operator [GBY_1643] (rows=1 width=12)
+ <-Reducer 102 [BROADCAST_EDGE] vectorized
+ BROADCAST [RS_1638]
+ Group By Operator [GBY_1637] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
- <-Map 102 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1641]
- Group By Operator [GBY_1636] (rows=1 width=12)
+ <-Map 96 [CUSTOM_SIMPLE_EDGE] vectorized
+ PARTITION_ONLY_SHUFFLE [RS_1635]
+ Group By Operator [GBY_1630] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1626] (rows=1957 width=4)
+ Select Operator [SEL_1620] (rows=1957 width=4)
Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1618]
+ Please refer to the previous Select Operator [SEL_1612]
<-Reducer 22 [CONTAINS]
- Reduce Output Operator [RS_1502]
- Group By Operator [GBY_1501] (rows=1 width=8)
+ Reduce Output Operator [RS_1496]
+ Group By Operator [GBY_1495] (rows=1 width=8)
Output:["_col0"],aggregations:["count(_col0)"]
- Select Operator [SEL_1500] (rows=26270325 width=1)
+ Select Operator [SEL_1494] (rows=26270325 width=1)
Output:["_col0"]
- Select Operator [SEL_1498] (rows=7676736 width=3)
+ Select Operator [SEL_1492] (rows=7676736 width=3)
Output:["_col0"]
- Merge Join Operator [MERGEJOIN_1497] (rows=7676736 width=3)
- Conds:RS_1800._col0=RS_1787._col0(Inner),Output:["_col1"]
+ Merge Join Operator [MERGEJOIN_1491] (rows=7676736 width=3)
+ Conds:RS_1776._col0=RS_1763._col0(Inner),Output:["_col1"]
<-Map 24 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1787]
+ SHUFFLE [RS_1763]
PartitionCols:_col0
- Select Operator [SEL_1782] (rows=1957 width=8)
+ Select Operator [SEL_1758] (rows=1957 width=8)
Output:["_col0"]
- Filter Operator [FIL_1781] (rows=1957 width=8)
+ Filter Operator [FIL_1757] (rows=1957 width=8)
predicate:(d_date_sk is not null and d_year BETWEEN 1998 AND 2000)
TableScan [TS_13] (rows=73049 width=8)
default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_year"]
<-Map 20 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1800]
+ SHUFFLE [RS_1776]
PartitionCols:_col0
- Select Operator [SEL_1798] (rows=286549727 width=7)
+ Select Operator [SEL_1774] (rows=286549727 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_1797] (rows=286549727 width=7)
+ Filter Operator [FIL_1773] (rows=286549727 width=7)
predicate:((cs_sold_date_sk BETWEEN DynamicValue(RS_17_date_dim_d_date_sk_min) AND DynamicValue(RS_17_date_dim_d_date_sk_max) and in_bloom_filter(cs_sold_date_sk, DynamicValue(RS_17_date_dim_d_date_sk_bloom_filter))) and cs_sold_date_sk is not null)
TableScan [TS_10] (rows=287989836 width=7)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["cs_sold_date_sk","cs_quantity"]
<-Reducer 25 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1796]
- Group By Operator [GBY_1795] (rows=1 width=12)
+ BROADCAST [RS_1772]
+ Group By Operator [GBY_1771] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
<-Map 24 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1793]
- Group By Operator [GBY_1791] (rows=1 width=12)
+ SHUFFLE [RS_1769]
+ Group By Operator [GBY_1767] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1784] (rows=1957 width=4)
+ Select Operator [SEL_1760] (rows=1957 width=4)
Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1782]
+ Please refer to the previous Select Operator [SEL_1758]
<-Reducer 38 [CONTAINS]
- Reduce Output Operator [RS_1538]
- Group By Operator [GBY_1537] (rows=1 width=8)
+ Reduce Output Operator [RS_1532]
+ Group By Operator [GBY_1531] (rows=1 width=8)
Output:["_col0"],aggregations:["count(_col0)"]
- Select Operator [SEL_1536] (rows=26270325 width=1)
+ Select Operator [SEL_1530] (rows=26270325 width=1)
Output:["_col0"]
- Select Operator [SEL_1534] (rows=3856907 width=3)
+ Select Operator [SEL_1528] (rows=3856907 width=3)
Output:["_col0"]
- Merge Join Operator [MERGEJOIN_1533] (rows=3856907 width=3)
- Conds:RS_1828._col0=RS_1815._col0(Inner),Output:["_col1"]
+ Merge Join Operator [MERGEJOIN_1527] (rows=3856907 width=3)
+ Conds:RS_1804._col0=RS_1791._col0(Inner),Output:["_col1"]
<-Map 40 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1815]
+ SHUFFLE [RS_1791]
PartitionCols:_col0
- Select Operator [SEL_1810] (rows=1957 width=8)
+ Select Operator [SEL_1786] (rows=1957 width=8)
Output:["_col0"]
- Filter Operator [FIL_1809] (rows=1957 width=8)
+ Filter Operator [FIL_1785] (rows=1957 width=8)
predicate:(d_date_sk is not null and d_year BETWEEN 1998 AND 2000)
TableScan [TS_24] (rows=73049 width=8)
default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE,Output:["d_date_sk","d_year"]
<-Map 36 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1828]
+ SHUFFLE [RS_1804]
PartitionCols:_col0
- Select Operator [SEL_1826] (rows=143966864 width=7)
+ Select Operator [SEL_1802] (rows=143966864 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_1825] (rows=143966864 width=7)
+ Filter Operator [FIL_1801] (rows=143966864 width=7)
predicate:((ws_sold_date_sk BETWEEN DynamicValue(RS_28_date_dim_d_date_sk_min) AND DynamicValue(RS_28_date_dim_d_date_sk_max) and in_bloom_filter(ws_sold_date_sk, DynamicValue(RS_28_date_dim_d_date_sk_bloom_filter))) and ws_sold_date_sk is not null)
TableScan [TS_21] (rows=144002668 width=7)
default@web_sales,web_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ws_sold_date_sk","ws_quantity"]
<-Reducer 41 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1824]
- Group By Operator [GBY_1823] (rows=1 width=12)
+ BROADCAST [RS_1800]
+ Group By Operator [GBY_1799] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
<-Map 40 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1821]
- Group By Operator [GBY_1819] (rows=1 width=12)
+ SHUFFLE [RS_1797]
+ Group By Operator [GBY_1795] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1812] (rows=1957 width=4)
+ Select Operator [SEL_1788] (rows=1957 width=4)
Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1810]
+ Please refer to the previous Select Operator [SEL_1786]
<-Reducer 32 [CUSTOM_SIMPLE_EDGE] vectorized
- PARTITION_ONLY_SHUFFLE [RS_1735]
- Select Operator [SEL_1734] (rows=1 width=112)
+ PARTITION_ONLY_SHUFFLE [RS_1721]
+ Select Operator [SEL_1720] (rows=1 width=112)
Output:["_col0"]
- Group By Operator [GBY_1733] (rows=1 width=120)
+ Group By Operator [GBY_1719] (rows=1 width=120)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"]
<-Union 31 [CUSTOM_SIMPLE_EDGE]
<-Reducer 30 [CONTAINS]
- Reduce Output Operator [RS_1520]
- Group By Operator [GBY_1519] (rows=1 width=120)
+ Reduce Output Operator [RS_1514]
+ Group By Operator [GBY_1513] (rows=1 width=120)
Output:["_col0","_col1"],aggregations:["sum(_col0)","count(_col0)"]
- Select Operator [SEL_1518] (rows=26270325 width=44)
+ Select Operator [SEL_1512] (rows=26270325 width=44)
Output:["_col0"]
- Select Operator [SEL_1516] (rows=7676736 width=94)
+ Select Operator [SEL_1510] (rows=7676736 width=94)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_1515] (rows=7676736 width=94)
- Conds:RS_1807._col0=RS_1788._col0(Inner),Output:["_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_1509] (rows=7676736 width=94)
+ Conds:RS_1783._col0=RS_1764._col0(Inner),Output:["_col1","_col2"]
<-Map 24 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1788]
+ SHUFFLE [RS_1764]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_1782]
+ Please refer to the previous Select Operator [SEL_1758]
<-Map 50 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1807]
+ SHUFFLE [RS_1783]
PartitionCols:_col0
- Select Operator [SEL_1805] (rows=286549727 width=119)
+ Select Operator [SEL_1781] (rows=286549727 width=119)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_1804] (rows=286549727 width=119)
+ Filter Operator [FIL_1780] (rows=286549727 width=119)
predicate:((cs_sold_date_sk BETWEEN DynamicValue(RS_62_date_dim_d_date_sk_min) AND DynamicValue(RS_62_date_dim_d_date_sk_max) and in_bloom_filter(cs_sold_date_sk, DynamicValue(RS_62_date_dim_d_date_sk_bloom_filter))) and cs_sold_date_sk is not null)
TableScan [TS_55] (rows=287989836 width=119)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"]
<-Reducer 29 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1803]
- Group By Operator [GBY_1802] (rows=1 width=12)
+ BROADCAST [RS_1779]
+ Group By Operator [GBY_1778] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
<-Map 24 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1794]
- Group By Operator [GBY_1792] (rows=1 width=12)
+ SHUFFLE [RS_1770]
+ Group By Operator [GBY_1768] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1786] (rows=1957 width=4)
+ Select Operator [SEL_1762] (rows=1957 width=4)
Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1782]
+ Please refer to the previous Select Operator [SEL_1758]
<-Reducer 44 [CONTAINS]
- Reduce Output Operator [RS_1556]
- Group By Operator [GBY_1555] (rows=1 width=120)
+ Reduce Output Operator [RS_1550]
+ Group By Operator [GBY_1549] (rows=1 width=120)
Output:["_col0","_col1"],aggregations:["sum(_col0)","count(_col0)"]
- Select Operator [SEL_1554] (rows=26270325 width=44)
+ Select Operator [SEL_1548] (rows=26270325 width=44)
Output:["_col0"]
- Select Operator [SEL_1552] (rows=3856907 width=114)
+ Select Operator [SEL_1546] (rows=3856907 width=114)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_1551] (rows=3856907 width=114)
- Conds:RS_1835._col0=RS_1816._col0(Inner),Output:["_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_1545] (rows=3856907 width=114)
+ Conds:RS_1811._col0=RS_1792._col0(Inner),Output:["_col1","_col2"]
<-Map 40 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1816]
+ SHUFFLE [RS_1792]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_1810]
+ Please refer to the previous Select Operator [SEL_1786]
<-Map 51 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1835]
+ SHUFFLE [RS_1811]
PartitionCols:_col0
- Select Operator [SEL_1833] (rows=143966864 width=119)
+ Select Operator [SEL_1809] (rows=143966864 width=119)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_1832] (rows=143966864 width=119)
+ Filter Operator [FIL_1808] (rows=143966864 width=119)
predicate:((ws_sold_date_sk BETWEEN DynamicValue(RS_73_date_dim_d_date_sk_min) AND DynamicValue(RS_73_date_dim_d_date_sk_max) and in_bloom_filter(ws_sold_date_sk, DynamicValue(RS_73_date_dim_d_date_sk_bloom_filter))) and ws_sold_date_sk is not null)
TableScan [TS_66] (rows=144002668 width=119)
default@web_sales,web_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"]
<-Reducer 43 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1831]
- Group By Operator [GBY_1830] (rows=1 width=12)
+ BROADCAST [RS_1807]
+ Group By Operator [GBY_1806] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
<-Map 40 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1822]
- Group By Operator [GBY_1820] (rows=1 width=12)
+ SHUFFLE [RS_1798]
+ Group By Operator [GBY_1796] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1814] (rows=1957 width=4)
+ Select Operator [SEL_1790] (rows=1957 width=4)
Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1810]
+ Please refer to the previous Select Operator [SEL_1786]
<-Reducer 48 [CONTAINS]
- Reduce Output Operator [RS_1574]
- Group By Operator [GBY_1573] (rows=1 width=120)
+ Reduce Output Operator [RS_1568]
+ Group By Operator [GBY_1567] (rows=1 width=120)
Output:["_col0","_col1"],aggregations:["sum(_col0)","count(_col0)"]
- Select Operator [SEL_1572] (rows=26270325 width=44)
+ Select Operator [SEL_1566] (rows=26270325 width=44)
Output:["_col0"]
- Select Operator [SEL_1570] (rows=14736682 width=0)
+ Select Operator [SEL_1564] (rows=14736682 width=0)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_1569] (rows=14736682 width=0)
- Conds:RS_1842._col0=RS_1630._col0(Inner),Output:["_col1","_col2"]
- <-Map 102 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1630]
+ Merge Join Operator [MERGEJOIN_1563] (rows=14736682 width=0)
+ Conds:RS_1818._col0=RS_1624._col0(Inner),Output:["_col1","_col2"]
+ <-Map 96 [SIMPLE_EDGE] vectorized
+ PARTITION_ONLY_SHUFFLE [RS_1624]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_1618]
+ Please refer to the previous Select Operator [SEL_1612]
<-Map 46 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1842]
+ SHUFFLE [RS_1818]
PartitionCols:_col0
- Select Operator [SEL_1840] (rows=550076554 width=114)
+ Select Operator [SEL_1816] (rows=550076554 width=114)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_1839] (rows=550076554 width=114)
+ Filter Operator [FIL_1815] (rows=550076554 width=114)
predicate:((ss_sold_date_sk BETWEEN DynamicValue(RS_52_date_dim_d_date_sk_min) AND DynamicValue(RS_52_date_dim_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_52_date_dim_d_date_sk_bloom_filter))) and ss_sold_date_sk is not null)
TableScan [TS_45] (rows=575995635 width=114)
default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"]
- <-Reducer 109 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1838]
- Group By Operator [GBY_1837] (rows=1 width=12)
+ <-Reducer 103 [BROADCAST_EDGE] vectorized
+ BROADCAST [RS_1814]
+ Group By Operator [GBY_1813] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
- <-Map 102 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1642]
- Group By Operator [GBY_1637] (rows=1 width=12)
+ <-Map 96 [CUSTOM_SIMPLE_EDGE] vectorized
+ PARTITION_ONLY_SHUFFLE [RS_1636]
+ Group By Operator [GBY_1631] (rows=1 width=12)
Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1628] (rows=1957 width=4)
+ Select Operator [SEL_1622] (rows=1957 width=4)
Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1618]
+ Please refer to the previous Select Operator [SEL_1612]
<-Reducer 62 [CUSTOM_SIMPLE_EDGE] vectorized
- PARTITION_ONLY_SHUFFLE [RS_1753]
- Group By Operator [GBY_1752] (rows=1 width=132)
+ PARTITION_ONLY_SHUFFLE [RS_1734]
+ Group By Operator [GBY_1733] (rows=1 width=132)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2
<-Reducer 61 [SIMPLE_EDGE]
- SHUFFLE [RS_375]
+ SHUFFLE [RS_371]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_374] (rows=1 width=132)
+ Group By Operator [GBY_370] (rows=1 width=132)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col3)","count()"],keys:_col0, _col1, _col2
- Select Operator [SEL_372] (rows=1 width=128)
+ Select Operator [SEL_368] (rows=1 width=128)
Output:["_col0","_col1","_col2","_col3"]
- Merge Join Operator [MERGEJOIN_1444] (rows=1 width=128)
- Conds:RS_369._col1=RS_1743._col0(Inner),Output:["_col2","_col3","_col8","_col9","_col10"]
- <-Reducer 82 [ONE_TO_ONE_EDGE] vectorized
- PARTITION_ONLY_SHUFFLE [RS_1743]
+ Merge Join Operator [MERGEJOIN_1438] (rows=1 width=128)
+ Conds:RS_365._col1=RS_366._col0(Left Semi),Output:["_col2","_col3","_col8","_col9","_col10"]
+ <-Reducer 79 [SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_366]
PartitionCols:_col0
- Group By Operator [GBY_1742] (rows=362 width=4)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 81 [SIMPLE_EDGE]
- SHUFFLE [RS_360]
- PartitionCols:_col0
- Group By Operator [GBY_359] (rows=362 width=4)
- Output:["_col0"],keys:_col0
- Merge Join Operator [MERGEJOIN_1427] (rows=724 width=4)
- Conds:RS_1698._col1, _col2, _col3=RS_1741._col0, _col1, _col2(Inner),Output:["_col0"]
- <-Map 69 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1698]
- PartitionCols:_col1, _col2, _col3
- Select Operator [SEL_1686] (rows=458612 width=15)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_1677] (rows=458612 width=15)
- predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
- TableScan [TS_91] (rows=462000 width=15)
- default@item,item,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"]
- <-Reducer 87 [ONE_TO_ONE_EDGE] vectorized
- FORWARD [RS_1741]
- PartitionCols:_col0, _col1, _col2
- Select Operator [SEL_1740] (rows=1 width=12)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_1739] (rows=1 width=20)
- predicate:(_col3 = 3L)
- Group By Operator [GBY_1738] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Union 86 [SIMPLE_EDGE]
- <-Reducer 85 [CONTAINS] vectorized
- Reduce Output Operator [RS_1870]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_1869] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
- Group By Operator [GBY_1868] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 84 [SIMPLE_EDGE]
- SHUFFLE [RS_304]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_303] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
- Merge Join Operator [MERGEJOIN_1422] (rows=14628613 width=11)
- Conds:RS_299._col1=RS_1699._col0(Inner),Output:["_col5","_col6","_col7"]
- <-Map 69 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1699]
+ Group By Operator [GBY_364] (rows=362 width=4)
+ Output:["_col0"],keys:_col0
+ Merge Join Operator [MERGEJOIN_1421] (rows=724 width=4)
+ Conds:RS_1693._col1, _col2, _col3=RS_1727._col0, _col1, _col2(Inner),Output:["_col0"]
+ <-Map 69 [SIMPLE_EDGE] vectorized
+ SHUFFLE [RS_1693]
+ PartitionCols:_col1, _col2, _col3
+ Select Operator [SEL_1684] (rows=458612 width=15)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_1675] (rows=458612 width=15)
+ predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
+ TableScan [TS_88] (rows=462000 width=15)
+ default@item,item,Tbl:COMPLETE,Col:COMPLETE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id"]
+ <-Reducer 78 [ONE_TO_ONE_EDGE] vectorized
+ FORWARD [RS_1727]
+ PartitionCols:_col0, _col1, _col2
+ Select Operator [SEL_1726] (rows=1 width=12)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_1725] (rows=1 width=20)
+ predicate:(_col3 = 3L)
+ Group By Operator [GBY_1724] (rows=121728 width=19)
+ Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
+ <-Union 77 [SIMPLE_EDGE]
+ <-Reducer 76 [CONTAINS] vectorized
+ Reduce Output Operator [RS_1830]
+ PartitionCols:_col0, _col1, _col2
+ Group By Operator [GBY_1829] (rows=121728 width=19)
+ Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
+ Group By Operator [GBY_1828] (rows=121728 width=19)
+ Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
+ <-Reducer 72 [SIMPLE_EDGE]
+ SHUFFLE [RS_302]
+ PartitionCols:_col0, _col1, _col2
+ Group By Operator [GBY_110] (rows=121728 width=19)
+ Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
+ Merge Join Operator [MERGEJOIN_1401] (rows=14628613 width=11)
+ Conds:RS_106._col1=RS_1689._col0(Inner),Output:["_col5","_col6","_col7"]
+ <-Map 69 [SIMPLE_EDGE] vectorized
+ SHUFFLE [RS_1689]
+ PartitionCols:_col0
+ Select Operator [SEL_1680] (rows=458612 width=15)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_1671] (rows=458612 width=15)
+ predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
+ Please refer to the previous TableScan [TS_88]
+ <-Reducer 95 [SIMPLE_EDGE]
+ SHUFFLE [RS_106]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_1400] (rows=14736682 width=4)
+ Conds:RS_1824._col0=RS_1613._col0(Inner),Output:["_col1"]
+ <-Map 96 [SIMPLE_EDGE] vectorized
+ PARTITION_ONLY_SHUFFLE [RS_1613]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_1612]
+ <-Map 94 [SIMPLE_EDGE] vectorized
+ SHUFFLE [RS_1824]
PartitionCols:_col0
- Select Operator [SEL_1687] (rows=458612 width=15)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_1678] (rows=458612 width=15)
- predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_91]
- <-Reducer 101 [SIMPLE_EDGE]
- SHUFFLE [RS_299]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_1406] (rows=14736682 width=4)
- Conds:RS_1848._col0=RS_1619._col0(Inner),Output:["_col1"]
- <-Map 102 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1619]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_1618]
- <-Map 100 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1848]
- PartitionCols:_col0
- Select Operator [SEL_1847] (rows=550076554 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_1846] (rows=550076554 width=7)
- predicate:((ss_sold_date_sk BETWEEN DynamicValue(RS_104_d1_d_date_sk_min) AND DynamicValue(RS_104_d1_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_104_d1_d_date_sk_bloom_filter))) and ss_item_sk is not null and ss_sold_date_sk is not null)
- TableScan [TS_94] (rows=575995635 width=7)
- default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_item_sk"]
- <-Reducer 103 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1845]
- Group By Operator [GBY_1844] (rows=1 width=12)
- Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
- <-Map 102 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1638]
- Group By Operator [GBY_1633] (rows=1 width=12)
- Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1620] (rows=1957 width=4)
- Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1618]
- <-Reducer 95 [CONTAINS] vectorized
- Reduce Output Operator [RS_1876]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_1875] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
- Group By Operator [GBY_1874] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 94 [SIMPLE_EDGE]
- SHUFFLE [RS_324]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_323] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
- Merge Join Operator [MERGEJOIN_1424] (rows=7620440 width=11)
- Conds:RS_319._col1=RS_1700._col0(Inner),Output:["_col5","_col6","_col7"]
- <-Map 69 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1700]
+ Select Operator [SEL_1823] (rows=550076554 width=7)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_1822] (rows=550076554 width=7)
+ predicate:((ss_sold_date_sk BETWEEN DynamicValue(RS_104_d1_d_date_sk_min) AND DynamicValue(RS_104_d1_d_date_sk_max) and in_bloom_filter(ss_sold_date_sk, DynamicValue(RS_104_d1_d_date_sk_bloom_filter))) and ss_item_sk is not null and ss_sold_date_sk is not null)
+ TableScan [TS_94] (rows=575995635 width=7)
+ default@store_sales,store_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_date_sk","ss_item_sk"]
+ <-Reducer 97 [BROADCAST_EDGE] vectorized
+ BROADCAST [RS_1821]
+ Group By Operator [GBY_1820] (rows=1 width=12)
+ Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
+ <-Map 96 [CUSTOM_SIMPLE_EDGE] vectorized
+ PARTITION_ONLY_SHUFFLE [RS_1632]
+ Group By Operator [GBY_1627] (rows=1 width=12)
+ Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
+ Select Operator [SEL_1614] (rows=1957 width=4)
+ Output:["_col0"]
+ Please refer to the previous Select Operator [SEL_1612]
+ <-Reducer 88 [CONTAINS] vectorized
+ Reduce Output Operator [RS_1844]
+ PartitionCols:_col0, _col1, _col2
+ Group By Operator [GBY_1843] (rows=121728 width=19)
+ Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
+ Group By Operator [GBY_1842] (rows=121728 width=19)
+ Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
+ <-Reducer 86 [SIMPLE_EDGE]
+ SHUFFLE [RS_322]
+ PartitionCols:_col0, _col1, _col2
+ Group By Operator [GBY_130] (rows=121728 width=19)
+ Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
+ Merge Join Operator [MERGEJOIN_1403] (rows=7620440 width=11)
+ Conds:RS_126._col1=RS_1690._col0(Inner),Output:["_col5","_col6","_col7"]
+ <-Map 69 [SIMPLE_EDGE] vectorized
+ SHUFFLE [RS_1690]
+ PartitionCols:_col0
+ Select Operator [SEL_1681] (rows=458612 width=15)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_1672] (rows=458612 width=15)
+ predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
+ Please refer to the previous TableScan [TS_88]
+ <-Reducer 98 [SIMPLE_EDGE]
+ SHUFFLE [RS_126]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_1402] (rows=7676736 width=4)
+ Conds:RS_1838._col0=RS_1615._col0(Inner),Output:["_col1"]
+ <-Map 96 [SIMPLE_EDGE] vectorized
+ PARTITION_ONLY_SHUFFLE [RS_1615]
PartitionCols:_col0
- Select Operator [SEL_1688] (rows=458612 width=15)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_1679] (rows=458612 width=15)
- predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_91]
- <-Reducer 104 [SIMPLE_EDGE]
- SHUFFLE [RS_319]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_1408] (rows=7676736 width=4)
- Conds:RS_1856._col0=RS_1621._col0(Inner),Output:["_col1"]
- <-Map 102 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1621]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_1618]
- <-Map 110 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1856]
- PartitionCols:_col0
- Select Operator [SEL_1855] (rows=286549727 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_1854] (rows=286549727 width=7)
- predicate:((cs_sold_date_sk BETWEEN DynamicValue(RS_124_d2_d_date_sk_min) AND DynamicValue(RS_124_d2_d_date_sk_max) and in_bloom_filter(cs_sold_date_sk, DynamicValue(RS_124_d2_d_date_sk_bloom_filter))) and cs_item_sk is not null and cs_sold_date_sk is not null)
- TableScan [TS_114] (rows=287989836 width=7)
- default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["cs_sold_date_sk","cs_item_sk"]
- <-Reducer 105 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1853]
- Group By Operator [GBY_1852] (rows=1 width=12)
- Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
- <-Map 102 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1639]
- Group By Operator [GBY_1634] (rows=1 width=12)
- Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Select Operator [SEL_1622] (rows=1957 width=4)
- Output:["_col0"]
- Please refer to the previous Select Operator [SEL_1618]
- <-Reducer 98 [CONTAINS] vectorized
- Reduce Output Operator [RS_1882]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_1881] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
- Group By Operator [GBY_1880] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 97 [SIMPLE_EDGE]
- SHUFFLE [RS_345]
- PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_344] (rows=121728 width=19)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
- Merge Join Operator [MERGEJOIN_1426] (rows=3828623 width=11)
- Conds:RS_340._col1=RS_1701._col0(Inner),Output:["_col5","_col6","_col7"]
- <-Map 69 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1701]
+ Please refer to the previous Select Operator [SEL_1612]
+ <-Map 104 [SIMPLE_EDGE] vectorized
+ SHUFFLE [RS_1838]
PartitionCols:_col0
- Select Operator [SEL_1689] (rows=458612 width=15)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_1680] (rows=458612 width=15)
- predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_91]
- <-Reducer 106 [SIMPLE_EDGE]
- SHUFFLE [RS_340]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_1410] (rows=3856907 width=4)
- Conds:RS_1864._col0=RS_1623._col0(Inner),Output:["_col1"]
- <-Map 102 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1623]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_1618]
- <-Map 111 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1864]
- PartitionCols:_col0
- Select Operator [SEL_1863] (rows=143966864 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_1862] (rows=143966864 width=7)
- predicate:((ws_sold_date_sk BETWEEN DynamicValue(RS_145_d3_d_date_sk_min) AND DynamicValue(RS_145_d3_d_date_sk_max) and in_bloom_filter(ws_sold_date_sk, DynamicValue(RS_145_d3_d_date_sk_bloom_filter))) and ws_item_sk is not null and ws_sold_date_sk is not null)
- TableScan [TS_135] (rows=144002668 width=7)
- default@web_sales,web_sales,Tbl:COMPLETE,Col:COMPLETE,Output:["ws_sold_date_sk","ws_item_sk"]
- <-Reducer 107 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_1861]
- Group By Operator [GBY_1860] (rows=1 width=12)
- Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)","bloom_filter(VALUE._col2, expectedEntries=1000000)"]
- <-Map 102 [CUSTOM_SIMPLE_EDGE] vectorized
- SHUFFLE [RS_1640]
- Group By Operator [GBY_1635] (rows=1 width=12)
- Output:["_col0","_col1","_col2"],aggregations:["min(_col0)","max(_col0)","bloom_filter(_col0, expectedEntries=1000000)"]
- Selec
<TRUNCATED>
[4/4] hive git commit: HIVE-20826 : Enhance HiveSemiJoin rule to
convert join + group by on left side to Left Semi Join (Vineet Garg via
Ashutosh Chauhan)
Posted by ha...@apache.org.
HIVE-20826 : Enhance HiveSemiJoin rule to convert join + group by on left side to Left Semi Join (Vineet Garg via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c049cace
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c049cace
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c049cace
Branch: refs/heads/master
Commit: c049caceb73fa0701b81c8735b6a879e37eaafd0
Parents: a1cb7e4
Author: Vineet Garg <vg...@apache.org>
Authored: Wed Oct 31 18:53:00 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Sat Nov 10 18:45:04 2018 -0800
----------------------------------------------------------------------
.../calcite/rules/HiveSemiJoinRule.java | 157 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +-
ql/src/test/queries/clientpositive/semijoin.q | 4 +
.../clientpositive/llap/explainuser_1.q.out | 54 +-
.../clientpositive/llap/optimize_join_ptp.q.out | 88 +-
.../results/clientpositive/llap/semijoin.q.out | 52 +
.../clientpositive/llap/subquery_in.q.out | 106 +-
.../clientpositive/llap/subquery_views.q.out | 122 +-
.../llap/vector_mapjoin_reduce.q.out | 45 +-
.../clientpositive/perf/tez/cbo_query14.q.out | 327 ++--
.../clientpositive/perf/tez/query14.q.out | 1556 +++++++++---------
.../results/clientpositive/spark/semijoin.q.out | 52 +
.../spark/spark_explainuser_1.q.out | 54 +-
.../clientpositive/spark/subquery_in.q.out | 90 +-
.../clientpositive/spark/subquery_views.q.out | 192 +--
.../spark/vector_mapjoin_reduce.q.out | 57 +-
16 files changed, 1461 insertions(+), 1497 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java
index 7799090..7ce924a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveSemiJoinRule.java
@@ -1,12 +1,13 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -29,8 +30,10 @@ import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.ImmutableIntList;
@@ -57,23 +60,26 @@ public abstract class HiveSemiJoinRule extends RelOptRule {
protected static final Logger LOG = LoggerFactory.getLogger(HiveSemiJoinRule.class);
public static final HiveProjectToSemiJoinRule INSTANCE_PROJECT =
- new HiveProjectToSemiJoinRule(HiveRelFactories.HIVE_BUILDER);
+ new HiveProjectToSemiJoinRule(HiveRelFactories.HIVE_BUILDER);
+
+ public static final HiveProjectToSemiJoinRuleSwapInputs INSTANCE_PROJECT_SWAPPED =
+ new HiveProjectToSemiJoinRuleSwapInputs(HiveRelFactories.HIVE_BUILDER);
public static final HiveAggregateToSemiJoinRule INSTANCE_AGGREGATE =
- new HiveAggregateToSemiJoinRule(HiveRelFactories.HIVE_BUILDER);
+ new HiveAggregateToSemiJoinRule(HiveRelFactories.HIVE_BUILDER);
private HiveSemiJoinRule(RelOptRuleOperand operand, RelBuilderFactory relBuilder) {
super(operand, relBuilder, null);
}
protected void perform(RelOptRuleCall call, ImmutableBitSet topRefs,
- RelNode topOperator, Join join, RelNode left, Aggregate aggregate) {
+ RelNode topOperator, Join join, RelNode left, Aggregate aggregate) {
LOG.debug("Matched HiveSemiJoinRule");
final RelOptCluster cluster = join.getCluster();
final RexBuilder rexBuilder = cluster.getRexBuilder();
final ImmutableBitSet rightBits =
ImmutableBitSet.range(left.getRowType().getFieldCount(),
- join.getRowType().getFieldCount());
+ join.getRowType().getFieldCount());
if (topRefs.intersects(rightBits)) {
return;
}
@@ -106,22 +112,22 @@ public abstract class HiveSemiJoinRule extends RelOptRule {
final RelNode newRight = aggregate.getInput();
final RexNode newCondition =
RelOptUtil.createEquiJoinCondition(left, joinInfo.leftKeys, newRight,
- newRightKeys, rexBuilder);
+ newRightKeys, rexBuilder);
RelNode semi = null;
//HIVE-15458: we need to add a Project on top of Join since SemiJoin with Join as it's right input
// is not expected further down the pipeline. see jira for more details
if(aggregate.getInput() instanceof HepRelVertex
- && ((HepRelVertex)aggregate.getInput()).getCurrentRel() instanceof Join) {
- Join rightJoin = (Join)(((HepRelVertex)aggregate.getInput()).getCurrentRel());
- List<RexNode> projects = new ArrayList<>();
- for(int i=0; i<rightJoin.getRowType().getFieldCount(); i++){
- projects.add(rexBuilder.makeInputRef(rightJoin, i));
- }
- RelNode topProject = call.builder().push(rightJoin).project(projects, rightJoin.getRowType().getFieldNames(), true).build();
+ && ((HepRelVertex)aggregate.getInput()).getCurrentRel() instanceof Join) {
+ Join rightJoin = (Join)(((HepRelVertex)aggregate.getInput()).getCurrentRel());
+ List<RexNode> projects = new ArrayList<>();
+ for(int i=0; i<rightJoin.getRowType().getFieldCount(); i++){
+ projects.add(rexBuilder.makeInputRef(rightJoin, i));
+ }
+ RelNode topProject = call.builder().push(rightJoin).project(projects, rightJoin.getRowType().getFieldNames(),
+ true).build();
semi = call.builder().push(left).push(topProject).semiJoin(newCondition).build();
- }
- else {
+ } else {
semi = call.builder().push(left).push(aggregate.getInput()).semiJoin(newCondition).build();
}
call.transformTo(topOperator.copy(topOperator.getTraitSet(), ImmutableList.of(semi)));
@@ -136,10 +142,10 @@ public abstract class HiveSemiJoinRule extends RelOptRule {
super(
operand(Project.class,
some(operand(Join.class,
- some(
- operand(RelNode.class, any()),
- operand(Aggregate.class, any()))))),
- relBuilder);
+ some(
+ operand(RelNode.class, any()),
+ operand(Aggregate.class, any()))))),
+ relBuilder);
}
@Override public void onMatch(RelOptRuleCall call) {
@@ -148,11 +154,102 @@ public abstract class HiveSemiJoinRule extends RelOptRule {
final RelNode left = call.rel(2);
final Aggregate aggregate = call.rel(3);
final ImmutableBitSet topRefs =
- RelOptUtil.InputFinder.bits(project.getChildExps(), null);
+ RelOptUtil.InputFinder.bits(project.getChildExps(), null);
perform(call, topRefs, project, join, left, aggregate);
}
}
+ /** SemiJoinRule that matches a Project on top of a Join with an Aggregate
+ * as its right child. */
+ public static class HiveProjectToSemiJoinRuleSwapInputs extends HiveSemiJoinRule {
+
+ /** Creates a HiveProjectToSemiJoinRule. */
+ public HiveProjectToSemiJoinRuleSwapInputs(RelBuilderFactory relBuilder) {
+ super(
+ operand(Project.class,
+ some(operand(Join.class,
+ some(
+ operand(Aggregate.class, any()),
+ operand(RelNode.class, any()))))),
+ relBuilder);
+ }
+
+ private Project swapInputs(Join join, Project topProject, RelBuilder builder) {
+ RexBuilder rexBuilder = join.getCluster().getRexBuilder();
+
+ int rightInputSize = join.getRight().getRowType().getFieldCount();
+ int leftInputSize = join.getLeft().getRowType().getFieldCount();
+ List<RelDataTypeField> joinFields = join.getRowType().getFieldList();
+
+ //swap the join inputs
+ //adjust join condition
+ int[] condAdjustments = new int[joinFields.size()];
+ for(int i=0; i<joinFields.size(); i++) {
+ if(i < leftInputSize) {
+ //left side refs need to be moved by right input size
+ condAdjustments[i] = rightInputSize;
+ } else {
+ //right side refs need to move "down" by left input size
+ condAdjustments[i] = -leftInputSize;
+ }
+ }
+ RexNode newJoinCond = join.getCondition().accept(new RelOptUtil.RexInputConverter(rexBuilder, joinFields,
+ joinFields, condAdjustments));
+ Join swappedJoin = (Join)builder.push(join.getRight()).push(join.getLeft()).join(join.getJoinType(),
+ newJoinCond).build();
+ // need to adjust project rex refs
+ List<RexNode> newProjects = new ArrayList<>();
+
+ List<RelDataTypeField> swappedJoinFeilds = swappedJoin.getRowType().getFieldList();
+ for(RexNode project:topProject.getProjects()) {
+ RexNode newProject = project.accept(new RelOptUtil.RexInputConverter(rexBuilder, swappedJoinFeilds,
+ swappedJoinFeilds, condAdjustments));
+ newProjects.add(newProject);
+ }
+ return (Project)builder.push(swappedJoin).project(newProjects).build();
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Project project = call.rel(0);
+ final Join join = call.rel(1);
+ final RelNode right = call.rel(3);
+ final Aggregate aggregate = call.rel(2);
+
+ // make sure the following conditions are met
+ // Join is INNER
+ // Join keys are same as gb keys
+ // project above is referring to inputs only from non-aggregate side
+ final JoinInfo joinInfo = join.analyzeCondition();
+ if(!joinInfo.isEqui()) {
+ return;
+ }
+ if (!joinInfo.leftSet().equals(
+ ImmutableBitSet.range(aggregate.getGroupCount()))) {
+ // Rule requires that aggregate key to be the same as the join key.
+ // By the way, neither a super-set nor a sub-set would work.
+ return;
+ }
+ final ImmutableBitSet topRefs =
+ RelOptUtil.InputFinder.bits(project.getChildExps(), null);
+
+ final ImmutableBitSet leftBits =
+ ImmutableBitSet.range(0, join.getLeft().getRowType().getFieldCount());
+
+ if (topRefs.intersects(leftBits)) {
+ return;
+ }
+ // it is safe to swap inputs
+ final Project swappedProject = swapInputs(join, project, call.builder());
+ final RelNode swappedJoin = swappedProject.getInput();
+ assert(swappedJoin instanceof Join);
+
+ final ImmutableBitSet swappedTopRefs =
+ RelOptUtil.InputFinder.bits(swappedProject.getChildExps(), null);
+
+ perform(call, swappedTopRefs, swappedProject, (Join)swappedJoin, right, aggregate);
+ }
+ }
+
/** SemiJoinRule that matches a Aggregate on top of a Join with an Aggregate
* as its right child. */
public static class HiveAggregateToSemiJoinRule extends HiveSemiJoinRule {
@@ -161,10 +258,10 @@ public abstract class HiveSemiJoinRule extends RelOptRule {
public HiveAggregateToSemiJoinRule(RelBuilderFactory relBuilder) {
super(
operand(Aggregate.class,
- some(operand(Join.class,
- some(
- operand(RelNode.class, any()),
- operand(Aggregate.class, any()))))),
+ some(operand(Join.class,
+ some(
+ operand(RelNode.class, any()),
+ operand(Aggregate.class, any()))))),
relBuilder);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 11c8f5f..35f2b0a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -1883,7 +1883,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
if (conf.getBoolVar(ConfVars.SEMIJOIN_CONVERSION)) {
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null,
- HiveSemiJoinRule.INSTANCE_PROJECT, HiveSemiJoinRule.INSTANCE_AGGREGATE);
+ HiveSemiJoinRule.INSTANCE_PROJECT, HiveSemiJoinRule.INSTANCE_PROJECT_SWAPPED, HiveSemiJoinRule.INSTANCE_AGGREGATE);
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Semijoin conversion");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/queries/clientpositive/semijoin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/semijoin.q b/ql/src/test/queries/clientpositive/semijoin.q
index 144069b..e1d31cc 100644
--- a/ql/src/test/queries/clientpositive/semijoin.q
+++ b/ql/src/test/queries/clientpositive/semijoin.q
@@ -1,4 +1,5 @@
--! qt:dataset:src
+--! qt:dataset:part
SET hive.vectorized.execution.enabled=false;
set hive.mapred.mode=nonstrict;
-- SORT_QUERY_RESULTS
@@ -86,3 +87,6 @@ explain select key, value from src outr left semi join
select key, value from src outr left semi join
(select a.key, b.value from src a join (select distinct value from src) b on a.value > b.value group by a.key, b.value) inr
on outr.key=inr.key and outr.value=inr.value;
+
+explain cbo select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name;
+select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name;
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
index a87890f..067a43c 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
@@ -2251,42 +2251,36 @@ POSTHOOK: Input: default@src_cbo
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE)
-Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:-1
Stage-1
- Reducer 3 llap
- File Output Operator [FS_14]
- Select Operator [SEL_13] (rows=83 width=178)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_29] (rows=83 width=178)
- Conds:RS_10._col0=RS_11._col0(Inner),Output:["_col1","_col2"]
- <-Map 4 [SIMPLE_EDGE] llap
- SHUFFLE [RS_11]
- PartitionCols:_col0
- Select Operator [SEL_9] (rows=166 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_18] (rows=166 width=178)
+ Reducer 2 llap
+ File Output Operator [FS_12]
+ Merge Join Operator [MERGEJOIN_27] (rows=83 width=178)
+ Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col0","_col1"]
+ <-Map 1 [SIMPLE_EDGE] llap
+ SHUFFLE [RS_8]
+ PartitionCols:_col0
+ Select Operator [SEL_2] (rows=166 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_15] (rows=166 width=178)
+ predicate:(key > '9')
+ TableScan [TS_0] (rows=500 width=178)
+ default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Map 3 [SIMPLE_EDGE] llap
+ SHUFFLE [RS_9]
+ PartitionCols:_col0
+ Group By Operator [GBY_7] (rows=83 width=87)
+ Output:["_col0"],keys:_col0
+ Select Operator [SEL_5] (rows=166 width=87)
+ Output:["_col0"]
+ Filter Operator [FIL_16] (rows=166 width=87)
predicate:(key > '9')
- TableScan [TS_7] (rows=500 width=178)
- default@src_cbo,src_cbo,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
- <-Reducer 2 [ONE_TO_ONE_EDGE] llap
- FORWARD [RS_10]
- PartitionCols:_col0
- Group By Operator [GBY_5] (rows=83 width=87)
- Output:["_col0"],keys:KEY._col0
- <-Map 1 [SIMPLE_EDGE] llap
- SHUFFLE [RS_4]
- PartitionCols:_col0
- Group By Operator [GBY_3] (rows=83 width=87)
- Output:["_col0"],keys:key
- Filter Operator [FIL_17] (rows=166 width=87)
- predicate:(key > '9')
- TableScan [TS_0] (rows=500 width=87)
- default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
+ TableScan [TS_3] (rows=500 width=87)
+ default@src_cbo,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
PREHOOK: query: explain select p.p_partkey, li.l_suppkey
from (select distinct l_partkey as p_partkey from lineitem) p join lineitem li on p.p_partkey = li.l_partkey
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/llap/optimize_join_ptp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/optimize_join_ptp.q.out b/ql/src/test/results/clientpositive/llap/optimize_join_ptp.q.out
index af55d4d..a68275b 100644
--- a/ql/src/test/results/clientpositive/llap/optimize_join_ptp.q.out
+++ b/ql/src/test/results/clientpositive/llap/optimize_join_ptp.q.out
@@ -55,13 +55,33 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
- Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
+ alias: t1_n97
+ filterExpr: (k < 15) (type: boolean)
+ Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (k < 15) (type: boolean)
+ Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: v (type: string), k (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: UDFToDouble(_col1) (type: double)
+ sort order: +
+ Map-reduce partition columns: UDFToDouble(_col1) (type: double)
+ Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col1 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: no inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
alias: t2_n60
filterExpr: ((v = 'people') and k is not null) (type: boolean)
Statistics: Num rows: 3 Data size: 297 Basic stats: COMPLETE Column stats: COMPLETE
@@ -70,10 +90,10 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: k (type: double)
- outputColumnNames: k
- Statistics: Num rows: 1 Data size: 99 Basic stats: COMPLETE Column stats: COMPLETE
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: k (type: double)
+ keys: _col0 (type: double)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -84,62 +104,24 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: no inputs
- Map 4
- Map Operator Tree:
- TableScan
- alias: t1_n97
- filterExpr: (k < 15) (type: boolean)
- Statistics: Num rows: 3 Data size: 285 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (k < 15) (type: boolean)
- Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: v (type: string), k (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: UDFToDouble(_col1) (type: double)
- sort order: +
- Map-reduce partition columns: UDFToDouble(_col1) (type: double)
- Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col1 (type: int)
- Execution mode: vectorized, llap
- LLAP IO: no inputs
Reducer 2
- Execution mode: vectorized, llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: double)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: double)
- sort order: +
- Map-reduce partition columns: _col0 (type: double)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 3
Execution mode: llap
Reduce Operator Tree:
Merge Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
- 0 _col0 (type: double)
- 1 UDFToDouble(_col1) (type: double)
- outputColumnNames: _col1, _col2
+ 0 UDFToDouble(_col1) (type: double)
+ 1 _col0 (type: double)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: string), _col2 (type: int)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 95 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/llap/semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/semijoin.q.out b/ql/src/test/results/clientpositive/llap/semijoin.q.out
index 531ef46..63a270e 100644
--- a/ql/src/test/results/clientpositive/llap/semijoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/semijoin.q.out
@@ -3076,3 +3076,55 @@ POSTHOOK: query: select key, value from src outr left semi join
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
+PREHOOK: query: explain cbo select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(p_partkey=[$0])
+ HiveSemiJoin(condition=[=($1, $3)], joinType=[inner])
+ HiveProject(p_partkey=[$0], p_name=[$1])
+ HiveFilter(condition=[IS NOT NULL($1)])
+ HiveTableScan(table=[[default, part]], table:alias=[pp])
+ HiveProject(p_partkey=[$0], p_name=[$1], p_mfgr=[$2], p_brand=[$3], p_type=[$4], p_size=[$5], p_container=[$6], p_retailprice=[$7], p_comment=[$8], BLOCK__OFFSET__INSIDE__FILE=[$9], INPUT__FILE__NAME=[$10], ROW__ID=[$11])
+ HiveFilter(condition=[IS NOT NULL($1)])
+ HiveTableScan(table=[[default, part]], table:alias=[part])
+
+PREHOOK: query: select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part
+#### A masked pattern was here ####
+POSTHOOK: query: select pp.p_partkey from (select distinct p_name from part) p join part pp on pp.p_name = p.p_name
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part
+#### A masked pattern was here ####
+105685
+110592
+112398
+121152
+121152
+132666
+144293
+146985
+15103
+155733
+17273
+17927
+191709
+192697
+195606
+33357
+40982
+42669
+45261
+48427
+49671
+65667
+78486
+85768
+86428
+90681
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/llap/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out
index 3222e2f..039b52a 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out
@@ -21,34 +21,12 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (SIMPLE_EDGE)
- Reducer 3 <- Map 4 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: s1
- filterExpr: (key > '9') (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (key > '9') (type: boolean)
- Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE
- Execution mode: vectorized, llap
- LLAP IO: no inputs
- Map 4
- Map Operator Tree:
- TableScan
alias: src
filterExpr: (key > '9') (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
@@ -67,41 +45,49 @@ STAGE PLANS:
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: no inputs
- Reducer 2
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: s1
+ filterExpr: (key > '9') (type: boolean)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (key > '9') (type: boolean)
+ Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 3
+ LLAP IO: no inputs
+ Reducer 2
Execution mode: llap
Reduce Operator Tree:
Merge Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: string), _col2 (type: string)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -1096,9 +1082,8 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
+ Reducer 3 <- Map 6 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 5 <- Map 4 (SIMPLE_EDGE)
- Reducer 7 <- Map 6 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1154,10 +1139,10 @@ STAGE PLANS:
Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: l_orderkey (type: int)
- outputColumnNames: l_orderkey
- Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE
+ outputColumnNames: _col0
+ Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: l_orderkey (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
@@ -1190,7 +1175,7 @@ STAGE PLANS:
Reduce Operator Tree:
Merge Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
@@ -1220,19 +1205,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 50 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 7
- Execution mode: vectorized, llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/llap/subquery_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
index 4c723dc..aa50b46 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
@@ -108,15 +108,14 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 11 <- Map 10 (SIMPLE_EDGE)
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE), Reducer 7 (SIMPLE_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
- Reducer 4 <- Reducer 3 (ONE_TO_ONE_EDGE), Reducer 8 (SIMPLE_EDGE)
+ Reducer 10 <- Map 9 (SIMPLE_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
+ Reducer 4 <- Map 1 (SIMPLE_EDGE)
Reducer 5 <- Map 1 (SIMPLE_EDGE)
- Reducer 6 <- Map 1 (SIMPLE_EDGE)
- Reducer 7 <- Reducer 11 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE)
- Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
- Reducer 9 <- Map 1 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
+ Reducer 8 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -196,7 +195,7 @@ STAGE PLANS:
value expressions: _col2 (type: bigint), _col3 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: no inputs
- Map 10
+ Map 9
Map Operator Tree:
TableScan
alias: b
@@ -219,7 +218,7 @@ STAGE PLANS:
Statistics: Num rows: 83 Data size: 7221 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: no inputs
- Reducer 11
+ Reducer 10
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
@@ -243,61 +242,40 @@ STAGE PLANS:
0 _col0 (type: string), _col1 (type: string)
1 _col0 (type: string), _col1 (type: string)
2 _col3 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col4, _col5, _col8
- Statistics: Num rows: 27 Data size: 2889 Basic stats: COMPLETE Column stats: COMPLETE
+ outputColumnNames: _col0, _col1, _col4, _col5, _col8
+ Statistics: Num rows: 27 Data size: 5346 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
- Statistics: Num rows: 13 Data size: 1391 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
+ Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 1391 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Reducer 3
- Execution mode: vectorized, llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 4
Execution mode: llap
Reduce Operator Tree:
Merge Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: string), _col2 (type: string)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 5
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 4
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
@@ -312,7 +290,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 27 Data size: 5238 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint), _col3 (type: bigint)
- Reducer 6
+ Reducer 5
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
@@ -330,7 +308,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 27 Data size: 4914 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string), _col2 (type: boolean)
- Reducer 7
+ Reducer 6
Execution mode: llap
Reduce Operator Tree:
Merge Join Operator
@@ -353,7 +331,7 @@ STAGE PLANS:
Map-reduce partition columns: _col3 (type: string), _col1 (type: string)
Statistics: Num rows: 27 Data size: 4914 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: boolean)
- Reducer 8
+ Reducer 7
Execution mode: llap
Reduce Operator Tree:
Merge Join Operator
@@ -364,22 +342,26 @@ STAGE PLANS:
0 _col0 (type: string), _col1 (type: string)
1 _col0 (type: string), _col1 (type: string)
2 _col3 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1, _col4, _col5, _col8
- Statistics: Num rows: 27 Data size: 5346 Basic stats: COMPLETE Column stats: COMPLETE
+ outputColumnNames: _col0, _col4, _col5, _col8
+ Statistics: Num rows: 27 Data size: 2889 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
- Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: CASE WHEN ((_col4 = 0L)) THEN (true) WHEN (_col4 is null) THEN (true) WHEN (_col8 is not null) THEN (false) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (false) ELSE (true) END (type: boolean)
+ Statistics: Num rows: 13 Data size: 1391 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 13 Data size: 2314 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: string)
- Reducer 9
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 13 Data size: 1131 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ Reducer 8
Execution mode: vectorized, llap
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/c049cace/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
index ad57fcd..aaf14ee 100644
--- a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
@@ -27,8 +27,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE)
- Reducer 5 <- Map 4 (SIMPLE_EDGE)
+ Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE), Map 4 (BROADCAST_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -139,12 +138,12 @@ STAGE PLANS:
Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: l_orderkey (type: int)
- outputColumnNames: l_orderkey
+ outputColumnNames: _col0
Select Vectorization:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0]
- Statistics: Num rows: 14 Data size: 1288 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 14 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
@@ -153,7 +152,7 @@ STAGE PLANS:
native: false
vectorProcessingMode: HASH
projectedOutputColumnNums: []
- keys: l_orderkey (type: int)
+ keys: _col0 (type: int)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
@@ -215,18 +214,18 @@ STAGE PLANS:
Statistics: Num rows: 14 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE
Map Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
Map Join Vectorization:
- className: VectorMapJoinInnerBigOnlyLongOperator
+ className: VectorMapJoinLeftSemiLongOperator
native: true
nativeConditionsMet: hive.mapjoin.optimized.hashtable IS true, hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS true, Small table vectorizes IS true, Optimized Table and Supports Key Types IS true
hashTableImplementationType: OPTIMIZED
outputColumnNames: _col1, _col2
input vertices:
- 1 Reducer 5
+ 1 Map 4
Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col1 (type: int), _col2 (type: int)
@@ -246,36 +245,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 5
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- Reduce Operator Tree:
- Group By Operator
- Group By Vectorization:
- className: VectorGroupByOperator
- groupByMode: MERGEPARTIAL
- keyExpressions: col 0:int
- native: false
- vectorProcessingMode: MERGE_PARTIAL
- projectedOutputColumnNums: []
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkLongOperator
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator