You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/01/05 02:44:41 UTC
[1/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 3f5148d6a -> 96a409e1c
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query90.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query90.q.out b/ql/src/test/results/clientpositive/perf/tez/query90.q.out
index d63ce1b..6f8f3c5 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query90.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query90.q.out
@@ -44,15 +44,14 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Reducer 10 (CUSTOM_SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
-Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 9 (CUSTOM_SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE)
+Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE)
-Reducer 6 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE)
+Reducer 6 <- Reducer 10 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
+Reducer 8 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 9 <- Map 13 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Stage-0
@@ -71,17 +70,17 @@ Stage-0
Output:["_col0"]
Merge Join Operator [MERGEJOIN_92] (rows=1 width=17)
Conds:(Inner),Output:["_col0","_col1"]
- <-Reducer 11 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 10 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_53]
Group By Operator [GBY_50] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Reducer 10 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 9 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_49]
Group By Operator [GBY_48] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
Merge Join Operator [MERGEJOIN_91] (rows=191667562 width=135)
Conds:RS_44._col1=RS_45._col0(Inner)
- <-Map 14 [SIMPLE_EDGE]
+ <-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_45]
PartitionCols:_col0
Select Operator [SEL_11] (rows=3600 width=107)
@@ -90,12 +89,12 @@ Stage-0
predicate:((hd_dep_count = 8) and hd_demo_sk is not null)
TableScan [TS_9] (rows=7200 width=107)
default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"]
- <-Reducer 9 [SIMPLE_EDGE]
+ <-Reducer 8 [SIMPLE_EDGE]
SHUFFLE [RS_44]
PartitionCols:_col1
Merge Join Operator [MERGEJOIN_90] (rows=174243235 width=135)
Conds:RS_41._col0=RS_42._col0(Inner),Output:["_col1"]
- <-Map 13 [SIMPLE_EDGE]
+ <-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_42]
PartitionCols:_col0
Select Operator [SEL_34] (rows=9600 width=471)
@@ -104,13 +103,13 @@ Stage-0
predicate:(t_hour BETWEEN 14 AND 15 and t_time_sk is not null)
TableScan [TS_6] (rows=86400 width=471)
default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_hour"]
- <-Reducer 8 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_41]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_89] (rows=158402938 width=135)
- Conds:RS_38._col2=RS_39._col0(Inner),Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_86] (rows=158402938 width=135)
+ Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_38]
+ SHUFFLE [RS_12]
PartitionCols:_col2
Select Operator [SEL_2] (rows=144002668 width=135)
Output:["_col0","_col1","_col2"]
@@ -118,8 +117,8 @@ Stage-0
predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null)
TableScan [TS_0] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_39]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
PartitionCols:_col0
Select Operator [SEL_5] (rows=511 width=585)
Output:["_col0"]
@@ -137,7 +136,7 @@ Stage-0
Output:["_col0"],aggregations:["count()"]
Merge Join Operator [MERGEJOIN_88] (rows=191667562 width=135)
Conds:RS_18._col1=RS_19._col0(Inner)
- <-Map 14 [SIMPLE_EDGE]
+ <-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_11]
@@ -146,7 +145,7 @@ Stage-0
PartitionCols:_col1
Merge Join Operator [MERGEJOIN_87] (rows=174243235 width=135)
Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"]
- <-Map 13 [SIMPLE_EDGE]
+ <-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
Select Operator [SEL_8] (rows=9600 width=471)
@@ -157,14 +156,5 @@ Stage-0
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_86] (rows=158402938 width=135)
- Conds:RS_12._col2=RS_13._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_12]
- PartitionCols:_col2
- Please refer to the previous Select Operator [SEL_2]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_13]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_5]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_86]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query92.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query92.q.out b/ql/src/test/results/clientpositive/perf/tez/query92.q.out
index c97c3cf..fbb4eb3 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query92.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query92.q.out
@@ -59,13 +59,12 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
-Reducer 6 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 10 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
+Reducer 6 <- Reducer 2 (SIMPLE_EDGE)
+Reducer 7 <- Map 9 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE)
Stage-0
Fetch Operator
@@ -107,7 +106,7 @@ Stage-0
predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
TableScan [TS_0] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_discount_amt"]
- <-Map 9 [SIMPLE_EDGE]
+ <-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
Select Operator [SEL_5] (rows=8116 width=1119)
@@ -116,12 +115,12 @@ Stage-0
predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
- <-Reducer 8 [ONE_TO_ONE_EDGE]
+ <-Reducer 7 [ONE_TO_ONE_EDGE]
FORWARD [RS_31]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_63] (rows=87121617 width=135)
Conds:RS_23._col1=RS_24._col0(Inner),Output:["_col0","_col2"]
- <-Map 10 [SIMPLE_EDGE]
+ <-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_24]
PartitionCols:_col0
Select Operator [SEL_22] (rows=231000 width=1436)
@@ -130,26 +129,17 @@ Stage-0
predicate:((i_manufact_id = 269) and i_item_sk is not null)
TableScan [TS_20] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"]
- <-Reducer 7 [ONE_TO_ONE_EDGE]
+ <-Reducer 6 [ONE_TO_ONE_EDGE]
FORWARD [RS_23]
PartitionCols:_col1
Select Operator [SEL_19] (rows=79201469 width=135)
Output:["_col0","_col1"]
Group By Operator [GBY_18] (rows=79201469 width=135)
Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0
- <-Reducer 6 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_17]
PartitionCols:_col0
Group By Operator [GBY_16] (rows=158402938 width=135)
Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1
- Merge Join Operator [MERGEJOIN_62] (rows=158402938 width=135)
- Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_12]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_2]
- <-Map 9 [SIMPLE_EDGE]
- SHUFFLE [RS_13]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_5]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_61]
[6/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/subquery_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
index af69569..9391362 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
@@ -124,17 +124,14 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 13 (ONE_TO_ONE_EDGE), Reducer 9 (SIMPLE_EDGE)
- Reducer 12 <- Map 11 (SIMPLE_EDGE)
- Reducer 13 <- Map 11 (SIMPLE_EDGE)
+ Reducer 10 <- Map 9 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE), Reducer 6 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
Reducer 4 <- Map 1 (SIMPLE_EDGE)
Reducer 5 <- Map 1 (SIMPLE_EDGE)
- Reducer 6 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE)
- Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
+ Reducer 6 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 7 <- Map 1 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
Reducer 8 <- Map 1 (SIMPLE_EDGE)
- Reducer 9 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -211,22 +208,9 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint), _col3 (type: bigint)
- Filter Operator
- predicate: ((value > 'val_11') and key is not null) (type: boolean)
- Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: key (type: string), value (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
- Map 11
+ Map 9
Map Operator Tree:
TableScan
alias: b
@@ -243,46 +227,11 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
Reducer 10
Execution mode: llap
Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col2, _col3, _col4
- Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col4 (type: string), _col2 (type: string)
- sort order: ++
- Map-reduce partition columns: _col4 (type: string), _col2 (type: string)
- Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col3 (type: boolean)
- Reducer 12
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 13
- Execution mode: llap
- Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: mergepartial
@@ -387,6 +336,12 @@ STAGE PLANS:
Map-reduce partition columns: _col4 (type: string), _col2 (type: string)
Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col3 (type: boolean)
+ Reduce Output Operator
+ key expressions: _col4 (type: string), _col2 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col4 (type: string), _col2 (type: string)
+ Statistics: Num rows: 67 Data size: 12194 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col3 (type: boolean)
Reducer 7
Execution mode: llap
Reduce Operator Tree:
@@ -432,24 +387,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 83 Data size: 16102 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint), _col3 (type: bigint)
- Reducer 9
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 83 Data size: 14774 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
- outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 83 Data size: 15106 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: string), _col3 (type: boolean)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
index 24c7ce7..e6075c7 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_id2.q.out
@@ -1073,9 +1073,8 @@ STAGE PLANS:
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
- Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
- Reducer 5 <- Map 1 (SIMPLE_EDGE)
- Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1118,18 +1117,6 @@ STAGE PLANS:
partitionColumnNums: [3]
valueColumnNums: []
Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
- sort order: +++
- Map-reduce partition columns: rand() (type: double)
- Reduce Sink Vectorization:
- className: VectorReduceSinkObjectHashOperator
- keyColumnNums: [0, 1, 2]
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumnNums: [4]
- valueColumnNums: []
- Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -1187,6 +1174,18 @@ STAGE PLANS:
partitionColumnNums: [0, 1]
valueColumnNums: []
Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumnNums: [0, 1, 2]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumnNums: [0, 1]
+ valueColumnNums: []
+ Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -1271,46 +1270,6 @@ STAGE PLANS:
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
- groupByMode: PARTIALS
- keyExpressions: col 0:int, col 1:int, col 2:int
- native: false
- vectorProcessingMode: STREAMING
- projectedOutputColumnNums: []
- keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int)
- mode: partials
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
- sort order: +++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkObjectHashOperator
- keyColumnNums: [0, 1, 2]
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumnNums: [0, 1]
- valueColumnNums: []
- Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
- Reducer 6
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- reduceColumnNullOrder: aaa
- reduceColumnSortOrder: +++
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- dataColumns: KEY._col0:int, KEY._col1:int, KEY._col2:int
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Operator Tree:
- Group By Operator
- Group By Vectorization:
- className: VectorGroupByOperator
groupByMode: FINAL
keyExpressions: col 0:int, col 1:int, col 2:int
native: false
@@ -1442,9 +1401,8 @@ STAGE PLANS:
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
- Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
- Reducer 5 <- Map 1 (SIMPLE_EDGE)
- Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1487,18 +1445,6 @@ STAGE PLANS:
partitionColumnNums: [3]
valueColumnNums: []
Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
- sort order: +++
- Map-reduce partition columns: rand() (type: double)
- Reduce Sink Vectorization:
- className: VectorReduceSinkObjectHashOperator
- keyColumnNums: [0, 1, 2]
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumnNums: [4]
- valueColumnNums: []
- Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -1556,6 +1502,18 @@ STAGE PLANS:
partitionColumnNums: [0, 1]
valueColumnNums: []
Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumnNums: [0, 1, 2]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumnNums: [0, 1]
+ valueColumnNums: []
+ Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -1640,46 +1598,6 @@ STAGE PLANS:
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
- groupByMode: PARTIALS
- keyExpressions: col 0:int, col 1:int, col 2:int
- native: false
- vectorProcessingMode: STREAMING
- projectedOutputColumnNums: []
- keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int)
- mode: partials
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
- sort order: +++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkObjectHashOperator
- keyColumnNums: [0, 1, 2]
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- partitionColumnNums: [0, 1]
- valueColumnNums: []
- Statistics: Num rows: 18 Data size: 144 Basic stats: COMPLETE Column stats: NONE
- Reducer 6
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- reduceColumnNullOrder: aaa
- reduceColumnSortOrder: +++
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- dataColumns: KEY._col0:int, KEY._col1:int, KEY._col2:int
- partitionColumnCount: 0
- scratchColumnTypeNames: []
- Reduce Operator Tree:
- Group By Operator
- Group By Vectorization:
- className: VectorGroupByOperator
groupByMode: FINAL
keyExpressions: col 0:int, col 1:int, col 2:int
native: false
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
index 957bc22..6a5e679 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
@@ -576,9 +576,8 @@ STAGE PLANS:
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
- Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
- Reducer 5 <- Map 1 (SIMPLE_EDGE)
- Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -622,18 +621,6 @@ STAGE PLANS:
valueColumnNums: [2]
Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint)
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Reduce Sink Vectorization:
- className: VectorReduceSinkMultiKeyOperator
- keyColumnNums: [0, 1]
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumnNums: [2]
- Statistics: Num rows: 2 Data size: 736 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -693,6 +680,18 @@ STAGE PLANS:
valueColumnNums: [3]
Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkMultiKeyOperator
+ keyColumnNums: [0, 1, 2]
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ valueColumnNums: [3]
+ Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: bigint)
Reducer 3
Execution mode: vectorized, llap
Reduce Vectorization:
@@ -774,48 +773,6 @@ STAGE PLANS:
Reduce Vectorization:
enabled: true
enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
- reduceColumnNullOrder: aa
- reduceColumnSortOrder: ++
- allNative: false
- usesVectorUDFAdaptor: false
- vectorized: true
- rowBatchContext:
- dataColumnCount: 3
- dataColumns: KEY._col0:string, KEY._col1:string, VALUE._col0:bigint
- partitionColumnCount: 0
- scratchColumnTypeNames: [bigint]
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- Group By Vectorization:
- aggregators: VectorUDAFCountMerge(col 2:bigint) -> bigint
- className: VectorGroupByOperator
- groupByMode: PARTIALS
- keyExpressions: col 0:string, col 1:string, ConstantVectorExpression(val 0) -> 3:int
- native: false
- vectorProcessingMode: STREAMING
- projectedOutputColumnNums: [0]
- keys: KEY._col0 (type: string), KEY._col1 (type: string), 0 (type: int)
- mode: partials
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
- sort order: +++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
- Reduce Sink Vectorization:
- className: VectorReduceSinkMultiKeyOperator
- keyColumnNums: [0, 1, 2]
- native: true
- nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- valueColumnNums: [3]
- Statistics: Num rows: 8 Data size: 2944 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: bigint)
- Reducer 6
- Execution mode: vectorized, llap
- Reduce Vectorization:
- enabled: true
- enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true
reduceColumnNullOrder: aaa
reduceColumnSortOrder: +++
allNative: false
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
index b2a45f1..ef44cdb 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out
@@ -212,12 +212,12 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 11 <- Union 10 (CUSTOM_SIMPLE_EDGE)
Reducer 12 <- Reducer 11 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE), Reducer 52 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 13 <- Map 1 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 14 (CONTAINS)
+Reducer 13 <- Map 1 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 14 (CONTAINS)
Reducer 15 <- Union 14 (CUSTOM_SIMPLE_EDGE)
Reducer 16 <- Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 30 (CUSTOM_SIMPLE_EDGE), Reducer 55 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS)
Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 3 (CONTAINS)
Reducer 19 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 10 (CONTAINS)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 3 (CONTAINS)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 3 (CONTAINS)
Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE), Union 14 (CONTAINS)
Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 43 (SIMPLE_EDGE), Union 23 (CONTAINS)
Reducer 24 <- Union 23 (CUSTOM_SIMPLE_EDGE)
@@ -232,58 +232,49 @@ Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE), Union 23 (CONTAINS)
Reducer 37 <- Map 35 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE), Union 26 (CONTAINS)
Reducer 38 <- Map 35 (SIMPLE_EDGE), Map 44 (SIMPLE_EDGE), Union 29 (CONTAINS)
Reducer 4 <- Union 3 (CUSTOM_SIMPLE_EDGE)
-Reducer 40 <- Map 39 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 23 (CONTAINS)
-Reducer 41 <- Map 39 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 26 (CONTAINS)
-Reducer 42 <- Map 39 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 29 (CONTAINS)
+Reducer 40 <- Map 39 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 23 (CONTAINS)
+Reducer 41 <- Map 39 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 26 (CONTAINS)
+Reducer 42 <- Map 39 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 29 (CONTAINS)
Reducer 46 <- Map 45 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE)
Reducer 47 <- Map 56 (SIMPLE_EDGE), Reducer 46 (SIMPLE_EDGE), Reducer 58 (ONE_TO_ONE_EDGE)
Reducer 48 <- Reducer 47 (SIMPLE_EDGE)
Reducer 5 <- Reducer 24 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 48 (CUSTOM_SIMPLE_EDGE), Union 6 (CONTAINS)
-Reducer 50 <- Map 100 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE)
+Reducer 50 <- Map 49 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE)
Reducer 51 <- Map 56 (SIMPLE_EDGE), Reducer 50 (SIMPLE_EDGE), Reducer 68 (ONE_TO_ONE_EDGE)
Reducer 52 <- Reducer 51 (SIMPLE_EDGE)
-Reducer 53 <- Map 101 (SIMPLE_EDGE), Map 49 (SIMPLE_EDGE)
-Reducer 54 <- Map 56 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE), Reducer 78 (ONE_TO_ONE_EDGE)
+Reducer 53 <- Map 49 (SIMPLE_EDGE), Map 92 (SIMPLE_EDGE)
+Reducer 54 <- Map 56 (SIMPLE_EDGE), Reducer 53 (SIMPLE_EDGE), Reducer 77 (ONE_TO_ONE_EDGE)
Reducer 55 <- Reducer 54 (SIMPLE_EDGE)
Reducer 57 <- Map 56 (SIMPLE_EDGE), Reducer 62 (ONE_TO_ONE_EDGE)
Reducer 58 <- Reducer 57 (SIMPLE_EDGE)
-Reducer 59 <- Map 56 (SIMPLE_EDGE), Reducer 88 (SIMPLE_EDGE)
+Reducer 59 <- Map 56 (SIMPLE_EDGE), Reducer 85 (SIMPLE_EDGE)
Reducer 60 <- Reducer 59 (SIMPLE_EDGE), Union 61 (CONTAINS)
Reducer 62 <- Union 61 (SIMPLE_EDGE)
-Reducer 63 <- Map 56 (SIMPLE_EDGE), Reducer 92 (SIMPLE_EDGE)
+Reducer 63 <- Map 56 (SIMPLE_EDGE), Reducer 87 (SIMPLE_EDGE)
Reducer 64 <- Reducer 63 (SIMPLE_EDGE), Union 61 (CONTAINS)
-Reducer 65 <- Map 56 (SIMPLE_EDGE), Reducer 93 (SIMPLE_EDGE)
+Reducer 65 <- Map 56 (SIMPLE_EDGE), Reducer 88 (SIMPLE_EDGE)
Reducer 66 <- Reducer 65 (SIMPLE_EDGE), Union 61 (CONTAINS)
Reducer 67 <- Map 56 (SIMPLE_EDGE), Reducer 72 (ONE_TO_ONE_EDGE)
Reducer 68 <- Reducer 67 (SIMPLE_EDGE)
-Reducer 69 <- Map 56 (SIMPLE_EDGE), Reducer 89 (SIMPLE_EDGE)
+Reducer 69 <- Map 56 (SIMPLE_EDGE), Reducer 85 (SIMPLE_EDGE)
Reducer 7 <- Union 6 (SIMPLE_EDGE)
Reducer 70 <- Reducer 69 (SIMPLE_EDGE), Union 71 (CONTAINS)
Reducer 72 <- Union 71 (SIMPLE_EDGE)
-Reducer 73 <- Map 56 (SIMPLE_EDGE), Reducer 94 (SIMPLE_EDGE)
-Reducer 74 <- Reducer 73 (SIMPLE_EDGE), Union 71 (CONTAINS)
-Reducer 75 <- Map 56 (SIMPLE_EDGE), Reducer 95 (SIMPLE_EDGE)
-Reducer 76 <- Reducer 75 (SIMPLE_EDGE), Union 71 (CONTAINS)
-Reducer 77 <- Map 56 (SIMPLE_EDGE), Reducer 82 (ONE_TO_ONE_EDGE)
-Reducer 78 <- Reducer 77 (SIMPLE_EDGE)
-Reducer 79 <- Map 56 (SIMPLE_EDGE), Reducer 90 (SIMPLE_EDGE)
+Reducer 73 <- Reducer 69 (SIMPLE_EDGE), Union 74 (CONTAINS)
+Reducer 75 <- Union 74 (SIMPLE_EDGE)
+Reducer 76 <- Map 56 (SIMPLE_EDGE), Reducer 75 (ONE_TO_ONE_EDGE)
+Reducer 77 <- Reducer 76 (SIMPLE_EDGE)
+Reducer 78 <- Map 56 (SIMPLE_EDGE), Reducer 87 (SIMPLE_EDGE)
+Reducer 79 <- Reducer 78 (SIMPLE_EDGE), Union 71 (CONTAINS)
Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
-Reducer 80 <- Reducer 79 (SIMPLE_EDGE), Union 81 (CONTAINS)
-Reducer 82 <- Union 81 (SIMPLE_EDGE)
-Reducer 83 <- Map 56 (SIMPLE_EDGE), Reducer 96 (SIMPLE_EDGE)
-Reducer 84 <- Reducer 83 (SIMPLE_EDGE), Union 81 (CONTAINS)
-Reducer 85 <- Map 56 (SIMPLE_EDGE), Reducer 97 (SIMPLE_EDGE)
-Reducer 86 <- Reducer 85 (SIMPLE_EDGE), Union 81 (CONTAINS)
-Reducer 88 <- Map 87 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE)
-Reducer 89 <- Map 87 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE)
-Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE), Union 10 (CONTAINS)
-Reducer 90 <- Map 87 (SIMPLE_EDGE), Map 91 (SIMPLE_EDGE)
-Reducer 92 <- Map 91 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE)
-Reducer 93 <- Map 91 (SIMPLE_EDGE), Map 99 (SIMPLE_EDGE)
-Reducer 94 <- Map 91 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE)
-Reducer 95 <- Map 91 (SIMPLE_EDGE), Map 99 (SIMPLE_EDGE)
-Reducer 96 <- Map 91 (SIMPLE_EDGE), Map 98 (SIMPLE_EDGE)
-Reducer 97 <- Map 91 (SIMPLE_EDGE), Map 99 (SIMPLE_EDGE)
+Reducer 80 <- Reducer 78 (SIMPLE_EDGE), Union 74 (CONTAINS)
+Reducer 81 <- Map 56 (SIMPLE_EDGE), Reducer 88 (SIMPLE_EDGE)
+Reducer 82 <- Reducer 81 (SIMPLE_EDGE), Union 71 (CONTAINS)
+Reducer 83 <- Reducer 81 (SIMPLE_EDGE), Union 74 (CONTAINS)
+Reducer 85 <- Map 84 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE)
+Reducer 87 <- Map 86 (SIMPLE_EDGE), Map 89 (SIMPLE_EDGE)
+Reducer 88 <- Map 86 (SIMPLE_EDGE), Map 90 (SIMPLE_EDGE)
+Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 86 (SIMPLE_EDGE), Union 10 (CONTAINS)
Stage-0
Fetch Operator
@@ -390,7 +381,7 @@ Stage-0
Output:["_col0"]
Merge Join Operator [MERGEJOIN_860] (rows=633595212 width=88)
Conds:RS_194._col0=RS_195._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
+ <-Map 86 [SIMPLE_EDGE]
SHUFFLE [RS_195]
PartitionCols:_col0
Select Operator [SEL_98] (rows=8116 width=1119)
@@ -469,7 +460,7 @@ Stage-0
Output:["_col0","_col1"]
Merge Join Operator [MERGEJOIN_863] (rows=633595212 width=88)
Conds:RS_238._col0=RS_239._col0(Inner),Output:["_col1","_col2"]
- <-Map 91 [SIMPLE_EDGE]
+ <-Map 86 [SIMPLE_EDGE]
SHUFFLE [RS_239]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
@@ -518,7 +509,7 @@ Stage-0
predicate:((d_moy = 11) and (d_year = 2000) and d_date_sk is not null)
TableScan [TS_84] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Map 100 [SIMPLE_EDGE]
+ <-Map 91 [SIMPLE_EDGE]
SHUFFLE [RS_357]
PartitionCols:_col0
Select Operator [SEL_271] (rows=287989836 width=135)
@@ -579,17 +570,17 @@ Stage-0
Filter Operator [FIL_813] (rows=462000 width=1436)
predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
Please refer to the previous TableScan [TS_90]
- <-Reducer 89 [SIMPLE_EDGE]
+ <-Reducer 85 [SIMPLE_EDGE]
SHUFFLE [RS_293]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_867] (rows=633595212 width=88)
- Conds:RS_290._col0=RS_291._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_291]
+ Merge Join Operator [MERGEJOIN_852] (rows=633595212 width=88)
+ Conds:RS_102._col0=RS_103._col0(Inner),Output:["_col1"]
+ <-Map 86 [SIMPLE_EDGE]
+ SHUFFLE [RS_103]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
- <-Map 87 [SIMPLE_EDGE]
- SHUFFLE [RS_290]
+ <-Map 84 [SIMPLE_EDGE]
+ SHUFFLE [RS_102]
PartitionCols:_col0
Select Operator [SEL_95] (rows=575995635 width=88)
Output:["_col0","_col1"]
@@ -597,14 +588,14 @@ Stage-0
predicate:(ss_item_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_93] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"]
- <-Reducer 74 [CONTAINS]
+ <-Reducer 79 [CONTAINS]
Reduce Output Operator [RS_345]
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_344] (rows=609832849 width=108)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
Group By Operator [GBY_319] (rows=174233858 width=135)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 73 [SIMPLE_EDGE]
+ <-Reducer 78 [SIMPLE_EDGE]
SHUFFLE [RS_318]
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_317] (rows=348467716 width=135)
@@ -619,17 +610,17 @@ Stage-0
Filter Operator [FIL_816] (rows=462000 width=1436)
predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
Please refer to the previous TableScan [TS_90]
- <-Reducer 94 [SIMPLE_EDGE]
+ <-Reducer 87 [SIMPLE_EDGE]
SHUFFLE [RS_313]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_869] (rows=316788826 width=135)
- Conds:RS_310._col0=RS_311._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_311]
+ Merge Join Operator [MERGEJOIN_854] (rows=316788826 width=135)
+ Conds:RS_122._col0=RS_123._col0(Inner),Output:["_col1"]
+ <-Map 86 [SIMPLE_EDGE]
+ SHUFFLE [RS_123]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
- <-Map 98 [SIMPLE_EDGE]
- SHUFFLE [RS_310]
+ <-Map 89 [SIMPLE_EDGE]
+ SHUFFLE [RS_122]
PartitionCols:_col0
Select Operator [SEL_115] (rows=287989836 width=135)
Output:["_col0","_col1"]
@@ -637,14 +628,14 @@ Stage-0
predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
TableScan [TS_113] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"]
- <-Reducer 76 [CONTAINS]
+ <-Reducer 82 [CONTAINS]
Reduce Output Operator [RS_345]
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_344] (rows=609832849 width=108)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
Group By Operator [GBY_340] (rows=87121617 width=135)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 75 [SIMPLE_EDGE]
+ <-Reducer 81 [SIMPLE_EDGE]
SHUFFLE [RS_339]
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_338] (rows=174243235 width=135)
@@ -659,17 +650,17 @@ Stage-0
Filter Operator [FIL_819] (rows=462000 width=1436)
predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
Please refer to the previous TableScan [TS_90]
- <-Reducer 95 [SIMPLE_EDGE]
+ <-Reducer 88 [SIMPLE_EDGE]
SHUFFLE [RS_334]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_871] (rows=158402938 width=135)
- Conds:RS_331._col0=RS_332._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_332]
+ Merge Join Operator [MERGEJOIN_856] (rows=158402938 width=135)
+ Conds:RS_143._col0=RS_144._col0(Inner),Output:["_col1"]
+ <-Map 86 [SIMPLE_EDGE]
+ SHUFFLE [RS_144]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
- <-Map 99 [SIMPLE_EDGE]
- SHUFFLE [RS_331]
+ <-Map 90 [SIMPLE_EDGE]
+ SHUFFLE [RS_143]
PartitionCols:_col0
Select Operator [SEL_136] (rows=144002668 width=135)
Output:["_col0","_col1"]
@@ -709,7 +700,7 @@ Stage-0
Output:["_col0"]
Merge Join Operator [MERGEJOIN_875] (rows=633595212 width=88)
Conds:RS_383._col0=RS_384._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
+ <-Map 86 [SIMPLE_EDGE]
SHUFFLE [RS_384]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
@@ -804,7 +795,7 @@ Stage-0
Output:["_col0","_col1"]
Merge Join Operator [MERGEJOIN_878] (rows=633595212 width=88)
Conds:RS_427._col0=RS_428._col0(Inner),Output:["_col1","_col2"]
- <-Map 91 [SIMPLE_EDGE]
+ <-Map 86 [SIMPLE_EDGE]
SHUFFLE [RS_428]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
@@ -842,7 +833,7 @@ Stage-0
SHUFFLE [RS_547]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_86]
- <-Map 101 [SIMPLE_EDGE]
+ <-Map 92 [SIMPLE_EDGE]
SHUFFLE [RS_546]
PartitionCols:_col0
Select Operator [SEL_460] (rows=144002668 width=135)
@@ -851,12 +842,12 @@ Stage-0
predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
TableScan [TS_458] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_quantity","ws_list_price"]
- <-Reducer 78 [ONE_TO_ONE_EDGE]
+ <-Reducer 77 [ONE_TO_ONE_EDGE]
FORWARD [RS_551]
PartitionCols:_col0
Group By Operator [GBY_544] (rows=254100 width=1436)
Output:["_col0"],keys:KEY._col0
- <-Reducer 77 [SIMPLE_EDGE]
+ <-Reducer 76 [SIMPLE_EDGE]
SHUFFLE [RS_543]
PartitionCols:_col0
Group By Operator [GBY_542] (rows=508200 width=1436)
@@ -871,7 +862,7 @@ Stage-0
Filter Operator [FIL_835] (rows=462000 width=1436)
predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
Please refer to the previous TableScan [TS_90]
- <-Reducer 82 [ONE_TO_ONE_EDGE]
+ <-Reducer 75 [ONE_TO_ONE_EDGE]
FORWARD [RS_539]
PartitionCols:_col0, _col1, _col2
Select Operator [SEL_537] (rows=1 width=108)
@@ -880,112 +871,40 @@ Stage-0
predicate:(_col3 = 3)
Group By Operator [GBY_535] (rows=304916424 width=108)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Union 81 [SIMPLE_EDGE]
- <-Reducer 80 [CONTAINS]
+ <-Union 74 [SIMPLE_EDGE]
+ <-Reducer 73 [CONTAINS]
Reduce Output Operator [RS_534]
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_533] (rows=609832849 width=108)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
Group By Operator [GBY_488] (rows=348477374 width=88)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 79 [SIMPLE_EDGE]
+ <-Reducer 69 [SIMPLE_EDGE]
SHUFFLE [RS_487]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_486] (rows=696954748 width=88)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
- Merge Join Operator [MERGEJOIN_883] (rows=696954748 width=88)
- Conds:RS_482._col1=RS_483._col0(Inner),Output:["_col5","_col6","_col7"]
- <-Map 56 [SIMPLE_EDGE]
- SHUFFLE [RS_483]
- PartitionCols:_col0
- Select Operator [SEL_478] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_838] (rows=462000 width=1436)
- predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_90]
- <-Reducer 90 [SIMPLE_EDGE]
- SHUFFLE [RS_482]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_882] (rows=633595212 width=88)
- Conds:RS_479._col0=RS_480._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_480]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_98]
- <-Map 87 [SIMPLE_EDGE]
- SHUFFLE [RS_479]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_95]
- <-Reducer 84 [CONTAINS]
+ Please refer to the previous Group By Operator [GBY_297]
+ <-Reducer 80 [CONTAINS]
Reduce Output Operator [RS_534]
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_533] (rows=609832849 width=108)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
Group By Operator [GBY_508] (rows=174233858 width=135)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 83 [SIMPLE_EDGE]
+ <-Reducer 78 [SIMPLE_EDGE]
SHUFFLE [RS_507]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_506] (rows=348467716 width=135)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
- Merge Join Operator [MERGEJOIN_885] (rows=348467716 width=135)
- Conds:RS_502._col1=RS_503._col0(Inner),Output:["_col5","_col6","_col7"]
- <-Map 56 [SIMPLE_EDGE]
- SHUFFLE [RS_503]
- PartitionCols:_col0
- Select Operator [SEL_498] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_841] (rows=462000 width=1436)
- predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_90]
- <-Reducer 96 [SIMPLE_EDGE]
- SHUFFLE [RS_502]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_884] (rows=316788826 width=135)
- Conds:RS_499._col0=RS_500._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_500]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_98]
- <-Map 98 [SIMPLE_EDGE]
- SHUFFLE [RS_499]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_115]
- <-Reducer 86 [CONTAINS]
+ Please refer to the previous Group By Operator [GBY_317]
+ <-Reducer 83 [CONTAINS]
Reduce Output Operator [RS_534]
PartitionCols:_col0, _col1, _col2
Group By Operator [GBY_533] (rows=609832849 width=108)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(_col3)"],keys:_col0, _col1, _col2
Group By Operator [GBY_529] (rows=87121617 width=135)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 85 [SIMPLE_EDGE]
+ <-Reducer 81 [SIMPLE_EDGE]
SHUFFLE [RS_528]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_527] (rows=174243235 width=135)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col5, _col6, _col7
- Merge Join Operator [MERGEJOIN_887] (rows=174243235 width=135)
- Conds:RS_523._col1=RS_524._col0(Inner),Output:["_col5","_col6","_col7"]
- <-Map 56 [SIMPLE_EDGE]
- SHUFFLE [RS_524]
- PartitionCols:_col0
- Select Operator [SEL_519] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_844] (rows=462000 width=1436)
- predicate:(i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_90]
- <-Reducer 97 [SIMPLE_EDGE]
- SHUFFLE [RS_523]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_886] (rows=158402938 width=135)
- Conds:RS_520._col0=RS_521._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_521]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_98]
- <-Map 99 [SIMPLE_EDGE]
- SHUFFLE [RS_520]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_136]
+ Please refer to the previous Group By Operator [GBY_338]
<-Reducer 5 [CONTAINS]
Reduce Output Operator [RS_568]
PartitionCols:_col0, _col1, _col2, _col3, _col4
@@ -1048,7 +967,7 @@ Stage-0
Output:["_col0","_col1"]
Merge Join Operator [MERGEJOIN_848] (rows=633595212 width=88)
Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col1","_col2"]
- <-Map 91 [SIMPLE_EDGE]
+ <-Map 86 [SIMPLE_EDGE]
SHUFFLE [RS_51]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
@@ -1095,7 +1014,7 @@ Stage-0
Output:["_col0"]
Merge Join Operator [MERGEJOIN_845] (rows=633595212 width=88)
Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
+ <-Map 86 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_98]
@@ -1208,19 +1127,10 @@ Stage-0
SHUFFLE [RS_106]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_92]
- <-Reducer 88 [SIMPLE_EDGE]
+ <-Reducer 85 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_852] (rows=633595212 width=88)
- Conds:RS_102._col0=RS_103._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_103]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_98]
- <-Map 87 [SIMPLE_EDGE]
- SHUFFLE [RS_102]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_95]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_852]
<-Reducer 64 [CONTAINS]
Reduce Output Operator [RS_157]
PartitionCols:_col0, _col1, _col2
@@ -1239,19 +1149,10 @@ Stage-0
SHUFFLE [RS_126]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_92]
- <-Reducer 92 [SIMPLE_EDGE]
+ <-Reducer 87 [SIMPLE_EDGE]
SHUFFLE [RS_125]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_854] (rows=316788826 width=135)
- Conds:RS_122._col0=RS_123._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_123]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_98]
- <-Map 98 [SIMPLE_EDGE]
- SHUFFLE [RS_122]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_115]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_854]
<-Reducer 66 [CONTAINS]
Reduce Output Operator [RS_157]
PartitionCols:_col0, _col1, _col2
@@ -1270,17 +1171,8 @@ Stage-0
SHUFFLE [RS_147]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_92]
- <-Reducer 93 [SIMPLE_EDGE]
+ <-Reducer 88 [SIMPLE_EDGE]
SHUFFLE [RS_146]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_856] (rows=158402938 width=135)
- Conds:RS_143._col0=RS_144._col0(Inner),Output:["_col1"]
- <-Map 91 [SIMPLE_EDGE]
- SHUFFLE [RS_144]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_98]
- <-Map 99 [SIMPLE_EDGE]
- SHUFFLE [RS_143]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_136]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_856]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query2.q.out b/ql/src/test/results/clientpositive/perf/tez/query2.q.out
index 23f8249..32d9e1d 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query2.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query2.q.out
@@ -120,17 +120,13 @@ Plan optimized by CBO.
Vertex dependency in root stage
Map 1 <- Union 2 (CONTAINS)
-Map 14 <- Union 15 (CONTAINS)
-Map 16 <- Union 15 (CONTAINS)
-Map 8 <- Union 2 (CONTAINS)
-Reducer 10 <- Map 9 (SIMPLE_EDGE), Union 15 (SIMPLE_EDGE)
-Reducer 11 <- Reducer 10 (SIMPLE_EDGE)
-Reducer 12 <- Map 13 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE)
-Reducer 3 <- Map 9 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE)
+Map 9 <- Union 2 (CONTAINS)
+Reducer 3 <- Map 10 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE)
-Reducer 6 <- Reducer 12 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE)
+Reducer 5 <- Map 11 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE)
+Reducer 6 <- Reducer 5 (ONE_TO_ONE_EDGE), Reducer 8 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
+Reducer 8 <- Map 11 (SIMPLE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE)
Stage-0
Fetch Operator
@@ -146,75 +142,20 @@ Stage-0
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
Merge Join Operator [MERGEJOIN_95] (rows=287491028 width=135)
Conds:RS_54._col0=RS_55.(_col0 - 53)(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col13","_col14","_col15"]
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_55]
- PartitionCols:(_col0 - 53)
- Merge Join Operator [MERGEJOIN_94] (rows=261355475 width=135)
- Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- <-Map 13 [SIMPLE_EDGE]
- SHUFFLE [RS_51]
- PartitionCols:_col0
- Select Operator [SEL_49] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_90] (rows=36524 width=1119)
- predicate:((d_year = 2002) and d_week_seq is not null)
- TableScan [TS_20] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_week_seq","d_year"]
- <-Reducer 11 [ONE_TO_ONE_EDGE]
- FORWARD [RS_50]
- PartitionCols:_col0
- Group By Operator [GBY_45] (rows=237595882 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0
- <-Reducer 10 [SIMPLE_EDGE]
- SHUFFLE [RS_44]
- PartitionCols:_col0
- Group By Operator [GBY_43] (rows=475191764 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col1)","sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)"],keys:_col0
- Select Operator [SEL_41] (rows=475191764 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Merge Join Operator [MERGEJOIN_93] (rows=475191764 width=135)
- Conds:Union 15._col0=RS_39._col0(Inner),Output:["_col1","_col3","_col4"]
- <-Map 9 [SIMPLE_EDGE]
- SHUFFLE [RS_39]
- PartitionCols:_col0
- Select Operator [SEL_10] (rows=73049 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_85] (rows=73049 width=1119)
- predicate:(d_date_sk is not null and d_week_seq is not null)
- TableScan [TS_8] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"]
- <-Union 15 [SIMPLE_EDGE]
- <-Map 14 [CONTAINS]
- Reduce Output Operator [RS_38]
- PartitionCols:_col0
- Select Operator [SEL_29] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_87] (rows=144002668 width=135)
- predicate:ws_sold_date_sk is not null
- TableScan [TS_27] (rows=144002668 width=135)
- Output:["ws_sold_date_sk","ws_ext_sales_price"]
- <-Map 16 [CONTAINS]
- Reduce Output Operator [RS_38]
- PartitionCols:_col0
- Select Operator [SEL_32] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_88] (rows=287989836 width=135)
- predicate:cs_sold_date_sk is not null
- TableScan [TS_30] (rows=287989836 width=135)
- Output:["cs_sold_date_sk","cs_ext_sales_price"]
<-Reducer 5 [ONE_TO_ONE_EDGE]
FORWARD [RS_54]
PartitionCols:_col0
Merge Join Operator [MERGEJOIN_92] (rows=261355475 width=135)
Conds:RS_23._col0=RS_24._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- <-Map 13 [SIMPLE_EDGE]
+ <-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_24]
PartitionCols:_col0
Select Operator [SEL_22] (rows=36524 width=1119)
Output:["_col0"]
Filter Operator [FIL_86] (rows=36524 width=1119)
predicate:((d_year = 2001) and d_week_seq is not null)
- Please refer to the previous TableScan [TS_20]
+ TableScan [TS_20] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_week_seq","d_year"]
<-Reducer 4 [ONE_TO_ONE_EDGE]
FORWARD [RS_23]
PartitionCols:_col0
@@ -229,10 +170,15 @@ Stage-0
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
Merge Join Operator [MERGEJOIN_91] (rows=475191764 width=135)
Conds:Union 2._col0=RS_12._col0(Inner),Output:["_col1","_col3","_col4"]
- <-Map 9 [SIMPLE_EDGE]
+ <-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_10]
+ Select Operator [SEL_10] (rows=73049 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_85] (rows=73049 width=1119)
+ predicate:(d_date_sk is not null and d_week_seq is not null)
+ TableScan [TS_8] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"]
<-Union 2 [SIMPLE_EDGE]
<-Map 1 [CONTAINS]
Reduce Output Operator [RS_11]
@@ -243,7 +189,7 @@ Stage-0
predicate:ws_sold_date_sk is not null
TableScan [TS_0] (rows=144002668 width=135)
Output:["ws_sold_date_sk","ws_ext_sales_price"]
- <-Map 8 [CONTAINS]
+ <-Map 9 [CONTAINS]
Reduce Output Operator [RS_11]
PartitionCols:_col0
Select Operator [SEL_5] (rows=287989836 width=135)
@@ -252,4 +198,21 @@ Stage-0
predicate:cs_sold_date_sk is not null
TableScan [TS_3] (rows=287989836 width=135)
Output:["cs_sold_date_sk","cs_ext_sales_price"]
+ <-Reducer 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_55]
+ PartitionCols:(_col0 - 53)
+ Merge Join Operator [MERGEJOIN_94] (rows=261355475 width=135)
+ Conds:RS_50._col0=RS_51._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_51]
+ PartitionCols:_col0
+ Select Operator [SEL_49] (rows=36524 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_90] (rows=36524 width=1119)
+ predicate:((d_year = 2002) and d_week_seq is not null)
+ Please refer to the previous TableScan [TS_20]
+ <-Reducer 4 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_50]
+ PartitionCols:_col0
+ Please refer to the previous Group By Operator [GBY_18]
[7/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
index 9cc1262..3779333 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
@@ -606,11 +606,10 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
Reducer 5 <- Map 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
- Reducer 7 <- Map 4 (SIMPLE_EDGE)
- Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
+ Reducer 7 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -639,12 +638,6 @@ STAGE PLANS:
Map-reduce partition columns: p_mfgr (type: string)
Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
- Reduce Output Operator
- key expressions: p_mfgr (type: string), p_size (type: int)
- sort order: ++
- Map-reduce partition columns: p_mfgr (type: string)
- Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
- TopN Hash Memory Usage: 0.1
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -733,30 +726,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
- Reducer 6
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- aggregations: avg(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(), count(_col0)
- mode: complete
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 7
- Execution mode: llap
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
- outputColumnNames: _col2, _col5
- Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
PTF Operator
Function definitions:
Input definition
@@ -794,7 +763,24 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
- Reducer 8
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: avg(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(), count(_col0)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Reducer 7
Execution mode: llap
Reduce Operator Tree:
Group By Operator
@@ -891,15 +877,13 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Map 5 (SIMPLE_EDGE)
- Reducer 11 <- Reducer 10 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
- Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
- Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (ONE_TO_ONE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
Reducer 6 <- Map 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
- Reducer 8 <- Map 5 (SIMPLE_EDGE)
- Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+ Reducer 8 <- Reducer 6 (SIMPLE_EDGE)
+ Reducer 9 <- Reducer 6 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -933,93 +917,8 @@ STAGE PLANS:
Map-reduce partition columns: p_mfgr (type: string)
Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
- Reduce Output Operator
- key expressions: p_mfgr (type: string), p_size (type: int)
- sort order: ++
- Map-reduce partition columns: p_mfgr (type: string)
- Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
- TopN Hash Memory Usage: 0.1
- Reduce Output Operator
- key expressions: p_mfgr (type: string), p_size (type: int)
- sort order: ++
- Map-reduce partition columns: p_mfgr (type: string)
- Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
- TopN Hash Memory Usage: 0.1
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
- outputColumnNames: _col2, _col5
- Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col2: string, _col5: int
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col5 ASC NULLS FIRST
- partition by: _col2
- raw input shape:
- window functions:
- window function definition
- alias: rank_window_0
- arguments: _col5
- name: rank
- window function: GenericUDAFRankEvaluator
- window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
- isPivotResult: true
- Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (rank_window_0 <= 2) (type: boolean)
- Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col2 (type: string), _col5 (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: min(_col1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: int)
- Reducer 11
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int), _col0 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: string), true (type: boolean)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col1 (type: string), _col0 (type: int)
- sort order: ++
- Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
- Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: boolean)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -1131,33 +1030,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 7
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count()
- keys: _col0 (type: string)
- mode: complete
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Reducer 8
- Execution mode: llap
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
- outputColumnNames: _col2, _col5
- Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
PTF Operator
Function definitions:
Input definition
@@ -1198,7 +1070,67 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Reducer 9
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col2: string, _col5: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col5 ASC NULLS FIRST
+ partition by: _col2
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col5
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (rank_window_0 <= 2) (type: boolean)
+ Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col2 (type: string), _col5 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 8 Data size: 2960 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: int)
+ Reducer 7
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 2 Data size: 196 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count()
+ keys: _col0 (type: string)
+ mode: complete
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: bigint)
+ Reducer 8
Execution mode: llap
Reduce Operator Tree:
Group By Operator
@@ -1219,6 +1151,32 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 228 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reducer 9
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col1 (type: int), _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: string), true (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col1 (type: string), _col0 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 212 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: boolean)
Stage: Stage-0
Fetch Operator
@@ -1841,12 +1799,11 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
Reducer 6 <- Map 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
- Reducer 8 <- Map 5 (SIMPLE_EDGE)
- Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+ Reducer 8 <- Reducer 6 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1885,12 +1842,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: int)
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -1969,6 +1920,16 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Group By Operator
+ keys: _col1 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 7
Execution mode: llap
Reduce Operator Tree:
@@ -1985,29 +1946,6 @@ STAGE PLANS:
Execution mode: llap
Reduce Operator Tree:
Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col1 (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 9
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
@@ -2069,11 +2007,10 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
Reducer 5 <- Map 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
- Reducer 7 <- Map 4 (SIMPLE_EDGE)
- Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+ Reducer 7 <- Reducer 5 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2112,12 +2049,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: int)
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -2184,6 +2115,16 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Group By Operator
+ keys: _col1 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 6
Execution mode: llap
Reduce Operator Tree:
@@ -2200,29 +2141,6 @@ STAGE PLANS:
Execution mode: llap
Reduce Operator Tree:
Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col1 (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 8
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
@@ -2518,11 +2436,10 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
Reducer 5 <- Map 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
- Reducer 7 <- Map 4 (SIMPLE_EDGE)
- Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+ Reducer 7 <- Reducer 5 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2561,12 +2478,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: double)
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: double)
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -2633,6 +2544,16 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Group By Operator
+ keys: _col0 (type: bigint)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: bigint)
+ Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 6
Execution mode: llap
Reduce Operator Tree:
@@ -2649,29 +2570,6 @@ STAGE PLANS:
Execution mode: llap
Reduce Operator Tree:
Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: floor(_col1) (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col0 (type: bigint)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: bigint)
- sort order: +
- Map-reduce partition columns: _col0 (type: bigint)
- Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 8
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
keys: KEY._col0 (type: bigint)
mode: mergepartial
outputColumnNames: _col0
@@ -2716,14 +2614,12 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Map 9 (SIMPLE_EDGE)
- Reducer 11 <- Map 9 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
- Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
+ Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
- Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE)
- Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+ Reducer 7 <- Reducer 5 (SIMPLE_EDGE)
+ Reducer 9 <- Map 8 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2758,15 +2654,9 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int), _col0 (type: int)
Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
- Reduce Output Operator
- key expressions: _col2 (type: int), _col0 (type: int)
- sort order: ++
- Map-reduce partition columns: _col2 (type: int), _col0 (type: int)
- Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: string)
Execution mode: llap
LLAP IO: no inputs
- Map 9
+ Map 8
Map Operator Tree:
TableScan
alias: part
@@ -2781,39 +2671,8 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int), (_col0 + 121150) (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int)
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 11
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int), (_col0 + 121150) (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int)
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -2879,6 +2738,16 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Group By Operator
+ keys: _col1 (type: string), _col3 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 6
Execution mode: llap
Reduce Operator Tree:
@@ -2893,29 +2762,8 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 6 Data size: 120 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Reducer 7
- Execution mode: llap
- Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int), _col0 (type: int)
- 1 _col0 (type: int), (_col0 + 121150) (type: int)
- outputColumnNames: _col1, _col3
- Statistics: Num rows: 13 Data size: 1625 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col1 (type: string), _col3 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
- Statistics: Num rows: 6 Data size: 750 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 8
+ value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Reducer 7
Execution mode: llap
Reduce Operator Tree:
Group By Operator
@@ -2936,6 +2784,19 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: boolean)
+ Reducer 9
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), (_col0 + 121150) (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), (_col0 + 121150) (type: int)
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator
@@ -3322,14 +3183,12 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Map 9 (SIMPLE_EDGE)
- Reducer 11 <- Map 9 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
- Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 10 (ONE_TO_ONE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
+ Reducer 5 <- Map 4 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
- Reducer 7 <- Map 4 (SIMPLE_EDGE), Reducer 11 (ONE_TO_ONE_EDGE)
- Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+ Reducer 7 <- Reducer 5 (SIMPLE_EDGE)
+ Reducer 9 <- Map 8 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3364,15 +3223,9 @@ STAGE PLANS:
Map-reduce partition columns: (_col1 + 1) (type: int)
Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Reduce Output Operator
- key expressions: (_col1 + 1) (type: int)
- sort order: +
- Map-reduce partition columns: (_col1 + 1) (type: int)
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
- Map 9
+ Map 8
Map Operator Tree:
TableScan
alias: part
@@ -3391,39 +3244,8 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 11
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -3489,6 +3311,16 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 8 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Group By Operator
+ keys: _col0 (type: string), _col2 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 6
Execution mode: llap
Reduce Operator Tree:
@@ -3507,27 +3339,6 @@ STAGE PLANS:
Reducer 7
Execution mode: llap
Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 (_col1 + 1) (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col2
- Statistics: Num rows: 16 Data size: 1728 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col0 (type: string), _col2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
- Statistics: Num rows: 8 Data size: 864 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 8
- Execution mode: llap
- Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string), KEY._col1 (type: int)
mode: mergepartial
@@ -3546,6 +3357,19 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
Statistics: Num rows: 8 Data size: 896 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: boolean)
+ Reducer 9
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator
@@ -4175,12 +3999,11 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
Reducer 6 <- Map 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
- Reducer 8 <- Map 5 (SIMPLE_EDGE)
- Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+ Reducer 8 <- Reducer 6 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4219,12 +4042,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: int)
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -4303,6 +4120,16 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Group By Operator
+ keys: _col1 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 7
Execution mode: llap
Reduce Operator Tree:
@@ -4319,29 +4146,6 @@ STAGE PLANS:
Execution mode: llap
Reduce Operator Tree:
Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col1 (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 9
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
@@ -4403,12 +4207,11 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
Reducer 6 <- Map 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
- Reducer 8 <- Map 5 (SIMPLE_EDGE)
- Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+ Reducer 8 <- Reducer 6 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -4447,12 +4250,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: int)
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -4535,6 +4332,16 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
+ Group By Operator
+ keys: _col1 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 7
Execution mode: llap
Reduce Operator Tree:
@@ -4551,29 +4358,6 @@ STAGE PLANS:
Execution mode: llap
Reduce Operator Tree:
Group By Operator
- aggregations: min(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col1 (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reducer 9
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0
@@ -5747,14 +5531,13 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
- Reducer 3 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 2 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
Reducer 5 <- Map 4 (SIMPLE_EDGE)
- Reducer 6 <- Map 11 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 6 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
- Reducer 8 <- Map 4 (SIMPLE_EDGE)
- Reducer 9 <- Map 11 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+ Reducer 8 <- Map 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -5774,7 +5557,7 @@ STAGE PLANS:
value expressions: _col0 (type: int)
Execution mode: llap
LLAP IO: no inputs
- Map 11
+ Map 10
Map Operator Tree:
TableScan
alias: t2
@@ -5812,34 +5595,8 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: char(100))
Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: char(100))
- sort order: +
- Map-reduce partition columns: _col0 (type: char(100))
- Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int), KEY._col1 (type: char(100))
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: char(100)), true (type: boolean)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: char(100))
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100))
- Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: boolean)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -5896,6 +5653,12 @@ STAGE PLANS:
Map-reduce partition columns: UDFToDouble(_col0) (type: double)
Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: char(100))
+ Reduce Output Operator
+ key expressions: UDFToDouble(_col0) (type: double)
+ sort order: +
+ Map-reduce partition columns: UDFToDouble(_col0) (type: double)
+ Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: char(100))
Reducer 6
Execution mode: llap
Reduce Operator Tree:
@@ -5937,20 +5700,6 @@ STAGE PLANS:
Reducer 8
Execution mode: llap
Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: char(100))
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: UDFToDouble(_col0) (type: double)
- sort order: +
- Map-reduce partition columns: UDFToDouble(_col0) (type: double)
- Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: char(100))
- Reducer 9
- Execution mode: llap
- Reduce Operator Tree:
Merge Join Operator
condition map:
Inner Join 0 to 1
@@ -5969,6 +5718,27 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100))
Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
+ Reducer 9
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int), KEY._col1 (type: char(100))
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 92 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: char(100)), true (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: char(100))
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: char(100))
+ Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: boolean)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
index 64337ed..c250d2f 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
@@ -946,12 +946,11 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE), Reducer 8 (XPROD_EDGE)
+ Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 6 (XPROD_EDGE), Reducer 7 (XPROD_EDGE)
Reducer 4 <- Map 3 (SIMPLE_EDGE)
Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (CUSTOM_SIMPLE_EDGE)
- Reducer 7 <- Map 3 (SIMPLE_EDGE)
- Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
+ Reducer 7 <- Reducer 4 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -979,11 +978,6 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: p_mfgr (type: string)
Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: p_mfgr (type: string), p_size (type: int)
- sort order: ++
- Map-reduce partition columns: p_mfgr (type: string)
- Statistics: Num rows: 26 Data size: 2652 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -1046,6 +1040,35 @@ STAGE PLANS:
key expressions: _col0 (type: int)
sort order: +
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col2: string, _col5: int
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col5 ASC NULLS FIRST
+ partition by: _col2
+ raw input shape:
+ window functions:
+ window function definition
+ alias: first_value_window_0
+ arguments: _col5
+ name: first_value
+ window function: GenericUDAFFirstValueEvaluator
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: first_value_window_0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
Reducer 5
Execution mode: llap
Reduce Operator Tree:
@@ -1083,42 +1106,6 @@ STAGE PLANS:
Execution mode: llap
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
- outputColumnNames: _col2, _col5
- Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col2: string, _col5: int
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col5 ASC NULLS FIRST
- partition by: _col2
- raw input shape:
- window functions:
- window function definition
- alias: first_value_window_0
- arguments: _col5
- name: first_value
- window function: GenericUDAFFirstValueEvaluator
- window frame: RANGE PRECEDING(MAX)~CURRENT
- Statistics: Num rows: 26 Data size: 9620 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: first_value_window_0 (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- TopN Hash Memory Usage: 0.1
- Reducer 8
- Execution mode: llap
- Reduce Operator Tree:
- Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
@@ -2770,18 +2757,16 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 11 (CONTAINS)
- Reducer 12 <- Union 11 (SIMPLE_EDGE)
- Reducer 14 <- Map 13 (CUSTOM_SIMPLE_EDGE)
- Reducer 15 <- Reducer 14 (SIMPLE_EDGE), Union 6 (CONTAINS)
- Reducer 16 <- Map 13 (CUSTOM_SIMPLE_EDGE)
- Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Union 11 (CONTAINS)
- Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 12 (XPROD_EDGE), Reducer 8 (XPROD_EDGE)
+ Reducer 11 <- Union 10 (SIMPLE_EDGE)
+ Reducer 13 <- Map 12 (CUSTOM_SIMPLE_EDGE)
+ Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Union 6 (CONTAINS)
+ Reducer 15 <- Reducer 13 (SIMPLE_EDGE), Union 10 (CONTAINS)
+ Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 11 (XPROD_EDGE), Reducer 8 (XPROD_EDGE)
Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE)
Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Union 6 (CONTAINS)
Reducer 7 <- Union 6 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
- Reducer 9 <- Map 3 (CUSTOM_SIMPLE_EDGE)
+ Reducer 9 <- Reducer 4 (SIMPLE_EDGE), Union 10 (CONTAINS)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2799,7 +2784,7 @@ STAGE PLANS:
value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
Execution mode: llap
LLAP IO: no inputs
- Map 13
+ Map 12
Map Operator Tree:
TableScan
alias: part
@@ -2817,10 +2802,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
Execution mode: llap
LLAP IO: no inputs
Map 3
@@ -2841,34 +2822,9 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: bigint)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(_col1)
- keys: _col0 (type: bigint)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: bigint)
- sort order: +
- Map-reduce partition columns: _col0 (type: bigint)
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Reducer 12
+ Reducer 11
Execution mode: llap
Reduce Operator Tree:
Group By Operator
@@ -2888,7 +2844,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Reducer 14
+ Reducer 13
Execution mode: llap
Reduce Operator Tree:
Group By Operator
@@ -2908,37 +2864,23 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: bigint)
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Reducer 15
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: bigint)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(_col1)
- keys: _col0 (type: bigint)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: bigint)
sort order: +
Map-reduce partition columns: _col0 (type: bigint)
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Reducer 16
+ Reducer 14
Execution mode: llap
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: bigint)
mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- aggregations: count()
+ aggregations: count(_col1)
keys: _col0 (type: bigint)
mode: hash
outputColumnNames: _col0, _col1
@@ -2949,7 +2891,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: bigint)
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Reducer 17
+ Reducer 15
Execution mode: llap
Reduce Operator Tree:
Group By Operator
@@ -3015,6 +2957,12 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: bigint)
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: bigint)
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: bigint)
Reducer 5
Execution mode: llap
Reduce Operator Tree:
@@ -3084,11 +3032,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: bigint)
mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- aggregations: count()
+ aggregations: count(_col1)
keys: _col0 (type: bigint)
mode: hash
outputColumnNames: _col0, _col1
@@ -3099,8 +3048,8 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: bigint)
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Union 11
- Vertex: Union 11
+ Union 10
+ Vertex: Union 10
Union 6
Vertex: Union 6
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/subquery_select.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
index b090cd0..4d67a85 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
@@ -3083,11 +3083,10 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE)
- Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Reducer 4 <- Reducer 10 (XPROD_EDGE), Reducer 3 (XPROD_EDGE)
- Reducer 5 <- Reducer 11 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 10 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
Reducer 7 <- Map 6 (SIMPLE_EDGE)
Reducer 8 <- Map 6 (SIMPLE_EDGE)
#### A masked pattern was here ####
@@ -3155,10 +3154,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string)
Execution mode: llap
LLAP IO: no inputs
Reducer 10
@@ -3178,14 +3173,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 11
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- aggregations: min(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: string), true (type: boolean)
outputColumnNames: _col0, _col1
@@ -3503,7 +3490,6 @@ POSTHOOK: Input: default@part
45 false
Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
-Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 9' is a cross product
PREHOOK: query: explain select p_size, (p_size IN
(select (select max(p_size) from part) as sb from part order by sb limit 1)) = true
from part
@@ -3521,15 +3507,13 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 7 (XPROD_EDGE)
- Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Reducer 4 <- Map 1 (CUSTOM_SIMPLE_EDGE)
- Reducer 5 <- Map 11 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE)
+ Reducer 5 <- Map 9 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
- Reducer 8 <- Map 1 (CUSTOM_SIMPLE_EDGE)
- Reducer 9 <- Map 11 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE)
+ Reducer 8 <- Reducer 5 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -3558,22 +3542,9 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int)
- Select Operator
- expressions: p_size (type: int)
- outputColumnNames: p_size
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: max(p_size)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: int)
Execution mode: llap
LLAP IO: no inputs
- Map 11
+ Map 9
Map Operator Tree:
TableScan
alias: part
@@ -3583,31 +3554,8 @@ STAGE PLANS:
Reduce Output Operator
sort order:
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: int), true (type: boolean)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: boolean)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -3678,6 +3626,11 @@ STAGE PLANS:
key expressions: _col0 (type: int)
sort order: +
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
+ TopN Hash Memory Usage: 0.1
Reducer 6
Execution mode: llap
Reduce Operator Tree:
@@ -3712,35 +3665,23 @@ STAGE PLANS:
Reducer 8
Execution mode: llap
Reduce Operator Tree:
- Group By Operator
- aggregations: max(VALUE._col0)
- mode: mergepartial
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: int)
- Reducer 9
- Execution mode: llap
- Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Left Outer Join 0 to 1
- keys:
- 0
- 1
- outputColumnNames: _col1
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- TopN Hash Memory Usage: 0.1
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), true (type: boolean)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: boolean)
Stage: Stage-0
Fetch Operator
@@ -3750,7 +3691,6 @@ STAGE PLANS:
Warning: Shuffle Join MERGEJOIN[50][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 5' is a cross product
Warning: Shuffle Join MERGEJOIN[51][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product
-Warning: Shuffle Join MERGEJOIN[52][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 9' is a cross product
PREHOOK: query: select p_size, (p_size IN
(select (select max(p_size) from part) as sb from part order by sb limit 1)) = true
from part
[8/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
HIVE-18361: Extend shared work optimizer to reuse computation beyond work boundaries (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/96a409e1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/96a409e1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/96a409e1
Branch: refs/heads/master
Commit: 96a409e1c6ec846eeb6c72b50bed60790ccc1836
Parents: 3f5148d
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Thu Dec 21 17:08:07 2017 -0800
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Thu Jan 4 18:44:15 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 4 +
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/optimizer/SharedWorkOptimizer.java | 407 ++++++-
.../test/queries/clientpositive/sharedworkext.q | 53 +
.../clientpositive/llap/explainuser_1.q.out | 47 +-
.../clientpositive/llap/sharedworkext.q.out | 1076 ++++++++++++++++++
.../clientpositive/llap/subquery_multi.q.out | 185 +--
.../clientpositive/llap/subquery_notin.q.out | 752 +++++-------
.../clientpositive/llap/subquery_scalar.q.out | 167 +--
.../clientpositive/llap/subquery_select.q.out | 110 +-
.../clientpositive/llap/subquery_views.q.out | 83 +-
.../llap/vector_groupby_grouping_id2.q.out | 138 +--
.../llap/vector_groupby_grouping_sets4.q.out | 71 +-
.../clientpositive/perf/tez/query14.q.out | 264 ++---
.../clientpositive/perf/tez/query2.q.out | 103 +-
.../clientpositive/perf/tez/query23.q.out | 360 ++----
.../clientpositive/perf/tez/query32.q.out | 30 +-
.../clientpositive/perf/tez/query33.q.out | 232 ++--
.../clientpositive/perf/tez/query44.q.out | 69 +-
.../clientpositive/perf/tez/query47.q.out | 224 ++--
.../clientpositive/perf/tez/query54.q.out | 34 +-
.../clientpositive/perf/tez/query56.q.out | 232 ++--
.../clientpositive/perf/tez/query57.q.out | 224 ++--
.../clientpositive/perf/tez/query58.q.out | 200 +---
.../clientpositive/perf/tez/query59.q.out | 52 +-
.../clientpositive/perf/tez/query60.q.out | 232 ++--
.../clientpositive/perf/tez/query61.q.out | 118 +-
.../clientpositive/perf/tez/query64.q.out | 468 +++-----
.../clientpositive/perf/tez/query70.q.out | 66 +-
.../clientpositive/perf/tez/query83.q.out | 116 +-
.../clientpositive/perf/tez/query90.q.out | 50 +-
.../clientpositive/perf/tez/query92.q.out | 30 +-
32 files changed, 3056 insertions(+), 3142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 841d075..6529da6 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1695,6 +1695,10 @@ public class HiveConf extends Configuration {
HIVE_SHARED_WORK_OPTIMIZATION("hive.optimize.shared.work", true,
"Whether to enable shared work optimizer. The optimizer finds scan operator over the same table\n" +
"and follow-up operators in the query plan and merges them if they meet some preconditions. Tez only."),
+ HIVE_SHARED_WORK_EXTENDED_OPTIMIZATION("hive.optimize.shared.work.extended", true,
+ "Whether to enable shared work extended optimizer. The optimizer tries to merge equal operators\n" +
+ "after a work boundary after shared work optimizer has been executed. Requires hive.optimize.shared.work\n" +
+ "to be set to true. Tez only."),
HIVE_COMBINE_EQUIVALENT_WORK_OPTIMIZATION("hive.combine.equivalent.work.optimization", true, "Whether to " +
"combine equivalent work objects during physical optimization.\n This optimization looks for equivalent " +
"work objects and combines them if they meet certain preconditions. Spark only."),
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 98e390c..ac81995 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -628,6 +628,7 @@ minillaplocal.query.files=\
semijoin6.q,\
semijoin7.q,\
semijoin_hint.q,\
+ sharedworkext.q,\
smb_cache.q,\
special_character_in_tabnames_1.q,\
sqlmerge.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index d4ddb75..08fec42 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hive.ql.optimizer;
import java.util.ArrayList;
@@ -36,6 +35,7 @@ import java.util.Set;
import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
@@ -79,6 +79,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multiset;
+import com.google.common.collect.Sets;
import com.google.common.collect.TreeMultiset;
/**
@@ -102,8 +103,8 @@ import com.google.common.collect.TreeMultiset;
* | | / \
* Op Op Op Op
*
- * <p>A limitation in the current implementation is that the optimizer does not
- * go beyond a work boundary.
+ * <p>If the extended version of the optimizer is enabled, it can go beyond
+ * a work boundary to find reutilization opportunities.
*
* <p>The optimization only works with the Tez execution engine.
*/
@@ -146,13 +147,13 @@ public class SharedWorkOptimizer extends Transform {
String tableName = tablePair.getKey();
for (TableScanOperator discardableTsOp : tableNameToOps.get(tableName)) {
if (removedOps.contains(discardableTsOp)) {
- LOG.debug("Skip {} as it has been already removed", discardableTsOp);
+ LOG.debug("Skip {} as it has already been removed", discardableTsOp);
continue;
}
Collection<TableScanOperator> prevTsOps = existingOps.get(tableName);
for (TableScanOperator retainableTsOp : prevTsOps) {
if (removedOps.contains(retainableTsOp)) {
- LOG.debug("Skip {} as it has been already removed", retainableTsOp);
+ LOG.debug("Skip {} as it has already been removed", retainableTsOp);
continue;
}
@@ -167,7 +168,7 @@ public class SharedWorkOptimizer extends Transform {
// Secondly, we extract information about the part of the tree that can be merged
// as well as some structural information (memory consumption) that needs to be
// used to determined whether the merge can happen
- SharedResult sr = extractSharedOptimizationInfo(
+ SharedResult sr = extractSharedOptimizationInfoForRoot(
pctx, optimizerCache, retainableTsOp, discardableTsOp);
// It seems these two operators can be merged.
@@ -197,7 +198,8 @@ public class SharedWorkOptimizer extends Transform {
}
}
- LOG.debug("Merging subtree starting at {} into subtree starting at {}", discardableTsOp, retainableTsOp);
+ LOG.debug("Merging subtree starting at {} into subtree starting at {}",
+ discardableTsOp, retainableTsOp);
} else {
// Only TS operator
ExprNodeGenericFuncDesc exprNode = null;
@@ -262,6 +264,7 @@ public class SharedWorkOptimizer extends Transform {
!sr.discardableInputOps.contains(sjbi.getTsOp())) {
GenTezUtils.removeSemiJoinOperator(
pctx, (ReduceSinkOperator) op, sjbi.getTsOp());
+ optimizerCache.tableScanToDPPSource.remove(sjbi.getTsOp(), op);
}
} else if (op instanceof AppMasterEventOperator) {
DynamicPruningEventDesc dped = (DynamicPruningEventDesc) op.getConf();
@@ -269,6 +272,7 @@ public class SharedWorkOptimizer extends Transform {
!sr.discardableInputOps.contains(dped.getTableScan())) {
GenTezUtils.removeSemiJoinOperator(
pctx, (AppMasterEventOperator) op, dped.getTableScan());
+ optimizerCache.tableScanToDPPSource.remove(dped.getTableScan(), op);
}
}
LOG.debug("Input operator removed: {}", op);
@@ -292,10 +296,12 @@ public class SharedWorkOptimizer extends Transform {
GenTezUtils.removeSemiJoinOperator(pctx,
(ReduceSinkOperator) dppSource,
(TableScanOperator) sr.retainableOps.get(0));
+ optimizerCache.tableScanToDPPSource.remove(sr.retainableOps.get(0), op);
} else if (dppSource instanceof AppMasterEventOperator) {
GenTezUtils.removeSemiJoinOperator(pctx,
(AppMasterEventOperator) dppSource,
(TableScanOperator) sr.retainableOps.get(0));
+ optimizerCache.tableScanToDPPSource.remove(sr.retainableOps.get(0), op);
}
}
}
@@ -328,6 +334,203 @@ public class SharedWorkOptimizer extends Transform {
LOG.debug("After SharedWorkOptimizer:\n" + Operator.toString(pctx.getTopOps().values()));
}
+ if(pctx.getConf().getBoolVar(ConfVars.HIVE_SHARED_WORK_EXTENDED_OPTIMIZATION)) {
+ // Gather RS operators that 1) belong to root works, i.e., works containing TS operators,
+ // and 2) share the same input operator.
+ // These will be the first target for extended shared work optimization
+ Multimap<Operator<?>, ReduceSinkOperator> parentToRsOps = ArrayListMultimap.create();
+ Set<Operator<?>> visited = new HashSet<>();
+ for (Entry<String, TableScanOperator> e : topOps.entrySet()) {
+ gatherReduceSinkOpsByInput(parentToRsOps, visited,
+ findWorkOperators(optimizerCache, e.getValue()));
+ }
+
+ while (!parentToRsOps.isEmpty()) {
+ // As above, we enforce a certain order when we do the reutilization.
+ // In particular, we use size of data in RS x number of uses.
+ List<Entry<Operator<?>, Long>> sortedRSGroups =
+ rankOpsByAccumulatedSize(parentToRsOps.keySet());
+ LOG.debug("Sorted operators by size: {}", sortedRSGroups);
+
+ // Execute extended optimization
+ // For each RS, check whether other RS in same work could be merge into this one.
+ // If they are merged, RS operators in the resulting work will be considered
+ // mergeable in next loop iteration.
+ Multimap<Operator<?>, ReduceSinkOperator> existingRsOps = ArrayListMultimap.create();
+ for (Entry<Operator<?>, Long> rsGroupInfo : sortedRSGroups) {
+ Operator<?> rsParent = rsGroupInfo.getKey();
+ for (ReduceSinkOperator discardableRsOp : parentToRsOps.get(rsParent)) {
+ if (removedOps.contains(discardableRsOp)) {
+ LOG.debug("Skip {} as it has already been removed", discardableRsOp);
+ continue;
+ }
+ Collection<ReduceSinkOperator> otherRsOps = existingRsOps.get(rsParent);
+ for (ReduceSinkOperator retainableRsOp : otherRsOps) {
+ if (removedOps.contains(retainableRsOp)) {
+ LOG.debug("Skip {} as it has already been removed", retainableRsOp);
+ continue;
+ }
+
+ // First we quickly check if the two RS operators can actually be merged.
+ // We already know that these two RS operators have the same parent, but
+ // we need to check whether both RS are actually equal. Further, we check
+ // whether their child is also equal. If any of these conditions are not
+ // met, we are not going to try to merge.
+ boolean mergeable = compareOperator(pctx, retainableRsOp, discardableRsOp) &&
+ compareOperator(pctx, retainableRsOp.getChildOperators().get(0),
+ discardableRsOp.getChildOperators().get(0));
+ if (!mergeable) {
+ // Skip
+ LOG.debug("{} and {} cannot be merged", retainableRsOp, discardableRsOp);
+ continue;
+ }
+
+ LOG.debug("Checking additional conditions for merging subtree starting at {}"
+ + " into subtree starting at {}", discardableRsOp, retainableRsOp);
+
+ // Secondly, we extract information about the part of the tree that can be merged
+ // as well as some structural information (memory consumption) that needs to be
+ // used to determined whether the merge can happen
+ Operator<?> retainableRsOpChild = retainableRsOp.getChildOperators().get(0);
+ Operator<?> discardableRsOpChild = discardableRsOp.getChildOperators().get(0);
+ SharedResult sr = extractSharedOptimizationInfo(
+ pctx, optimizerCache, retainableRsOp, discardableRsOp,
+ retainableRsOpChild, discardableRsOpChild);
+
+ // It seems these two operators can be merged.
+ // Check that plan meets some preconditions before doing it.
+ // In particular, in the presence of map joins in the upstream plan:
+ // - we cannot exceed the noconditional task size, and
+ // - if we already merged the big table, we cannot merge the broadcast
+ // tables.
+ if (sr.retainableOps.isEmpty() || !validPreConditions(pctx, optimizerCache, sr)) {
+ // Skip
+ LOG.debug("{} and {} do not meet preconditions", retainableRsOp, discardableRsOp);
+ continue;
+ }
+
+ // We can merge
+ Operator<?> lastRetainableOp = sr.retainableOps.get(sr.retainableOps.size() - 1);
+ Operator<?> lastDiscardableOp = sr.discardableOps.get(sr.discardableOps.size() - 1);
+ if (lastDiscardableOp.getNumChild() != 0) {
+ List<Operator<? extends OperatorDesc>> allChildren =
+ Lists.newArrayList(lastDiscardableOp.getChildOperators());
+ for (Operator<? extends OperatorDesc> op : allChildren) {
+ lastDiscardableOp.getChildOperators().remove(op);
+ op.replaceParent(lastDiscardableOp, lastRetainableOp);
+ lastRetainableOp.getChildOperators().add(op);
+ }
+ }
+
+ LOG.debug("Merging subtree starting at {} into subtree starting at {}",
+ discardableRsOp, retainableRsOp);
+
+ // First we remove the input operators of the expression that
+ // we are going to eliminate
+ for (Operator<?> op : sr.discardableInputOps) {
+ OperatorUtils.removeOperator(op);
+ optimizerCache.removeOp(op);
+ removedOps.add(op);
+ // Remove DPP predicates
+ if (op instanceof ReduceSinkOperator) {
+ SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op);
+ if (sjbi != null && !sr.discardableOps.contains(sjbi.getTsOp()) &&
+ !sr.discardableInputOps.contains(sjbi.getTsOp())) {
+ GenTezUtils.removeSemiJoinOperator(
+ pctx, (ReduceSinkOperator) op, sjbi.getTsOp());
+ optimizerCache.tableScanToDPPSource.remove(sjbi.getTsOp(), op);
+ }
+ } else if (op instanceof AppMasterEventOperator) {
+ DynamicPruningEventDesc dped = (DynamicPruningEventDesc) op.getConf();
+ if (!sr.discardableOps.contains(dped.getTableScan()) &&
+ !sr.discardableInputOps.contains(dped.getTableScan())) {
+ GenTezUtils.removeSemiJoinOperator(
+ pctx, (AppMasterEventOperator) op, dped.getTableScan());
+ optimizerCache.tableScanToDPPSource.remove(dped.getTableScan(), op);
+ }
+ }
+ LOG.debug("Input operator removed: {}", op);
+ }
+ // We remove the discardable RS operator
+ OperatorUtils.removeOperator(discardableRsOp);
+ optimizerCache.removeOp(discardableRsOp);
+ removedOps.add(discardableRsOp);
+ LOG.debug("Operator removed: {}", discardableRsOp);
+ // Then we merge the operators of the works we are going to merge
+ optimizerCache.removeOpAndCombineWork(discardableRsOpChild, retainableRsOpChild);
+ // Finally we remove the rest of the expression from the tree
+ for (Operator<?> op : sr.discardableOps) {
+ OperatorUtils.removeOperator(op);
+ optimizerCache.removeOp(op);
+ removedOps.add(op);
+ LOG.debug("Operator removed: {}", op);
+ }
+
+ break;
+ }
+
+ if (removedOps.contains(discardableRsOp)) {
+ // This operator has been removed, remove it from the list of existing operators
+ existingRsOps.remove(rsParent, discardableRsOp);
+ } else {
+ // This operator has not been removed, include it in the list of existing operators
+ existingRsOps.put(rsParent, discardableRsOp);
+ }
+ }
+ }
+
+ // We gather the operators that will be used for next iteration of extended optimization
+ // (if any)
+ parentToRsOps = ArrayListMultimap.create();
+ visited = new HashSet<>();
+ for (Entry<Operator<?>, ReduceSinkOperator> e : existingRsOps.entries()) {
+ if (removedOps.contains(e.getValue()) || e.getValue().getNumChild() < 1) {
+ // If 1) RS has been removed, or 2) it does not have a child (for instance, it is a
+ // semijoin RS), we can quickly skip this one
+ continue;
+ }
+ gatherReduceSinkOpsByInput(parentToRsOps, visited,
+ findWorkOperators(optimizerCache, e.getValue().getChildOperators().get(0)));
+ }
+ }
+
+ // Remove unused table scan operators
+ it = topOps.entrySet().iterator();
+ while (it.hasNext()) {
+ Entry<String, TableScanOperator> e = it.next();
+ if (e.getValue().getNumChild() == 0) {
+ it.remove();
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("After SharedWorkExtendedOptimizer:\n"
+ + Operator.toString(pctx.getTopOps().values()));
+ }
+ }
+
+ // If we are running tests, we are going to verify that the contents of the cache
+ // correspond with the contents of the plan, and otherwise we fail.
+ // This check always run when we are running in test mode, independently on whether
+ // we use the basic or the extended version of the optimizer.
+ if (pctx.getConf().getBoolVar(ConfVars.HIVE_IN_TEST)) {
+ Set<Operator<?>> visited = new HashSet<>();
+ it = topOps.entrySet().iterator();
+ while (it.hasNext()) {
+ Entry<String, TableScanOperator> e = it.next();
+ for (Operator<?> op : OperatorUtils.findOperators(e.getValue(), Operator.class)) {
+ if (!visited.contains(op)) {
+ if (!findWorkOperators(optimizerCache, op).equals(
+ findWorkOperators(op, new HashSet<Operator<?>>()))) {
+ throw new SemanticException("Error in shared work optimizer: operator cache contents"
+ + "and actual plan differ");
+ }
+ visited.add(op);
+ }
+ }
+ }
+ }
+
return pctx;
}
@@ -392,16 +595,59 @@ public class SharedWorkOptimizer extends Transform {
}
}
List<Entry<String, Long>> sortedTables =
- new LinkedList<>(tableToTotalSize.entrySet());
+ new LinkedList<>(tableToTotalSize.entrySet());
Collections.sort(sortedTables, Collections.reverseOrder(
- new Comparator<Map.Entry<String, Long>>() {
- public int compare(Map.Entry<String, Long> o1, Map.Entry<String, Long> o2) {
- return (o1.getValue()).compareTo(o2.getValue());
- }
- }));
+ new Comparator<Map.Entry<String, Long>>() {
+ public int compare(Map.Entry<String, Long> o1, Map.Entry<String, Long> o2) {
+ return (o1.getValue()).compareTo(o2.getValue());
+ }
+ }));
return sortedTables;
}
+ private static void gatherReduceSinkOpsByInput(Multimap<Operator<?>,
+ ReduceSinkOperator> parentToRsOps, Set<Operator<?>> visited, Set<Operator<?>> ops) {
+ for (Operator<?> op : ops) {
+ // If the RS has other RS siblings, we will add it to be considered in next iteration
+ if (op instanceof ReduceSinkOperator && !visited.contains(op)) {
+ Operator<?> parent = op.getParentOperators().get(0);
+ Set<ReduceSinkOperator> s = new LinkedHashSet<>();
+ for (Operator<?> c : parent.getChildOperators()) {
+ if (c instanceof ReduceSinkOperator) {
+ s.add((ReduceSinkOperator) c);
+ visited.add(c);
+ }
+ }
+ if (s.size() > 1) {
+ parentToRsOps.putAll(parent, s);
+ }
+ }
+ }
+ }
+
+ private static List<Entry<Operator<?>, Long>> rankOpsByAccumulatedSize(Set<Operator<?>> opsSet) {
+ Map<Operator<?>, Long> opToTotalSize = new HashMap<>();
+ for (Operator<?> op : opsSet) {
+ long size = op.getStatistics() != null ?
+ op.getStatistics().getDataSize() : 0L;
+ opToTotalSize.put(op,
+ StatsUtils.safeMult(op.getChildOperators().size(), size));
+ }
+ List<Entry<Operator<?>, Long>> sortedOps =
+ new LinkedList<>(opToTotalSize.entrySet());
+ Collections.sort(sortedOps, Collections.reverseOrder(
+ new Comparator<Map.Entry<Operator<?>, Long>>() {
+ public int compare(Map.Entry<Operator<?>, Long> o1, Map.Entry<Operator<?>, Long> o2) {
+ int valCmp = o1.getValue().compareTo(o2.getValue());
+ if (valCmp == 0) {
+ return o1.getKey().toString().compareTo(o2.getKey().toString());
+ }
+ return valCmp;
+ }
+ }));
+ return sortedOps;
+ }
+
private static boolean areMergeable(ParseContext pctx, SharedWorkOptimizerCache optimizerCache,
TableScanOperator tsOp1, TableScanOperator tsOp2) throws SemanticException {
// First we check if the two table scan operators can actually be merged
@@ -486,15 +732,15 @@ public class SharedWorkOptimizer extends Transform {
return true;
}
- private static SharedResult extractSharedOptimizationInfo(ParseContext pctx,
+ private static SharedResult extractSharedOptimizationInfoForRoot(ParseContext pctx,
SharedWorkOptimizerCache optimizerCache,
TableScanOperator retainableTsOp,
TableScanOperator discardableTsOp) throws SemanticException {
- Set<Operator<?>> retainableOps = new LinkedHashSet<>();
- Set<Operator<?>> discardableOps = new LinkedHashSet<>();
+ LinkedHashSet<Operator<?>> retainableOps = new LinkedHashSet<>();
+ LinkedHashSet<Operator<?>> discardableOps = new LinkedHashSet<>();
Set<Operator<?>> discardableInputOps = new HashSet<>();
- long dataSize = 0l;
- long maxDataSize = 0l;
+ long dataSize = 0L;
+ long maxDataSize = 0L;
retainableOps.add(retainableTsOp);
discardableOps.add(discardableTsOp);
@@ -503,7 +749,8 @@ public class SharedWorkOptimizer extends Transform {
if (equalOp1.getNumChild() > 1 || equalOp2.getNumChild() > 1) {
// TODO: Support checking multiple child operators to merge further.
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
- return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps,
+ dataSize, maxDataSize);
}
Operator<?> currentOp1 = retainableTsOp.getChildOperators().get(0);
Operator<?> currentOp2 = discardableTsOp.getChildOperators().get(0);
@@ -532,19 +779,54 @@ public class SharedWorkOptimizer extends Transform {
currentOp2.getChildOperators().size() > 1) {
// TODO: Support checking multiple child operators to merge further.
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
- discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
- return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps,
+ discardableInputOps));
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps,
+ dataSize, maxDataSize);
}
currentOp1 = currentOp1.getChildOperators().get(0);
currentOp2 = currentOp2.getChildOperators().get(0);
} else {
// Bail out
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
- discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
- return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps,
+ discardableInputOps));
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps,
+ dataSize, maxDataSize);
}
}
+ return extractSharedOptimizationInfo(pctx, optimizerCache, equalOp1, equalOp2,
+ currentOp1, currentOp2, retainableOps, discardableOps, discardableInputOps, false);
+ }
+
+ private static SharedResult extractSharedOptimizationInfo(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache,
+ Operator<?> retainableOpEqualParent,
+ Operator<?> discardableOpEqualParent,
+ Operator<?> retainableOp,
+ Operator<?> discardableOp) throws SemanticException {
+ return extractSharedOptimizationInfo(pctx, optimizerCache,
+ retainableOpEqualParent, discardableOpEqualParent, retainableOp, discardableOp,
+ new LinkedHashSet<>(), new LinkedHashSet<>(), new HashSet<>(), true);
+ }
+
+ private static SharedResult extractSharedOptimizationInfo(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache,
+ Operator<?> retainableOpEqualParent,
+ Operator<?> discardableOpEqualParent,
+ Operator<?> retainableOp,
+ Operator<?> discardableOp,
+ LinkedHashSet<Operator<?>> retainableOps,
+ LinkedHashSet<Operator<?>> discardableOps,
+ Set<Operator<?>> discardableInputOps,
+ boolean removeInputBranch) throws SemanticException {
+ Operator<?> equalOp1 = retainableOpEqualParent;
+ Operator<?> equalOp2 = discardableOpEqualParent;
+ Operator<?> currentOp1 = retainableOp;
+ Operator<?> currentOp2 = discardableOp;
+ long dataSize = 0L;
+ long maxDataSize = 0L;
// Try to merge rest of operators
while (!(currentOp1 instanceof ReduceSinkOperator)) {
// Check whether current operators are equal
@@ -563,7 +845,7 @@ public class SharedWorkOptimizer extends Transform {
for (; idx < currentOp1.getParentOperators().size(); idx++) {
Operator<?> parentOp1 = currentOp1.getParentOperators().get(idx);
Operator<?> parentOp2 = currentOp2.getParentOperators().get(idx);
- if (parentOp1 == equalOp1 && parentOp2 == equalOp2) {
+ if (parentOp1 == equalOp1 && parentOp2 == equalOp2 && !removeInputBranch) {
continue;
}
if ((parentOp1 == equalOp1 && parentOp2 != equalOp2) ||
@@ -572,7 +854,8 @@ public class SharedWorkOptimizer extends Transform {
break;
}
// Compare input
- List<Operator<?>> removeOpsForCurrentInput = compareAndGatherOps(pctx, parentOp1, parentOp2);
+ List<Operator<?>> removeOpsForCurrentInput =
+ compareAndGatherOps(pctx, parentOp1, parentOp2);
if (removeOpsForCurrentInput == null) {
// Inputs are not the same, bail out
break;
@@ -626,8 +909,10 @@ public class SharedWorkOptimizer extends Transform {
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps));
discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
- discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
- return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps,
+ discardableInputOps));
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps,
+ dataSize, maxDataSize);
}
private static Multiset<String> extractConjsIgnoringDPPPreds(ExprNodeDesc predicate) {
@@ -656,7 +941,8 @@ public class SharedWorkOptimizer extends Transform {
Set<Operator<?>> dppBranches = new HashSet<>();
for (Operator<?> op : ops) {
if (op instanceof TableScanOperator) {
- Collection<Operator<?>> c = optimizerCache.tableScanToDPPSource.get((TableScanOperator) op);
+ Collection<Operator<?>> c = optimizerCache.tableScanToDPPSource
+ .get((TableScanOperator) op);
for (Operator<?> dppSource : c) {
// Remove the branches
Operator<?> currentOp = dppSource;
@@ -676,7 +962,8 @@ public class SharedWorkOptimizer extends Transform {
Set<Operator<?>> dppBranches = new HashSet<>();
for (Operator<?> op : ops) {
if (op instanceof TableScanOperator) {
- Collection<Operator<?>> c = optimizerCache.tableScanToDPPSource.get((TableScanOperator) op);
+ Collection<Operator<?>> c = optimizerCache.tableScanToDPPSource
+ .get((TableScanOperator) op);
for (Operator<?> dppSource : c) {
Set<Operator<?>> ascendants =
findAscendantWorkOperators(pctx, optimizerCache, dppSource);
@@ -711,6 +998,11 @@ public class SharedWorkOptimizer extends Transform {
return false;
}
+ if (gather && op2.getChildOperators().size() > 1) {
+ // If the second operator has more than one child, we stop gathering
+ gather = false;
+ }
+
if (gather) {
result.add(op2);
}
@@ -724,12 +1016,8 @@ public class SharedWorkOptimizer extends Transform {
for (int i = 0; i < op1ParentOperators.size(); i++) {
Operator<?> op1ParentOp = op1ParentOperators.get(i);
Operator<?> op2ParentOp = op2ParentOperators.get(i);
- boolean mergeable;
- if (gather && op2ParentOp.getChildOperators().size() < 2) {
- mergeable = compareAndGatherOps(pctx, op1ParentOp, op2ParentOp, result, true);
- } else {
- mergeable = compareAndGatherOps(pctx, op1ParentOp, op2ParentOp, result, false);
- }
+ boolean mergeable =
+ compareAndGatherOps(pctx, op1ParentOp, op2ParentOp, result, gather);
if (!mergeable) {
return false;
}
@@ -741,7 +1029,6 @@ public class SharedWorkOptimizer extends Transform {
return true;
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
private static boolean compareOperator(ParseContext pctx, Operator<?> op1, Operator<?> op2)
throws SemanticException {
if (!op1.getClass().getName().equals(op2.getClass().getName())) {
@@ -809,21 +1096,21 @@ public class SharedWorkOptimizer extends Transform {
return false;
}
- TableScanOperator tsOp1 = (TableScanOperator) sr.retainableOps.get(0);
- TableScanOperator tsOp2 = (TableScanOperator) sr.discardableOps.get(0);
+ Operator<?> op1 = sr.retainableOps.get(0);
+ Operator<?> op2 = sr.discardableOps.get(0);
- // 1) The set of operators in the works of the TS operators need to meet
+ // 1) The set of operators in the works that we are merging need to meet
// some requirements. In particular:
- // 1.1. None of the works that contain the TS operators can contain a Union
+ // 1.1. None of the works that we are merging can contain a Union
// operator. This is not supported yet as we might end up with cycles in
// the Tez DAG.
// 1.2. There cannot be more than one DummyStore operator in the new resulting
- // work when the TS operators are merged. This is due to an assumption in
+ // work when the operators are merged. This is due to an assumption in
// MergeJoinProc that needs to be further explored.
// If any of these conditions are not met, we cannot merge.
// TODO: Extend rule so it can be applied for these cases.
- final Set<Operator<?>> workOps1 = findWorkOperators(optimizerCache, tsOp1);
- final Set<Operator<?>> workOps2 = findWorkOperators(optimizerCache, tsOp2);
+ final Set<Operator<?>> workOps1 = findWorkOperators(optimizerCache, op1);
+ final Set<Operator<?>> workOps2 = findWorkOperators(optimizerCache, op2);
boolean foundDummyStoreOp = false;
for (Operator<?> op : workOps1) {
if (op instanceof UnionOperator) {
@@ -853,8 +1140,8 @@ public class SharedWorkOptimizer extends Transform {
// If we do, we cannot merge. The reason is that Tez currently does
// not support parallel edges, i.e., multiple edges from same work x
// into same work y.
- final Set<Operator<?>> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, tsOp1);
- final Set<Operator<?>> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, tsOp2);
+ final Set<Operator<?>> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, op1);
+ final Set<Operator<?>> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, op2);
if (!Collections.disjoint(outputWorksOps1, outputWorksOps2)) {
// We cannot merge
return false;
@@ -866,10 +1153,19 @@ public class SharedWorkOptimizer extends Transform {
// Work2 Work3 Work2
//
// If we do, we cannot merge. The reason is the same as above, currently
- // Tez currently does not support parallel edges.
- final Set<Operator<?>> inputWorksOps1 = findParentWorkOperators(pctx, optimizerCache, tsOp1);
+ // Tez does not support parallel edges.
+ //
+ // In the check, we exclude the inputs to the root operator that we are trying
+ // to merge (only useful for extended merging as TS do not have inputs).
+ final Set<Operator<?>> excludeOps1 = sr.retainableOps.get(0).getNumParent() > 0 ?
+ ImmutableSet.copyOf(sr.retainableOps.get(0).getParentOperators()) : ImmutableSet.of();
+ final Set<Operator<?>> inputWorksOps1 =
+ findParentWorkOperators(pctx, optimizerCache, op1, excludeOps1);
+ final Set<Operator<?>> excludeOps2 = sr.discardableOps.get(0).getNumParent() > 0 ?
+ Sets.union(ImmutableSet.copyOf(sr.discardableOps.get(0).getParentOperators()), sr.discardableInputOps) :
+ sr.discardableInputOps;
final Set<Operator<?>> inputWorksOps2 =
- findParentWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps);
+ findParentWorkOperators(pctx, optimizerCache, op2, excludeOps2);
if (!Collections.disjoint(inputWorksOps1, inputWorksOps2)) {
// We cannot merge
return false;
@@ -885,9 +1181,9 @@ public class SharedWorkOptimizer extends Transform {
//
// If we do, we cannot merge, as we would end up with a cycle in the DAG.
final Set<Operator<?>> descendantWorksOps1 =
- findDescendantWorkOperators(pctx, optimizerCache, tsOp1, sr.discardableInputOps);
+ findDescendantWorkOperators(pctx, optimizerCache, op1, sr.discardableInputOps);
final Set<Operator<?>> descendantWorksOps2 =
- findDescendantWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps);
+ findDescendantWorkOperators(pctx, optimizerCache, op2, sr.discardableInputOps);
if (!Collections.disjoint(descendantWorksOps1, workOps2)
|| !Collections.disjoint(workOps1, descendantWorksOps2)) {
return false;
@@ -1120,6 +1416,12 @@ public class SharedWorkOptimizer extends Transform {
this.dataSize = dataSize;
this.maxDataSize = maxDataSize;
}
+
+ @Override
+ public String toString() {
+ return "SharedResult { " + this.retainableOps + "; " + this.discardableOps + "; "
+ + this.discardableInputOps + "};";
+ }
}
/** Cache to accelerate optimization */
@@ -1173,6 +1475,11 @@ public class SharedWorkOptimizer extends Transform {
}
}
}
+
+ @Override
+ public String toString() {
+ return "SharedWorkOptimizerCache { \n" + operatorToWorkOperators.toString() + "\n };";
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/queries/clientpositive/sharedworkext.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/sharedworkext.q b/ql/src/test/queries/clientpositive/sharedworkext.q
new file mode 100644
index 0000000..b1801ea
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/sharedworkext.q
@@ -0,0 +1,53 @@
+EXPLAIN
+SELECT a.key FROM
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) a
+JOIN
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) b
+ON a.key = b.key;
+
+SELECT a.key FROM
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) a
+JOIN
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) b
+ON a.key = b.key;
+
+EXPLAIN
+SELECT a.key FROM
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) a
+) a
+JOIN
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) b
+) b
+ON a.key = b.key;
+
+SELECT a.key FROM
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) a
+) a
+JOIN
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) b
+) b
+ON a.key = b.key;
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
index 8ab5e3a..2fb1854 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
@@ -2363,10 +2363,9 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
Reducer 6 <- Map 5 (CUSTOM_SIMPLE_EDGE)
-Reducer 7 <- Map 5 (CUSTOM_SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -2384,41 +2383,37 @@ Stage-0
predicate:((_col2 = 0) or (_col5 is null and _col1 is not null and (_col3 >= _col2)))
Merge Join Operator [MERGEJOIN_37] (rows=26 width=141)
Conds:RS_24.UDFToDouble(_col1)=RS_25._col0(Left Outer),Output:["_col0","_col1","_col2","_col3","_col5"]
+ <-Reducer 6 [SIMPLE_EDGE] llap
+ SHUFFLE [RS_25]
+ PartitionCols:_col0
+ Select Operator [SEL_20] (rows=1 width=12)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_7] (rows=1 width=8)
+ Output:["_col0"],aggregations:["avg(VALUE._col0)"]
+ <-Map 5 [CUSTOM_SIMPLE_EDGE] llap
+ PARTITION_ONLY_SHUFFLE [RS_6]
+ Group By Operator [GBY_5] (rows=1 width=76)
+ Output:["_col0"],aggregations:["avg(p_size)"]
+ Filter Operator [FIL_33] (rows=8 width=4)
+ predicate:(p_size < 10)
+ TableScan [TS_2] (rows=26 width=4)
+ default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"]
<-Reducer 2 [SIMPLE_EDGE] llap
SHUFFLE [RS_24]
PartitionCols:UDFToDouble(_col1)
Merge Join Operator [MERGEJOIN_36] (rows=26 width=141)
Conds:(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap
+ SHUFFLE [RS_22]
+ Group By Operator [GBY_12] (rows=1 width=16)
+ Output:["_col0","_col1"],aggregations:["count()","count(_col0)"]
+ Please refer to the previous Group By Operator [GBY_7]
<-Map 1 [CUSTOM_SIMPLE_EDGE] llap
PARTITION_ONLY_SHUFFLE [RS_21]
Select Operator [SEL_1] (rows=26 width=125)
Output:["_col0","_col1"]
TableScan [TS_0] (rows=26 width=125)
default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_size"]
- <-Reducer 6 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_22]
- Group By Operator [GBY_12] (rows=1 width=16)
- Output:["_col0","_col1"],aggregations:["count()","count(_col0)"]
- Group By Operator [GBY_7] (rows=1 width=8)
- Output:["_col0"],aggregations:["avg(VALUE._col0)"]
- <-Map 5 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_6]
- Group By Operator [GBY_5] (rows=1 width=76)
- Output:["_col0"],aggregations:["avg(p_size)"]
- Filter Operator [FIL_33] (rows=8 width=4)
- predicate:(p_size < 10)
- TableScan [TS_2] (rows=26 width=4)
- default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_size"]
- <-Reducer 7 [SIMPLE_EDGE] llap
- SHUFFLE [RS_25]
- PartitionCols:_col0
- Select Operator [SEL_20] (rows=1 width=12)
- Output:["_col0","_col1"]
- Group By Operator [GBY_19] (rows=1 width=8)
- Output:["_col0"],aggregations:["avg(VALUE._col0)"]
- <-Map 5 [CUSTOM_SIMPLE_EDGE] llap
- PARTITION_ONLY_SHUFFLE [RS_18]
- Please refer to the previous Group By Operator [GBY_5]
PREHOOK: query: explain select b.p_mfgr, min(p_retailprice)
from part b
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/sharedworkext.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/sharedworkext.q.out b/ql/src/test/results/clientpositive/llap/sharedworkext.q.out
new file mode 100644
index 0000000..e56b1ce
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/sharedworkext.q.out
@@ -0,0 +1,1076 @@
+PREHOOK: query: EXPLAIN
+SELECT a.key FROM
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) a
+JOIN
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) b
+ON a.key = b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT a.key FROM
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) a
+JOIN
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) b
+ON a.key = b.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a2
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 6
+ Map Operator Tree:
+ TableScan
+ alias: a1
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col2 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 528 Data size: 45936 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 528 Data size: 45936 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT a.key FROM
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) a
+JOIN
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) b
+ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a.key FROM
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) a
+JOIN
+(
+ SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value
+) b
+ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0
+10
+100
+103
+104
+105
+11
+111
+113
+114
+116
+118
+119
+12
+120
+125
+126
+128
+129
+131
+133
+134
+136
+137
+138
+143
+145
+146
+149
+15
+150
+152
+153
+155
+156
+157
+158
+160
+162
+163
+164
+165
+166
+167
+168
+169
+17
+170
+172
+174
+175
+176
+177
+178
+179
+18
+180
+181
+183
+186
+187
+189
+19
+190
+191
+192
+193
+194
+195
+196
+197
+199
+2
+20
+200
+201
+202
+203
+205
+207
+208
+209
+213
+214
+216
+217
+218
+219
+221
+222
+223
+224
+226
+228
+229
+230
+233
+235
+237
+238
+239
+24
+241
+242
+244
+247
+248
+249
+252
+255
+256
+257
+258
+26
+260
+262
+263
+265
+266
+27
+272
+273
+274
+275
+277
+278
+28
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+291
+292
+296
+298
+30
+302
+305
+306
+307
+308
+309
+310
+311
+315
+316
+317
+318
+321
+322
+323
+325
+327
+33
+331
+332
+333
+335
+336
+338
+339
+34
+341
+342
+344
+345
+348
+35
+351
+353
+356
+360
+362
+364
+365
+366
+367
+368
+369
+37
+373
+374
+375
+377
+378
+379
+382
+384
+386
+389
+392
+393
+394
+395
+396
+397
+399
+4
+400
+401
+402
+403
+404
+406
+407
+409
+41
+411
+413
+414
+417
+418
+419
+42
+421
+424
+427
+429
+43
+430
+431
+432
+435
+436
+437
+438
+439
+44
+443
+444
+446
+448
+449
+452
+453
+454
+455
+457
+458
+459
+460
+462
+463
+466
+467
+468
+469
+47
+470
+472
+475
+477
+478
+479
+480
+481
+482
+483
+484
+485
+487
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+5
+51
+53
+54
+57
+58
+64
+65
+66
+67
+69
+70
+72
+74
+76
+77
+78
+8
+80
+82
+83
+84
+85
+86
+87
+9
+90
+92
+95
+96
+97
+98
+PREHOOK: query: EXPLAIN
+SELECT a.key FROM
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) a
+) a
+JOIN
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) b
+) b
+ON a.key = b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT a.key FROM
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) a
+) a
+JOIN
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) b
+) b
+ON a.key = b.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 5 <- Reducer 4 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+ Reducer 6 <- Reducer 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a2
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: a1
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 809 Data size: 144002 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: _col1 (type: string), _col2 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reducer 3
+ Execution mode: llap
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: 0 (type: int), _col0 (type: string)
+ sort order: ++
+ Map-reduce partition columns: 0 (type: int)
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: 0 (type: int), _col0 (type: string)
+ sort order: ++
+ Map-reduce partition columns: 0 (type: int)
+ Statistics: Num rows: 404 Data size: 71912 Basic stats: COMPLETE Column stats: COMPLETE
+ Reducer 4
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: rank_window_0 is not null (type: boolean)
+ Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: rank_window_0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE
+ Reducer 5
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 6
+ Execution mode: llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: rank_window_0
+ arguments: _col0
+ name: rank
+ window function: GenericUDAFRankEvaluator
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ isPivotResult: true
+ Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: rank_window_0 is not null (type: boolean)
+ Statistics: Num rows: 404 Data size: 35148 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: rank_window_0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 404 Data size: 1616 Basic stats: COMPLETE Column stats: COMPLETE
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT a.key FROM
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) a
+) a
+JOIN
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) b
+) b
+ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a.key FROM
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) a
+) a
+JOIN
+(
+ SELECT rank() OVER (ORDER BY key) AS key FROM
+ (SELECT a1.key, a2.value FROM src a1 JOIN src a2 ON (a1.key = a2.key)
+ GROUP BY a1.key, a2.value) b
+) b
+ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
+100
+101
+102
+103
+104
+105
+106
+107
+108
+109
+110
+111
+112
+113
+114
+115
+116
+117
+118
+119
+120
+121
+122
+123
+124
+125
+126
+127
+128
+129
+130
+131
+132
+133
+134
+135
+136
+137
+138
+139
+140
+141
+142
+143
+144
+145
+146
+147
+148
+149
+150
+151
+152
+153
+154
+155
+156
+157
+158
+159
+160
+161
+162
+163
+164
+165
+166
+167
+168
+169
+170
+171
+172
+173
+174
+175
+176
+177
+178
+179
+180
+181
+182
+183
+184
+185
+186
+187
+188
+189
+190
+191
+192
+193
+194
+195
+196
+197
+198
+199
+200
+201
+202
+203
+204
+205
+206
+207
+208
+209
+210
+211
+212
+213
+214
+215
+216
+217
+218
+219
+220
+221
+222
+223
+224
+225
+226
+227
+228
+229
+230
+231
+232
+233
+234
+235
+236
+237
+238
+239
+240
+241
+242
+243
+244
+245
+246
+247
+248
+249
+250
+251
+252
+253
+254
+255
+256
+257
+258
+259
+260
+261
+262
+263
+264
+265
+266
+267
+268
+269
+270
+271
+272
+273
+274
+275
+276
+277
+278
+279
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+290
+291
+292
+293
+294
+295
+296
+297
+298
+299
+300
+301
+302
+303
+304
+305
+306
+307
+308
+309
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
index 975fd13..45f698b 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
@@ -1663,13 +1663,12 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
- Reducer 4 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE)
- Reducer 7 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
+ Reducer 7 <- Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
- Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 9 <- Reducer 7 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1692,7 +1691,7 @@ STAGE PLANS:
value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string)
Execution mode: llap
LLAP IO: no inputs
- Map 11
+ Map 10
Map Operator Tree:
TableScan
alias: pp
@@ -1714,11 +1713,6 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
Map 5
@@ -1763,35 +1757,8 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string), _col0 (type: string)
Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: string)
- Reduce Output Operator
- key expressions: _col1 (type: string), _col0 (type: string)
- sort order: ++
- Map-reduce partition columns: _col1 (type: string), _col0 (type: string)
- Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: string)
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: boolean)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -1874,6 +1841,16 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Group By Operator
+ keys: _col2 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 8
Execution mode: llap
Reduce Operator Tree:
@@ -1892,24 +1869,24 @@ STAGE PLANS:
Reducer 9
Execution mode: llap
Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Left Semi Join 0 to 1
- keys:
- 0 _col1 (type: string), _col0 (type: string)
- 1 _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col2 (type: string), _col1 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: boolean)
Stage: Stage-0
Fetch Operator
@@ -2143,13 +2120,12 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 8 (ONE_TO_ONE_EDGE)
- Reducer 4 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE)
- Reducer 7 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 9 (ONE_TO_ONE_EDGE)
+ Reducer 7 <- Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
- Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
+ Reducer 9 <- Reducer 7 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2172,7 +2148,7 @@ STAGE PLANS:
value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col5 (type: int), _col7 (type: double), _col8 (type: string)
Execution mode: llap
LLAP IO: no inputs
- Map 11
+ Map 10
Map Operator Tree:
TableScan
alias: pp
@@ -2194,11 +2170,6 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
Map 5
@@ -2243,35 +2214,8 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string), _col0 (type: string)
Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: string)
- Reduce Output Operator
- key expressions: _col1 (type: string), _col0 (type: string)
- sort order: ++
- Map-reduce partition columns: _col1 (type: string), _col0 (type: string)
- Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: string)
Execution mode: llap
LLAP IO: no inputs
- Reducer 10
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: string), KEY._col1 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: boolean)
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -2354,6 +2298,16 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 7 Data size: 840 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
+ Group By Operator
+ keys: _col2 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
Reducer 8
Execution mode: llap
Reduce Operator Tree:
@@ -2372,24 +2326,24 @@ STAGE PLANS:
Reducer 9
Execution mode: llap
Reduce Operator Tree:
- Merge Join Operator
- condition map:
- Left Semi Join 0 to 1
- keys:
- 0 _col1 (type: string), _col0 (type: string)
- 1 _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 14 Data size: 2744 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col2 (type: string), _col1 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
+ Group By Operator
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 7 Data size: 1372 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), true (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 7 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col2 (type: boolean)
Stage: Stage-0
Fetch Operator
@@ -2855,12 +2809,11 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 10 <- Map 9 (CUSTOM_SIMPLE_EDGE)
- Reducer 11 <- Map 9 (CUSTOM_SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Map 5 (SIMPLE_EDGE), Reducer 2 (ONE_TO_ONE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Reducer 7 <- Map 6 (XPROD_EDGE), Reducer 10 (XPROD_EDGE)
- Reducer 8 <- Reducer 11 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
+ Reducer 8 <- Reducer 10 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2939,10 +2892,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:double>)
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: struct<count:bigint,sum:double,input:double>)
Execution mode: llap
LLAP IO: no inputs
Reducer 10
@@ -2962,14 +2911,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 11
- Execution mode: llap
- Reduce Operator Tree:
- Group By Operator
- aggregations: avg(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: double), true (type: boolean)
outputColumnNames: _col0, _col1
[5/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query23.q.out b/ql/src/test/results/clientpositive/perf/tez/query23.q.out
index 7112de6..c7a5041 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query23.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query23.q.out
@@ -1,5 +1,5 @@
-Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 25' is a cross product
-Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 30' is a cross product
+Warning: Shuffle Join MERGEJOIN[367][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 22' is a cross product
+Warning: Shuffle Join MERGEJOIN[369][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 23' is a cross product
PREHOOK: query: explain
with frequent_ss_items as
(select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt
@@ -105,41 +105,28 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Reducer 30 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE)
-Reducer 13 <- Map 19 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 23 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE)
+Reducer 13 <- Map 16 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE)
Reducer 14 <- Reducer 13 (SIMPLE_EDGE)
-Reducer 15 <- Map 11 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE)
-Reducer 16 <- Map 19 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE)
+Reducer 19 <- Map 33 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 21 <- Map 20 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE)
-Reducer 22 <- Map 46 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE)
-Reducer 23 <- Reducer 22 (SIMPLE_EDGE)
-Reducer 24 <- Reducer 23 (CUSTOM_SIMPLE_EDGE)
-Reducer 25 <- Reducer 24 (CUSTOM_SIMPLE_EDGE), Reducer 35 (CUSTOM_SIMPLE_EDGE), Reducer 43 (CUSTOM_SIMPLE_EDGE)
-Reducer 26 <- Map 20 (SIMPLE_EDGE), Map 31 (SIMPLE_EDGE)
-Reducer 27 <- Map 46 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE)
-Reducer 28 <- Reducer 27 (SIMPLE_EDGE)
-Reducer 29 <- Reducer 28 (CUSTOM_SIMPLE_EDGE)
+Reducer 20 <- Reducer 19 (SIMPLE_EDGE)
+Reducer 21 <- Reducer 20 (CUSTOM_SIMPLE_EDGE)
+Reducer 22 <- Reducer 21 (CUSTOM_SIMPLE_EDGE), Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 32 (CUSTOM_SIMPLE_EDGE)
+Reducer 23 <- Reducer 21 (CUSTOM_SIMPLE_EDGE), Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 32 (CUSTOM_SIMPLE_EDGE)
+Reducer 25 <- Map 24 (SIMPLE_EDGE), Map 29 (SIMPLE_EDGE)
+Reducer 26 <- Map 33 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE)
+Reducer 27 <- Reducer 26 (SIMPLE_EDGE)
+Reducer 28 <- Reducer 27 (CUSTOM_SIMPLE_EDGE)
Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 30 <- Reducer 29 (CUSTOM_SIMPLE_EDGE), Reducer 39 (CUSTOM_SIMPLE_EDGE), Reducer 45 (CUSTOM_SIMPLE_EDGE)
-Reducer 32 <- Map 31 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE)
-Reducer 33 <- Map 46 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE)
-Reducer 34 <- Reducer 33 (SIMPLE_EDGE)
-Reducer 35 <- Reducer 34 (CUSTOM_SIMPLE_EDGE)
-Reducer 36 <- Map 31 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE)
-Reducer 37 <- Map 46 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE)
-Reducer 38 <- Reducer 37 (SIMPLE_EDGE)
-Reducer 39 <- Reducer 38 (CUSTOM_SIMPLE_EDGE)
-Reducer 4 <- Reducer 25 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 42 <- Map 41 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE)
-Reducer 43 <- Reducer 42 (SIMPLE_EDGE)
-Reducer 44 <- Map 41 (SIMPLE_EDGE), Map 46 (SIMPLE_EDGE)
-Reducer 45 <- Reducer 44 (SIMPLE_EDGE)
+Reducer 31 <- Map 30 (SIMPLE_EDGE), Map 33 (SIMPLE_EDGE)
+Reducer 32 <- Reducer 31 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
Reducer 6 <- Union 5 (CUSTOM_SIMPLE_EDGE)
-Reducer 8 <- Map 47 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 9 <- Reducer 17 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Map 34 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -160,7 +147,7 @@ Stage-0
Output:["_col0"]
Merge Join Operator [MERGEJOIN_370] (rows=191667562 width=135)
Conds:RS_234._col2=RS_235._col0(Inner),Output:["_col3","_col4"]
- <-Reducer 30 [SIMPLE_EDGE]
+ <-Reducer 23 [SIMPLE_EDGE]
SHUFFLE [RS_235]
PartitionCols:_col0
Select Operator [SEL_227] (rows=105599202 width=321)
@@ -169,31 +156,31 @@ Stage-0
predicate:(_col3 > (0.95 * _col1))
Merge Join Operator [MERGEJOIN_369] (rows=316797606 width=321)
Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"]
- <-Reducer 29 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 21 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_222]
- Select Operator [SEL_180] (rows=1 width=8)
- Filter Operator [FIL_179] (rows=1 width=8)
+ Select Operator [SEL_61] (rows=1 width=8)
+ Filter Operator [FIL_60] (rows=1 width=8)
predicate:(sq_count_check(_col0) <= 1)
- Group By Operator [GBY_177] (rows=1 width=8)
+ Group By Operator [GBY_58] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Reducer 28 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_176]
- Group By Operator [GBY_175] (rows=1 width=8)
+ <-Reducer 20 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_57]
+ Group By Operator [GBY_56] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_172] (rows=348477374 width=88)
- Group By Operator [GBY_171] (rows=348477374 width=88)
+ Select Operator [SEL_53] (rows=348477374 width=88)
+ Group By Operator [GBY_52] (rows=348477374 width=88)
Output:["_col0"],keys:KEY._col0
- <-Reducer 27 [SIMPLE_EDGE]
- SHUFFLE [RS_170]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_51]
PartitionCols:_col0
- Group By Operator [GBY_169] (rows=696954748 width=88)
+ Group By Operator [GBY_50] (rows=696954748 width=88)
Output:["_col0"],keys:_col0
- Select Operator [SEL_167] (rows=696954748 width=88)
+ Select Operator [SEL_48] (rows=696954748 width=88)
Output:["_col0"]
- Merge Join Operator [MERGEJOIN_361] (rows=696954748 width=88)
- Conds:RS_164._col1=RS_165._col0(Inner),Output:["_col6"]
- <-Map 46 [SIMPLE_EDGE]
- SHUFFLE [RS_165]
+ Merge Join Operator [MERGEJOIN_353] (rows=696954748 width=88)
+ Conds:RS_45._col1=RS_46._col0(Inner),Output:["_col6"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_46]
PartitionCols:_col0
Select Operator [SEL_93] (rows=80000000 width=860)
Output:["_col0"]
@@ -201,13 +188,13 @@ Stage-0
predicate:c_customer_sk is not null
TableScan [TS_91] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"]
- <-Reducer 26 [SIMPLE_EDGE]
- SHUFFLE [RS_164]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_360] (rows=633595212 width=88)
- Conds:RS_161._col0=RS_162._col0(Inner),Output:["_col1"]
- <-Map 31 [SIMPLE_EDGE]
- SHUFFLE [RS_162]
+ Merge Join Operator [MERGEJOIN_352] (rows=633595212 width=88)
+ Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1"]
+ <-Map 24 [SIMPLE_EDGE]
+ SHUFFLE [RS_43]
PartitionCols:_col0
Select Operator [SEL_38] (rows=36525 width=1119)
Output:["_col0"]
@@ -215,8 +202,8 @@ Stage-0
predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null)
TableScan [TS_36] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_161]
+ <-Map 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_42]
PartitionCols:_col0
Select Operator [SEL_35] (rows=575995635 width=88)
Output:["_col0","_col1"]
@@ -224,42 +211,42 @@ Stage-0
predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_33] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"]
- <-Reducer 39 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 28 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_223]
- Group By Operator [GBY_205] (rows=1 width=224)
+ Group By Operator [GBY_86] (rows=1 width=224)
Output:["_col0"],aggregations:["max(VALUE._col0)"]
- <-Reducer 38 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_204]
- Group By Operator [GBY_203] (rows=1 width=224)
+ <-Reducer 27 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_85]
+ Group By Operator [GBY_84] (rows=1 width=224)
Output:["_col0"],aggregations:["max(_col1)"]
- Select Operator [SEL_201] (rows=348477374 width=88)
+ Select Operator [SEL_82] (rows=348477374 width=88)
Output:["_col1"]
- Group By Operator [GBY_200] (rows=348477374 width=88)
+ Group By Operator [GBY_81] (rows=348477374 width=88)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 37 [SIMPLE_EDGE]
- SHUFFLE [RS_199]
+ <-Reducer 26 [SIMPLE_EDGE]
+ SHUFFLE [RS_80]
PartitionCols:_col0
- Group By Operator [GBY_198] (rows=696954748 width=88)
+ Group By Operator [GBY_79] (rows=696954748 width=88)
Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Select Operator [SEL_196] (rows=696954748 width=88)
+ Select Operator [SEL_77] (rows=696954748 width=88)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_363] (rows=696954748 width=88)
- Conds:RS_193._col1=RS_194._col0(Inner),Output:["_col2","_col3","_col6"]
- <-Map 46 [SIMPLE_EDGE]
- SHUFFLE [RS_194]
+ Merge Join Operator [MERGEJOIN_355] (rows=696954748 width=88)
+ Conds:RS_74._col1=RS_75._col0(Inner),Output:["_col2","_col3","_col6"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_75]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_93]
- <-Reducer 36 [SIMPLE_EDGE]
- SHUFFLE [RS_193]
+ <-Reducer 25 [SIMPLE_EDGE]
+ SHUFFLE [RS_74]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_362] (rows=633595212 width=88)
- Conds:RS_190._col0=RS_191._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 31 [SIMPLE_EDGE]
- SHUFFLE [RS_191]
+ Merge Join Operator [MERGEJOIN_354] (rows=633595212 width=88)
+ Conds:RS_71._col0=RS_72._col0(Inner),Output:["_col1","_col2","_col3"]
+ <-Map 24 [SIMPLE_EDGE]
+ SHUFFLE [RS_72]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_38]
- <-Map 40 [SIMPLE_EDGE]
- SHUFFLE [RS_190]
+ <-Map 29 [SIMPLE_EDGE]
+ SHUFFLE [RS_71]
PartitionCols:_col0
Select Operator [SEL_64] (rows=575995635 width=88)
Output:["_col0","_col1","_col2","_col3"]
@@ -267,25 +254,25 @@ Stage-0
predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_62] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"]
- <-Reducer 45 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 32 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_224]
- Group By Operator [GBY_220] (rows=316797606 width=88)
+ Group By Operator [GBY_101] (rows=316797606 width=88)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 44 [SIMPLE_EDGE]
- SHUFFLE [RS_219]
+ <-Reducer 31 [SIMPLE_EDGE]
+ SHUFFLE [RS_100]
PartitionCols:_col0
- Group By Operator [GBY_218] (rows=633595212 width=88)
+ Group By Operator [GBY_99] (rows=633595212 width=88)
Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Select Operator [SEL_216] (rows=633595212 width=88)
+ Select Operator [SEL_97] (rows=633595212 width=88)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_364] (rows=633595212 width=88)
- Conds:RS_213._col0=RS_214._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 46 [SIMPLE_EDGE]
- SHUFFLE [RS_214]
+ Merge Join Operator [MERGEJOIN_356] (rows=633595212 width=88)
+ Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_95]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_93]
- <-Map 41 [SIMPLE_EDGE]
- SHUFFLE [RS_213]
+ <-Map 30 [SIMPLE_EDGE]
+ SHUFFLE [RS_94]
PartitionCols:_col0
Select Operator [SEL_90] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
@@ -298,30 +285,30 @@ Stage-0
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_366] (rows=174243235 width=135)
Conds:RS_231._col1=RS_232._col0(Inner),Output:["_col2","_col3","_col4"]
- <-Reducer 17 [SIMPLE_EDGE]
+ <-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_232]
PartitionCols:_col0
- Group By Operator [GBY_150] (rows=58079562 width=88)
+ Group By Operator [GBY_31] (rows=58079562 width=88)
Output:["_col0"],keys:_col1
- Select Operator [SEL_146] (rows=116159124 width=88)
+ Select Operator [SEL_27] (rows=116159124 width=88)
Output:["_col1"]
- Filter Operator [FIL_145] (rows=116159124 width=88)
+ Filter Operator [FIL_26] (rows=116159124 width=88)
predicate:(_col3 > 4)
- Select Operator [SEL_348] (rows=348477374 width=88)
+ Select Operator [SEL_347] (rows=348477374 width=88)
Output:["_col0","_col3"]
- Group By Operator [GBY_144] (rows=348477374 width=88)
+ Group By Operator [GBY_25] (rows=348477374 width=88)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_143]
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_24]
PartitionCols:_col0
- Group By Operator [GBY_142] (rows=696954748 width=88)
+ Group By Operator [GBY_23] (rows=696954748 width=88)
Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2
- Select Operator [SEL_140] (rows=696954748 width=88)
+ Select Operator [SEL_21] (rows=696954748 width=88)
Output:["_col0","_col1","_col2"]
- Merge Join Operator [MERGEJOIN_359] (rows=696954748 width=88)
- Conds:RS_137._col1=RS_138._col0(Inner),Output:["_col3","_col5","_col6"]
- <-Map 19 [SIMPLE_EDGE]
- SHUFFLE [RS_138]
+ Merge Join Operator [MERGEJOIN_351] (rows=696954748 width=88)
+ Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col5","_col6"]
+ <-Map 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_19]
PartitionCols:_col0
Select Operator [SEL_14] (rows=462000 width=1436)
Output:["_col0","_col1"]
@@ -329,13 +316,13 @@ Stage-0
predicate:i_item_sk is not null
TableScan [TS_12] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"]
- <-Reducer 15 [SIMPLE_EDGE]
- SHUFFLE [RS_137]
+ <-Reducer 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_18]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_358] (rows=633595212 width=88)
- Conds:RS_134._col0=RS_135._col0(Inner),Output:["_col1","_col3"]
+ Merge Join Operator [MERGEJOIN_350] (rows=633595212 width=88)
+ Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col3"]
<-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_134]
+ SHUFFLE [RS_15]
PartitionCols:_col0
Select Operator [SEL_8] (rows=575995635 width=88)
Output:["_col0","_col1"]
@@ -343,8 +330,8 @@ Stage-0
predicate:(ss_item_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_6] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_135]
+ <-Map 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
PartitionCols:_col0
Select Operator [SEL_11] (rows=36525 width=1119)
Output:["_col0","_col1"]
@@ -366,7 +353,7 @@ Stage-0
predicate:((d_moy = 1) and (d_year = 1999) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Map 47 [SIMPLE_EDGE]
+ <-Map 34 [SIMPLE_EDGE]
SHUFFLE [RS_228]
PartitionCols:_col0
Select Operator [SEL_121] (rows=144002668 width=135)
@@ -383,7 +370,7 @@ Stage-0
Output:["_col0"]
Merge Join Operator [MERGEJOIN_368] (rows=383314495 width=135)
Conds:RS_115._col1=RS_116._col0(Inner),Output:["_col3","_col4"]
- <-Reducer 25 [SIMPLE_EDGE]
+ <-Reducer 22 [SIMPLE_EDGE]
SHUFFLE [RS_116]
PartitionCols:_col0
Select Operator [SEL_108] (rows=105599202 width=321)
@@ -392,105 +379,15 @@ Stage-0
predicate:(_col3 > (0.95 * _col1))
Merge Join Operator [MERGEJOIN_367] (rows=316797606 width=321)
Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"]
- <-Reducer 24 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 21 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_103]
- Select Operator [SEL_61] (rows=1 width=8)
- Filter Operator [FIL_60] (rows=1 width=8)
- predicate:(sq_count_check(_col0) <= 1)
- Group By Operator [GBY_58] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Reducer 23 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_57]
- Group By Operator [GBY_56] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_53] (rows=348477374 width=88)
- Group By Operator [GBY_52] (rows=348477374 width=88)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 22 [SIMPLE_EDGE]
- SHUFFLE [RS_51]
- PartitionCols:_col0
- Group By Operator [GBY_50] (rows=696954748 width=88)
- Output:["_col0"],keys:_col0
- Select Operator [SEL_48] (rows=696954748 width=88)
- Output:["_col0"]
- Merge Join Operator [MERGEJOIN_353] (rows=696954748 width=88)
- Conds:RS_45._col1=RS_46._col0(Inner),Output:["_col6"]
- <-Map 46 [SIMPLE_EDGE]
- SHUFFLE [RS_46]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_93]
- <-Reducer 21 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_352] (rows=633595212 width=88)
- Conds:RS_42._col0=RS_43._col0(Inner),Output:["_col1"]
- <-Map 31 [SIMPLE_EDGE]
- SHUFFLE [RS_43]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_38]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_42]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_35]
- <-Reducer 35 [CUSTOM_SIMPLE_EDGE]
+ Please refer to the previous Select Operator [SEL_61]
+ <-Reducer 28 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_104]
- Group By Operator [GBY_86] (rows=1 width=224)
- Output:["_col0"],aggregations:["max(VALUE._col0)"]
- <-Reducer 34 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_85]
- Group By Operator [GBY_84] (rows=1 width=224)
- Output:["_col0"],aggregations:["max(_col1)"]
- Select Operator [SEL_82] (rows=348477374 width=88)
- Output:["_col1"]
- Group By Operator [GBY_81] (rows=348477374 width=88)
- Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 33 [SIMPLE_EDGE]
- SHUFFLE [RS_80]
- PartitionCols:_col0
- Group By Operator [GBY_79] (rows=696954748 width=88)
- Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Select Operator [SEL_77] (rows=696954748 width=88)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_355] (rows=696954748 width=88)
- Conds:RS_74._col1=RS_75._col0(Inner),Output:["_col2","_col3","_col6"]
- <-Map 46 [SIMPLE_EDGE]
- SHUFFLE [RS_75]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_93]
- <-Reducer 32 [SIMPLE_EDGE]
- SHUFFLE [RS_74]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_354] (rows=633595212 width=88)
- Conds:RS_71._col0=RS_72._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 31 [SIMPLE_EDGE]
- SHUFFLE [RS_72]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_38]
- <-Map 40 [SIMPLE_EDGE]
- SHUFFLE [RS_71]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_64]
- <-Reducer 43 [CUSTOM_SIMPLE_EDGE]
+ Please refer to the previous Group By Operator [GBY_86]
+ <-Reducer 32 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_105]
- Group By Operator [GBY_101] (rows=316797606 width=88)
- Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 42 [SIMPLE_EDGE]
- SHUFFLE [RS_100]
- PartitionCols:_col0
- Group By Operator [GBY_99] (rows=633595212 width=88)
- Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Select Operator [SEL_97] (rows=633595212 width=88)
- Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_356] (rows=633595212 width=88)
- Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 46 [SIMPLE_EDGE]
- SHUFFLE [RS_95]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_93]
- <-Map 41 [SIMPLE_EDGE]
- SHUFFLE [RS_94]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_90]
+ Please refer to the previous Group By Operator [GBY_101]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_115]
PartitionCols:_col1
@@ -499,42 +396,7 @@ Stage-0
<-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_113]
PartitionCols:_col0
- Group By Operator [GBY_31] (rows=58079562 width=88)
- Output:["_col0"],keys:_col1
- Select Operator [SEL_27] (rows=116159124 width=88)
- Output:["_col1"]
- Filter Operator [FIL_26] (rows=116159124 width=88)
- predicate:(_col3 > 4)
- Select Operator [SEL_347] (rows=348477374 width=88)
- Output:["_col0","_col3"]
- Group By Operator [GBY_25] (rows=348477374 width=88)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_24]
- PartitionCols:_col0
- Group By Operator [GBY_23] (rows=696954748 width=88)
- Output:["_col0","_col1","_col2","_col3"],aggregations:["count()"],keys:_col1, _col0, _col2
- Select Operator [SEL_21] (rows=696954748 width=88)
- Output:["_col0","_col1","_col2"]
- Merge Join Operator [MERGEJOIN_351] (rows=696954748 width=88)
- Conds:RS_18._col1=RS_19._col0(Inner),Output:["_col3","_col5","_col6"]
- <-Map 19 [SIMPLE_EDGE]
- SHUFFLE [RS_19]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_14]
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_18]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_350] (rows=633595212 width=88)
- Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1","_col3"]
- <-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_15]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_8]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_11]
+ Please refer to the previous Group By Operator [GBY_31]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_112]
PartitionCols:_col2
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query32.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query32.q.out b/ql/src/test/results/clientpositive/perf/tez/query32.q.out
index bba1bd8..3ad9595 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query32.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query32.q.out
@@ -55,12 +55,11 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 7 (ONE_TO_ONE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE)
Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
-Reducer 5 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE)
-Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Map 9 (SIMPLE_EDGE), Reducer 6 (ONE_TO_ONE_EDGE)
+Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
+Reducer 6 <- Map 8 (SIMPLE_EDGE), Reducer 5 (ONE_TO_ONE_EDGE)
Stage-0
Fetch Operator
@@ -96,7 +95,7 @@ Stage-0
predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
TableScan [TS_0] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_ext_discount_amt"]
- <-Map 8 [SIMPLE_EDGE]
+ <-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
Select Operator [SEL_5] (rows=8116 width=1119)
@@ -105,12 +104,12 @@ Stage-0
predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
- <-Reducer 7 [ONE_TO_ONE_EDGE]
+ <-Reducer 6 [ONE_TO_ONE_EDGE]
FORWARD [RS_31]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_60] (rows=174233858 width=135)
Conds:RS_23._col1=RS_24._col0(Inner),Output:["_col0","_col2"]
- <-Map 9 [SIMPLE_EDGE]
+ <-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_24]
PartitionCols:_col0
Select Operator [SEL_22] (rows=231000 width=1436)
@@ -119,26 +118,17 @@ Stage-0
predicate:((i_manufact_id = 269) and i_item_sk is not null)
TableScan [TS_20] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"]
- <-Reducer 6 [ONE_TO_ONE_EDGE]
+ <-Reducer 5 [ONE_TO_ONE_EDGE]
FORWARD [RS_23]
PartitionCols:_col1
Select Operator [SEL_19] (rows=158394413 width=135)
Output:["_col0","_col1"]
Group By Operator [GBY_18] (rows=158394413 width=135)
Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0
- <-Reducer 5 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_17]
PartitionCols:_col0
Group By Operator [GBY_16] (rows=316788826 width=135)
Output:["_col0","_col1"],aggregations:["avg(_col2)"],keys:_col1
- Merge Join Operator [MERGEJOIN_59] (rows=316788826 width=135)
- Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_12]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_2]
- <-Map 8 [SIMPLE_EDGE]
- SHUFFLE [RS_13]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_5]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_58]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query33.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query33.q.out b/ql/src/test/results/clientpositive/perf/tez/query33.q.out
index 0810c28..a966865 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query33.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query33.q.out
@@ -149,26 +149,22 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE)
-Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 15 <- Map 14 (SIMPLE_EDGE)
-Reducer 16 <- Map 14 (SIMPLE_EDGE)
-Reducer 17 <- Map 14 (SIMPLE_EDGE)
-Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE)
-Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
-Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE)
-Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE)
-Reducer 24 <- Map 21 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
-Reducer 25 <- Map 26 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 2 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE)
+Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 13 <- Map 12 (SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
+Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
+Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE)
+Reducer 19 <- Map 22 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE)
+Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE)
+Reducer 21 <- Map 22 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 16 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
Reducer 6 <- Union 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE)
-Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 19 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 5 (CONTAINS)
Stage-0
Fetch Operator
@@ -185,66 +181,27 @@ Stage-0
Group By Operator [GBY_116] (rows=335408073 width=108)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
<-Union 5 [SIMPLE_EDGE]
- <-Reducer 10 [CONTAINS]
+ <-Reducer 11 [CONTAINS]
Reduce Output Operator [RS_115]
PartitionCols:_col0
Group By Operator [GBY_114] (rows=670816147 width=108)
Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Group By Operator [GBY_72] (rows=191657247 width=135)
+ Group By Operator [GBY_110] (rows=95833781 width=135)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 9 [SIMPLE_EDGE]
- SHUFFLE [RS_71]
+ <-Reducer 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_109]
PartitionCols:_col0
- Group By Operator [GBY_70] (rows=383314495 width=135)
+ Group By Operator [GBY_108] (rows=191667562 width=135)
Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
- Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135)
- Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"]
- <-Reducer 23 [SIMPLE_EDGE]
- SHUFFLE [RS_67]
- PartitionCols:_col4
- Select Operator [SEL_62] (rows=348467716 width=135)
- Output:["_col4","_col5"]
- Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135)
- Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"]
- <-Map 26 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_162] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
- TableScan [TS_16] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
- <-Reducer 22 [SIMPLE_EDGE]
- SHUFFLE [RS_59]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135)
- Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
- PartitionCols:_col0
- Select Operator [SEL_15] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_161] (rows=18262 width=1119)
- predicate:((d_moy = 3) and (d_year = 1999) and d_date_sk is not null)
- TableScan [TS_13] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Map 27 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
- PartitionCols:_col0
- Select Operator [SEL_49] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_165] (rows=287989836 width=135)
- predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
- TableScan [TS_47] (rows=287989836 width=135)
- default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"]
- <-Reducer 8 [SIMPLE_EDGE]
- SHUFFLE [RS_66]
+ Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135)
+ Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_104]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_176] (rows=508200 width=1436)
- Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436)
+ Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_63]
+ SHUFFLE [RS_26]
PartitionCols:_col1
Select Operator [SEL_2] (rows=462000 width=1436)
Output:["_col0","_col1"]
@@ -252,13 +209,13 @@ Stage-0
predicate:(i_item_sk is not null and i_manufact_id is not null)
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"]
- <-Reducer 16 [ONE_TO_ONE_EDGE]
- FORWARD [RS_64]
+ <-Reducer 13 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_27]
PartitionCols:_col0
- Group By Operator [GBY_45] (rows=115500 width=1436)
+ Group By Operator [GBY_8] (rows=115500 width=1436)
Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_44]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
PartitionCols:_col0
Group By Operator [GBY_6] (rows=231000 width=1436)
Output:["_col0"],keys:i_manufact_id
@@ -268,59 +225,37 @@ Stage-0
predicate:((i_category) IN ('Books') and i_manufact_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"]
- <-Reducer 13 [CONTAINS]
- Reduce Output Operator [RS_115]
- PartitionCols:_col0
- Group By Operator [GBY_114] (rows=670816147 width=108)
- Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Group By Operator [GBY_110] (rows=95833781 width=135)
- Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_109]
- PartitionCols:_col0
- Group By Operator [GBY_108] (rows=191667562 width=135)
- Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
- Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135)
- Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"]
- <-Reducer 11 [SIMPLE_EDGE]
- SHUFFLE [RS_104]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_179] (rows=508200 width=1436)
- Conds:RS_101._col1=RS_102._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_101]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_2]
- <-Reducer 17 [ONE_TO_ONE_EDGE]
- FORWARD [RS_102]
- PartitionCols:_col0
- Group By Operator [GBY_83] (rows=115500 width=1436)
- Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_82]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_6]
- <-Reducer 25 [SIMPLE_EDGE]
+ <-Reducer 21 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col3
Select Operator [SEL_100] (rows=174243235 width=135)
Output:["_col3","_col5"]
Merge Join Operator [MERGEJOIN_181] (rows=174243235 width=135)
Conds:RS_97._col2=RS_98._col0(Inner),Output:["_col1","_col3"]
- <-Map 26 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_98]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_18]
- <-Reducer 24 [SIMPLE_EDGE]
+ Select Operator [SEL_18] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_162] (rows=20000000 width=1014)
+ predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
+ TableScan [TS_16] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
+ <-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_180] (rows=158402938 width=135)
Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_95]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_15]
- <-Map 28 [SIMPLE_EDGE]
+ Select Operator [SEL_15] (rows=18262 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_161] (rows=18262 width=1119)
+ predicate:((d_moy = 3) and (d_year = 1999) and d_date_sk is not null)
+ TableScan [TS_13] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
+ <-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
Select Operator [SEL_87] (rows=144002668 width=135)
@@ -346,42 +281,28 @@ Stage-0
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436)
- Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_26]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_2]
- <-Reducer 15 [ONE_TO_ONE_EDGE]
- FORWARD [RS_27]
- PartitionCols:_col0
- Group By Operator [GBY_8] (rows=115500 width=1436)
- Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_6]
- <-Reducer 20 [SIMPLE_EDGE]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_173]
+ <-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
Select Operator [SEL_25] (rows=696954748 width=88)
Output:["_col3","_col5"]
Merge Join Operator [MERGEJOIN_175] (rows=696954748 width=88)
Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"]
- <-Map 26 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_18]
- <-Reducer 19 [SIMPLE_EDGE]
+ <-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_174] (rows=633595212 width=88)
Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_20]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_15]
- <-Map 18 [SIMPLE_EDGE]
+ <-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
Select Operator [SEL_12] (rows=575995635 width=88)
@@ -390,4 +311,51 @@ Stage-0
predicate:(ss_addr_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_10] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"]
+ <-Reducer 9 [CONTAINS]
+ Reduce Output Operator [RS_115]
+ PartitionCols:_col0
+ Group By Operator [GBY_114] (rows=670816147 width=108)
+ Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
+ Group By Operator [GBY_72] (rows=191657247 width=135)
+ Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
+ <-Reducer 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_71]
+ PartitionCols:_col0
+ Group By Operator [GBY_70] (rows=383314495 width=135)
+ Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
+ Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135)
+ Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_66]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_173]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_67]
+ PartitionCols:_col4
+ Select Operator [SEL_62] (rows=348467716 width=135)
+ Output:["_col4","_col5"]
+ Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135)
+ Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_60]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_18]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135)
+ Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"]
+ <-Map 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_57]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_15]
+ <-Map 23 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
+ PartitionCols:_col0
+ Select Operator [SEL_49] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_165] (rows=287989836 width=135)
+ predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
+ TableScan [TS_47] (rows=287989836 width=135)
+ default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query44.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query44.q.out b/ql/src/test/results/clientpositive/perf/tez/query44.q.out
index 07187b1..b982de0 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query44.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query44.q.out
@@ -1,5 +1,4 @@
Warning: Shuffle Join MERGEJOIN[131][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 9' is a cross product
-Warning: Shuffle Join MERGEJOIN[133][tables = [$hdt$_4, $hdt$_5, $hdt$_3]] in Stage 'Reducer 13' is a cross product
PREHOOK: query: explain
select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing
from(select *
@@ -72,21 +71,16 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Map 6 (SIMPLE_EDGE)
-Reducer 12 <- Reducer 11 (CUSTOM_SIMPLE_EDGE)
-Reducer 13 <- Reducer 12 (CUSTOM_SIMPLE_EDGE), Reducer 17 (CUSTOM_SIMPLE_EDGE), Reducer 20 (CUSTOM_SIMPLE_EDGE)
-Reducer 14 <- Reducer 13 (SIMPLE_EDGE)
-Reducer 16 <- Map 15 (SIMPLE_EDGE)
-Reducer 17 <- Map 15 (SIMPLE_EDGE)
-Reducer 19 <- Map 18 (SIMPLE_EDGE)
+Reducer 11 <- Reducer 9 (SIMPLE_EDGE)
+Reducer 13 <- Map 12 (SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
-Reducer 20 <- Map 18 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Map 1 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE)
+Reducer 5 <- Map 1 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE)
Reducer 7 <- Map 6 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
-Reducer 9 <- Reducer 16 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE)
+Reducer 9 <- Reducer 13 (CUSTOM_SIMPLE_EDGE), Reducer 15 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -136,13 +130,13 @@ Stage-0
predicate:(_col3 > (0.9 * _col1))
Merge Join Operator [MERGEJOIN_131] (rows=10367842752596232 width=185)
Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"]
- <-Reducer 16 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 13 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_34]
Select Operator [SEL_25] (rows=71999454 width=88)
Output:["_col0"]
Group By Operator [GBY_24] (rows=71999454 width=88)
Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0
- <-Map 15 [SIMPLE_EDGE]
+ <-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
Group By Operator [GBY_22] (rows=143998908 width=88)
@@ -153,11 +147,11 @@ Stage-0
predicate:((ss_store_sk = 410) and ss_hdemo_sk is null)
TableScan [TS_18] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_hdemo_sk","ss_store_sk","ss_net_profit"]
- <-Reducer 19 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 15 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_35]
Group By Operator [GBY_31] (rows=143998908 width=88)
Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0
- <-Map 18 [SIMPLE_EDGE]
+ <-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col0
Group By Operator [GBY_29] (rows=287997817 width=88)
@@ -201,7 +195,7 @@ Stage-0
SHUFFLE [RS_90]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_2]
- <-Reducer 14 [SIMPLE_EDGE]
+ <-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_91]
PartitionCols:_col0
Select Operator [SEL_86] (rows=1151982528066248 width=185)
@@ -212,47 +206,8 @@ Stage-0
Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 DESC NULLS LAST","partition by:":"0"}]
Select Operator [SEL_84] (rows=3455947584198744 width=185)
Output:["_col2","_col3"]
- <-Reducer 13 [SIMPLE_EDGE]
+ <-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_83]
PartitionCols:0
- Filter Operator [FIL_82] (rows=3455947584198744 width=185)
- predicate:(_col3 > (0.9 * _col1))
- Merge Join Operator [MERGEJOIN_133] (rows=10367842752596232 width=185)
- Conds:(Inner),(Inner),Output:["_col1","_col2","_col3"]
- <-Reducer 12 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_78]
- Select Operator [SEL_62] (rows=1 width=8)
- Filter Operator [FIL_61] (rows=1 width=8)
- predicate:(sq_count_check(_col0) <= 1)
- Group By Operator [GBY_59] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Reducer 11 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_58]
- Group By Operator [GBY_57] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_55] (rows=71999454 width=88)
- Group By Operator [GBY_54] (rows=71999454 width=88)
- Output:["_col0"],keys:KEY._col0
- <-Map 6 [SIMPLE_EDGE]
- SHUFFLE [RS_53]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_7]
- <-Reducer 17 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_79]
- Select Operator [SEL_70] (rows=71999454 width=88)
- Output:["_col0"]
- Group By Operator [GBY_69] (rows=71999454 width=88)
- Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0
- <-Map 15 [SIMPLE_EDGE]
- SHUFFLE [RS_68]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_22]
- <-Reducer 20 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_80]
- Group By Operator [GBY_76] (rows=143998908 width=88)
- Output:["_col0","_col1"],aggregations:["avg(VALUE._col0)"],keys:KEY._col0
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_75]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_29]
+ Please refer to the previous Filter Operator [FIL_37]
[4/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query47.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query47.q.out b/ql/src/test/results/clientpositive/perf/tez/query47.q.out
index 1288408..04354b9 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query47.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query47.q.out
@@ -101,24 +101,16 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Map 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Map 22 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE)
-Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
-Reducer 14 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
-Reducer 15 <- Map 21 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE)
-Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
-Reducer 18 <- Reducer 17 (SIMPLE_EDGE)
-Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
-Reducer 3 <- Map 21 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 5 (SIMPLE_EDGE)
+Reducer 11 <- Reducer 10 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
+Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
-Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 5 (SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -136,78 +128,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
Merge Join Operator [MERGEJOIN_189] (rows=843315280 width=88)
Conds:RS_103._col0, _col1, _col2, _col3, (_col7 + 1)=RS_104._col0, _col1, _col2, _col3, _col8(Inner),RS_104._col0, _col1, _col2, _col3, _col8=RS_105._col0, _col1, _col2, _col3, (_col7 - 1)(Inner),Output:["_col6","_col8","_col12","_col13","_col14","_col15","_col23"]
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_103]
- PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1)
- Select Operator [SEL_29] (rows=383325119 width=88)
- Output:["_col0","_col1","_col2","_col3","_col6","_col7"]
- Filter Operator [FIL_164] (rows=383325119 width=88)
- predicate:rank_window_0 is not null
- PTF Operator [PTF_28] (rows=383325119 width=88)
- Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}]
- Select Operator [SEL_27] (rows=383325119 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_26]
- PartitionCols:_col0, _col1, _col2, _col3
- Select Operator [SEL_25] (rows=383325119 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Group By Operator [GBY_24] (rows=383325119 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5
- <-Reducer 11 [SIMPLE_EDGE]
- SHUFFLE [RS_23]
- PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5
- Group By Operator [GBY_22] (rows=766650239 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12
- Merge Join Operator [MERGEJOIN_182] (rows=766650239 width=88)
- Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"]
- <-Map 22 [SIMPLE_EDGE]
- SHUFFLE [RS_19]
- PartitionCols:_col0
- Select Operator [SEL_81] (rows=1704 width=1910)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_179] (rows=1704 width=1910)
- predicate:(s_company_name is not null and s_store_name is not null and s_store_sk is not null)
- TableScan [TS_79] (rows=1704 width=1910)
- default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"]
- <-Reducer 10 [SIMPLE_EDGE]
- SHUFFLE [RS_18]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_181] (rows=696954748 width=88)
- Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
- PartitionCols:_col0
- Select Operator [SEL_78] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_178] (rows=462000 width=1436)
- predicate:(i_brand is not null and i_category is not null and i_item_sk is not null)
- TableScan [TS_76] (rows=462000 width=1436)
- default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
- <-Reducer 9 [SIMPLE_EDGE]
- SHUFFLE [RS_15]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_180] (rows=633595212 width=88)
- Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_12]
- PartitionCols:_col0
- Select Operator [SEL_72] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_176] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
- TableScan [TS_70] (rows=575995635 width=88)
- default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_13]
- PartitionCols:_col0
- Select Operator [SEL_75] (rows=73048 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_177] (rows=73048 width=1119)
- predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
- TableScan [TS_73] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Reducer 19 [SIMPLE_EDGE]
+ <-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_104]
PartitionCols:_col0, _col1, _col2, _col3, _col8
Select Operator [SEL_67] (rows=31943759 width=88)
@@ -222,7 +143,7 @@ Stage-0
Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col5 ASC NULLS FIRST, _col6 ASC NULLS FIRST","partition by:":"_col1, _col2, _col3, _col4"}]
Select Operator [SEL_64] (rows=383325119 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- <-Reducer 18 [SIMPLE_EDGE]
+ <-Reducer 10 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col0, _col1, _col2, _col3
Select Operator [SEL_62] (rows=383325119 width=88)
@@ -231,46 +152,66 @@ Stage-0
Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3, _col4"}]
Select Operator [SEL_60] (rows=383325119 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- <-Reducer 17 [SIMPLE_EDGE]
+ <-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_59]
PartitionCols:_col0, _col1, _col2, _col3, _col4
- Select Operator [SEL_58] (rows=383325119 width=88)
+ Select Operator [SEL_95] (rows=383325119 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Group By Operator [GBY_57] (rows=383325119 width=88)
+ Group By Operator [GBY_94] (rows=383325119 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_93]
PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5
- Group By Operator [GBY_55] (rows=766650239 width=88)
+ Group By Operator [GBY_92] (rows=766650239 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12
- Merge Join Operator [MERGEJOIN_185] (rows=766650239 width=88)
- Conds:RS_51._col2=RS_52._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"]
- <-Map 22 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
+ Merge Join Operator [MERGEJOIN_188] (rows=766650239 width=88)
+ Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"]
+ <-Map 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_89]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_81]
- <-Reducer 15 [SIMPLE_EDGE]
- SHUFFLE [RS_51]
+ Select Operator [SEL_81] (rows=1704 width=1910)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_179] (rows=1704 width=1910)
+ predicate:(s_company_name is not null and s_store_name is not null and s_store_sk is not null)
+ TableScan [TS_79] (rows=1704 width=1910)
+ default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_88]
PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_184] (rows=696954748 width=88)
- Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
+ Merge Join Operator [MERGEJOIN_187] (rows=696954748 width=88)
+ Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"]
+ <-Map 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_86]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_78]
- <-Reducer 14 [SIMPLE_EDGE]
- SHUFFLE [RS_48]
+ Select Operator [SEL_78] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_178] (rows=462000 width=1436)
+ predicate:(i_brand is not null and i_category is not null and i_item_sk is not null)
+ TableScan [TS_76] (rows=462000 width=1436)
+ default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_85]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_183] (rows=633595212 width=88)
- Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ Merge Join Operator [MERGEJOIN_186] (rows=633595212 width=88)
+ Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
+ SHUFFLE [RS_82]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_72]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_46]
+ Select Operator [SEL_72] (rows=575995635 width=88)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_176] (rows=575995635 width=88)
+ predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
+ TableScan [TS_70] (rows=575995635 width=88)
+ default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_83]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_75]
+ Select Operator [SEL_75] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_177] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ TableScan [TS_73] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col0, _col1, _col2, _col3, (_col7 - 1)
@@ -285,41 +226,20 @@ Stage-0
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_96]
PartitionCols:_col0, _col1, _col2, _col3
- Select Operator [SEL_95] (rows=383325119 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Group By Operator [GBY_94] (rows=383325119 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_93]
- PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5
- Group By Operator [GBY_92] (rows=766650239 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col9, _col11, _col12
- Merge Join Operator [MERGEJOIN_188] (rows=766650239 width=88)
- Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col9","_col11","_col12"]
- <-Map 22 [SIMPLE_EDGE]
- SHUFFLE [RS_89]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_81]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_88]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_187] (rows=696954748 width=88)
- Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8","_col9"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_86]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_78]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_85]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_186] (rows=633595212 width=88)
- Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_82]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_72]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_83]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_75]
+ Please refer to the previous Select Operator [SEL_95]
+ <-Reducer 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_103]
+ PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1)
+ Select Operator [SEL_29] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col6","_col7"]
+ Filter Operator [FIL_164] (rows=383325119 width=88)
+ predicate:rank_window_0 is not null
+ PTF Operator [PTF_28] (rows=383325119 width=88)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}]
+ Select Operator [SEL_27] (rows=383325119 width=88)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_26]
+ PartitionCols:_col0, _col1, _col2, _col3
+ Please refer to the previous Select Operator [SEL_95]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query54.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query54.q.out b/ql/src/test/results/clientpositive/perf/tez/query54.q.out
index e6b9b28..a674acc 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query54.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query54.q.out
@@ -1,6 +1,6 @@
Warning: Shuffle Join MERGEJOIN[184][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Stage 'Reducer 4' is a cross product
Warning: Shuffle Join MERGEJOIN[185][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Stage 'Reducer 5' is a cross product
-Warning: Shuffle Join MERGEJOIN[183][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 30' is a cross product
+Warning: Shuffle Join MERGEJOIN[183][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 29' is a cross product
Warning: Shuffle Join MERGEJOIN[186][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 6' is a cross product
PREHOOK: query: explain
with my_customers as (
@@ -127,14 +127,13 @@ Reducer 20 <- Reducer 19 (SIMPLE_EDGE)
Reducer 26 <- Map 25 (SIMPLE_EDGE)
Reducer 27 <- Reducer 26 (CUSTOM_SIMPLE_EDGE)
Reducer 28 <- Map 25 (SIMPLE_EDGE)
-Reducer 29 <- Map 25 (SIMPLE_EDGE)
+Reducer 29 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 31 (CUSTOM_SIMPLE_EDGE)
Reducer 3 <- Reducer 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 30 <- Reducer 29 (CUSTOM_SIMPLE_EDGE), Reducer 32 (CUSTOM_SIMPLE_EDGE)
-Reducer 31 <- Map 25 (SIMPLE_EDGE)
-Reducer 32 <- Reducer 31 (CUSTOM_SIMPLE_EDGE)
+Reducer 30 <- Map 25 (SIMPLE_EDGE)
+Reducer 31 <- Reducer 30 (CUSTOM_SIMPLE_EDGE)
Reducer 4 <- Reducer 27 (CUSTOM_SIMPLE_EDGE), Reducer 3 (CUSTOM_SIMPLE_EDGE)
-Reducer 5 <- Reducer 28 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE)
-Reducer 6 <- Reducer 30 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE)
+Reducer 5 <- Reducer 26 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE)
+Reducer 6 <- Reducer 29 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
@@ -177,11 +176,11 @@ Stage-0
Output:["_col0","_col4","_col11","_col13","_col15"]
Merge Join Operator [MERGEJOIN_186] (rows=58108714324214428 width=158)
Conds:(Inner),Output:["_col0","_col2","_col6","_col13","_col15"]
- <-Reducer 30 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 29 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_112]
Merge Join Operator [MERGEJOIN_183] (rows=9131 width=1128)
Conds:(Right Outer),Output:["_col0"]
- <-Reducer 29 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 28 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_109]
Group By Operator [GBY_6] (rows=9131 width=1119)
Output:["_col0"],keys:KEY._col0
@@ -196,14 +195,14 @@ Stage-0
predicate:((d_moy = 3) and (d_year = 1999))
TableScan [TS_73] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"]
- <-Reducer 32 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 31 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_110]
Select Operator [SEL_22] (rows=1 width=8)
Filter Operator [FIL_21] (rows=1 width=8)
predicate:(sq_count_check(_col0) <= 1)
Group By Operator [GBY_19] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Reducer 31 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 30 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_18]
Group By Operator [GBY_17] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
@@ -224,12 +223,12 @@ Stage-0
Output:["_col0","_col4","_col11","_col13"]
Merge Join Operator [MERGEJOIN_185] (rows=6363893803988 width=1217)
Conds:(Left Outer),Output:["_col2","_col4","_col10","_col13"]
- <-Reducer 28 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 26 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_106]
- Group By Operator [GBY_94] (rows=9131 width=1119)
+ Group By Operator [GBY_79] (rows=9131 width=1119)
Output:["_col0"],keys:KEY._col0
<-Map 25 [SIMPLE_EDGE]
- SHUFFLE [RS_93]
+ SHUFFLE [RS_78]
PartitionCols:_col0
Group By Operator [GBY_77] (rows=18262 width=1119)
Output:["_col0"],keys:_col0
@@ -252,12 +251,7 @@ Stage-0
Group By Operator [GBY_82] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
Select Operator [SEL_80] (rows=9131 width=1119)
- Group By Operator [GBY_79] (rows=9131 width=1119)
- Output:["_col0"],keys:KEY._col0
- <-Map 25 [SIMPLE_EDGE]
- SHUFFLE [RS_78]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_77]
+ Please refer to the previous Group By Operator [GBY_79]
<-Reducer 3 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_102]
Merge Join Operator [MERGEJOIN_182] (rows=696954748 width=88)
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query56.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query56.q.out b/ql/src/test/results/clientpositive/perf/tez/query56.q.out
index 4041740..93f8a1d 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query56.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query56.q.out
@@ -135,26 +135,22 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE)
-Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 15 <- Map 14 (SIMPLE_EDGE)
-Reducer 16 <- Map 14 (SIMPLE_EDGE)
-Reducer 17 <- Map 14 (SIMPLE_EDGE)
-Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE)
-Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
-Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE)
-Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE)
-Reducer 24 <- Map 21 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
-Reducer 25 <- Map 26 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 2 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE)
+Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 13 <- Map 12 (SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
+Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
+Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE)
+Reducer 19 <- Map 22 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE)
+Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE)
+Reducer 21 <- Map 22 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 16 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
Reducer 6 <- Union 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE)
-Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 19 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 5 (CONTAINS)
Stage-0
Fetch Operator
@@ -171,66 +167,27 @@ Stage-0
Group By Operator [GBY_116] (rows=335408073 width=108)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
<-Union 5 [SIMPLE_EDGE]
- <-Reducer 10 [CONTAINS]
+ <-Reducer 11 [CONTAINS]
Reduce Output Operator [RS_115]
PartitionCols:_col0
Group By Operator [GBY_114] (rows=670816147 width=108)
Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Group By Operator [GBY_72] (rows=191657247 width=135)
+ Group By Operator [GBY_110] (rows=95833781 width=135)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 9 [SIMPLE_EDGE]
- SHUFFLE [RS_71]
+ <-Reducer 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_109]
PartitionCols:_col0
- Group By Operator [GBY_70] (rows=383314495 width=135)
+ Group By Operator [GBY_108] (rows=191667562 width=135)
Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
- Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135)
- Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"]
- <-Reducer 23 [SIMPLE_EDGE]
- SHUFFLE [RS_67]
- PartitionCols:_col4
- Select Operator [SEL_62] (rows=348467716 width=135)
- Output:["_col4","_col5"]
- Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135)
- Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"]
- <-Map 26 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_162] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -8) and ca_address_sk is not null)
- TableScan [TS_16] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
- <-Reducer 22 [SIMPLE_EDGE]
- SHUFFLE [RS_59]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135)
- Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
- PartitionCols:_col0
- Select Operator [SEL_15] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_161] (rows=18262 width=1119)
- predicate:((d_moy = 1) and (d_year = 2000) and d_date_sk is not null)
- TableScan [TS_13] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Map 27 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
- PartitionCols:_col0
- Select Operator [SEL_49] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_165] (rows=287989836 width=135)
- predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
- TableScan [TS_47] (rows=287989836 width=135)
- default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"]
- <-Reducer 8 [SIMPLE_EDGE]
- SHUFFLE [RS_66]
+ Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135)
+ Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_104]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_176] (rows=508200 width=1436)
- Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436)
+ Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_63]
+ SHUFFLE [RS_26]
PartitionCols:_col1
Select Operator [SEL_2] (rows=462000 width=1436)
Output:["_col0","_col1"]
@@ -238,13 +195,13 @@ Stage-0
predicate:(i_item_id is not null and i_item_sk is not null)
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
- <-Reducer 16 [ONE_TO_ONE_EDGE]
- FORWARD [RS_64]
+ <-Reducer 13 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_27]
PartitionCols:_col0
- Group By Operator [GBY_45] (rows=115500 width=1436)
+ Group By Operator [GBY_8] (rows=115500 width=1436)
Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_44]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
PartitionCols:_col0
Group By Operator [GBY_6] (rows=231000 width=1436)
Output:["_col0"],keys:i_item_id
@@ -254,59 +211,37 @@ Stage-0
predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"]
- <-Reducer 13 [CONTAINS]
- Reduce Output Operator [RS_115]
- PartitionCols:_col0
- Group By Operator [GBY_114] (rows=670816147 width=108)
- Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Group By Operator [GBY_110] (rows=95833781 width=135)
- Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_109]
- PartitionCols:_col0
- Group By Operator [GBY_108] (rows=191667562 width=135)
- Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
- Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135)
- Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"]
- <-Reducer 11 [SIMPLE_EDGE]
- SHUFFLE [RS_104]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_179] (rows=508200 width=1436)
- Conds:RS_101._col1=RS_102._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_101]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_2]
- <-Reducer 17 [ONE_TO_ONE_EDGE]
- FORWARD [RS_102]
- PartitionCols:_col0
- Group By Operator [GBY_83] (rows=115500 width=1436)
- Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_82]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_6]
- <-Reducer 25 [SIMPLE_EDGE]
+ <-Reducer 21 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col3
Select Operator [SEL_100] (rows=174243235 width=135)
Output:["_col3","_col5"]
Merge Join Operator [MERGEJOIN_181] (rows=174243235 width=135)
Conds:RS_97._col2=RS_98._col0(Inner),Output:["_col1","_col3"]
- <-Map 26 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_98]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_18]
- <-Reducer 24 [SIMPLE_EDGE]
+ Select Operator [SEL_18] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_162] (rows=20000000 width=1014)
+ predicate:((ca_gmt_offset = -8) and ca_address_sk is not null)
+ TableScan [TS_16] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
+ <-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_180] (rows=158402938 width=135)
Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_95]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_15]
- <-Map 28 [SIMPLE_EDGE]
+ Select Operator [SEL_15] (rows=18262 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_161] (rows=18262 width=1119)
+ predicate:((d_moy = 1) and (d_year = 2000) and d_date_sk is not null)
+ TableScan [TS_13] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
+ <-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
Select Operator [SEL_87] (rows=144002668 width=135)
@@ -332,42 +267,28 @@ Stage-0
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436)
- Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_26]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_2]
- <-Reducer 15 [ONE_TO_ONE_EDGE]
- FORWARD [RS_27]
- PartitionCols:_col0
- Group By Operator [GBY_8] (rows=115500 width=1436)
- Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_6]
- <-Reducer 20 [SIMPLE_EDGE]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_173]
+ <-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
Select Operator [SEL_25] (rows=696954748 width=88)
Output:["_col3","_col5"]
Merge Join Operator [MERGEJOIN_175] (rows=696954748 width=88)
Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"]
- <-Map 26 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_18]
- <-Reducer 19 [SIMPLE_EDGE]
+ <-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_174] (rows=633595212 width=88)
Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_20]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_15]
- <-Map 18 [SIMPLE_EDGE]
+ <-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
Select Operator [SEL_12] (rows=575995635 width=88)
@@ -376,4 +297,51 @@ Stage-0
predicate:(ss_addr_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_10] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"]
+ <-Reducer 9 [CONTAINS]
+ Reduce Output Operator [RS_115]
+ PartitionCols:_col0
+ Group By Operator [GBY_114] (rows=670816147 width=108)
+ Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
+ Group By Operator [GBY_72] (rows=191657247 width=135)
+ Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
+ <-Reducer 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_71]
+ PartitionCols:_col0
+ Group By Operator [GBY_70] (rows=383314495 width=135)
+ Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
+ Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135)
+ Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_66]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_173]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_67]
+ PartitionCols:_col4
+ Select Operator [SEL_62] (rows=348467716 width=135)
+ Output:["_col4","_col5"]
+ Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135)
+ Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_60]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_18]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135)
+ Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"]
+ <-Map 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_57]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_15]
+ <-Map 23 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
+ PartitionCols:_col0
+ Select Operator [SEL_49] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_165] (rows=287989836 width=135)
+ predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
+ TableScan [TS_47] (rows=287989836 width=135)
+ default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query57.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query57.q.out b/ql/src/test/results/clientpositive/perf/tez/query57.q.out
index 1a1bbf1..7a275d2 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query57.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query57.q.out
@@ -95,24 +95,16 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Map 21 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Map 22 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE)
-Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
-Reducer 14 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
-Reducer 15 <- Map 21 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE)
-Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
-Reducer 18 <- Reducer 17 (SIMPLE_EDGE)
-Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
-Reducer 3 <- Map 21 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 22 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 5 (SIMPLE_EDGE)
+Reducer 11 <- Reducer 10 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
+Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 14 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 13 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 11 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
-Reducer 9 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 5 (SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -130,78 +122,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
Merge Join Operator [MERGEJOIN_189] (rows=421645952 width=135)
Conds:RS_103._col0, _col2, _col1, (_col6 + 1)=RS_104._col0, _col2, _col1, _col7(Inner),RS_104._col0, _col2, _col1, _col7=RS_105._col0, _col2, _col1, (_col6 - 1)(Inner),Output:["_col5","_col7","_col8","_col10","_col11","_col12","_col13","_col20"]
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_103]
- PartitionCols:_col0, _col2, _col1, (_col6 + 1)
- Select Operator [SEL_29] (rows=191657247 width=135)
- Output:["_col0","_col1","_col2","_col5","_col6"]
- Filter Operator [FIL_164] (rows=191657247 width=135)
- predicate:rank_window_0 is not null
- PTF Operator [PTF_28] (rows=191657247 width=135)
- Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2"}]
- Select Operator [SEL_27] (rows=191657247 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_26]
- PartitionCols:_col0, _col1, _col2
- Select Operator [SEL_25] (rows=191657247 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Group By Operator [GBY_24] (rows=191657247 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
- <-Reducer 11 [SIMPLE_EDGE]
- SHUFFLE [RS_23]
- PartitionCols:_col0, _col1, _col2, _col3, _col4
- Group By Operator [GBY_22] (rows=383314495 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11
- Merge Join Operator [MERGEJOIN_182] (rows=383314495 width=135)
- Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"]
- <-Map 22 [SIMPLE_EDGE]
- SHUFFLE [RS_19]
- PartitionCols:_col0
- Select Operator [SEL_81] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_179] (rows=462000 width=1436)
- predicate:(i_brand is not null and i_category is not null and i_item_sk is not null)
- TableScan [TS_79] (rows=462000 width=1436)
- default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
- <-Reducer 10 [SIMPLE_EDGE]
- SHUFFLE [RS_18]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_181] (rows=348467716 width=135)
- Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
- PartitionCols:_col0
- Select Operator [SEL_78] (rows=60 width=2045)
- Output:["_col0","_col1"]
- Filter Operator [FIL_178] (rows=60 width=2045)
- predicate:(cc_call_center_sk is not null and cc_name is not null)
- TableScan [TS_76] (rows=60 width=2045)
- default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_name"]
- <-Reducer 9 [SIMPLE_EDGE]
- SHUFFLE [RS_15]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_180] (rows=316788826 width=135)
- Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_12]
- PartitionCols:_col0
- Select Operator [SEL_72] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_176] (rows=287989836 width=135)
- predicate:(cs_call_center_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
- TableScan [TS_70] (rows=287989836 width=135)
- default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_call_center_sk","cs_item_sk","cs_sales_price"]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_13]
- PartitionCols:_col0
- Select Operator [SEL_75] (rows=73048 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_177] (rows=73048 width=1119)
- predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
- TableScan [TS_73] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Reducer 19 [SIMPLE_EDGE]
+ <-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_104]
PartitionCols:_col0, _col2, _col1, _col7
Select Operator [SEL_67] (rows=15971437 width=135)
@@ -216,7 +137,7 @@ Stage-0
Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST","partition by:":"_col1, _col2, _col3"}]
Select Operator [SEL_64] (rows=191657247 width=135)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- <-Reducer 18 [SIMPLE_EDGE]
+ <-Reducer 10 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col0, _col1, _col2
Select Operator [SEL_62] (rows=191657247 width=135)
@@ -225,46 +146,66 @@ Stage-0
Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2, _col3"}]
Select Operator [SEL_60] (rows=191657247 width=135)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- <-Reducer 17 [SIMPLE_EDGE]
+ <-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_59]
PartitionCols:_col0, _col1, _col2, _col3
- Select Operator [SEL_58] (rows=191657247 width=135)
+ Select Operator [SEL_95] (rows=191657247 width=135)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Group By Operator [GBY_57] (rows=191657247 width=135)
+ Group By Operator [GBY_94] (rows=191657247 width=135)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_93]
PartitionCols:_col0, _col1, _col2, _col3, _col4
- Group By Operator [GBY_55] (rows=383314495 width=135)
+ Group By Operator [GBY_92] (rows=383314495 width=135)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11
- Merge Join Operator [MERGEJOIN_185] (rows=383314495 width=135)
- Conds:RS_51._col2=RS_52._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"]
- <-Map 22 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
+ Merge Join Operator [MERGEJOIN_188] (rows=383314495 width=135)
+ Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"]
+ <-Map 14 [SIMPLE_EDGE]
+ SHUFFLE [RS_89]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_81]
- <-Reducer 15 [SIMPLE_EDGE]
- SHUFFLE [RS_51]
+ Select Operator [SEL_81] (rows=462000 width=1436)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_179] (rows=462000 width=1436)
+ predicate:(i_brand is not null and i_category is not null and i_item_sk is not null)
+ TableScan [TS_79] (rows=462000 width=1436)
+ default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_88]
PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_184] (rows=348467716 width=135)
- Conds:RS_48._col1=RS_49._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
+ Merge Join Operator [MERGEJOIN_187] (rows=348467716 width=135)
+ Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"]
+ <-Map 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_86]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_78]
- <-Reducer 14 [SIMPLE_EDGE]
- SHUFFLE [RS_48]
+ Select Operator [SEL_78] (rows=60 width=2045)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_178] (rows=60 width=2045)
+ predicate:(cc_call_center_sk is not null and cc_name is not null)
+ TableScan [TS_76] (rows=60 width=2045)
+ default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_name"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_85]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_183] (rows=316788826 width=135)
- Conds:RS_45._col0=RS_46._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
+ Merge Join Operator [MERGEJOIN_186] (rows=316788826 width=135)
+ Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
+ SHUFFLE [RS_82]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_72]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_46]
+ Select Operator [SEL_72] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_176] (rows=287989836 width=135)
+ predicate:(cs_call_center_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
+ TableScan [TS_70] (rows=287989836 width=135)
+ default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_call_center_sk","cs_item_sk","cs_sales_price"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_83]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_75]
+ Select Operator [SEL_75] (rows=73048 width=1119)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_177] (rows=73048 width=1119)
+ predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
+ TableScan [TS_73] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col0, _col2, _col1, (_col6 - 1)
@@ -279,41 +220,20 @@ Stage-0
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_96]
PartitionCols:_col0, _col1, _col2
- Select Operator [SEL_95] (rows=191657247 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Group By Operator [GBY_94] (rows=191657247 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_93]
- PartitionCols:_col0, _col1, _col2, _col3, _col4
- Group By Operator [GBY_92] (rows=383314495 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col3)"],keys:_col5, _col6, _col8, _col10, _col11
- Merge Join Operator [MERGEJOIN_188] (rows=383314495 width=135)
- Conds:RS_88._col2=RS_89._col0(Inner),Output:["_col3","_col5","_col6","_col8","_col10","_col11"]
- <-Map 22 [SIMPLE_EDGE]
- SHUFFLE [RS_89]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_81]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_88]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_187] (rows=348467716 width=135)
- Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col2","_col3","_col5","_col6","_col8"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_86]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_78]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_85]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_186] (rows=316788826 width=135)
- Conds:RS_82._col0=RS_83._col0(Inner),Output:["_col1","_col2","_col3","_col5","_col6"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_82]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_72]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_83]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_75]
+ Please refer to the previous Select Operator [SEL_95]
+ <-Reducer 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_103]
+ PartitionCols:_col0, _col2, _col1, (_col6 + 1)
+ Select Operator [SEL_29] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col5","_col6"]
+ Filter Operator [FIL_164] (rows=191657247 width=135)
+ predicate:rank_window_0 is not null
+ PTF Operator [PTF_28] (rows=191657247 width=135)
+ Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST","partition by:":"_col0, _col1, _col2"}]
+ Select Operator [SEL_27] (rows=191657247 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_26]
+ PartitionCols:_col0, _col1, _col2
+ Please refer to the previous Select Operator [SEL_95]
[3/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query58.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query58.q.out b/ql/src/test/results/clientpositive/perf/tez/query58.q.out
index d5ef23b..769af24 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query58.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query58.q.out
@@ -1,6 +1,4 @@
-Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 20' is a cross product
-Warning: Shuffle Join MERGEJOIN[269][tables = [$hdt$_5, $hdt$_6]] in Stage 'Reducer 24' is a cross product
-Warning: Shuffle Join MERGEJOIN[273][tables = [$hdt$_6, $hdt$_7]] in Stage 'Reducer 28' is a cross product
+Warning: Shuffle Join MERGEJOIN[265][tables = [$hdt$_4, $hdt$_5]] in Stage 'Reducer 18' is a cross product
PREHOOK: query: explain
with ss_items as
(select i_item_id item_id
@@ -133,31 +131,21 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Map 33 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE)
+Reducer 11 <- Map 23 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
-Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE)
-Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 26 (ONE_TO_ONE_EDGE)
-Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 30 (ONE_TO_ONE_EDGE)
-Reducer 19 <- Map 18 (CUSTOM_SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (ONE_TO_ONE_EDGE)
+Reducer 17 <- Map 16 (CUSTOM_SIMPLE_EDGE)
+Reducer 18 <- Map 21 (CUSTOM_SIMPLE_EDGE), Reducer 17 (CUSTOM_SIMPLE_EDGE)
+Reducer 19 <- Map 21 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 20 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 19 (CUSTOM_SIMPLE_EDGE)
-Reducer 21 <- Map 31 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
-Reducer 22 <- Reducer 21 (SIMPLE_EDGE)
-Reducer 23 <- Map 18 (CUSTOM_SIMPLE_EDGE)
-Reducer 24 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 23 (CUSTOM_SIMPLE_EDGE)
-Reducer 25 <- Map 31 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
-Reducer 26 <- Reducer 25 (SIMPLE_EDGE)
-Reducer 27 <- Map 18 (CUSTOM_SIMPLE_EDGE)
-Reducer 28 <- Map 31 (CUSTOM_SIMPLE_EDGE), Reducer 27 (CUSTOM_SIMPLE_EDGE)
-Reducer 29 <- Map 31 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE)
+Reducer 20 <- Reducer 19 (SIMPLE_EDGE)
Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 30 <- Reducer 29 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
-Reducer 8 <- Map 32 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 9 <- Reducer 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Map 22 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 15 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -189,13 +177,13 @@ Stage-0
Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4
Merge Join Operator [MERGEJOIN_277] (rows=696954748 width=88)
Conds:RS_92._col0=RS_93._col0(Inner),Output:["_col2","_col4"]
- <-Reducer 16 [SIMPLE_EDGE]
+ <-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_93]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_271] (rows=80353 width=1119)
- Conds:RS_85._col1=RS_86._col0(Inner),Output:["_col0"]
+ Merge Join Operator [MERGEJOIN_267] (rows=80353 width=1119)
+ Conds:RS_35._col1=RS_36._col0(Inner),Output:["_col0"]
<-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_85]
+ SHUFFLE [RS_35]
PartitionCols:_col1
Select Operator [SEL_8] (rows=73049 width=1119)
Output:["_col0","_col1"]
@@ -203,48 +191,48 @@ Stage-0
predicate:(d_date is not null and d_date_sk is not null)
TableScan [TS_6] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
- <-Reducer 26 [ONE_TO_ONE_EDGE]
- FORWARD [RS_86]
+ <-Reducer 20 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_36]
PartitionCols:_col0
- Group By Operator [GBY_83] (rows=40176 width=1119)
+ Group By Operator [GBY_33] (rows=40176 width=1119)
Output:["_col0"],keys:KEY._col0
- <-Reducer 25 [SIMPLE_EDGE]
- SHUFFLE [RS_82]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_32]
PartitionCols:_col0
- Group By Operator [GBY_81] (rows=80353 width=1119)
+ Group By Operator [GBY_31] (rows=80353 width=1119)
Output:["_col0"],keys:_col2
- Merge Join Operator [MERGEJOIN_270] (rows=80353 width=1119)
- Conds:RS_77._col1=RS_78._col1(Inner),Output:["_col2"]
- <-Map 31 [SIMPLE_EDGE]
- SHUFFLE [RS_78]
+ Merge Join Operator [MERGEJOIN_266] (rows=80353 width=1119)
+ Conds:RS_27._col1=RS_28._col1(Inner),Output:["_col2"]
+ <-Map 21 [SIMPLE_EDGE]
+ SHUFFLE [RS_28]
PartitionCols:_col1
- Select Operator [SEL_73] (rows=73049 width=1119)
+ Select Operator [SEL_23] (rows=73049 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_257] (rows=73049 width=1119)
+ Filter Operator [FIL_251] (rows=73049 width=1119)
predicate:(d_date is not null and d_week_seq is not null)
TableScan [TS_21] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"]
- <-Reducer 24 [SIMPLE_EDGE]
- SHUFFLE [RS_77]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_27]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_269] (rows=36524 width=1128)
+ Merge Join Operator [MERGEJOIN_265] (rows=36524 width=1128)
Conds:(Inner),Output:["_col1"]
- <-Map 31 [CUSTOM_SIMPLE_EDGE]
- SHUFFLE [RS_75]
- Select Operator [SEL_70] (rows=36524 width=1119)
+ <-Map 21 [CUSTOM_SIMPLE_EDGE]
+ SHUFFLE [RS_25]
+ Select Operator [SEL_20] (rows=36524 width=1119)
Output:["_col0"]
- Filter Operator [FIL_256] (rows=36524 width=1119)
+ Filter Operator [FIL_250] (rows=36524 width=1119)
predicate:((d_date = '1998-02-19') and d_week_seq is not null)
Please refer to the previous TableScan [TS_21]
- <-Reducer 23 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_74]
- Select Operator [SEL_67] (rows=1 width=8)
- Filter Operator [FIL_66] (rows=1 width=8)
+ <-Reducer 17 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_24]
+ Select Operator [SEL_17] (rows=1 width=8)
+ Filter Operator [FIL_16] (rows=1 width=8)
predicate:(sq_count_check(_col0) <= 1)
- Group By Operator [GBY_64] (rows=1 width=8)
+ Group By Operator [GBY_14] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 18 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_63]
+ <-Map 16 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_13]
Group By Operator [GBY_12] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
Select Operator [SEL_11] (rows=36524 width=1119)
@@ -266,7 +254,7 @@ Stage-0
predicate:(i_item_id is not null and i_item_sk is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
- <-Map 32 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_89]
PartitionCols:_col1
Select Operator [SEL_52] (rows=575995635 width=88)
@@ -287,6 +275,10 @@ Stage-0
Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4
Merge Join Operator [MERGEJOIN_278] (rows=174243235 width=135)
Conds:RS_142._col0=RS_143._col0(Inner),Output:["_col2","_col4"]
+ <-Reducer 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_143]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_267]
<-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_142]
PartitionCols:_col0
@@ -296,7 +288,7 @@ Stage-0
SHUFFLE [RS_140]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_5]
- <-Map 33 [SIMPLE_EDGE]
+ <-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_139]
PartitionCols:_col1
Select Operator [SEL_102] (rows=144002668 width=135)
@@ -305,57 +297,6 @@ Stage-0
predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
TableScan [TS_100] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_ext_sales_price"]
- <-Reducer 17 [SIMPLE_EDGE]
- SHUFFLE [RS_143]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_275] (rows=80353 width=1119)
- Conds:RS_135._col1=RS_136._col0(Inner),Output:["_col0"]
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_135]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_8]
- <-Reducer 30 [ONE_TO_ONE_EDGE]
- FORWARD [RS_136]
- PartitionCols:_col0
- Group By Operator [GBY_133] (rows=40176 width=1119)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 29 [SIMPLE_EDGE]
- SHUFFLE [RS_132]
- PartitionCols:_col0
- Group By Operator [GBY_131] (rows=80353 width=1119)
- Output:["_col0"],keys:_col2
- Merge Join Operator [MERGEJOIN_274] (rows=80353 width=1119)
- Conds:RS_127._col1=RS_128._col1(Inner),Output:["_col2"]
- <-Map 31 [SIMPLE_EDGE]
- SHUFFLE [RS_128]
- PartitionCols:_col1
- Select Operator [SEL_123] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_263] (rows=73049 width=1119)
- predicate:(d_date is not null and d_week_seq is not null)
- Please refer to the previous TableScan [TS_21]
- <-Reducer 28 [SIMPLE_EDGE]
- SHUFFLE [RS_127]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_273] (rows=36524 width=1128)
- Conds:(Inner),Output:["_col1"]
- <-Map 31 [CUSTOM_SIMPLE_EDGE]
- SHUFFLE [RS_125]
- Select Operator [SEL_120] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_262] (rows=36524 width=1119)
- predicate:((d_date = '1998-02-19') and d_week_seq is not null)
- Please refer to the previous TableScan [TS_21]
- <-Reducer 27 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_124]
- Select Operator [SEL_117] (rows=1 width=8)
- Filter Operator [FIL_116] (rows=1 width=8)
- predicate:(sq_count_check(_col0) <= 1)
- Group By Operator [GBY_114] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 18 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_113]
- Please refer to the previous Group By Operator [GBY_12]
<-Reducer 4 [ONE_TO_ONE_EDGE]
FORWARD [RS_150]
PartitionCols:_col0
@@ -371,54 +312,7 @@ Stage-0
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_43]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_267] (rows=80353 width=1119)
- Conds:RS_35._col1=RS_36._col0(Inner),Output:["_col0"]
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_35]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_8]
- <-Reducer 22 [ONE_TO_ONE_EDGE]
- FORWARD [RS_36]
- PartitionCols:_col0
- Group By Operator [GBY_33] (rows=40176 width=1119)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 21 [SIMPLE_EDGE]
- SHUFFLE [RS_32]
- PartitionCols:_col0
- Group By Operator [GBY_31] (rows=80353 width=1119)
- Output:["_col0"],keys:_col2
- Merge Join Operator [MERGEJOIN_266] (rows=80353 width=1119)
- Conds:RS_27._col1=RS_28._col1(Inner),Output:["_col2"]
- <-Map 31 [SIMPLE_EDGE]
- SHUFFLE [RS_28]
- PartitionCols:_col1
- Select Operator [SEL_23] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_251] (rows=73049 width=1119)
- predicate:(d_date is not null and d_week_seq is not null)
- Please refer to the previous TableScan [TS_21]
- <-Reducer 20 [SIMPLE_EDGE]
- SHUFFLE [RS_27]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_265] (rows=36524 width=1128)
- Conds:(Inner),Output:["_col1"]
- <-Map 31 [CUSTOM_SIMPLE_EDGE]
- SHUFFLE [RS_25]
- Select Operator [SEL_20] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_250] (rows=36524 width=1119)
- predicate:((d_date = '1998-02-19') and d_week_seq is not null)
- Please refer to the previous TableScan [TS_21]
- <-Reducer 19 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_24]
- Select Operator [SEL_17] (rows=1 width=8)
- Filter Operator [FIL_16] (rows=1 width=8)
- predicate:(sq_count_check(_col0) <= 1)
- Group By Operator [GBY_14] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 18 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_13]
- Please refer to the previous Group By Operator [GBY_12]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_267]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_42]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query59.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query59.q.out b/ql/src/test/results/clientpositive/perf/tez/query59.q.out
index 37c2cc0..61818a8 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query59.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query59.q.out
@@ -87,16 +87,15 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Map 13 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Map 15 (SIMPLE_EDGE), Reducer 10 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
+Reducer 10 <- Map 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 13 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Map 14 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
-Reducer 6 <- Reducer 11 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Map 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
-Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 2 (SIMPLE_EDGE)
+Reducer 9 <- Map 12 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -114,14 +113,14 @@ Stage-0
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"]
Merge Join Operator [MERGEJOIN_104] (rows=421657640 width=88)
Conds:RS_56._col2, _col1=RS_57._col1, (_col0 - 52)(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col12","_col13","_col14","_col15","_col16","_col17"]
- <-Reducer 11 [SIMPLE_EDGE]
+ <-Reducer 10 [SIMPLE_EDGE]
SHUFFLE [RS_57]
PartitionCols:_col1, (_col0 - 52)
Select Operator [SEL_55] (rows=383325119 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
Merge Join Operator [MERGEJOIN_103] (rows=383325119 width=88)
Conds:RS_52._col1=RS_53._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col11"]
- <-Map 15 [SIMPLE_EDGE]
+ <-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_53]
PartitionCols:_col0
Select Operator [SEL_48] (rows=1704 width=1910)
@@ -130,12 +129,12 @@ Stage-0
predicate:(s_store_id is not null and s_store_sk is not null)
TableScan [TS_46] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id"]
- <-Reducer 10 [SIMPLE_EDGE]
+ <-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_52]
PartitionCols:_col1
Merge Join Operator [MERGEJOIN_102] (rows=348477374 width=88)
Conds:RS_49._col0=RS_50._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- <-Map 13 [SIMPLE_EDGE]
+ <-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_50]
PartitionCols:_col1
Select Operator [SEL_45] (rows=8116 width=1119)
@@ -144,22 +143,22 @@ Stage-0
predicate:(d_month_seq BETWEEN 1197 AND 1208 and d_week_seq is not null)
TableScan [TS_15] (rows=73049 width=1119)
default@date_dim,d,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_week_seq"]
- <-Reducer 9 [SIMPLE_EDGE]
+ <-Reducer 8 [SIMPLE_EDGE]
SHUFFLE [RS_49]
PartitionCols:_col0
Group By Operator [GBY_41] (rows=316797606 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)"],keys:KEY._col0, KEY._col1
- <-Reducer 8 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_40]
PartitionCols:_col0, _col1
Group By Operator [GBY_39] (rows=633595212 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(_col2)","sum(_col3)","sum(_col5)","sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col0, _col1
Select Operator [SEL_37] (rows=633595212 width=88)
Output:["_col0","_col1","_col2","_col3","_col5","_col6","_col7","_col8"]
- Merge Join Operator [MERGEJOIN_101] (rows=633595212 width=88)
- Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col1","_col2","_col4","_col5"]
+ Merge Join Operator [MERGEJOIN_98] (rows=633595212 width=88)
+ Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col4","_col5"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_34]
+ SHUFFLE [RS_6]
PartitionCols:_col0
Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
@@ -167,8 +166,8 @@ Stage-0
predicate:(ss_sold_date_sk is not null and ss_store_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_sales_price"]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_35]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
PartitionCols:_col0
Select Operator [SEL_5] (rows=73049 width=1119)
Output:["_col0","_col1","_col2"]
@@ -183,7 +182,7 @@ Stage-0
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"]
Merge Join Operator [MERGEJOIN_100] (rows=383325119 width=88)
Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col12","_col13"]
- <-Map 14 [SIMPLE_EDGE]
+ <-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_25]
PartitionCols:_col0
Select Operator [SEL_20] (rows=1704 width=1910)
@@ -197,7 +196,7 @@ Stage-0
PartitionCols:_col1
Merge Join Operator [MERGEJOIN_99] (rows=348477374 width=88)
Conds:RS_21._col0=RS_22._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
- <-Map 13 [SIMPLE_EDGE]
+ <-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col1
Select Operator [SEL_17] (rows=8116 width=1119)
@@ -217,14 +216,5 @@ Stage-0
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"],aggregations:["sum(_col2)","sum(_col3)","sum(_col4)","sum(_col5)","sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col0, _col1
Select Operator [SEL_9] (rows=633595212 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
- Merge Join Operator [MERGEJOIN_98] (rows=633595212 width=88)
- Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col1","_col2","_col4","_col5"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_6]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_2]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_5]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_98]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query60.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query60.q.out b/ql/src/test/results/clientpositive/perf/tez/query60.q.out
index 84a9ada..67102b3 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query60.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query60.q.out
@@ -155,26 +155,22 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Reducer 9 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 11 <- Map 1 (SIMPLE_EDGE), Reducer 17 (ONE_TO_ONE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE)
-Reducer 13 <- Reducer 12 (SIMPLE_EDGE), Union 5 (CONTAINS)
-Reducer 15 <- Map 14 (SIMPLE_EDGE)
-Reducer 16 <- Map 14 (SIMPLE_EDGE)
-Reducer 17 <- Map 14 (SIMPLE_EDGE)
-Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 21 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE)
-Reducer 20 <- Map 26 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
-Reducer 22 <- Map 21 (SIMPLE_EDGE), Map 27 (SIMPLE_EDGE)
-Reducer 23 <- Map 26 (SIMPLE_EDGE), Reducer 22 (SIMPLE_EDGE)
-Reducer 24 <- Map 21 (SIMPLE_EDGE), Map 28 (SIMPLE_EDGE)
-Reducer 25 <- Map 26 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 2 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE)
+Reducer 11 <- Reducer 10 (SIMPLE_EDGE), Union 5 (CONTAINS)
+Reducer 13 <- Map 12 (SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
+Reducer 16 <- Map 22 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
+Reducer 18 <- Map 17 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE)
+Reducer 19 <- Map 22 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE)
+Reducer 20 <- Map 17 (SIMPLE_EDGE), Map 24 (SIMPLE_EDGE)
+Reducer 21 <- Map 22 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 16 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Union 5 (CONTAINS)
Reducer 6 <- Union 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 1 (SIMPLE_EDGE), Reducer 16 (ONE_TO_ONE_EDGE)
-Reducer 9 <- Reducer 23 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 19 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 8 (SIMPLE_EDGE), Union 5 (CONTAINS)
Stage-0
Fetch Operator
@@ -191,66 +187,27 @@ Stage-0
Group By Operator [GBY_116] (rows=335408073 width=108)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
<-Union 5 [SIMPLE_EDGE]
- <-Reducer 10 [CONTAINS]
+ <-Reducer 11 [CONTAINS]
Reduce Output Operator [RS_115]
PartitionCols:_col0
Group By Operator [GBY_114] (rows=670816147 width=108)
Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Group By Operator [GBY_72] (rows=191657247 width=135)
+ Group By Operator [GBY_110] (rows=95833781 width=135)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 9 [SIMPLE_EDGE]
- SHUFFLE [RS_71]
+ <-Reducer 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_109]
PartitionCols:_col0
- Group By Operator [GBY_70] (rows=383314495 width=135)
+ Group By Operator [GBY_108] (rows=191667562 width=135)
Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
- Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135)
- Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"]
- <-Reducer 23 [SIMPLE_EDGE]
- SHUFFLE [RS_67]
- PartitionCols:_col4
- Select Operator [SEL_62] (rows=348467716 width=135)
- Output:["_col4","_col5"]
- Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135)
- Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"]
- <-Map 26 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col0
- Select Operator [SEL_18] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_162] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
- TableScan [TS_16] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
- <-Reducer 22 [SIMPLE_EDGE]
- SHUFFLE [RS_59]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135)
- Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
- PartitionCols:_col0
- Select Operator [SEL_15] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_161] (rows=18262 width=1119)
- predicate:((d_moy = 9) and (d_year = 1999) and d_date_sk is not null)
- TableScan [TS_13] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Map 27 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
- PartitionCols:_col0
- Select Operator [SEL_49] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_165] (rows=287989836 width=135)
- predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
- TableScan [TS_47] (rows=287989836 width=135)
- default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"]
- <-Reducer 8 [SIMPLE_EDGE]
- SHUFFLE [RS_66]
+ Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135)
+ Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_104]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_176] (rows=508200 width=1436)
- Conds:RS_63._col1=RS_64._col0(Inner),Output:["_col0","_col1"]
+ Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436)
+ Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_63]
+ SHUFFLE [RS_26]
PartitionCols:_col1
Select Operator [SEL_2] (rows=462000 width=1436)
Output:["_col0","_col1"]
@@ -258,13 +215,13 @@ Stage-0
predicate:(i_item_id is not null and i_item_sk is not null)
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
- <-Reducer 16 [ONE_TO_ONE_EDGE]
- FORWARD [RS_64]
+ <-Reducer 13 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_27]
PartitionCols:_col0
- Group By Operator [GBY_45] (rows=115500 width=1436)
+ Group By Operator [GBY_8] (rows=115500 width=1436)
Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_44]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_7]
PartitionCols:_col0
Group By Operator [GBY_6] (rows=231000 width=1436)
Output:["_col0"],keys:i_item_id
@@ -274,59 +231,37 @@ Stage-0
predicate:((i_category) IN ('Children') and i_item_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"]
- <-Reducer 13 [CONTAINS]
- Reduce Output Operator [RS_115]
- PartitionCols:_col0
- Group By Operator [GBY_114] (rows=670816147 width=108)
- Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Group By Operator [GBY_110] (rows=95833781 width=135)
- Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_109]
- PartitionCols:_col0
- Group By Operator [GBY_108] (rows=191667562 width=135)
- Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
- Merge Join Operator [MERGEJOIN_184] (rows=191667562 width=135)
- Conds:RS_104._col0=RS_105._col3(Inner),Output:["_col1","_col8"]
- <-Reducer 11 [SIMPLE_EDGE]
- SHUFFLE [RS_104]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_179] (rows=508200 width=1436)
- Conds:RS_101._col1=RS_102._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_101]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_2]
- <-Reducer 17 [ONE_TO_ONE_EDGE]
- FORWARD [RS_102]
- PartitionCols:_col0
- Group By Operator [GBY_83] (rows=115500 width=1436)
- Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_82]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_6]
- <-Reducer 25 [SIMPLE_EDGE]
+ <-Reducer 21 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col3
Select Operator [SEL_100] (rows=174243235 width=135)
Output:["_col3","_col5"]
Merge Join Operator [MERGEJOIN_181] (rows=174243235 width=135)
Conds:RS_97._col2=RS_98._col0(Inner),Output:["_col1","_col3"]
- <-Map 26 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_98]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_18]
- <-Reducer 24 [SIMPLE_EDGE]
+ Select Operator [SEL_18] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_162] (rows=20000000 width=1014)
+ predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
+ TableScan [TS_16] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
+ <-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_180] (rows=158402938 width=135)
Conds:RS_94._col0=RS_95._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_95]
PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_15]
- <-Map 28 [SIMPLE_EDGE]
+ Select Operator [SEL_15] (rows=18262 width=1119)
+ Output:["_col0"]
+ Filter Operator [FIL_161] (rows=18262 width=1119)
+ predicate:((d_moy = 9) and (d_year = 1999) and d_date_sk is not null)
+ TableScan [TS_13] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
+ <-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
Select Operator [SEL_87] (rows=144002668 width=135)
@@ -352,42 +287,28 @@ Stage-0
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_173] (rows=508200 width=1436)
- Conds:RS_26._col1=RS_27._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_26]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_2]
- <-Reducer 15 [ONE_TO_ONE_EDGE]
- FORWARD [RS_27]
- PartitionCols:_col0
- Group By Operator [GBY_8] (rows=115500 width=1436)
- Output:["_col0"],keys:KEY._col0
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_7]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_6]
- <-Reducer 20 [SIMPLE_EDGE]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_173]
+ <-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
Select Operator [SEL_25] (rows=696954748 width=88)
Output:["_col3","_col5"]
Merge Join Operator [MERGEJOIN_175] (rows=696954748 width=88)
Conds:RS_22._col2=RS_23._col0(Inner),Output:["_col1","_col3"]
- <-Map 26 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_18]
- <-Reducer 19 [SIMPLE_EDGE]
+ <-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_174] (rows=633595212 width=88)
Conds:RS_19._col0=RS_20._col0(Inner),Output:["_col1","_col2","_col3"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_20]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_15]
- <-Map 18 [SIMPLE_EDGE]
+ <-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
Select Operator [SEL_12] (rows=575995635 width=88)
@@ -396,4 +317,51 @@ Stage-0
predicate:(ss_addr_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_10] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_addr_sk","ss_ext_sales_price"]
+ <-Reducer 9 [CONTAINS]
+ Reduce Output Operator [RS_115]
+ PartitionCols:_col0
+ Group By Operator [GBY_114] (rows=670816147 width=108)
+ Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
+ Group By Operator [GBY_72] (rows=191657247 width=135)
+ Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
+ <-Reducer 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_71]
+ PartitionCols:_col0
+ Group By Operator [GBY_70] (rows=383314495 width=135)
+ Output:["_col0","_col1"],aggregations:["sum(_col8)"],keys:_col1
+ Merge Join Operator [MERGEJOIN_183] (rows=383314495 width=135)
+ Conds:RS_66._col0=RS_67._col4(Inner),Output:["_col1","_col8"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_66]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_173]
+ <-Reducer 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_67]
+ PartitionCols:_col4
+ Select Operator [SEL_62] (rows=348467716 width=135)
+ Output:["_col4","_col5"]
+ Merge Join Operator [MERGEJOIN_178] (rows=348467716 width=135)
+ Conds:RS_59._col1=RS_60._col0(Inner),Output:["_col2","_col3"]
+ <-Map 22 [SIMPLE_EDGE]
+ SHUFFLE [RS_60]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_18]
+ <-Reducer 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_177] (rows=316788826 width=135)
+ Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1","_col2","_col3"]
+ <-Map 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_57]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_15]
+ <-Map 23 [SIMPLE_EDGE]
+ SHUFFLE [RS_56]
+ PartitionCols:_col0
+ Select Operator [SEL_49] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2","_col3"]
+ Filter Operator [FIL_165] (rows=287989836 width=135)
+ predicate:(cs_bill_addr_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)
+ TableScan [TS_47] (rows=287989836 width=135)
+ default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_addr_sk","cs_item_sk","cs_ext_sales_price"]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query61.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query61.q.out b/ql/src/test/results/clientpositive/perf/tez/query61.q.out
index 0b4f5fd..7ac7c0f 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query61.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query61.q.out
@@ -88,21 +88,20 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 12 <- Map 11 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE)
+Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE)
+Reducer 12 <- Map 19 (SIMPLE_EDGE), Reducer 11 (SIMPLE_EDGE)
Reducer 13 <- Map 20 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE)
Reducer 14 <- Map 21 (SIMPLE_EDGE), Reducer 13 (SIMPLE_EDGE)
-Reducer 15 <- Map 22 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE)
-Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 23 (SIMPLE_EDGE)
+Reducer 16 <- Map 15 (SIMPLE_EDGE), Map 22 (SIMPLE_EDGE)
+Reducer 17 <- Map 19 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
Reducer 18 <- Map 20 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE)
-Reducer 19 <- Map 21 (SIMPLE_EDGE), Reducer 18 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE)
-Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
+Reducer 3 <- Reducer 14 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE)
-Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 9 (CUSTOM_SIMPLE_EDGE)
+Reducer 5 <- Reducer 4 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE)
-Reducer 8 <- Reducer 19 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
-Reducer 9 <- Reducer 8 (CUSTOM_SIMPLE_EDGE)
+Reducer 7 <- Reducer 18 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -130,12 +129,35 @@ Stage-0
Output:["_col0"],aggregations:["sum(_col9)"]
Merge Join Operator [MERGEJOIN_154] (rows=927646829 width=88)
Conds:RS_37._col0=RS_38._col2(Inner),Output:["_col9"]
- <-Reducer 15 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_37]
+ PartitionCols:_col0
+ Merge Join Operator [MERGEJOIN_145] (rows=88000001 width=860)
+ Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col0"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_34]
+ PartitionCols:_col1
+ Select Operator [SEL_2] (rows=80000000 width=860)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_132] (rows=80000000 width=860)
+ predicate:(c_current_addr_sk is not null and c_customer_sk is not null)
+ TableScan [TS_0] (rows=80000000 width=860)
+ default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"]
+ <-Map 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_35]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_133] (rows=20000000 width=1014)
+ predicate:((ca_gmt_offset = -7) and ca_address_sk is not null)
+ TableScan [TS_3] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
+ <-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_38]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_149] (rows=843315281 width=88)
Conds:RS_30._col4=RS_31._col0(Inner),Output:["_col2","_col5"]
- <-Map 22 [SIMPLE_EDGE]
+ <-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_31]
PartitionCols:_col0
Select Operator [SEL_20] (rows=2300 width=1179)
@@ -144,12 +166,12 @@ Stage-0
predicate:(((p_channel_dmail = 'Y') or (p_channel_email = 'Y') or (p_channel_tv = 'Y')) and p_promo_sk is not null)
TableScan [TS_18] (rows=2300 width=1179)
default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_dmail","p_channel_email","p_channel_tv"]
- <-Reducer 14 [SIMPLE_EDGE]
+ <-Reducer 13 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col4
Merge Join Operator [MERGEJOIN_148] (rows=766650239 width=88)
Conds:RS_27._col3=RS_28._col0(Inner),Output:["_col2","_col4","_col5"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
Select Operator [SEL_17] (rows=852 width=1910)
@@ -158,12 +180,12 @@ Stage-0
predicate:((s_gmt_offset = -7) and s_store_sk is not null)
TableScan [TS_15] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_gmt_offset"]
- <-Reducer 13 [SIMPLE_EDGE]
+ <-Reducer 12 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col3
Merge Join Operator [MERGEJOIN_147] (rows=696954748 width=88)
Conds:RS_24._col1=RS_25._col0(Inner),Output:["_col2","_col3","_col4","_col5"]
- <-Map 20 [SIMPLE_EDGE]
+ <-Map 19 [SIMPLE_EDGE]
SHUFFLE [RS_25]
PartitionCols:_col0
Select Operator [SEL_14] (rows=231000 width=1436)
@@ -172,12 +194,12 @@ Stage-0
predicate:((i_category = 'Electronics') and i_item_sk is not null)
TableScan [TS_12] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"]
- <-Reducer 12 [SIMPLE_EDGE]
+ <-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_24]
PartitionCols:_col1
Merge Join Operator [MERGEJOIN_146] (rows=633595212 width=88)
Conds:RS_21._col0=RS_22._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5"]
- <-Map 16 [SIMPLE_EDGE]
+ <-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col0
Select Operator [SEL_11] (rows=18262 width=1119)
@@ -186,7 +208,7 @@ Stage-0
predicate:((d_moy = 11) and (d_year = 1999) and d_date_sk is not null)
TableScan [TS_9] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Map 11 [SIMPLE_EDGE]
+ <-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_21]
PartitionCols:_col0
Select Operator [SEL_8] (rows=575995635 width=88)
@@ -195,67 +217,48 @@ Stage-0
predicate:(ss_customer_sk is not null and ss_item_sk is not null and ss_promo_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
TableScan [TS_6] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_promo_sk","ss_ext_sales_price"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_37]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_145] (rows=88000001 width=860)
- Conds:RS_34._col1=RS_35._col0(Inner),Output:["_col0"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_34]
- PartitionCols:_col1
- Select Operator [SEL_2] (rows=80000000 width=860)
- Output:["_col0","_col1"]
- Filter Operator [FIL_132] (rows=80000000 width=860)
- predicate:(c_current_addr_sk is not null and c_customer_sk is not null)
- TableScan [TS_0] (rows=80000000 width=860)
- default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_addr_sk"]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_35]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_133] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -7) and ca_address_sk is not null)
- TableScan [TS_3] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
- <-Reducer 9 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 8 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_85]
Group By Operator [GBY_82] (rows=1 width=112)
Output:["_col0"],aggregations:["sum(VALUE._col0)"]
- <-Reducer 8 [CUSTOM_SIMPLE_EDGE]
+ <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_81]
Group By Operator [GBY_80] (rows=1 width=112)
Output:["_col0"],aggregations:["sum(_col8)"]
Merge Join Operator [MERGEJOIN_155] (rows=843315281 width=88)
Conds:RS_76._col0=RS_77._col2(Inner),Output:["_col8"]
- <-Reducer 19 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_76]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_145]
+ <-Reducer 18 [SIMPLE_EDGE]
SHUFFLE [RS_77]
PartitionCols:_col2
Merge Join Operator [MERGEJOIN_153] (rows=766650239 width=88)
Conds:RS_69._col3=RS_70._col0(Inner),Output:["_col2","_col4"]
- <-Map 21 [SIMPLE_EDGE]
+ <-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_70]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_17]
- <-Reducer 18 [SIMPLE_EDGE]
+ <-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_69]
PartitionCols:_col3
Merge Join Operator [MERGEJOIN_152] (rows=696954748 width=88)
Conds:RS_66._col1=RS_67._col0(Inner),Output:["_col2","_col3","_col4"]
- <-Map 20 [SIMPLE_EDGE]
+ <-Map 19 [SIMPLE_EDGE]
SHUFFLE [RS_67]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_14]
- <-Reducer 17 [SIMPLE_EDGE]
+ <-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_66]
PartitionCols:_col1
Merge Join Operator [MERGEJOIN_151] (rows=633595212 width=88)
Conds:RS_63._col0=RS_64._col0(Inner),Output:["_col1","_col2","_col3","_col4"]
- <-Map 16 [SIMPLE_EDGE]
+ <-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_64]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_11]
- <-Map 23 [SIMPLE_EDGE]
+ <-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col0
Select Operator [SEL_53] (rows=575995635 width=88)
@@ -264,17 +267,4 @@ Stage-0
predicate:(ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
TableScan [TS_51] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ext_sales_price"]
- <-Reducer 7 [SIMPLE_EDGE]
- SHUFFLE [RS_76]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_150] (rows=88000001 width=860)
- Conds:RS_73._col1=RS_74._col0(Inner),Output:["_col0"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_73]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_2]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_74]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_5]
[2/8] hive git commit: HIVE-18361: Extend shared work optimizer to
reuse computation beyond work boundaries (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query64.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query64.q.out b/ql/src/test/results/clientpositive/perf/tez/query64.q.out
index e009eaf..cd76cc6 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query64.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query64.q.out
@@ -237,47 +237,36 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 10 <- Reducer 19 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
+Reducer 10 <- Reducer 14 (SIMPLE_EDGE), Reducer 9 (SIMPLE_EDGE)
Reducer 11 <- Reducer 10 (SIMPLE_EDGE)
-Reducer 12 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
-Reducer 13 <- Map 20 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE)
-Reducer 14 <- Reducer 13 (SIMPLE_EDGE), Reducer 38 (SIMPLE_EDGE)
-Reducer 15 <- Map 41 (SIMPLE_EDGE), Reducer 14 (SIMPLE_EDGE)
-Reducer 16 <- Map 55 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE), Reducer 34 (SIMPLE_EDGE)
-Reducer 18 <- Map 55 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE)
-Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 20 (SIMPLE_EDGE)
-Reducer 21 <- Map 20 (SIMPLE_EDGE), Reducer 43 (SIMPLE_EDGE)
-Reducer 22 <- Map 46 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE)
-Reducer 23 <- Reducer 22 (SIMPLE_EDGE), Reducer 37 (SIMPLE_EDGE)
-Reducer 24 <- Reducer 23 (SIMPLE_EDGE), Reducer 49 (ONE_TO_ONE_EDGE)
-Reducer 25 <- Map 53 (SIMPLE_EDGE), Reducer 24 (SIMPLE_EDGE)
-Reducer 26 <- Map 41 (SIMPLE_EDGE), Reducer 25 (SIMPLE_EDGE)
-Reducer 27 <- Map 54 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE)
-Reducer 28 <- Map 20 (SIMPLE_EDGE), Reducer 44 (SIMPLE_EDGE)
-Reducer 29 <- Map 46 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE)
-Reducer 3 <- Map 20 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 30 <- Reducer 29 (SIMPLE_EDGE), Reducer 39 (SIMPLE_EDGE)
-Reducer 31 <- Reducer 30 (SIMPLE_EDGE), Reducer 51 (ONE_TO_ONE_EDGE)
-Reducer 32 <- Map 53 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE)
-Reducer 33 <- Map 41 (SIMPLE_EDGE), Reducer 32 (SIMPLE_EDGE)
-Reducer 34 <- Map 54 (SIMPLE_EDGE), Reducer 33 (SIMPLE_EDGE)
-Reducer 36 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE)
-Reducer 37 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE)
-Reducer 38 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE)
-Reducer 39 <- Map 35 (SIMPLE_EDGE), Map 40 (SIMPLE_EDGE)
-Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 36 (SIMPLE_EDGE)
-Reducer 43 <- Map 42 (SIMPLE_EDGE), Map 45 (SIMPLE_EDGE)
-Reducer 44 <- Map 42 (SIMPLE_EDGE), Map 45 (SIMPLE_EDGE)
-Reducer 48 <- Map 47 (SIMPLE_EDGE), Map 52 (SIMPLE_EDGE)
-Reducer 49 <- Reducer 48 (SIMPLE_EDGE)
-Reducer 5 <- Map 41 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
-Reducer 50 <- Map 47 (SIMPLE_EDGE), Map 52 (SIMPLE_EDGE)
-Reducer 51 <- Reducer 50 (SIMPLE_EDGE)
-Reducer 6 <- Map 55 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 27 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 55 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
+Reducer 12 <- Reducer 29 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+Reducer 13 <- Map 44 (SIMPLE_EDGE), Reducer 12 (SIMPLE_EDGE)
+Reducer 14 <- Reducer 13 (SIMPLE_EDGE)
+Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE)
+Reducer 17 <- Map 37 (SIMPLE_EDGE), Reducer 16 (SIMPLE_EDGE)
+Reducer 18 <- Reducer 17 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE)
+Reducer 19 <- Reducer 18 (SIMPLE_EDGE), Reducer 40 (ONE_TO_ONE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE)
+Reducer 20 <- Map 42 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
+Reducer 21 <- Map 33 (SIMPLE_EDGE), Reducer 20 (SIMPLE_EDGE)
+Reducer 22 <- Map 43 (SIMPLE_EDGE), Reducer 21 (SIMPLE_EDGE)
+Reducer 23 <- Map 15 (SIMPLE_EDGE), Reducer 35 (SIMPLE_EDGE)
+Reducer 24 <- Map 37 (SIMPLE_EDGE), Reducer 23 (SIMPLE_EDGE)
+Reducer 25 <- Reducer 24 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE)
+Reducer 26 <- Reducer 25 (SIMPLE_EDGE), Reducer 40 (ONE_TO_ONE_EDGE)
+Reducer 27 <- Map 42 (SIMPLE_EDGE), Reducer 26 (SIMPLE_EDGE)
+Reducer 28 <- Map 33 (SIMPLE_EDGE), Reducer 27 (SIMPLE_EDGE)
+Reducer 29 <- Map 43 (SIMPLE_EDGE), Reducer 28 (SIMPLE_EDGE)
+Reducer 3 <- Map 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 31 <- Map 30 (SIMPLE_EDGE), Map 32 (SIMPLE_EDGE)
+Reducer 35 <- Map 34 (SIMPLE_EDGE), Map 36 (SIMPLE_EDGE)
+Reducer 39 <- Map 38 (SIMPLE_EDGE), Map 41 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 3 (SIMPLE_EDGE), Reducer 31 (SIMPLE_EDGE)
+Reducer 40 <- Reducer 39 (SIMPLE_EDGE)
+Reducer 5 <- Map 33 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Map 44 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 22 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE)
+Reducer 8 <- Map 44 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)
Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
Stage-0
@@ -296,14 +285,14 @@ Stage-0
predicate:(_col30 <= _col13)
Merge Join Operator [MERGEJOIN_615] (rows=821691577 width=88)
Conds:RS_256._col2, _col1, _col3=RS_257._col2, _col1, _col3(Inner),Output:["_col0","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col13","_col14","_col15","_col16","_col30","_col31","_col32","_col33"]
- <-Reducer 19 [SIMPLE_EDGE]
+ <-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_257]
PartitionCols:_col2, _col1, _col3
Select Operator [SEL_254] (rows=746992327 width=88)
Output:["_col1","_col2","_col3","_col13","_col14","_col15","_col16"]
Group By Operator [GBY_253] (rows=746992327 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5, KEY._col6, KEY._col7, KEY._col8, KEY._col9, KEY._col10, KEY._col11, KEY._col12, KEY._col13
- <-Reducer 18 [SIMPLE_EDGE]
+ <-Reducer 13 [SIMPLE_EDGE]
SHUFFLE [RS_252]
PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Group By Operator [GBY_251] (rows=1493984654 width=88)
@@ -314,7 +303,7 @@ Stage-0
predicate:(_col56 <> _col19)
Merge Join Operator [MERGEJOIN_614] (rows=1493984654 width=88)
Conds:RS_246._col37=RS_247._col0(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col43","_col44","_col45","_col46","_col49","_col56"]
- <-Map 55 [SIMPLE_EDGE]
+ <-Map 44 [SIMPLE_EDGE]
SHUFFLE [RS_247]
PartitionCols:_col0
Select Operator [SEL_99] (rows=1861800 width=385)
@@ -323,27 +312,27 @@ Stage-0
predicate:cd_demo_sk is not null
TableScan [TS_97] (rows=1861800 width=385)
default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"]
- <-Reducer 17 [SIMPLE_EDGE]
+ <-Reducer 12 [SIMPLE_EDGE]
SHUFFLE [RS_246]
PartitionCols:_col37
Merge Join Operator [MERGEJOIN_613] (rows=1358167838 width=88)
Conds:RS_243._col0=RS_244._col16(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col37","_col43","_col44","_col45","_col46","_col49"]
- <-Reducer 16 [SIMPLE_EDGE]
+ <-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_243]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_602] (rows=128840811 width=860)
- Conds:RS_240._col1=RS_241._col0(Inner),Output:["_col0","_col7","_col9","_col14","_col15","_col16","_col17","_col19"]
- <-Map 55 [SIMPLE_EDGE]
- SHUFFLE [RS_241]
+ Merge Join Operator [MERGEJOIN_584] (rows=128840811 width=860)
+ Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col0","_col7","_col9","_col14","_col15","_col16","_col17","_col19"]
+ <-Map 44 [SIMPLE_EDGE]
+ SHUFFLE [RS_113]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_99]
- <-Reducer 15 [SIMPLE_EDGE]
- SHUFFLE [RS_240]
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_112]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_601] (rows=117128008 width=860)
- Conds:RS_237._col3=RS_238._col0(Inner),Output:["_col0","_col1","_col7","_col9","_col14","_col15","_col16","_col17"]
- <-Map 41 [SIMPLE_EDGE]
- SHUFFLE [RS_238]
+ Merge Join Operator [MERGEJOIN_583] (rows=117128008 width=860)
+ Conds:RS_109._col3=RS_110._col0(Inner),Output:["_col0","_col1","_col7","_col9","_col14","_col15","_col16","_col17"]
+ <-Map 33 [SIMPLE_EDGE]
+ SHUFFLE [RS_110]
PartitionCols:_col0
Select Operator [SEL_21] (rows=40000000 width=1014)
Output:["_col0","_col1","_col2","_col3","_col4"]
@@ -351,40 +340,59 @@ Stage-0
predicate:ca_address_sk is not null
TableScan [TS_19] (rows=40000000 width=1014)
default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"]
- <-Reducer 14 [SIMPLE_EDGE]
- SHUFFLE [RS_237]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_109]
PartitionCols:_col3
- Merge Join Operator [MERGEJOIN_600] (rows=106480005 width=860)
- Conds:RS_234._col2=RS_235._col0(Inner),Output:["_col0","_col1","_col3","_col7","_col9"]
- <-Reducer 13 [SIMPLE_EDGE]
- SHUFFLE [RS_234]
+ Merge Join Operator [MERGEJOIN_582] (rows=106480005 width=860)
+ Conds:RS_106._col2=RS_107._col0(Inner),Output:["_col0","_col1","_col3","_col7","_col9"]
+ <-Reducer 31 [SIMPLE_EDGE]
+ SHUFFLE [RS_107]
+ PartitionCols:_col0
+ Merge Join Operator [MERGEJOIN_581] (rows=7920 width=107)
+ Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0"]
+ <-Map 30 [SIMPLE_EDGE]
+ SHUFFLE [RS_15]
+ PartitionCols:_col1
+ Select Operator [SEL_11] (rows=7200 width=107)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_544] (rows=7200 width=107)
+ predicate:(hd_demo_sk is not null and hd_income_band_sk is not null)
+ TableScan [TS_9] (rows=7200 width=107)
+ default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"]
+ <-Map 32 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
+ PartitionCols:_col0
+ Select Operator [SEL_14] (rows=20 width=12)
+ Output:["_col0"]
+ Filter Operator [FIL_545] (rows=20 width=12)
+ predicate:ib_income_band_sk is not null
+ TableScan [TS_12] (rows=20 width=12)
+ default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_106]
PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_598] (rows=96800003 width=860)
- Conds:RS_231._col4=RS_232._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_232]
+ Merge Join Operator [MERGEJOIN_580] (rows=96800003 width=860)
+ Conds:RS_103._col4=RS_104._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"]
+ <-Map 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_104]
PartitionCols:_col0
- Select Operator [SEL_136] (rows=73049 width=1119)
+ Select Operator [SEL_5] (rows=73049 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_562] (rows=73049 width=1119)
+ Filter Operator [FIL_542] (rows=73049 width=1119)
predicate:d_date_sk is not null
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
- <-Reducer 12 [SIMPLE_EDGE]
- SHUFFLE [RS_231]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_103]
PartitionCols:_col4
- Merge Join Operator [MERGEJOIN_597] (rows=88000001 width=860)
- Conds:RS_228._col5=RS_229._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_229]
+ Merge Join Operator [MERGEJOIN_579] (rows=88000001 width=860)
+ Conds:RS_100._col5=RS_101._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"]
+ <-Map 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_101]
PartitionCols:_col0
- Select Operator [SEL_133] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_561] (rows=73049 width=1119)
- predicate:d_date_sk is not null
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_228]
+ SHUFFLE [RS_100]
PartitionCols:_col5
Select Operator [SEL_2] (rows=80000000 width=860)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
@@ -392,37 +400,14 @@ Stage-0
predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_customer_sk is not null and c_first_sales_date_sk is not null and c_first_shipto_date_sk is not null)
TableScan [TS_0] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"]
- <-Reducer 38 [SIMPLE_EDGE]
- SHUFFLE [RS_235]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_599] (rows=7920 width=107)
- Conds:RS_143._col1=RS_144._col0(Inner),Output:["_col0"]
- <-Map 35 [SIMPLE_EDGE]
- SHUFFLE [RS_143]
- PartitionCols:_col1
- Select Operator [SEL_11] (rows=7200 width=107)
- Output:["_col0","_col1"]
- Filter Operator [FIL_544] (rows=7200 width=107)
- predicate:(hd_demo_sk is not null and hd_income_band_sk is not null)
- TableScan [TS_9] (rows=7200 width=107)
- default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"]
- <-Map 40 [SIMPLE_EDGE]
- SHUFFLE [RS_144]
- PartitionCols:_col0
- Select Operator [SEL_14] (rows=20 width=12)
- Output:["_col0"]
- Filter Operator [FIL_545] (rows=20 width=12)
- predicate:ib_income_band_sk is not null
- TableScan [TS_12] (rows=20 width=12)
- default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"]
- <-Reducer 34 [SIMPLE_EDGE]
+ <-Reducer 29 [SIMPLE_EDGE]
SHUFFLE [RS_244]
PartitionCols:_col16
Select Operator [SEL_224] (rows=1234698008 width=88)
Output:["_col3","_col4","_col5","_col6","_col8","_col9","_col16","_col17","_col23","_col24","_col25","_col26","_col29"]
Merge Join Operator [MERGEJOIN_612] (rows=1234698008 width=88)
Conds:RS_221._col5, _col12=RS_222._col0, _col1(Inner),Output:["_col6","_col7","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"]
- <-Map 54 [SIMPLE_EDGE]
+ <-Map 43 [SIMPLE_EDGE]
SHUFFLE [RS_222]
PartitionCols:_col0, _col1
Select Operator [SEL_77] (rows=57591150 width=77)
@@ -431,21 +416,21 @@ Stage-0
predicate:(sr_item_sk is not null and sr_ticket_number is not null)
TableScan [TS_75] (rows=57591150 width=77)
default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"]
- <-Reducer 33 [SIMPLE_EDGE]
+ <-Reducer 28 [SIMPLE_EDGE]
SHUFFLE [RS_221]
PartitionCols:_col5, _col12
Merge Join Operator [MERGEJOIN_611] (rows=1122452711 width=88)
Conds:RS_218._col9=RS_219._col0(Inner),Output:["_col5","_col6","_col7","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"]
- <-Map 41 [SIMPLE_EDGE]
+ <-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_219]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_21]
- <-Reducer 32 [SIMPLE_EDGE]
+ <-Reducer 27 [SIMPLE_EDGE]
SHUFFLE [RS_218]
PartitionCols:_col9
Merge Join Operator [MERGEJOIN_610] (rows=1020411534 width=88)
Conds:RS_215._col10=RS_216._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27"]
- <-Map 53 [SIMPLE_EDGE]
+ <-Map 42 [SIMPLE_EDGE]
SHUFFLE [RS_216]
PartitionCols:_col0
Select Operator [SEL_71] (rows=1704 width=1910)
@@ -454,24 +439,64 @@ Stage-0
predicate:(s_store_name is not null and s_store_sk is not null and s_zip is not null)
TableScan [TS_69] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"]
- <-Reducer 31 [SIMPLE_EDGE]
+ <-Reducer 26 [SIMPLE_EDGE]
SHUFFLE [RS_215]
PartitionCols:_col10
Merge Join Operator [MERGEJOIN_609] (rows=927646829 width=88)
Conds:RS_212._col5=RS_213._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"]
- <-Reducer 30 [SIMPLE_EDGE]
+ <-Reducer 40 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_213]
+ PartitionCols:_col0
+ Select Operator [SEL_68] (rows=52798137 width=135)
+ Output:["_col0"]
+ Filter Operator [FIL_67] (rows=52798137 width=135)
+ predicate:(_col1 > (2 * _col2))
+ Group By Operator [GBY_66] (rows=158394413 width=135)
+ Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
+ <-Reducer 39 [SIMPLE_EDGE]
+ SHUFFLE [RS_65]
+ PartitionCols:_col0
+ Group By Operator [GBY_64] (rows=316788826 width=135)
+ Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0
+ Select Operator [SEL_62] (rows=316788826 width=135)
+ Output:["_col0","_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_589] (rows=316788826 width=135)
+ Conds:RS_59._col0, _col1=RS_60._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"]
+ <-Map 38 [SIMPLE_EDGE]
+ SHUFFLE [RS_59]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_55] (rows=287989836 width=135)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_554] (rows=287989836 width=135)
+ predicate:(cs_item_sk is not null and cs_order_number is not null)
+ TableScan [TS_53] (rows=287989836 width=135)
+ default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"]
+ <-Map 41 [SIMPLE_EDGE]
+ SHUFFLE [RS_60]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_58] (rows=28798881 width=106)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ Filter Operator [FIL_555] (rows=28798881 width=106)
+ predicate:(cr_item_sk is not null and cr_order_number is not null)
+ TableScan [TS_56] (rows=28798881 width=106)
+ default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"]
+ <-Reducer 25 [SIMPLE_EDGE]
SHUFFLE [RS_212]
PartitionCols:_col5
Merge Join Operator [MERGEJOIN_608] (rows=843315281 width=88)
Conds:RS_209._col0=RS_210._col5(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"]
- <-Reducer 29 [SIMPLE_EDGE]
+ <-Reducer 31 [SIMPLE_EDGE]
+ SHUFFLE [RS_209]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_581]
+ <-Reducer 24 [SIMPLE_EDGE]
SHUFFLE [RS_210]
PartitionCols:_col5
Select Operator [SEL_180] (rows=766650239 width=88)
Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col13","_col16"]
Merge Join Operator [MERGEJOIN_606] (rows=766650239 width=88)
Conds:RS_177._col7=RS_178._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col8","_col9","_col10","_col11","_col12","_col15"]
- <-Map 46 [SIMPLE_EDGE]
+ <-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_178]
PartitionCols:_col0
Select Operator [SEL_42] (rows=2300 width=1179)
@@ -480,12 +505,12 @@ Stage-0
predicate:p_promo_sk is not null
TableScan [TS_40] (rows=2300 width=1179)
default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"]
- <-Reducer 28 [SIMPLE_EDGE]
+ <-Reducer 23 [SIMPLE_EDGE]
SHUFFLE [RS_177]
PartitionCols:_col7
Merge Join Operator [MERGEJOIN_605] (rows=696954748 width=88)
Conds:RS_174._col0=RS_175._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"]
- <-Map 20 [SIMPLE_EDGE]
+ <-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_175]
PartitionCols:_col0
Select Operator [SEL_167] (rows=36524 width=1119)
@@ -493,13 +518,13 @@ Stage-0
Filter Operator [FIL_571] (rows=36524 width=1119)
predicate:((d_year = 2001) and d_date_sk is not null)
Please refer to the previous TableScan [TS_3]
- <-Reducer 44 [SIMPLE_EDGE]
+ <-Reducer 35 [SIMPLE_EDGE]
SHUFFLE [RS_174]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_604] (rows=633595212 width=88)
- Conds:RS_171._col1=RS_172._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"]
- <-Map 42 [SIMPLE_EDGE]
- SHUFFLE [RS_171]
+ Merge Join Operator [MERGEJOIN_586] (rows=633595212 width=88)
+ Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"]
+ <-Map 34 [SIMPLE_EDGE]
+ SHUFFLE [RS_43]
PartitionCols:_col1
Select Operator [SEL_33] (rows=575995635 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
@@ -507,8 +532,8 @@ Stage-0
predicate:(ss_addr_sk is not null and ss_cdemo_sk is not null and ss_customer_sk is not null and ss_hdemo_sk is not null and ss_item_sk is not null and ss_promo_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null)
TableScan [TS_31] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
- <-Map 45 [SIMPLE_EDGE]
- SHUFFLE [RS_172]
+ <-Map 36 [SIMPLE_EDGE]
+ SHUFFLE [RS_44]
PartitionCols:_col0
Select Operator [SEL_36] (rows=2851 width=1436)
Output:["_col0","_col3"]
@@ -516,55 +541,6 @@ Stage-0
predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null)
TableScan [TS_34] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"]
- <-Reducer 39 [SIMPLE_EDGE]
- SHUFFLE [RS_209]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_603] (rows=7920 width=107)
- Conds:RS_206._col1=RS_207._col0(Inner),Output:["_col0"]
- <-Map 35 [SIMPLE_EDGE]
- SHUFFLE [RS_206]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_11]
- <-Map 40 [SIMPLE_EDGE]
- SHUFFLE [RS_207]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_14]
- <-Reducer 51 [ONE_TO_ONE_EDGE]
- FORWARD [RS_213]
- PartitionCols:_col0
- Select Operator [SEL_196] (rows=52798137 width=135)
- Output:["_col0"]
- Filter Operator [FIL_195] (rows=52798137 width=135)
- predicate:(_col1 > (2 * _col2))
- Group By Operator [GBY_194] (rows=158394413 width=135)
- Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
- <-Reducer 50 [SIMPLE_EDGE]
- SHUFFLE [RS_193]
- PartitionCols:_col0
- Group By Operator [GBY_192] (rows=316788826 width=135)
- Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0
- Select Operator [SEL_190] (rows=316788826 width=135)
- Output:["_col0","_col1","_col2"]
- Merge Join Operator [MERGEJOIN_607] (rows=316788826 width=135)
- Conds:RS_187._col0, _col1=RS_188._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"]
- <-Map 47 [SIMPLE_EDGE]
- SHUFFLE [RS_187]
- PartitionCols:_col0, _col1
- Select Operator [SEL_55] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_554] (rows=287989836 width=135)
- predicate:(cs_item_sk is not null and cs_order_number is not null)
- TableScan [TS_53] (rows=287989836 width=135)
- default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"]
- <-Map 52 [SIMPLE_EDGE]
- SHUFFLE [RS_188]
- PartitionCols:_col0, _col1
- Select Operator [SEL_58] (rows=28798881 width=106)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_555] (rows=28798881 width=106)
- predicate:(cr_item_sk is not null and cr_order_number is not null)
- TableScan [TS_56] (rows=28798881 width=106)
- default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"]
<-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_256]
PartitionCols:_col2, _col1, _col3
@@ -583,7 +559,7 @@ Stage-0
predicate:(_col56 <> _col19)
Merge Join Operator [MERGEJOIN_596] (rows=1493984654 width=88)
Conds:RS_118._col37=RS_119._col0(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col43","_col44","_col45","_col46","_col49","_col56"]
- <-Map 55 [SIMPLE_EDGE]
+ <-Map 44 [SIMPLE_EDGE]
SHUFFLE [RS_119]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_99]
@@ -592,62 +568,74 @@ Stage-0
PartitionCols:_col37
Merge Join Operator [MERGEJOIN_595] (rows=1358167838 width=88)
Conds:RS_115._col0=RS_116._col16(Inner),Output:["_col7","_col9","_col14","_col15","_col16","_col17","_col19","_col23","_col24","_col25","_col26","_col28","_col29","_col37","_col43","_col44","_col45","_col46","_col49"]
- <-Reducer 27 [SIMPLE_EDGE]
+ <-Reducer 6 [SIMPLE_EDGE]
+ SHUFFLE [RS_115]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_584]
+ <-Reducer 22 [SIMPLE_EDGE]
SHUFFLE [RS_116]
PartitionCols:_col16
Select Operator [SEL_96] (rows=1234698008 width=88)
Output:["_col3","_col4","_col5","_col6","_col8","_col9","_col16","_col17","_col23","_col24","_col25","_col26","_col29"]
Merge Join Operator [MERGEJOIN_594] (rows=1234698008 width=88)
Conds:RS_93._col5, _col12=RS_94._col0, _col1(Inner),Output:["_col6","_col7","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"]
- <-Map 54 [SIMPLE_EDGE]
+ <-Map 43 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0, _col1
Please refer to the previous Select Operator [SEL_77]
- <-Reducer 26 [SIMPLE_EDGE]
+ <-Reducer 21 [SIMPLE_EDGE]
SHUFFLE [RS_93]
PartitionCols:_col5, _col12
Merge Join Operator [MERGEJOIN_593] (rows=1122452711 width=88)
Conds:RS_90._col9=RS_91._col0(Inner),Output:["_col5","_col6","_col7","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27","_col29","_col30","_col31","_col32"]
- <-Map 41 [SIMPLE_EDGE]
+ <-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_91]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_21]
- <-Reducer 25 [SIMPLE_EDGE]
+ <-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_90]
PartitionCols:_col9
Merge Join Operator [MERGEJOIN_592] (rows=1020411534 width=88)
Conds:RS_87._col10=RS_88._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col12","_col13","_col14","_col15","_col16","_col19","_col26","_col27"]
- <-Map 53 [SIMPLE_EDGE]
+ <-Map 42 [SIMPLE_EDGE]
SHUFFLE [RS_88]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_71]
- <-Reducer 24 [SIMPLE_EDGE]
+ <-Reducer 19 [SIMPLE_EDGE]
SHUFFLE [RS_87]
PartitionCols:_col10
Merge Join Operator [MERGEJOIN_591] (rows=927646829 width=88)
Conds:RS_84._col5=RS_85._col0(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"]
- <-Reducer 23 [SIMPLE_EDGE]
+ <-Reducer 40 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_85]
+ PartitionCols:_col0
+ Please refer to the previous Select Operator [SEL_68]
+ <-Reducer 18 [SIMPLE_EDGE]
SHUFFLE [RS_84]
PartitionCols:_col5
Merge Join Operator [MERGEJOIN_590] (rows=843315281 width=88)
Conds:RS_81._col0=RS_82._col5(Inner),Output:["_col5","_col6","_col7","_col9","_col10","_col12","_col13","_col14","_col15","_col16","_col19"]
- <-Reducer 22 [SIMPLE_EDGE]
+ <-Reducer 31 [SIMPLE_EDGE]
+ SHUFFLE [RS_81]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_581]
+ <-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col5
Select Operator [SEL_52] (rows=766650239 width=88)
Output:["_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10","_col11","_col12","_col13","_col16"]
Merge Join Operator [MERGEJOIN_588] (rows=766650239 width=88)
Conds:RS_49._col7=RS_50._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col8","_col9","_col10","_col11","_col12","_col15"]
- <-Map 46 [SIMPLE_EDGE]
+ <-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_50]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_42]
- <-Reducer 21 [SIMPLE_EDGE]
+ <-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_49]
PartitionCols:_col7
Merge Join Operator [MERGEJOIN_587] (rows=696954748 width=88)
Conds:RS_46._col0=RS_47._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"]
- <-Map 20 [SIMPLE_EDGE]
+ <-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_47]
PartitionCols:_col0
Select Operator [SEL_39] (rows=36524 width=1119)
@@ -655,118 +643,8 @@ Stage-0
Filter Operator [FIL_552] (rows=36524 width=1119)
predicate:((d_year = 2000) and d_date_sk is not null)
Please refer to the previous TableScan [TS_3]
- <-Reducer 43 [SIMPLE_EDGE]
+ <-Reducer 35 [SIMPLE_EDGE]
SHUFFLE [RS_46]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_586] (rows=633595212 width=88)
- Conds:RS_43._col1=RS_44._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col15"]
- <-Map 42 [SIMPLE_EDGE]
- SHUFFLE [RS_43]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_33]
- <-Map 45 [SIMPLE_EDGE]
- SHUFFLE [RS_44]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_36]
- <-Reducer 37 [SIMPLE_EDGE]
- SHUFFLE [RS_81]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_585] (rows=7920 width=107)
- Conds:RS_78._col1=RS_79._col0(Inner),Output:["_col0"]
- <-Map 35 [SIMPLE_EDGE]
- SHUFFLE [RS_78]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_11]
- <-Map 40 [SIMPLE_EDGE]
- SHUFFLE [RS_79]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_14]
- <-Reducer 49 [ONE_TO_ONE_EDGE]
- FORWARD [RS_85]
- PartitionCols:_col0
- Select Operator [SEL_68] (rows=52798137 width=135)
- Output:["_col0"]
- Filter Operator [FIL_67] (rows=52798137 width=135)
- predicate:(_col1 > (2 * _col2))
- Group By Operator [GBY_66] (rows=158394413 width=135)
- Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
- <-Reducer 48 [SIMPLE_EDGE]
- SHUFFLE [RS_65]
- PartitionCols:_col0
- Group By Operator [GBY_64] (rows=316788826 width=135)
- Output:["_col0","_col1","_col2"],aggregations:["sum(_col1)","sum(_col2)"],keys:_col0
- Select Operator [SEL_62] (rows=316788826 width=135)
- Output:["_col0","_col1","_col2"]
- Merge Join Operator [MERGEJOIN_589] (rows=316788826 width=135)
- Conds:RS_59._col0, _col1=RS_60._col0, _col1(Inner),Output:["_col0","_col2","_col5","_col6","_col7"]
- <-Map 47 [SIMPLE_EDGE]
- SHUFFLE [RS_59]
- PartitionCols:_col0, _col1
- Please refer to the previous Select Operator [SEL_55]
- <-Map 52 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col0, _col1
- Please refer to the previous Select Operator [SEL_58]
- <-Reducer 6 [SIMPLE_EDGE]
- SHUFFLE [RS_115]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_584] (rows=128840811 width=860)
- Conds:RS_112._col1=RS_113._col0(Inner),Output:["_col0","_col7","_col9","_col14","_col15","_col16","_col17","_col19"]
- <-Map 55 [SIMPLE_EDGE]
- SHUFFLE [RS_113]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_99]
- <-Reducer 5 [SIMPLE_EDGE]
- SHUFFLE [RS_112]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_583] (rows=117128008 width=860)
- Conds:RS_109._col3=RS_110._col0(Inner),Output:["_col0","_col1","_col7","_col9","_col14","_col15","_col16","_col17"]
- <-Map 41 [SIMPLE_EDGE]
- SHUFFLE [RS_110]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_21]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_109]
- PartitionCols:_col3
- Merge Join Operator [MERGEJOIN_582] (rows=106480005 width=860)
- Conds:RS_106._col2=RS_107._col0(Inner),Output:["_col0","_col1","_col3","_col7","_col9"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_106]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_580] (rows=96800003 width=860)
- Conds:RS_103._col4=RS_104._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col7","_col9"]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_104]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_542] (rows=73049 width=1119)
- predicate:d_date_sk is not null
- Please refer to the previous TableScan [TS_3]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_103]
- PartitionCols:_col4
- Merge Join Operator [MERGEJOIN_579] (rows=88000001 width=860)
- Conds:RS_100._col5=RS_101._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col7"]
- <-Map 20 [SIMPLE_EDGE]
- SHUFFLE [RS_101]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_5]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_100]
- PartitionCols:_col5
- Please refer to the previous Select Operator [SEL_2]
- <-Reducer 36 [SIMPLE_EDGE]
- SHUFFLE [RS_107]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_581] (rows=7920 width=107)
- Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col0"]
- <-Map 35 [SIMPLE_EDGE]
- SHUFFLE [RS_15]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_11]
- <-Map 40 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_14]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_586]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query70.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query70.q.out b/ql/src/test/results/clientpositive/perf/tez/query70.q.out
index 0f6891e..d5ad6e5 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query70.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query70.q.out
@@ -76,15 +76,14 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Reducer 10 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
-Reducer 3 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 11 (SIMPLE_EDGE)
+Reducer 3 <- Map 12 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 10 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Map 1 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
-Reducer 9 <- Map 14 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Map 13 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -120,7 +119,7 @@ Stage-0
Output:["_col0","_col1","_col2"]
Merge Join Operator [MERGEJOIN_89] (rows=766650239 width=88)
Conds:RS_43._col7=RS_44._col0(Inner),Output:["_col2","_col6","_col7"]
- <-Reducer 11 [SIMPLE_EDGE]
+ <-Reducer 10 [SIMPLE_EDGE]
SHUFFLE [RS_44]
PartitionCols:_col0
Select Operator [SEL_32] (rows=116159124 width=88)
@@ -131,34 +130,25 @@ Stage-0
Function definitions:[{},{"name:":"windowingtablefunction","order by:":"_col1 DESC NULLS LAST","partition by:":"_col0"}]
Select Operator [SEL_30] (rows=348477374 width=88)
Output:["_col0","_col1"]
- <-Reducer 10 [SIMPLE_EDGE]
+ <-Reducer 9 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
Group By Operator [GBY_27] (rows=348477374 width=88)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
- <-Reducer 9 [SIMPLE_EDGE]
+ <-Reducer 8 [SIMPLE_EDGE]
SHUFFLE [RS_26]
PartitionCols:_col0
Group By Operator [GBY_25] (rows=696954748 width=88)
Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col6
Merge Join Operator [MERGEJOIN_88] (rows=696954748 width=88)
Conds:RS_21._col1=RS_22._col0(Inner),Output:["_col2","_col6"]
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_22]
- PartitionCols:_col0
- Select Operator [SEL_17] (rows=1704 width=1910)
- Output:["_col0","_col1"]
- Filter Operator [FIL_84] (rows=1704 width=1910)
- predicate:(s_state is not null and s_store_sk is not null)
- TableScan [TS_15] (rows=1704 width=1910)
- default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"]
- <-Reducer 8 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_21]
PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_87] (rows=633595212 width=88)
- Conds:RS_18._col0=RS_19._col0(Inner),Output:["_col1","_col2"]
+ Merge Join Operator [MERGEJOIN_85] (rows=633595212 width=88)
+ Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2"]
<-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_18]
+ SHUFFLE [RS_37]
PartitionCols:_col0
Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
@@ -166,8 +156,8 @@ Stage-0
predicate:(ss_sold_date_sk is not null and ss_store_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_19]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_38]
PartitionCols:_col0
Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0"]
@@ -175,12 +165,25 @@ Stage-0
predicate:(d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"]
+ <-Map 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_22]
+ PartitionCols:_col0
+ Select Operator [SEL_17] (rows=1704 width=1910)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_84] (rows=1704 width=1910)
+ predicate:(s_state is not null and s_store_sk is not null)
+ TableScan [TS_15] (rows=1704 width=1910)
+ default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_state"]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_43]
PartitionCols:_col7
Merge Join Operator [MERGEJOIN_86] (rows=696954748 width=88)
Conds:RS_40._col1=RS_41._col0(Inner),Output:["_col2","_col6","_col7"]
- <-Map 13 [SIMPLE_EDGE]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_40]
+ PartitionCols:_col1
+ Please refer to the previous Merge Join Operator [MERGEJOIN_85]
+ <-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_41]
PartitionCols:_col0
Select Operator [SEL_8] (rows=1704 width=1910)
@@ -189,17 +192,4 @@ Stage-0
predicate:(s_state is not null and s_store_sk is not null)
TableScan [TS_6] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_county","s_state"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_40]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_85] (rows=633595212 width=88)
- Conds:RS_37._col0=RS_38._col0(Inner),Output:["_col1","_col2"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_37]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_2]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_38]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/96a409e1/ql/src/test/results/clientpositive/perf/tez/query83.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query83.q.out b/ql/src/test/results/clientpositive/perf/tez/query83.q.out
index 2a3a946..9b4e7e4 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query83.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query83.q.out
@@ -134,25 +134,19 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 10 <- Reducer 9 (SIMPLE_EDGE)
-Reducer 11 <- Map 27 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 17 (SIMPLE_EDGE)
+Reducer 11 <- Map 21 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+Reducer 12 <- Reducer 11 (SIMPLE_EDGE), Reducer 15 (SIMPLE_EDGE)
Reducer 13 <- Reducer 12 (SIMPLE_EDGE)
-Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 20 (ONE_TO_ONE_EDGE)
-Reducer 16 <- Map 14 (SIMPLE_EDGE), Reducer 22 (ONE_TO_ONE_EDGE)
-Reducer 17 <- Map 14 (SIMPLE_EDGE), Reducer 24 (ONE_TO_ONE_EDGE)
-Reducer 19 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE)
+Reducer 15 <- Map 14 (SIMPLE_EDGE), Reducer 18 (ONE_TO_ONE_EDGE)
+Reducer 17 <- Map 16 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE)
+Reducer 18 <- Reducer 17 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 20 <- Reducer 19 (SIMPLE_EDGE)
-Reducer 21 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE)
-Reducer 22 <- Reducer 21 (SIMPLE_EDGE)
-Reducer 23 <- Map 18 (SIMPLE_EDGE), Map 25 (SIMPLE_EDGE)
-Reducer 24 <- Reducer 23 (SIMPLE_EDGE)
Reducer 3 <- Reducer 15 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
Reducer 5 <- Reducer 10 (ONE_TO_ONE_EDGE), Reducer 13 (ONE_TO_ONE_EDGE), Reducer 4 (ONE_TO_ONE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
-Reducer 8 <- Map 26 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
-Reducer 9 <- Reducer 16 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+Reducer 8 <- Map 20 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE)
+Reducer 9 <- Reducer 15 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
Stage-0
Fetch Operator
@@ -182,13 +176,13 @@ Stage-0
Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4
Merge Join Operator [MERGEJOIN_220] (rows=69685294 width=77)
Conds:RS_72._col0=RS_73._col0(Inner),Output:["_col2","_col4"]
- <-Reducer 16 [SIMPLE_EDGE]
+ <-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_73]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_215] (rows=80353 width=1119)
- Conds:RS_65._col1=RS_66._col0(Inner),Output:["_col0"]
+ Merge Join Operator [MERGEJOIN_212] (rows=80353 width=1119)
+ Conds:RS_25._col1=RS_26._col0(Inner),Output:["_col0"]
<-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_65]
+ SHUFFLE [RS_25]
PartitionCols:_col1
Select Operator [SEL_8] (rows=73049 width=1119)
Output:["_col0","_col1"]
@@ -196,20 +190,20 @@ Stage-0
predicate:(d_date is not null and d_date_sk is not null)
TableScan [TS_6] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
- <-Reducer 22 [ONE_TO_ONE_EDGE]
- FORWARD [RS_66]
+ <-Reducer 18 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_26]
PartitionCols:_col0
- Group By Operator [GBY_63] (rows=40176 width=1119)
+ Group By Operator [GBY_23] (rows=40176 width=1119)
Output:["_col0"],keys:KEY._col0
- <-Reducer 21 [SIMPLE_EDGE]
- SHUFFLE [RS_62]
+ <-Reducer 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_22]
PartitionCols:_col0
- Group By Operator [GBY_61] (rows=80353 width=1119)
+ Group By Operator [GBY_21] (rows=80353 width=1119)
Output:["_col0"],keys:_col0
- Merge Join Operator [MERGEJOIN_214] (rows=80353 width=1119)
- Conds:RS_57._col1=RS_58._col0(Left Semi),Output:["_col0"]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
+ Merge Join Operator [MERGEJOIN_211] (rows=80353 width=1119)
+ Conds:RS_17._col1=RS_18._col0(Left Semi),Output:["_col0"]
+ <-Map 16 [SIMPLE_EDGE]
+ SHUFFLE [RS_17]
PartitionCols:_col1
Select Operator [SEL_11] (rows=73049 width=1119)
Output:["_col0","_col1"]
@@ -217,8 +211,8 @@ Stage-0
predicate:(d_date is not null and d_week_seq is not null)
TableScan [TS_9] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date","d_week_seq"]
- <-Map 25 [SIMPLE_EDGE]
- SHUFFLE [RS_58]
+ <-Map 19 [SIMPLE_EDGE]
+ SHUFFLE [RS_18]
PartitionCols:_col0
Group By Operator [GBY_16] (rows=36525 width=1119)
Output:["_col0"],keys:_col0
@@ -242,7 +236,7 @@ Stage-0
predicate:(i_item_id is not null and i_item_sk is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
- <-Map 26 [SIMPLE_EDGE]
+ <-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_69]
PartitionCols:_col1
Select Operator [SEL_42] (rows=57591150 width=77)
@@ -263,6 +257,10 @@ Stage-0
Output:["_col0","_col1"],aggregations:["sum(_col2)"],keys:_col4
Merge Join Operator [MERGEJOIN_221] (rows=17422145 width=92)
Conds:RS_112._col0=RS_113._col0(Inner),Output:["_col2","_col4"]
+ <-Reducer 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_113]
+ PartitionCols:_col0
+ Please refer to the previous Merge Join Operator [MERGEJOIN_212]
<-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_112]
PartitionCols:_col0
@@ -272,7 +270,7 @@ Stage-0
SHUFFLE [RS_110]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_5]
- <-Map 27 [SIMPLE_EDGE]
+ <-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_109]
PartitionCols:_col1
Select Operator [SEL_82] (rows=14398467 width=92)
@@ -281,35 +279,6 @@ Stage-0
predicate:(wr_item_sk is not null and wr_returned_date_sk is not null)
TableScan [TS_80] (rows=14398467 width=92)
default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_returned_date_sk","wr_item_sk","wr_return_quantity"]
- <-Reducer 17 [SIMPLE_EDGE]
- SHUFFLE [RS_113]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_218] (rows=80353 width=1119)
- Conds:RS_105._col1=RS_106._col0(Inner),Output:["_col0"]
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_105]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_8]
- <-Reducer 24 [ONE_TO_ONE_EDGE]
- FORWARD [RS_106]
- PartitionCols:_col0
- Group By Operator [GBY_103] (rows=40176 width=1119)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 23 [SIMPLE_EDGE]
- SHUFFLE [RS_102]
- PartitionCols:_col0
- Group By Operator [GBY_101] (rows=80353 width=1119)
- Output:["_col0"],keys:_col0
- Merge Join Operator [MERGEJOIN_217] (rows=80353 width=1119)
- Conds:RS_97._col1=RS_98._col0(Left Semi),Output:["_col0"]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_97]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_11]
- <-Map 25 [SIMPLE_EDGE]
- SHUFFLE [RS_98]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_16]
<-Reducer 4 [ONE_TO_ONE_EDGE]
FORWARD [RS_120]
PartitionCols:_col0
@@ -325,32 +294,7 @@ Stage-0
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_33]
PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_212] (rows=80353 width=1119)
- Conds:RS_25._col1=RS_26._col0(Inner),Output:["_col0"]
- <-Map 14 [SIMPLE_EDGE]
- SHUFFLE [RS_25]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_8]
- <-Reducer 20 [ONE_TO_ONE_EDGE]
- FORWARD [RS_26]
- PartitionCols:_col0
- Group By Operator [GBY_23] (rows=40176 width=1119)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 19 [SIMPLE_EDGE]
- SHUFFLE [RS_22]
- PartitionCols:_col0
- Group By Operator [GBY_21] (rows=80353 width=1119)
- Output:["_col0"],keys:_col0
- Merge Join Operator [MERGEJOIN_211] (rows=80353 width=1119)
- Conds:RS_17._col1=RS_18._col0(Left Semi),Output:["_col0"]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_17]
- PartitionCols:_col1
- Please refer to the previous Select Operator [SEL_11]
- <-Map 25 [SIMPLE_EDGE]
- SHUFFLE [RS_18]
- PartitionCols:_col0
- Please refer to the previous Group By Operator [GBY_16]
+ Please refer to the previous Merge Join Operator [MERGEJOIN_212]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_32]
PartitionCols:_col0