You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2017/06/22 15:12:53 UTC
[1/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 71f52d8ad -> 7819cd366
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query76.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query76.q.out b/ql/src/test/results/clientpositive/perf/query76.q.out
index e7fa6df..de59732 100644
--- a/ql/src/test/results/clientpositive/perf/query76.q.out
+++ b/ql/src/test/results/clientpositive/perf/query76.q.out
@@ -97,9 +97,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_43]
PartitionCols:_col0
- Select Operator [SEL_38] (rows=462000 width=1436)
+ Select Operator [SEL_2] (rows=462000 width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_81] (rows=462000 width=1436)
+ Filter Operator [FIL_74] (rows=462000 width=1436)
predicate:i_item_sk is not null
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_category"]
@@ -138,11 +138,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_74] (rows=462000 width=1436)
- predicate:i_item_sk is not null
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col1
@@ -178,11 +174,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_26]
PartitionCols:_col0
- Select Operator [SEL_21] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_78] (rows=462000 width=1436)
- predicate:i_item_sk is not null
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_25]
PartitionCols:_col1
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query77.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query77.q.out b/ql/src/test/results/clientpositive/perf/query77.q.out
index 04aa84b..db42b18 100644
--- a/ql/src/test/results/clientpositive/perf/query77.q.out
+++ b/ql/src/test/results/clientpositive/perf/query77.q.out
@@ -276,9 +276,9 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_51]
PartitionCols:_col0
- Select Operator [SEL_49] (rows=8116 width=1119)
+ Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_168] (rows=8116 width=1119)
+ Filter Operator [FIL_162] (rows=8116 width=1119)
predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
@@ -304,11 +304,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_65]
PartitionCols:_col0
- Select Operator [SEL_63] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_170] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 28 [SIMPLE_EDGE]
SHUFFLE [RS_64]
PartitionCols:_col0
@@ -356,11 +352,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_87]
PartitionCols:_col0
- Select Operator [SEL_82] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_172] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 29 [SIMPLE_EDGE]
SHUFFLE [RS_86]
PartitionCols:_col0
@@ -385,11 +377,7 @@ Stage-0
<-Map 30 [SIMPLE_EDGE]
SHUFFLE [RS_110]
PartitionCols:_col0
- Select Operator [SEL_105] (rows=4602 width=585)
- Output:["_col0"]
- Filter Operator [FIL_176] (rows=4602 width=585)
- predicate:wp_web_page_sk is not null
- Please refer to the previous TableScan [TS_83]
+ Please refer to the previous Select Operator [SEL_85]
<-Reducer 22 [SIMPLE_EDGE]
SHUFFLE [RS_109]
PartitionCols:_col1
@@ -398,11 +386,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_107]
PartitionCols:_col0
- Select Operator [SEL_102] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_175] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_106]
PartitionCols:_col0
@@ -436,9 +420,9 @@ Stage-0
<-Map 25 [SIMPLE_EDGE]
SHUFFLE [RS_33]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=1704 width=1910)
+ Select Operator [SEL_8] (rows=1704 width=1910)
Output:["_col0"]
- Filter Operator [FIL_166] (rows=1704 width=1910)
+ Filter Operator [FIL_163] (rows=1704 width=1910)
predicate:s_store_sk is not null
TableScan [TS_6] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk"]
@@ -450,11 +434,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col0
- Select Operator [SEL_25] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_165] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
@@ -479,11 +459,7 @@ Stage-0
<-Map 25 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=1704 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_163] (rows=1704 width=1910)
- predicate:s_store_sk is not null
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col1
@@ -492,11 +468,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_162] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query78.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query78.q.out b/ql/src/test/results/clientpositive/perf/query78.q.out
index b49103f..2fccac2 100644
--- a/ql/src/test/results/clientpositive/perf/query78.q.out
+++ b/ql/src/test/results/clientpositive/perf/query78.q.out
@@ -165,9 +165,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_60]
PartitionCols:_col0
- Select Operator [SEL_49] (rows=36524 width=1119)
+ Select Operator [SEL_2] (rows=36524 width=1119)
Output:["_col0"]
- Filter Operator [FIL_103] (rows=36524 width=1119)
+ Filter Operator [FIL_97] (rows=36524 width=1119)
predicate:((d_year = 2000) and d_date_sk is not null)
TableScan [TS_0] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
@@ -220,11 +220,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_97] (rows=36524 width=1119)
- predicate:((d_year = 2000) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 12 [SIMPLE_EDGE]
SHUFFLE [RS_14]
PartitionCols:_col0
@@ -267,11 +263,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_34]
PartitionCols:_col0
- Select Operator [SEL_23] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_100] (rows=36524 width=1119)
- predicate:((d_year = 2000) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_35]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query80.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query80.q.out b/ql/src/test/results/clientpositive/perf/query80.q.out
index e37b50d..b717bc5 100644
--- a/ql/src/test/results/clientpositive/perf/query80.q.out
+++ b/ql/src/test/results/clientpositive/perf/query80.q.out
@@ -264,9 +264,9 @@ Stage-0
<-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_67]
PartitionCols:_col0
- Select Operator [SEL_53] (rows=1150 width=1179)
+ Select Operator [SEL_14] (rows=1150 width=1179)
Output:["_col0"]
- Filter Operator [FIL_195] (rows=1150 width=1179)
+ Filter Operator [FIL_189] (rows=1150 width=1179)
predicate:((p_channel_tv = 'N') and p_promo_sk is not null)
TableScan [TS_12] (rows=2300 width=1179)
default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk","p_channel_tv"]
@@ -278,9 +278,9 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_64]
PartitionCols:_col0
- Select Operator [SEL_50] (rows=154000 width=1436)
+ Select Operator [SEL_11] (rows=154000 width=1436)
Output:["_col0"]
- Filter Operator [FIL_194] (rows=154000 width=1436)
+ Filter Operator [FIL_188] (rows=154000 width=1436)
predicate:((i_current_price > 50) and i_item_sk is not null)
TableScan [TS_9] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price"]
@@ -292,9 +292,9 @@ Stage-0
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_61]
PartitionCols:_col0
- Select Operator [SEL_47] (rows=8116 width=1119)
+ Select Operator [SEL_8] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_193] (rows=8116 width=1119)
+ Filter Operator [FIL_187] (rows=8116 width=1119)
predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
TableScan [TS_6] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
@@ -356,11 +356,7 @@ Stage-0
<-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_107]
PartitionCols:_col0
- Select Operator [SEL_93] (rows=1150 width=1179)
- Output:["_col0"]
- Filter Operator [FIL_201] (rows=1150 width=1179)
- predicate:((p_channel_tv = 'N') and p_promo_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 19 [SIMPLE_EDGE]
SHUFFLE [RS_106]
PartitionCols:_col3
@@ -369,11 +365,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_104]
PartitionCols:_col0
- Select Operator [SEL_90] (rows=154000 width=1436)
- Output:["_col0"]
- Filter Operator [FIL_200] (rows=154000 width=1436)
- predicate:((i_current_price > 50) and i_item_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 18 [SIMPLE_EDGE]
SHUFFLE [RS_103]
PartitionCols:_col1
@@ -382,11 +374,7 @@ Stage-0
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_101]
PartitionCols:_col0
- Select Operator [SEL_87] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_199] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 31 [SIMPLE_EDGE]
SHUFFLE [RS_100]
PartitionCols:_col0
@@ -445,11 +433,7 @@ Stage-0
<-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=1150 width=1179)
- Output:["_col0"]
- Filter Operator [FIL_189] (rows=1150 width=1179)
- predicate:((p_channel_tv = 'N') and p_promo_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col3
@@ -458,11 +442,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_25]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=154000 width=1436)
- Output:["_col0"]
- Filter Operator [FIL_188] (rows=154000 width=1436)
- predicate:((i_current_price > 50) and i_item_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_24]
PartitionCols:_col1
@@ -471,11 +451,7 @@ Stage-0
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_187] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-09-03 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_21]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query81.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query81.q.out b/ql/src/test/results/clientpositive/perf/query81.q.out
index abeb577..c7571fe 100644
--- a/ql/src/test/results/clientpositive/perf/query81.q.out
+++ b/ql/src/test/results/clientpositive/perf/query81.q.out
@@ -134,9 +134,9 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_36]
PartitionCols:_col0
- Select Operator [SEL_31] (rows=36524 width=1119)
+ Select Operator [SEL_11] (rows=36524 width=1119)
Output:["_col0"]
- Filter Operator [FIL_98] (rows=36524 width=1119)
+ Filter Operator [FIL_95] (rows=36524 width=1119)
predicate:((d_year = 1998) and d_date_sk is not null)
TableScan [TS_9] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
@@ -179,11 +179,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_95] (rows=36524 width=1119)
- predicate:((d_year = 1998) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Map 6 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query83.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query83.q.out b/ql/src/test/results/clientpositive/perf/query83.q.out
index 396c423..a82dbb8 100644
--- a/ql/src/test/results/clientpositive/perf/query83.q.out
+++ b/ql/src/test/results/clientpositive/perf/query83.q.out
@@ -193,9 +193,9 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_69]
PartitionCols:_col1
- Select Operator [SEL_50] (rows=73049 width=1119)
+ Select Operator [SEL_8] (rows=73049 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_208] (rows=73049 width=1119)
+ Filter Operator [FIL_203] (rows=73049 width=1119)
predicate:(d_date is not null and d_date_sk is not null)
TableScan [TS_6] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
@@ -243,9 +243,9 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_74]
PartitionCols:_col0
- Select Operator [SEL_47] (rows=462000 width=1436)
+ Select Operator [SEL_5] (rows=462000 width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_207] (rows=462000 width=1436)
+ Filter Operator [FIL_202] (rows=462000 width=1436)
predicate:(i_item_sk is not null and i_item_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
@@ -278,11 +278,7 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_116]
PartitionCols:_col0
- Select Operator [SEL_89] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_212] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 29 [SIMPLE_EDGE]
SHUFFLE [RS_115]
PartitionCols:_col1
@@ -300,11 +296,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_111]
PartitionCols:_col1
- Select Operator [SEL_92] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_213] (rows=73049 width=1119)
- predicate:(d_date is not null and d_date_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 26 [SIMPLE_EDGE]
SHUFFLE [RS_112]
PartitionCols:_col0
@@ -360,11 +352,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col1
- Select Operator [SEL_8] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_203] (rows=73049 width=1119)
- predicate:(d_date is not null and d_date_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
@@ -408,11 +396,7 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_32]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_202] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_31]
PartitionCols:_col1
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query85.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query85.q.out b/ql/src/test/results/clientpositive/perf/query85.q.out
index 86b961b..07e2e48 100644
--- a/ql/src/test/results/clientpositive/perf/query85.q.out
+++ b/ql/src/test/results/clientpositive/perf/query85.q.out
@@ -203,9 +203,9 @@ Stage-0
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_45]
PartitionCols:_col0, _col1, _col2
- Select Operator [SEL_23] (rows=1861800 width=385)
+ Select Operator [SEL_20] (rows=1861800 width=385)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_99] (rows=1861800 width=385)
+ Filter Operator [FIL_98] (rows=1861800 width=385)
predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null)
TableScan [TS_18] (rows=1861800 width=385)
default@customer_demographics,cd1,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status","cd_education_status"]
@@ -219,11 +219,7 @@ Stage-0
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_41]
PartitionCols:_col0
- Select Operator [SEL_20] (rows=1861800 width=385)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_98] (rows=1861800 width=385)
- predicate:(((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_marital_status is not null and cd_education_status is not null)
- Please refer to the previous TableScan [TS_18]
+ Please refer to the previous Select Operator [SEL_20]
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_40]
PartitionCols:_col11
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query87.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query87.q.out b/ql/src/test/results/clientpositive/perf/query87.q.out
index 58a33d9..88dc197 100644
--- a/ql/src/test/results/clientpositive/perf/query87.q.out
+++ b/ql/src/test/results/clientpositive/perf/query87.q.out
@@ -99,9 +99,9 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_78]
PartitionCols:_col0
- Select Operator [SEL_73] (rows=80000000 width=860)
+ Select Operator [SEL_8] (rows=80000000 width=860)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_128] (rows=80000000 width=860)
+ Filter Operator [FIL_122] (rows=80000000 width=860)
predicate:c_customer_sk is not null
TableScan [TS_6] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"]
@@ -113,9 +113,9 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_75]
PartitionCols:_col0
- Select Operator [SEL_70] (rows=8116 width=1119)
+ Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_127] (rows=8116 width=1119)
+ Filter Operator [FIL_121] (rows=8116 width=1119)
predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"]
@@ -169,11 +169,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_39]
PartitionCols:_col0
- Select Operator [SEL_34] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_125] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_38]
PartitionCols:_col1
@@ -182,11 +178,7 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_36]
PartitionCols:_col0
- Select Operator [SEL_31] (rows=8116 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_124] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_35]
PartitionCols:_col0
@@ -219,11 +211,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_122] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col1
@@ -232,11 +220,7 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_121] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query88.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query88.q.out b/ql/src/test/results/clientpositive/perf/query88.q.out
index 18d0a77..6bad30c 100644
--- a/ql/src/test/results/clientpositive/perf/query88.q.out
+++ b/ql/src/test/results/clientpositive/perf/query88.q.out
@@ -243,9 +243,9 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_45]
PartitionCols:_col0
- Select Operator [SEL_37] (rows=852 width=1910)
+ Select Operator [SEL_11] (rows=852 width=1910)
Output:["_col0"]
- Filter Operator [FIL_298] (rows=852 width=1910)
+ Filter Operator [FIL_294] (rows=852 width=1910)
predicate:((s_store_name = 'ese') and s_store_sk is not null)
TableScan [TS_9] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name"]
@@ -257,9 +257,9 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_42]
PartitionCols:_col0
- Select Operator [SEL_34] (rows=3600 width=107)
+ Select Operator [SEL_8] (rows=3600 width=107)
Output:["_col0"]
- Filter Operator [FIL_297] (rows=3600 width=107)
+ Filter Operator [FIL_293] (rows=3600 width=107)
predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
TableScan [TS_6] (rows=7200 width=107)
default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count","hd_vehicle_count"]
@@ -271,9 +271,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_38]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=575995635 width=88)
+ Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_295] (rows=575995635 width=88)
+ Filter Operator [FIL_291] (rows=575995635 width=88)
predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_time_sk","ss_hdemo_sk","ss_store_sk"]
@@ -299,11 +299,7 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_71]
PartitionCols:_col0
- Select Operator [SEL_63] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_302] (rows=852 width=1910)
- predicate:((s_store_name = 'ese') and s_store_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 12 [SIMPLE_EDGE]
SHUFFLE [RS_70]
PartitionCols:_col2
@@ -312,11 +308,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_68]
PartitionCols:_col0
- Select Operator [SEL_60] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_301] (rows=3600 width=107)
- predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_67]
PartitionCols:_col1
@@ -325,11 +317,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_64]
PartitionCols:_col0
- Select Operator [SEL_54] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_299] (rows=575995635 width=88)
- predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_65]
PartitionCols:_col0
@@ -351,11 +339,7 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col0
- Select Operator [SEL_89] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_306] (rows=852 width=1910)
- predicate:((s_store_name = 'ese') and s_store_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_96]
PartitionCols:_col2
@@ -364,11 +348,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
- Select Operator [SEL_86] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_305] (rows=3600 width=107)
- predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_93]
PartitionCols:_col1
@@ -377,11 +357,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_90]
PartitionCols:_col0
- Select Operator [SEL_80] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_303] (rows=575995635 width=88)
- predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_91]
PartitionCols:_col0
@@ -403,11 +379,7 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_123]
PartitionCols:_col0
- Select Operator [SEL_115] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_310] (rows=852 width=1910)
- predicate:((s_store_name = 'ese') and s_store_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_122]
PartitionCols:_col2
@@ -416,11 +388,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_120]
PartitionCols:_col0
- Select Operator [SEL_112] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_309] (rows=3600 width=107)
- predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 19 [SIMPLE_EDGE]
SHUFFLE [RS_119]
PartitionCols:_col1
@@ -429,11 +397,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_116]
PartitionCols:_col0
- Select Operator [SEL_106] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_307] (rows=575995635 width=88)
- predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_117]
PartitionCols:_col0
@@ -455,11 +419,7 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_149]
PartitionCols:_col0
- Select Operator [SEL_141] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_314] (rows=852 width=1910)
- predicate:((s_store_name = 'ese') and s_store_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 24 [SIMPLE_EDGE]
SHUFFLE [RS_148]
PartitionCols:_col2
@@ -468,11 +428,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_146]
PartitionCols:_col0
- Select Operator [SEL_138] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_313] (rows=3600 width=107)
- predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 23 [SIMPLE_EDGE]
SHUFFLE [RS_145]
PartitionCols:_col1
@@ -481,11 +437,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_142]
PartitionCols:_col0
- Select Operator [SEL_132] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_311] (rows=575995635 width=88)
- predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_143]
PartitionCols:_col0
@@ -507,11 +459,7 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_175]
PartitionCols:_col0
- Select Operator [SEL_167] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_318] (rows=852 width=1910)
- predicate:((s_store_name = 'ese') and s_store_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 28 [SIMPLE_EDGE]
SHUFFLE [RS_174]
PartitionCols:_col2
@@ -520,11 +468,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_172]
PartitionCols:_col0
- Select Operator [SEL_164] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_317] (rows=3600 width=107)
- predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 27 [SIMPLE_EDGE]
SHUFFLE [RS_171]
PartitionCols:_col1
@@ -533,11 +477,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_168]
PartitionCols:_col0
- Select Operator [SEL_158] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_315] (rows=575995635 width=88)
- predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_169]
PartitionCols:_col0
@@ -559,11 +499,7 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_201]
PartitionCols:_col0
- Select Operator [SEL_193] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_322] (rows=852 width=1910)
- predicate:((s_store_name = 'ese') and s_store_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 32 [SIMPLE_EDGE]
SHUFFLE [RS_200]
PartitionCols:_col2
@@ -572,11 +508,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_198]
PartitionCols:_col0
- Select Operator [SEL_190] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_321] (rows=3600 width=107)
- predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 31 [SIMPLE_EDGE]
SHUFFLE [RS_197]
PartitionCols:_col1
@@ -585,11 +517,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_194]
PartitionCols:_col0
- Select Operator [SEL_184] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_319] (rows=575995635 width=88)
- predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_195]
PartitionCols:_col0
@@ -611,11 +539,7 @@ Stage-0
<-Map 37 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_294] (rows=852 width=1910)
- predicate:((s_store_name = 'ese') and s_store_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col2
@@ -624,11 +548,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_293] (rows=3600 width=107)
- predicate:((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col1
@@ -637,11 +557,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_291] (rows=575995635 width=88)
- predicate:(ss_hdemo_sk is not null and ss_sold_time_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query90.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query90.q.out b/ql/src/test/results/clientpositive/perf/query90.q.out
index b3468ec..d63ce1b 100644
--- a/ql/src/test/results/clientpositive/perf/query90.q.out
+++ b/ql/src/test/results/clientpositive/perf/query90.q.out
@@ -84,9 +84,9 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_45]
PartitionCols:_col0
- Select Operator [SEL_37] (rows=3600 width=107)
+ Select Operator [SEL_11] (rows=3600 width=107)
Output:["_col0"]
- Filter Operator [FIL_85] (rows=3600 width=107)
+ Filter Operator [FIL_81] (rows=3600 width=107)
predicate:((hd_dep_count = 8) and hd_demo_sk is not null)
TableScan [TS_9] (rows=7200 width=107)
default@household_demographics,household_demographics,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_dep_count"]
@@ -112,18 +112,18 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_38]
PartitionCols:_col2
- Select Operator [SEL_28] (rows=144002668 width=135)
+ Select Operator [SEL_2] (rows=144002668 width=135)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_82] (rows=144002668 width=135)
+ Filter Operator [FIL_78] (rows=144002668 width=135)
predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null)
TableScan [TS_0] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_time_sk","ws_ship_hdemo_sk","ws_web_page_sk"]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_39]
PartitionCols:_col0
- Select Operator [SEL_31] (rows=511 width=585)
+ Select Operator [SEL_5] (rows=511 width=585)
Output:["_col0"]
- Filter Operator [FIL_83] (rows=511 width=585)
+ Filter Operator [FIL_79] (rows=511 width=585)
predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null)
TableScan [TS_3] (rows=4602 width=585)
default@web_page,web_page,Tbl:COMPLETE,Col:NONE,Output:["wp_web_page_sk","wp_char_count"]
@@ -140,11 +140,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=3600 width=107)
- Output:["_col0"]
- Filter Operator [FIL_81] (rows=3600 width=107)
- predicate:((hd_dep_count = 8) and hd_demo_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col1
@@ -166,17 +162,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col2
- Select Operator [SEL_2] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_78] (rows=144002668 width=135)
- predicate:(ws_ship_hdemo_sk is not null and ws_sold_time_sk is not null and ws_web_page_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=511 width=585)
- Output:["_col0"]
- Filter Operator [FIL_79] (rows=511 width=585)
- predicate:(wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query92.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query92.q.out b/ql/src/test/results/clientpositive/perf/query92.q.out
index ec4fbb9..761753f 100644
--- a/ql/src/test/results/clientpositive/perf/query92.q.out
+++ b/ql/src/test/results/clientpositive/perf/query92.q.out
@@ -147,17 +147,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_58] (rows=144002668 width=135)
- predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_59] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query94.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query94.q.out b/ql/src/test/results/clientpositive/perf/query94.q.out
index c5fc9e7..9cc934f 100644
--- a/ql/src/test/results/clientpositive/perf/query94.q.out
+++ b/ql/src/test/results/clientpositive/perf/query94.q.out
@@ -146,9 +146,7 @@ Stage-0
Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
<-Map 15 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_32]
- Select Operator [SEL_28] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Please refer to the previous TableScan [TS_19]
+ Please refer to the previous Select Operator [SEL_20]
<-Map 20 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_29]
Select Operator [SEL_22] (rows=73049 width=4)
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query95.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query95.q.out b/ql/src/test/results/clientpositive/perf/query95.q.out
index 332bef8..28db5a6 100644
--- a/ql/src/test/results/clientpositive/perf/query95.q.out
+++ b/ql/src/test/results/clientpositive/perf/query95.q.out
@@ -172,19 +172,11 @@ Stage-0
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_33]
PartitionCols:_col1
- Select Operator [SEL_29] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_112] (rows=144002668 width=135)
- predicate:ws_order_number is not null
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_34]
PartitionCols:_col1
- Select Operator [SEL_32] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_113] (rows=144002668 width=135)
- predicate:ws_order_number is not null
- Please refer to the previous TableScan [TS_15]
+ Please refer to the previous Select Operator [SEL_17]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_58]
PartitionCols:_col3
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query97.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query97.q.out b/ql/src/test/results/clientpositive/perf/query97.q.out
index 2d5129a..51dd341 100644
--- a/ql/src/test/results/clientpositive/perf/query97.q.out
+++ b/ql/src/test/results/clientpositive/perf/query97.q.out
@@ -119,11 +119,7 @@ Stage-0
<-Map 6 [SIMPLE_EDGE]
SHUFFLE [RS_21]
PartitionCols:_col0
- Select Operator [SEL_19] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_46] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_20]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
index ec87daa..9ced7ad 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_2.q.out
@@ -86,9 +86,9 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_44]
PartitionCols:_col1
- Select Operator [SEL_39] (rows=25/25 width=175)
+ Select Operator [SEL_14] (rows=25/25 width=175)
Output:["_col0","_col1"]
- Filter Operator [FIL_77] (rows=25/25 width=175)
+ Filter Operator [FIL_73] (rows=25/25 width=175)
predicate:(key is not null and value is not null)
TableScan [TS_12] (rows=25/25 width=175)
default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -142,11 +142,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col1
- Select Operator [SEL_14] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_73] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col1
@@ -276,9 +272,9 @@ Stage-0
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col0
- Select Operator [SEL_100] (rows=500/500 width=178)
+ Select Operator [SEL_17] (rows=500/500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_156] (rows=500/500 width=178)
+ Filter Operator [FIL_145] (rows=500/500 width=178)
predicate:key is not null
TableScan [TS_15] (rows=500/500 width=178)
default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -290,9 +286,9 @@ Stage-0
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_102]
PartitionCols:_col1
- Select Operator [SEL_97] (rows=25/25 width=175)
+ Select Operator [SEL_14] (rows=25/25 width=175)
Output:["_col0","_col1"]
- Filter Operator [FIL_155] (rows=25/25 width=175)
+ Filter Operator [FIL_144] (rows=25/25 width=175)
predicate:(key is not null and value is not null)
TableScan [TS_12] (rows=25/25 width=175)
default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -386,11 +382,7 @@ Stage-0
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_53]
PartitionCols:_col1
- Select Operator [SEL_48] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_149] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 21 [SIMPLE_EDGE]
SHUFFLE [RS_52]
PartitionCols:_col1
@@ -444,11 +436,7 @@ Stage-0
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col0
- Select Operator [SEL_17] (rows=500/500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_145] (rows=500/500 width=178)
- predicate:key is not null
- Please refer to the previous TableScan [TS_15]
+ Please refer to the previous Select Operator [SEL_17]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_21]
PartitionCols:_col2
@@ -457,11 +445,7 @@ Stage-0
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col1
- Select Operator [SEL_14] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_144] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col1
@@ -1280,11 +1264,7 @@ Stage-5
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_66]
PartitionCols:_col0
- Select Operator [SEL_48] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_113] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_8]
+ Please refer to the previous Select Operator [SEL_10]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_67]
PartitionCols:_col0
@@ -1478,9 +1458,9 @@ Stage-5
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_101]
PartitionCols:_col0
- Select Operator [SEL_67] (rows=25/25 width=175)
+ Select Operator [SEL_14] (rows=25/25 width=175)
Output:["_col0","_col1"]
- Filter Operator [FIL_153] (rows=25/25 width=175)
+ Filter Operator [FIL_146] (rows=25/25 width=175)
predicate:(key is not null and value is not null)
TableScan [TS_12] (rows=25/25 width=175)
default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -1655,11 +1635,7 @@ Stage-5
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col1
- Select Operator [SEL_14] (rows=25/25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_146] (rows=25/25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col1
[7/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out
index 3e71546..21de860 100644
--- a/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out
+++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer3.q.out
@@ -45,13 +45,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -273,13 +266,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -716,13 +702,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -944,13 +923,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out
index 82dae9a..c09a076 100644
--- a/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out
+++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer6.q.out
@@ -1815,13 +1815,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -2040,13 +2033,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -2265,13 +2251,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -2490,13 +2469,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
index 15f8499..8bac1f5 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out
@@ -2239,27 +2239,21 @@ STAGE PLANS:
alias: srcpart
filterExpr: (ds = '2008-04-08') (type: boolean)
Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (ds = '2008-04-08') (type: boolean)
- Select Operator
+ Select Operator
+ Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: '2008-04-08' (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
- sort order:
- Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (ds = '2008-04-08') (type: boolean)
- Select Operator
- Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: '2008-04-08' (type: string)
- mode: hash
- outputColumnNames: _col0
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
Reducer 2
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/except_distinct.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/except_distinct.q.out b/ql/src/test/results/clientpositive/llap/except_distinct.q.out
index e4c2941..4dd596e 100644
--- a/ql/src/test/results/clientpositive/llap/except_distinct.q.out
+++ b/ql/src/test/results/clientpositive/llap/except_distinct.q.out
@@ -398,10 +398,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(1)
keys: _col0 (type: string), _col1 (type: string)
@@ -414,10 +410,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 250 Data size: 46500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(1)
keys: _col0 (type: string), _col1 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
index b68ec63..64a4821 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
@@ -2432,11 +2432,7 @@ Stage-0
Output:["_col0"],aggregations:["avg(VALUE._col0)"]
<-Map 5 [CUSTOM_SIMPLE_EDGE] llap
PARTITION_ONLY_SHUFFLE [RS_18]
- Group By Operator [GBY_17] (rows=1 width=76)
- Output:["_col0"],aggregations:["avg(p_size)"]
- Filter Operator [FIL_35] (rows=8 width=4)
- predicate:(p_size < 10)
- Please refer to the previous TableScan [TS_2]
+ Please refer to the previous Group By Operator [GBY_5]
PREHOOK: query: explain select b.p_mfgr, min(p_retailprice)
from part b
@@ -2525,9 +2521,7 @@ Stage-0
PartitionCols:_col0
Group By Operator [GBY_8] (rows=5 width=114)
Output:["_col0","_col1","_col2"],aggregations:["min(p_retailprice)","max(p_retailprice)"],keys:p_mfgr
- Select Operator [SEL_7] (rows=26 width=106)
- Output:["p_mfgr","p_retailprice"]
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_1]
<-Reducer 8 [SIMPLE_EDGE] llap
SHUFFLE [RS_31]
PartitionCols:_col0, _col1
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
index a250fd6..42d5f85 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
@@ -490,9 +490,9 @@ Stage-0
<-Map 9 [SIMPLE_EDGE] llap
SHUFFLE [RS_46]
PartitionCols:_col1
- Select Operator [SEL_41] (rows=25 width=175)
+ Select Operator [SEL_15] (rows=25 width=175)
Output:["_col0","_col1"]
- Filter Operator [FIL_80] (rows=25 width=175)
+ Filter Operator [FIL_76] (rows=25 width=175)
predicate:(key is not null and value is not null)
TableScan [TS_13] (rows=25 width=175)
default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -552,11 +552,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE] llap
SHUFFLE [RS_20]
PartitionCols:_col1
- Select Operator [SEL_15] (rows=25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_76] (rows=25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Reducer 3 [SIMPLE_EDGE] llap
SHUFFLE [RS_19]
PartitionCols:_col1
@@ -662,9 +658,9 @@ Stage-0
<-Map 16 [SIMPLE_EDGE] llap
SHUFFLE [RS_112]
PartitionCols:_col0
- Select Operator [SEL_107] (rows=500 width=178)
+ Select Operator [SEL_18] (rows=500 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_164] (rows=500 width=178)
+ Filter Operator [FIL_153] (rows=500 width=178)
predicate:key is not null
TableScan [TS_16] (rows=500 width=178)
default@src,y,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -676,9 +672,9 @@ Stage-0
<-Map 11 [SIMPLE_EDGE] llap
SHUFFLE [RS_109]
PartitionCols:_col1
- Select Operator [SEL_104] (rows=25 width=175)
+ Select Operator [SEL_15] (rows=25 width=175)
Output:["_col0","_col1"]
- Filter Operator [FIL_163] (rows=25 width=175)
+ Filter Operator [FIL_152] (rows=25 width=175)
predicate:(key is not null and value is not null)
TableScan [TS_13] (rows=25 width=175)
default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -788,11 +784,7 @@ Stage-0
<-Map 11 [SIMPLE_EDGE] llap
SHUFFLE [RS_56]
PartitionCols:_col1
- Select Operator [SEL_51] (rows=25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_157] (rows=25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Reducer 21 [SIMPLE_EDGE] llap
SHUFFLE [RS_55]
PartitionCols:_col1
@@ -856,11 +848,7 @@ Stage-0
<-Map 16 [SIMPLE_EDGE] llap
SHUFFLE [RS_23]
PartitionCols:_col0
- Select Operator [SEL_18] (rows=500 width=178)
- Output:["_col0","_col1"]
- Filter Operator [FIL_153] (rows=500 width=178)
- predicate:key is not null
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Reducer 4 [SIMPLE_EDGE] llap
SHUFFLE [RS_22]
PartitionCols:_col2
@@ -869,11 +857,7 @@ Stage-0
<-Map 11 [SIMPLE_EDGE] llap
SHUFFLE [RS_20]
PartitionCols:_col1
- Select Operator [SEL_15] (rows=25 width=175)
- Output:["_col0","_col1"]
- Filter Operator [FIL_152] (rows=25 width=175)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Reducer 3 [SIMPLE_EDGE] llap
SHUFFLE [RS_19]
PartitionCols:_col1
@@ -1346,9 +1330,9 @@ Stage-0
<-Map 10 [BROADCAST_EDGE] llap
BROADCAST [RS_112]
PartitionCols:_col0
- Select Operator [SEL_107] (rows=500 width=10)
+ Select Operator [SEL_18] (rows=500 width=10)
Output:["_col0","_col1"]
- Filter Operator [FIL_164] (rows=500 width=10)
+ Filter Operator [FIL_153] (rows=500 width=10)
predicate:key is not null
TableScan [TS_16] (rows=500 width=10)
default@src,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
@@ -1357,9 +1341,9 @@ Stage-0
<-Map 9 [BROADCAST_EDGE] llap
BROADCAST [RS_109]
PartitionCols:_col1
- Select Operator [SEL_104] (rows=25 width=7)
+ Select Operator [SEL_15] (rows=25 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_163] (rows=25 width=7)
+ Filter Operator [FIL_152] (rows=25 width=7)
predicate:(key is not null and value is not null)
TableScan [TS_13] (rows=25 width=7)
default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
@@ -1529,21 +1513,13 @@ Stage-0
<-Map 10 [BROADCAST_EDGE] llap
BROADCAST [RS_23]
PartitionCols:_col0
- Select Operator [SEL_18] (rows=500 width=10)
- Output:["_col0","_col1"]
- Filter Operator [FIL_153] (rows=500 width=10)
- predicate:key is not null
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Map Join Operator [MAPJOIN_165] (rows=288 width=10)
Conds:SEL_12._col1=RS_20._col1(Inner),Output:["_col2"]
<-Map 9 [BROADCAST_EDGE] llap
BROADCAST [RS_20]
PartitionCols:_col1
- Select Operator [SEL_15] (rows=25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_152] (rows=25 width=7)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Select Operator [SEL_12] (rows=262 width=10)
Output:["_col1"]
Group By Operator [GBY_11] (rows=262 width=10)
@@ -2068,7 +2044,7 @@ Stage-0
BROADCAST [RS_69]
PartitionCols:_col1
Map Join Operator [MAPJOIN_118] (rows=27 width=7)
- Conds:SEL_48._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"]
+ Conds:SEL_10._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"]
<-Map 15 [BROADCAST_EDGE] llap
BROADCAST [RS_67]
PartitionCols:_col0
@@ -2078,9 +2054,9 @@ Stage-0
predicate:key is not null
TableScan [TS_49] (rows=25 width=7)
default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_48] (rows=25 width=7)
+ <-Select Operator [SEL_10] (rows=25 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_108] (rows=25 width=7)
+ Filter Operator [FIL_101] (rows=25 width=7)
predicate:(key is not null and value is not null)
TableScan [TS_8] (rows=25 width=7)
default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
@@ -2162,11 +2138,7 @@ Stage-0
<-Map 6 [BROADCAST_EDGE] llap
BROADCAST [RS_15]
PartitionCols:_col1
- Select Operator [SEL_10] (rows=25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_101] (rows=25 width=7)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_8]
+ Please refer to the previous Select Operator [SEL_10]
<-Select Operator [SEL_2] (rows=25 width=7)
Output:["_col0"]
Filter Operator [FIL_99] (rows=25 width=7)
@@ -2318,7 +2290,7 @@ Stage-0
BROADCAST [RS_111]
PartitionCols:_col1
Map Join Operator [MAPJOIN_166] (rows=27 width=7)
- Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"]
+ Conds:SEL_15._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"]
<-Map 22 [BROADCAST_EDGE] llap
BROADCAST [RS_109]
PartitionCols:_col0
@@ -2328,9 +2300,9 @@ Stage-0
predicate:key is not null
TableScan [TS_72] (rows=25 width=7)
default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_71] (rows=25 width=7)
+ <-Select Operator [SEL_15] (rows=25 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_156] (rows=25 width=7)
+ Filter Operator [FIL_149] (rows=25 width=7)
predicate:(key is not null and value is not null)
TableScan [TS_13] (rows=25 width=7)
default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
@@ -2519,11 +2491,7 @@ Stage-0
<-Map 10 [BROADCAST_EDGE] llap
BROADCAST [RS_20]
PartitionCols:_col1
- Select Operator [SEL_15] (rows=25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_149] (rows=25 width=7)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Select Operator [SEL_12] (rows=262 width=10)
Output:["_col1"]
Group By Operator [GBY_11] (rows=262 width=10)
@@ -2651,7 +2619,7 @@ Stage-5
BROADCAST [RS_69]
PartitionCols:_col1
Map Join Operator [MAPJOIN_123] (rows=27 width=7)
- Conds:SEL_48._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"]
+ Conds:SEL_10._col0=RS_67._col0(Inner),Output:["_col0","_col1","_col3"]
<-Map 15 [BROADCAST_EDGE] llap
BROADCAST [RS_67]
PartitionCols:_col0
@@ -2661,9 +2629,9 @@ Stage-5
predicate:key is not null
TableScan [TS_49] (rows=25 width=7)
default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_48] (rows=25 width=7)
+ <-Select Operator [SEL_10] (rows=25 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_113] (rows=25 width=7)
+ Filter Operator [FIL_106] (rows=25 width=7)
predicate:(key is not null and value is not null)
TableScan [TS_8] (rows=25 width=7)
default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
@@ -2773,11 +2741,7 @@ Stage-5
<-Map 6 [BROADCAST_EDGE] llap
BROADCAST [RS_15]
PartitionCols:_col1
- Select Operator [SEL_10] (rows=25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_106] (rows=25 width=7)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_8]
+ Please refer to the previous Select Operator [SEL_10]
<-Select Operator [SEL_2] (rows=25 width=7)
Output:["_col0"]
Filter Operator [FIL_104] (rows=25 width=7)
@@ -2965,7 +2929,7 @@ Stage-5
BROADCAST [RS_111]
PartitionCols:_col1
Map Join Operator [MAPJOIN_171] (rows=27 width=7)
- Conds:SEL_71._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"]
+ Conds:SEL_15._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col3"]
<-Map 22 [BROADCAST_EDGE] llap
BROADCAST [RS_109]
PartitionCols:_col0
@@ -2975,9 +2939,9 @@ Stage-5
predicate:key is not null
TableScan [TS_72] (rows=25 width=7)
default@src1,y,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
- <-Select Operator [SEL_71] (rows=25 width=7)
+ <-Select Operator [SEL_15] (rows=25 width=7)
Output:["_col0","_col1"]
- Filter Operator [FIL_161] (rows=25 width=7)
+ Filter Operator [FIL_154] (rows=25 width=7)
predicate:(key is not null and value is not null)
TableScan [TS_13] (rows=25 width=7)
default@src1,x,Tbl:COMPLETE,Col:NONE,Output:["key","value"]
@@ -3166,11 +3130,7 @@ Stage-5
<-Map 10 [BROADCAST_EDGE] llap
BROADCAST [RS_20]
PartitionCols:_col1
- Select Operator [SEL_15] (rows=25 width=7)
- Output:["_col0","_col1"]
- Filter Operator [FIL_154] (rows=25 width=7)
- predicate:(key is not null and value is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Select Operator [SEL_12] (rows=262 width=10)
Output:["_col1"]
Group By Operator [GBY_11] (rows=262 width=10)
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/intersect_merge.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/intersect_merge.q.out b/ql/src/test/results/clientpositive/llap/intersect_merge.q.out
index a312966..cf86a32 100644
--- a/ql/src/test/results/clientpositive/llap/intersect_merge.q.out
+++ b/ql/src/test/results/clientpositive/llap/intersect_merge.q.out
@@ -1764,16 +1764,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint)
- Select Operator
- expressions: key (type: int), value (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: int), _col1 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out
index 57594e0..d1d6664 100644
--- a/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/limit_pushdown.q.out
@@ -932,16 +932,6 @@ STAGE PLANS:
Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.3
value expressions: _col1 (type: bigint)
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/mrr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mrr.q.out b/ql/src/test/results/clientpositive/llap/mrr.q.out
index 726349c..7866e37 100644
--- a/ql/src/test/results/clientpositive/llap/mrr.q.out
+++ b/ql/src/test/results/clientpositive/llap/mrr.q.out
@@ -1322,30 +1322,12 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(value)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(value)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out b/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out
index b4b0e93..daf0189 100644
--- a/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out
+++ b/ql/src/test/results/clientpositive/llap/multiMapJoin2.q.out
@@ -722,7 +722,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Map 1 <- Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE)
+ Map 1 <- Map 6 (BROADCAST_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
@@ -761,28 +761,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0
- input vertices:
- 1 Map 7
- Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -809,25 +787,6 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
- Map 7
- Map Operator Tree:
- TableScan
- alias: y1
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Execution mode: llap
- LLAP IO: no inputs
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -982,7 +941,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Map 1 <- Map 6 (BROADCAST_EDGE), Map 7 (BROADCAST_EDGE)
+ Map 1 <- Map 6 (BROADCAST_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE)
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
@@ -1021,28 +980,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0
- input vertices:
- 1 Map 7
- Statistics: Num rows: 60 Data size: 5220 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -1069,25 +1006,6 @@ STAGE PLANS:
Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: no inputs
- Map 7
- Map Operator Tree:
- TableScan
- alias: y1
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Execution mode: llap
- LLAP IO: no inputs
Reducer 2
Execution mode: llap
Reduce Operator Tree:
@@ -1300,9 +1218,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: key (type: string)
mode: hash
@@ -1527,9 +1442,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 25 Data size: 2150 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: key (type: string)
mode: hash
@@ -1933,13 +1845,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Map Join Operator
condition map:
Inner Join 0 to 1
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
index c89ca6b..4b3a846 100644
--- a/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
+++ b/ql/src/test/results/clientpositive/llap/offset_limit_ppd_optimizer.q.out
@@ -1319,10 +1319,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 2.0E-5
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out b/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
index 34ba87c..bd3d247 100644
--- a/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
+++ b/ql/src/test/results/clientpositive/llap/partition_shared_scan.q.out
@@ -203,13 +203,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 10 Data size: 170 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: i is not null (type: boolean)
- Statistics: Num rows: 10 Data size: 170 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: i (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 170 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/subquery_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_in.q.out b/ql/src/test/results/clientpositive/llap/subquery_in.q.out
index f2bc558..d01b515 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_in.q.out
@@ -4024,9 +4024,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: p_type (type: string)
@@ -4492,9 +4489,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1352 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: p_type (type: string)
@@ -4507,9 +4501,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 2704 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: p_type (type: string)
@@ -4773,15 +4764,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: struct<count:bigint,sum:double,input:int>)
- Filter Operator
- predicate: p_partkey is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: avg(p_size)
- keys: p_partkey (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1040 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
index 29516ef..43f8a6f 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
@@ -1710,14 +1710,6 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: p_type (type: string), p_brand (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
@@ -1763,13 +1755,6 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string), _col0 (type: string)
Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: string)
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: p_brand (type: string), p_type (type: string), p_container (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: string)
sort order: ++
@@ -2248,14 +2233,6 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: p_type (type: string), p_brand (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 2548 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
@@ -2301,13 +2278,6 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string), _col0 (type: string)
Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: string)
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: p_brand (type: string), p_type (type: string), p_container (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 26 Data size: 7488 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string), _col0 (type: string)
sort order: ++
@@ -2631,9 +2601,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 260 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Filter Operator
- predicate: p_size is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: p_type (type: string), p_size (type: int)
mode: hash
@@ -3080,15 +3047,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:double>)
- Select Operator
- expressions: l_quantity (type: double)
- outputColumnNames: l_quantity
- Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: avg(l_quantity)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
index 7b9785f..d10fd84 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out
@@ -933,18 +933,12 @@ STAGE PLANS:
Map-reduce partition columns: p_mfgr (type: string)
Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
- Filter Operator
- predicate: (p_mfgr = p_mfgr) (type: boolean)
- Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: p_mfgr (type: string), p_size (type: int)
sort order: ++
Map-reduce partition columns: p_mfgr (type: string)
Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
TopN Hash Memory Usage: 0.1
- Filter Operator
- predicate: (p_mfgr = p_mfgr) (type: boolean)
- Statistics: Num rows: 13 Data size: 1326 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: p_mfgr (type: string), p_size (type: int)
sort order: ++
@@ -1507,9 +1501,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Filter Operator
- predicate: (key < '11') (type: boolean)
- Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: CASE WHEN ((key > '104')) THEN (null) ELSE (key) END (type: string)
outputColumnNames: _col0
@@ -1699,9 +1690,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1560 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: (p_size * p_size) (type: int), p_type (type: string)
outputColumnNames: _col0, _col1
@@ -1896,16 +1884,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Select Operator
- expressions: p_type (type: string), p_size (type: int)
- outputColumnNames: p_type, p_size
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: min(p_size)
- keys: p_type (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -2121,16 +2099,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Select Operator
- expressions: p_partkey (type: int), p_type (type: string)
- outputColumnNames: p_partkey, p_type
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: min(p_partkey)
- keys: p_type (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -2356,9 +2324,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1781 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Filter Operator
- predicate: p_name is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 3250 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: p_partkey (type: int), p_name (type: string)
mode: hash
@@ -2583,16 +2548,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: double)
- Select Operator
- expressions: p_type (type: string), p_retailprice (type: double)
- outputColumnNames: p_type, p_retailprice
- Statistics: Num rows: 26 Data size: 2912 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: min(p_retailprice)
- keys: p_type (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -2790,10 +2745,6 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int), _col0 (type: int)
Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
- Select Operator
- expressions: p_partkey (type: int), p_name (type: string), p_size (type: int)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 26 Data size: 3354 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col2 (type: int), _col0 (type: int)
sort order: ++
@@ -2817,11 +2768,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: p_size (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
@@ -3072,9 +3018,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 3 Data size: 72 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: bigint), _col3 (type: bigint)
- Filter Operator
- predicate: ((p_size = p_size) and (p_partkey = p_partkey)) (type: boolean)
- Statistics: Num rows: 6 Data size: 774 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: p_partkey (type: int), p_name (type: string), p_size (type: int)
mode: hash
@@ -3244,9 +3187,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Filter Operator
- predicate: p_brand is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: (UDFToDouble(p_type) + 2.0) (type: double), p_brand (type: string)
outputColumnNames: _col0, _col1
@@ -3411,10 +3351,6 @@ STAGE PLANS:
Map-reduce partition columns: (_col1 + 1) (type: int)
Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Select Operator
- expressions: p_type (type: string), p_size (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: (_col1 + 1) (type: int)
sort order: +
@@ -3442,15 +3378,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: (p_size + 1) (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: _col0 (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
@@ -4279,16 +4206,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Select Operator
- expressions: p_type (type: string), p_size (type: int)
- outputColumnNames: p_type, p_size
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: min(p_size)
- keys: p_type (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -4517,16 +4434,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Select Operator
- expressions: p_type (type: string), p_size (type: int)
- outputColumnNames: p_type, p_size
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: min(p_size)
- keys: p_type (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -5462,9 +5369,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 12 Data size: 1296 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Filter Operator
- predicate: p_brand is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 5096 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: p_brand (type: string), p_type (type: string)
mode: hash
@@ -5872,10 +5776,6 @@ STAGE PLANS:
Map-reduce partition columns: UDFToDouble(_col0) (type: double)
Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
- Select Operator
- expressions: c1 (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 3 Data size: 6 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: UDFToDouble(_col0) (type: double)
sort order: +
@@ -5899,11 +5799,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: char(100))
Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: c2 (type: char(100))
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 313 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: char(100))
sort order: +
@@ -6184,9 +6079,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Filter Operator
- predicate: b is not null (type: boolean)
- Statistics: Num rows: 3 Data size: 10 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: b (type: int), a (type: int)
mode: hash
@@ -6403,9 +6295,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint), _col2 (type: bigint)
- Filter Operator
- predicate: j is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: i (type: int), j (type: int)
mode: hash
@@ -7319,11 +7208,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: value (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 214 Data size: 19474 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out b/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out
index bff2781..dde2384 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_null_agg.q.out
@@ -71,8 +71,6 @@ STAGE PLANS:
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
@@ -85,8 +83,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Select Operator
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
Filter Operator
predicate: false (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
index f17977b..4bd2ed0 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out
@@ -372,11 +372,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Filter Operator
- predicate: p_name is null (type: boolean)
- Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Statistics: Num rows: 16 Data size: 1628 Basic stats: COMPLETE Column stats: NONE
@@ -826,10 +821,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int)
- Select Operator
- expressions: p_size (type: int)
- outputColumnNames: p_size
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: avg(p_size)
mode: hash
@@ -2486,11 +2477,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 86 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (key = '90') (type: boolean)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: '90' (type: string)
@@ -2855,15 +2841,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Select Operator
- expressions: p_brand (type: string)
- outputColumnNames: p_brand
- Statistics: Num rows: 26 Data size: 2392 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(p_brand)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -2888,15 +2865,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Select Operator
- expressions: p_name (type: string)
- outputColumnNames: p_name
- Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(p_name)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/subquery_select.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
index afb746e..440e9d7 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out
@@ -2913,10 +2913,6 @@ STAGE PLANS:
key expressions: _col0 (type: int)
sort order: +
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: p_size (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
@@ -3118,15 +3114,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: int)
- Filter Operator
- predicate: p_type is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 2808 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: max(p_size)
- keys: p_type (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13 Data size: 1404 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -3153,15 +3140,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Select Operator
- expressions: p_name (type: string)
- outputColumnNames: p_name
- Statistics: Num rows: 26 Data size: 3146 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: min(p_name)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE
@@ -3590,8 +3568,6 @@ STAGE PLANS:
Reduce Output Operator
sort order:
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
@@ -4447,10 +4423,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int)
- Select Operator
- expressions: p_size (type: int)
- outputColumnNames: p_size
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(p_size)
mode: hash
@@ -4460,10 +4432,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int)
- Select Operator
- expressions: p_size (type: int)
- outputColumnNames: p_size
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: avg(p_size)
mode: hash
@@ -4473,10 +4441,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 76 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: struct<count:bigint,sum:double,input:int>)
- Select Operator
- expressions: p_size (type: int)
- outputColumnNames: p_size
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(p_size)
mode: hash
@@ -5241,9 +5205,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: p_partkey is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
keys: p_partkey (type: int)
@@ -5256,9 +5217,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 13 Data size: 156 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: bigint)
- Filter Operator
- predicate: p_partkey is not null (type: boolean)
- Statistics: Num rows: 26 Data size: 104 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: p_partkey (type: int)
mode: hash
[8/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
new file mode 100644
index 0000000..8070c2a
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -0,0 +1,1176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
+import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
+import org.apache.hadoop.hive.ql.exec.FilterOperator;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorFactory;
+import org.apache.hadoop.hive.ql.exec.OperatorUtils;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UnionOperator;
+import org.apache.hadoop.hive.ql.parse.GenTezUtils;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo;
+import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicListDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
+import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.ql.stats.StatsUtils;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInBloomFilter;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multiset;
+import com.google.common.collect.TreeMultiset;
+
+/**
+ * Shared computation optimizer.
+ *
+ * <p>Originally, this rule would find scan operators over the same table
+ * in the query plan and merge them if they met some preconditions.
+ *
+ * TS TS TS
+ * | | -> / \
+ * Op Op Op Op
+ *
+ * <p>Now the rule has been extended to find opportunities to other operators
+ * downstream, not only a single table scan.
+ *
+ * TS1 TS2 TS1 TS2 TS1 TS2
+ * | | | | | |
+ * | RS | RS | RS
+ * \ / \ / -> \ /
+ * MapJoin MapJoin MapJoin
+ * | | / \
+ * Op Op Op Op
+ *
+ * <p>A limitation in the current implementation is that the optimizer does not
+ * go beyond a work boundary.
+ *
+ * <p>The optimization only works with the Tez execution engine.
+ */
+public class SharedWorkOptimizer extends Transform {
+
+ private final static Logger LOG = LoggerFactory.getLogger(SharedWorkOptimizer.class);
+
+ @Override
+ public ParseContext transform(ParseContext pctx) throws SemanticException {
+
+ final Map<String, TableScanOperator> topOps = pctx.getTopOps();
+ if (topOps.size() < 2) {
+ // Nothing to do, bail out
+ return pctx;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Before SharedWorkOptimizer:\n" + Operator.toString(pctx.getTopOps().values()));
+ }
+
+ // Cache to use during optimization
+ SharedWorkOptimizerCache optimizerCache = new SharedWorkOptimizerCache();
+
+ // Gather information about the DPP table scans and store it in the cache
+ gatherDPPTableScanOps(pctx, optimizerCache);
+
+ // Map of dbName.TblName -> TSOperator
+ Multimap<String, TableScanOperator> tableNameToOps = splitTableScanOpsByTable(pctx);
+
+ // We enforce a certain order when we do the reutilization.
+ // In particular, we use size of table x number of reads to
+ // rank the tables.
+ List<Entry<String, Long>> sortedTables = rankTablesByAccumulatedSize(pctx);
+ LOG.debug("Sorted tables by size: {}", sortedTables);
+
+ // Execute optimization
+ Multimap<String, TableScanOperator> existingOps = ArrayListMultimap.create();
+ Set<Operator<?>> removedOps = new HashSet<>();
+ for (Entry<String, Long> tablePair : sortedTables) {
+ String tableName = tablePair.getKey();
+ for (TableScanOperator discardableTsOp : tableNameToOps.get(tableName)) {
+ if (removedOps.contains(discardableTsOp)) {
+ LOG.debug("Skip {} as it has been already removed", discardableTsOp);
+ continue;
+ }
+ Collection<TableScanOperator> prevTsOps = existingOps.get(tableName);
+ for (TableScanOperator retainableTsOp : prevTsOps) {
+ if (removedOps.contains(retainableTsOp)) {
+ LOG.debug("Skip {} as it has been already removed", retainableTsOp);
+ continue;
+ }
+
+ // First we quickly check if the two table scan operators can actually be merged
+ boolean mergeable = areMergeable(pctx, optimizerCache, retainableTsOp, discardableTsOp);
+ if (!mergeable) {
+ // Skip
+ LOG.debug("{} and {} cannot be merged", retainableTsOp, discardableTsOp);
+ continue;
+ }
+
+ // Secondly, we extract information about the part of the tree that can be merged
+ // as well as some structural information (memory consumption) that needs to be
+ // used to determined whether the merge can happen
+ SharedResult sr = extractSharedOptimizationInfo(
+ pctx, optimizerCache, retainableTsOp, discardableTsOp);
+
+ // It seems these two operators can be merged.
+ // Check that plan meets some preconditions before doing it.
+ // In particular, in the presence of map joins in the upstream plan:
+ // - we cannot exceed the noconditional task size, and
+ // - if we already merged the big table, we cannot merge the broadcast
+ // tables.
+ if (!validPreConditions(pctx, optimizerCache, sr)) {
+ // Skip
+ LOG.debug("{} and {} do not meet preconditions", retainableTsOp, discardableTsOp);
+ continue;
+ }
+
+ // We can merge
+ if (sr.retainableOps.size() > 1) {
+ // More than TS operator
+ Operator<?> lastRetainableOp = sr.retainableOps.get(sr.retainableOps.size() - 1);
+ Operator<?> lastDiscardableOp = sr.discardableOps.get(sr.discardableOps.size() - 1);
+ if (lastDiscardableOp.getNumChild() != 0) {
+ List<Operator<? extends OperatorDesc>> allChildren =
+ Lists.newArrayList(lastDiscardableOp.getChildOperators());
+ for (Operator<? extends OperatorDesc> op : allChildren) {
+ lastDiscardableOp.getChildOperators().remove(op);
+ op.replaceParent(lastDiscardableOp, lastRetainableOp);
+ lastRetainableOp.getChildOperators().add(op);
+ }
+ }
+
+ LOG.debug("Merging subtree starting at {} into subtree starting at {}", discardableTsOp, retainableTsOp);
+ } else {
+ // Only TS operator
+ ExprNodeGenericFuncDesc exprNode = null;
+ if (retainableTsOp.getConf().getFilterExpr() != null) {
+ // Push filter on top of children
+ pushFilterToTopOfTableScan(optimizerCache, retainableTsOp);
+ // Clone to push to table scan
+ exprNode = (ExprNodeGenericFuncDesc) retainableTsOp.getConf().getFilterExpr();
+ }
+ if (discardableTsOp.getConf().getFilterExpr() != null) {
+ // Push filter on top
+ pushFilterToTopOfTableScan(optimizerCache, discardableTsOp);
+ ExprNodeGenericFuncDesc tsExprNode = discardableTsOp.getConf().getFilterExpr();
+ if (exprNode != null && !exprNode.isSame(tsExprNode)) {
+ // We merge filters from previous scan by ORing with filters from current scan
+ if (exprNode.getGenericUDF() instanceof GenericUDFOPOr) {
+ List<ExprNodeDesc> newChildren = new ArrayList<>(exprNode.getChildren().size() + 1);
+ for (ExprNodeDesc childExprNode : exprNode.getChildren()) {
+ if (childExprNode.isSame(tsExprNode)) {
+ // We do not need to do anything, it is in the OR expression
+ break;
+ }
+ newChildren.add(childExprNode);
+ }
+ if (exprNode.getChildren().size() == newChildren.size()) {
+ newChildren.add(tsExprNode);
+ exprNode = ExprNodeGenericFuncDesc.newInstance(
+ new GenericUDFOPOr(),
+ newChildren);
+ }
+ } else {
+ exprNode = ExprNodeGenericFuncDesc.newInstance(
+ new GenericUDFOPOr(),
+ Arrays.<ExprNodeDesc>asList(exprNode, tsExprNode));
+ }
+ }
+ }
+ // Replace filter
+ retainableTsOp.getConf().setFilterExpr(exprNode);
+ // Replace table scan operator
+ List<Operator<? extends OperatorDesc>> allChildren =
+ Lists.newArrayList(discardableTsOp.getChildOperators());
+ for (Operator<? extends OperatorDesc> op : allChildren) {
+ discardableTsOp.getChildOperators().remove(op);
+ op.replaceParent(discardableTsOp, retainableTsOp);
+ retainableTsOp.getChildOperators().add(op);
+ }
+
+ LOG.debug("Merging {} into {}", discardableTsOp, retainableTsOp);
+ }
+
+ // First we remove the input operators of the expression that
+ // we are going to eliminate
+ for (Operator<?> op : sr.discardableInputOps) {
+ OperatorUtils.removeOperator(op);
+ optimizerCache.removeOp(op);
+ removedOps.add(op);
+ // Remove DPP predicates
+ if (op instanceof ReduceSinkOperator) {
+ SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op);
+ if (sjbi != null && !sr.discardableOps.contains(sjbi.getTsOp()) &&
+ !sr.discardableInputOps.contains(sjbi.getTsOp())) {
+ GenTezUtils.removeSemiJoinOperator(
+ pctx, (ReduceSinkOperator) op, sjbi.getTsOp());
+ }
+ } else if (op instanceof AppMasterEventOperator) {
+ DynamicPruningEventDesc dped = (DynamicPruningEventDesc) op.getConf();
+ if (!sr.discardableOps.contains(dped.getTableScan()) &&
+ !sr.discardableInputOps.contains(dped.getTableScan())) {
+ GenTezUtils.removeSemiJoinOperator(
+ pctx, (AppMasterEventOperator) op, dped.getTableScan());
+ }
+ }
+ LOG.debug("Input operator removed: {}", op);
+ }
+ // Then we merge the operators of the works we are going to merge
+ optimizerCache.removeOpAndCombineWork(discardableTsOp, retainableTsOp);
+ removedOps.add(discardableTsOp);
+ // Finally we remove the expression from the tree
+ for (Operator<?> op : sr.discardableOps) {
+ OperatorUtils.removeOperator(op);
+ optimizerCache.removeOp(op);
+ removedOps.add(op);
+ if (sr.discardableOps.size() == 1) {
+ // If there is a single discardable operator, it is a TableScanOperator
+ // and it means that we have merged filter expressions for it. Thus, we
+ // might need to remove DPP predicates from the retainable TableScanOperator
+ Collection<Operator<?>> c =
+ optimizerCache.tableScanToDPPSource.get((TableScanOperator) op);
+ for (Operator<?> dppSource : c) {
+ if (dppSource instanceof ReduceSinkOperator) {
+ GenTezUtils.removeSemiJoinOperator(pctx,
+ (ReduceSinkOperator) dppSource,
+ (TableScanOperator) sr.retainableOps.get(0));
+ } else if (dppSource instanceof AppMasterEventOperator) {
+ GenTezUtils.removeSemiJoinOperator(pctx,
+ (AppMasterEventOperator) dppSource,
+ (TableScanOperator) sr.retainableOps.get(0));
+ }
+ }
+ }
+ LOG.debug("Operator removed: {}", op);
+ }
+
+ break;
+ }
+
+ if (removedOps.contains(discardableTsOp)) {
+ // This operator has been removed, remove it from the list of existing operators
+ existingOps.remove(tableName, discardableTsOp);
+ } else {
+ // This operator has not been removed, include it in the list of existing operators
+ existingOps.put(tableName, discardableTsOp);
+ }
+ }
+ }
+
+ // Remove unused table scan operators
+ Iterator<Entry<String, TableScanOperator>> it = topOps.entrySet().iterator();
+ while (it.hasNext()) {
+ Entry<String, TableScanOperator> e = it.next();
+ if (e.getValue().getNumChild() == 0) {
+ it.remove();
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("After SharedWorkOptimizer:\n" + Operator.toString(pctx.getTopOps().values()));
+ }
+
+ return pctx;
+ }
+
+ /**
+ * This method gathers the TS operators with DPP from the context and
+ * stores them into the input optimization cache.
+ */
+ private static void gatherDPPTableScanOps(
+ ParseContext pctx, SharedWorkOptimizerCache optimizerCache) throws SemanticException {
+ // Find TS operators with partition pruning enabled in plan
+ // because these TS may potentially read different data for
+ // different pipeline.
+ // These can be:
+ // 1) TS with DPP.
+ // 2) TS with semijoin DPP.
+ Map<String, TableScanOperator> topOps = pctx.getTopOps();
+ Collection<Operator<? extends OperatorDesc>> tableScanOps =
+ Lists.<Operator<?>>newArrayList(topOps.values());
+ Set<AppMasterEventOperator> s =
+ OperatorUtils.findOperators(tableScanOps, AppMasterEventOperator.class);
+ for (AppMasterEventOperator a : s) {
+ if (a.getConf() instanceof DynamicPruningEventDesc) {
+ DynamicPruningEventDesc dped = (DynamicPruningEventDesc) a.getConf();
+ optimizerCache.tableScanToDPPSource.put(dped.getTableScan(), a);
+ }
+ }
+ for (Entry<ReduceSinkOperator, SemiJoinBranchInfo> e
+ : pctx.getRsToSemiJoinBranchInfo().entrySet()) {
+ optimizerCache.tableScanToDPPSource.put(e.getValue().getTsOp(), e.getKey());
+ }
+ LOG.debug("DPP information stored in the cache: {}", optimizerCache.tableScanToDPPSource);
+ }
+
+ private static Multimap<String, TableScanOperator> splitTableScanOpsByTable(
+ ParseContext pctx) {
+ Multimap<String, TableScanOperator> tableNameToOps = ArrayListMultimap.create();
+ for (Entry<String, TableScanOperator> e : pctx.getTopOps().entrySet()) {
+ TableScanOperator tsOp = e.getValue();
+ tableNameToOps.put(
+ tsOp.getConf().getTableMetadata().getDbName() + "."
+ + tsOp.getConf().getTableMetadata().getTableName(), tsOp);
+ }
+ return tableNameToOps;
+ }
+
+ private static List<Entry<String, Long>> rankTablesByAccumulatedSize(ParseContext pctx) {
+ Map<String, Long> tableToTotalSize = new HashMap<>();
+ for (Entry<String, TableScanOperator> e : pctx.getTopOps().entrySet()) {
+ TableScanOperator tsOp = e.getValue();
+ String tableName = tsOp.getConf().getTableMetadata().getDbName() + "."
+ + tsOp.getConf().getTableMetadata().getTableName();
+ long tableSize = tsOp.getStatistics() != null ?
+ tsOp.getStatistics().getDataSize() : 0L;
+ Long totalSize = tableToTotalSize.get(tableName);
+ if (totalSize != null) {
+ tableToTotalSize.put(tableName,
+ StatsUtils.safeAdd(totalSize, tableSize));
+ } else {
+ tableToTotalSize.put(tableName, tableSize);
+ }
+ }
+ List<Entry<String, Long>> sortedTables =
+ new LinkedList<>(tableToTotalSize.entrySet());
+ Collections.sort(sortedTables, Collections.reverseOrder(
+ new Comparator<Map.Entry<String, Long>>() {
+ public int compare(Map.Entry<String, Long> o1, Map.Entry<String, Long> o2) {
+ return (o1.getValue()).compareTo(o2.getValue());
+ }
+ }));
+ return sortedTables;
+ }
+
+ private static boolean areMergeable(ParseContext pctx, SharedWorkOptimizerCache optimizerCache,
+ TableScanOperator tsOp1, TableScanOperator tsOp2) throws SemanticException {
+ // First we check if the two table scan operators can actually be merged
+ // If schemas do not match, we currently do not merge
+ List<String> prevTsOpNeededColumns = tsOp1.getNeededColumns();
+ List<String> tsOpNeededColumns = tsOp2.getNeededColumns();
+ if (prevTsOpNeededColumns.size() != tsOpNeededColumns.size()) {
+ return false;
+ }
+ boolean notEqual = false;
+ for (int i = 0; i < prevTsOpNeededColumns.size(); i++) {
+ if (!prevTsOpNeededColumns.get(i).equals(tsOpNeededColumns.get(i))) {
+ notEqual = true;
+ break;
+ }
+ }
+ if (notEqual) {
+ return false;
+ }
+ // If row limit does not match, we currently do not merge
+ if (tsOp1.getConf().getRowLimit() != tsOp2.getConf().getRowLimit()) {
+ return false;
+ }
+ // If partitions do not match, we currently do not merge
+ PrunedPartitionList prevTsOpPPList = pctx.getPrunedPartitions(tsOp1);
+ PrunedPartitionList tsOpPPList = pctx.getPrunedPartitions(tsOp2);
+ if (!prevTsOpPPList.getPartitions().equals(tsOpPPList.getPartitions())) {
+ return false;
+ }
+ // If is a DPP, check if actually it refers to same target, column, etc.
+ // Further, the DPP value needs to be generated from same subtree
+ List<Operator<?>> dppsOp1 = new ArrayList<>(optimizerCache.tableScanToDPPSource.get(tsOp1));
+ List<Operator<?>> dppsOp2 = new ArrayList<>(optimizerCache.tableScanToDPPSource.get(tsOp2));
+ if (dppsOp1.isEmpty() && dppsOp2.isEmpty()) {
+ return true;
+ }
+ for (int i = 0; i < dppsOp1.size(); i++) {
+ Operator<?> op = dppsOp1.get(i);
+ if (op instanceof ReduceSinkOperator) {
+ Set<Operator<?>> ascendants =
+ findAscendantWorkOperators(pctx, optimizerCache, op);
+ if (ascendants.contains(tsOp2)) {
+ dppsOp1.remove(i);
+ i--;
+ }
+ }
+ }
+ for (int i = 0; i < dppsOp2.size(); i++) {
+ Operator<?> op = dppsOp2.get(i);
+ if (op instanceof ReduceSinkOperator) {
+ Set<Operator<?>> ascendants =
+ findAscendantWorkOperators(pctx, optimizerCache, op);
+ if (ascendants.contains(tsOp1)) {
+ dppsOp2.remove(i);
+ i--;
+ }
+ }
+ }
+ if (dppsOp1.size() != dppsOp2.size()) {
+ // Only first or second operator contains DPP pruning
+ return false;
+ }
+ // Check if DPP branches are equal
+ for (int i = 0; i < dppsOp1.size(); i++) {
+ Operator<?> dppOp1 = dppsOp1.get(i);
+ BitSet bs = new BitSet();
+ for (int j = 0; j < dppsOp2.size(); j++) {
+ if (!bs.get(j)) {
+ // If not visited yet
+ Operator<?> dppOp2 = dppsOp2.get(j);
+ if (compareAndGatherOps(pctx, dppOp1, dppOp2) != null) {
+ // The DPP operator/branch are equal
+ bs.set(j);
+ break;
+ }
+ }
+ }
+ if (bs.cardinality() == i) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static SharedResult extractSharedOptimizationInfo(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache,
+ TableScanOperator retainableTsOp,
+ TableScanOperator discardableTsOp) throws SemanticException {
+ Set<Operator<?>> retainableOps = new LinkedHashSet<>();
+ Set<Operator<?>> discardableOps = new LinkedHashSet<>();
+ Set<Operator<?>> discardableInputOps = new HashSet<>();
+ long dataSize = 0l;
+ long maxDataSize = 0l;
+
+ retainableOps.add(retainableTsOp);
+ discardableOps.add(discardableTsOp);
+ Operator<?> equalOp1 = retainableTsOp;
+ Operator<?> equalOp2 = discardableTsOp;
+ if (equalOp1.getNumChild() > 1 || equalOp2.getNumChild() > 1) {
+ // TODO: Support checking multiple child operators to merge further.
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ }
+ Operator<?> currentOp1 = retainableTsOp.getChildOperators().get(0);
+ Operator<?> currentOp2 = discardableTsOp.getChildOperators().get(0);
+
+ // Special treatment for Filter operator that ignores the DPP predicates
+ if (currentOp1 instanceof FilterOperator && currentOp2 instanceof FilterOperator) {
+ boolean equalFilters = false;
+ FilterDesc op1Conf = ((FilterOperator) currentOp1).getConf();
+ FilterDesc op2Conf = ((FilterOperator) currentOp2).getConf();
+
+ if (op1Conf.getIsSamplingPred() == op2Conf.getIsSamplingPred() &&
+ StringUtils.equals(op1Conf.getSampleDescExpr(), op2Conf.getSampleDescExpr())) {
+ Multiset<String> conjsOp1String = extractConjsIgnoringDPPPreds(op1Conf.getPredicate());
+ Multiset<String> conjsOp2String = extractConjsIgnoringDPPPreds(op2Conf.getPredicate());
+ if (conjsOp1String.equals(conjsOp2String)) {
+ equalFilters = true;
+ }
+ }
+
+ if (equalFilters) {
+ equalOp1 = currentOp1;
+ equalOp2 = currentOp2;
+ retainableOps.add(equalOp1);
+ discardableOps.add(equalOp2);
+ if (currentOp1.getChildOperators().size() > 1 ||
+ currentOp2.getChildOperators().size() > 1) {
+ // TODO: Support checking multiple child operators to merge further.
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps));
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ }
+ currentOp1 = currentOp1.getChildOperators().get(0);
+ currentOp2 = currentOp2.getChildOperators().get(0);
+ } else {
+ // Bail out
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps));
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ }
+ }
+
+ // Try to merge rest of operators
+ while (!(currentOp1 instanceof ReduceSinkOperator)) {
+ // Check whether current operators are equal
+ if (!compareOperator(pctx, currentOp1, currentOp2)) {
+ // If they are not equal, we could zip up till here
+ break;
+ }
+ if (currentOp1.getParentOperators().size() !=
+ currentOp2.getParentOperators().size()) {
+ // If they are not equal, we could zip up till here
+ break;
+ }
+ if (currentOp1.getParentOperators().size() > 1) {
+ List<Operator<?>> discardableOpsForCurrentOp = new ArrayList<>();
+ int idx = 0;
+ for (; idx < currentOp1.getParentOperators().size(); idx++) {
+ Operator<?> parentOp1 = currentOp1.getParentOperators().get(idx);
+ Operator<?> parentOp2 = currentOp2.getParentOperators().get(idx);
+ if (parentOp1 == equalOp1 && parentOp2 == equalOp2) {
+ continue;
+ }
+ if ((parentOp1 == equalOp1 && parentOp2 != equalOp2) ||
+ (parentOp1 != equalOp1 && parentOp2 == equalOp2)) {
+ // Input operator is not in the same position
+ break;
+ }
+ // Compare input
+ List<Operator<?>> removeOpsForCurrentInput = compareAndGatherOps(pctx, parentOp1, parentOp2);
+ if (removeOpsForCurrentInput == null) {
+ // Inputs are not the same, bail out
+ break;
+ }
+ // Add inputs to ops to remove
+ discardableOpsForCurrentOp.addAll(removeOpsForCurrentInput);
+ }
+ if (idx != currentOp1.getParentOperators().size()) {
+ // If inputs are not equal, we could zip up till here
+ break;
+ }
+ discardableInputOps.addAll(discardableOpsForCurrentOp);
+ }
+
+ equalOp1 = currentOp1;
+ equalOp2 = currentOp2;
+ retainableOps.add(equalOp1);
+ discardableOps.add(equalOp2);
+ if (equalOp1 instanceof MapJoinOperator) {
+ MapJoinOperator mop = (MapJoinOperator) equalOp1;
+ dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize());
+ maxDataSize = mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize();
+ }
+ if (currentOp1.getChildOperators().size() > 1 ||
+ currentOp2.getChildOperators().size() > 1) {
+ // TODO: Support checking multiple child operators to merge further.
+ break;
+ }
+ // Update for next iteration
+ currentOp1 = currentOp1.getChildOperators().get(0);
+ currentOp2 = currentOp2.getChildOperators().get(0);
+ }
+
+ // Add the rest to the memory consumption
+ Set<Operator<?>> opsWork1 = findWorkOperators(optimizerCache, currentOp1);
+ for (Operator<?> op : opsWork1) {
+ if (op instanceof MapJoinOperator && !retainableOps.contains(op)) {
+ MapJoinOperator mop = (MapJoinOperator) op;
+ dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize());
+ maxDataSize = mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize();
+ }
+ }
+ Set<Operator<?>> opsWork2 = findWorkOperators(optimizerCache, currentOp2);
+ for (Operator<?> op : opsWork2) {
+ if (op instanceof MapJoinOperator && !discardableOps.contains(op)) {
+ MapJoinOperator mop = (MapJoinOperator) op;
+ dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize());
+ maxDataSize = mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize();
+ }
+ }
+
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableInputOps));
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, discardableOps));
+ discardableInputOps.addAll(gatherDPPBranchOps(pctx, optimizerCache, retainableOps, discardableInputOps));
+ return new SharedResult(retainableOps, discardableOps, discardableInputOps, dataSize, maxDataSize);
+ }
+
+ private static Multiset<String> extractConjsIgnoringDPPPreds(ExprNodeDesc predicate) {
+ List<ExprNodeDesc> conjsOp = ExprNodeDescUtils.split(predicate);
+ Multiset<String> conjsOpString = TreeMultiset.create();
+ for (int i = 0; i < conjsOp.size(); i++) {
+ if (conjsOp.get(i) instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc func = (ExprNodeGenericFuncDesc) conjsOp.get(i);
+ if (GenericUDFInBloomFilter.class == func.getGenericUDF().getClass()) {
+ continue;
+ } else if (GenericUDFBetween.class == func.getGenericUDF().getClass() &&
+ (func.getChildren().get(2) instanceof ExprNodeDynamicValueDesc ||
+ func.getChildren().get(3) instanceof ExprNodeDynamicValueDesc)) {
+ continue;
+ }
+ } else if(conjsOp.get(i) instanceof ExprNodeDynamicListDesc) {
+ continue;
+ }
+ conjsOpString.add(conjsOp.get(i).toString());
+ }
+ return conjsOpString;
+ }
+
+ private static Set<Operator<?>> gatherDPPBranchOps(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache, Set<Operator<?>> ops) {
+ Set<Operator<?>> dppBranches = new HashSet<>();
+ for (Operator<?> op : ops) {
+ if (op instanceof TableScanOperator) {
+ Collection<Operator<?>> c = optimizerCache.tableScanToDPPSource.get((TableScanOperator) op);
+ for (Operator<?> dppSource : c) {
+ // Remove the branches
+ Operator<?> currentOp = dppSource;
+ while (currentOp.getNumChild() <= 1) {
+ dppBranches.add(currentOp);
+ currentOp = currentOp.getParentOperators().get(0);
+ }
+ }
+ }
+ }
+ return dppBranches;
+ }
+
+ private static Set<Operator<?>> gatherDPPBranchOps(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache, Set<Operator<?>> ops,
+ Set<Operator<?>> discardedOps) {
+ Set<Operator<?>> dppBranches = new HashSet<>();
+ for (Operator<?> op : ops) {
+ if (op instanceof TableScanOperator) {
+ Collection<Operator<?>> c = optimizerCache.tableScanToDPPSource.get((TableScanOperator) op);
+ for (Operator<?> dppSource : c) {
+ Set<Operator<?>> ascendants =
+ findAscendantWorkOperators(pctx, optimizerCache, dppSource);
+ if (!Collections.disjoint(ascendants, discardedOps)) {
+ // Remove branch
+ Operator<?> currentOp = dppSource;
+ while (currentOp.getNumChild() <= 1) {
+ dppBranches.add(currentOp);
+ currentOp = currentOp.getParentOperators().get(0);
+ }
+ }
+ }
+ }
+ }
+ return dppBranches;
+ }
+
+ private static List<Operator<?>> compareAndGatherOps(ParseContext pctx,
+ Operator<?> op1, Operator<?> op2) throws SemanticException {
+ List<Operator<?>> result = new ArrayList<>();
+ boolean mergeable = compareAndGatherOps(pctx, op1, op2, result, true);
+ if (!mergeable) {
+ return null;
+ }
+ return result;
+ }
+
+ private static boolean compareAndGatherOps(ParseContext pctx, Operator<?> op1, Operator<?> op2,
+ List<Operator<?>> result, boolean gather) throws SemanticException {
+ if (!compareOperator(pctx, op1, op2)) {
+ LOG.debug("Operators not equal: {} and {}", op1, op2);
+ return false;
+ }
+
+ if (gather) {
+ result.add(op2);
+ }
+
+ List<Operator<? extends OperatorDesc>> op1ParentOperators = op1.getParentOperators();
+ List<Operator<? extends OperatorDesc>> op2ParentOperators = op2.getParentOperators();
+ if (op1ParentOperators != null && op2ParentOperators != null) {
+ if (op1ParentOperators.size() != op2ParentOperators.size()) {
+ return false;
+ }
+ for (int i = 0; i < op1ParentOperators.size(); i++) {
+ Operator<?> op1ParentOp = op1ParentOperators.get(i);
+ Operator<?> op2ParentOp = op2ParentOperators.get(i);
+ boolean mergeable;
+ if (gather && op2ParentOp.getChildOperators().size() < 2) {
+ mergeable = compareAndGatherOps(pctx, op1ParentOp, op2ParentOp, result, true);
+ } else {
+ mergeable = compareAndGatherOps(pctx, op1ParentOp, op2ParentOp, result, false);
+ }
+ if (!mergeable) {
+ return false;
+ }
+ }
+ } else if (op1ParentOperators != null || op2ParentOperators != null) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private static boolean compareOperator(ParseContext pctx, Operator<?> op1, Operator<?> op2)
+ throws SemanticException {
+ if (!op1.getClass().getName().equals(op2.getClass().getName())) {
+ return false;
+ }
+
+ // We handle ReduceSinkOperator here as we can safely ignore table alias
+ // and the current comparator implementation does not.
+ // We can ignore table alias since when we compare ReduceSinkOperator, all
+ // its ancestors need to match (down to table scan), thus we make sure that
+ // both plans are the same.
+ if (op1 instanceof ReduceSinkOperator) {
+ ReduceSinkDesc op1Conf = ((ReduceSinkOperator) op1).getConf();
+ ReduceSinkDesc op2Conf = ((ReduceSinkOperator) op2).getConf();
+
+ if (StringUtils.equals(op1Conf.getKeyColString(), op2Conf.getKeyColString()) &&
+ StringUtils.equals(op1Conf.getValueColsString(), op2Conf.getValueColsString()) &&
+ StringUtils.equals(op1Conf.getParitionColsString(), op2Conf.getParitionColsString()) &&
+ op1Conf.getTag() == op2Conf.getTag() &&
+ StringUtils.equals(op1Conf.getOrder(), op2Conf.getOrder()) &&
+ op1Conf.getTopN() == op2Conf.getTopN() &&
+ op1Conf.isAutoParallel() == op2Conf.isAutoParallel()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ // We handle TableScanOperator here as we can safely ignore table alias
+ // and the current comparator implementation does not.
+ if (op1 instanceof TableScanOperator) {
+ TableScanOperator tsOp1 = (TableScanOperator) op1;
+ TableScanOperator tsOp2 = (TableScanOperator) op2;
+ TableScanDesc op1Conf = tsOp1.getConf();
+ TableScanDesc op2Conf = tsOp2.getConf();
+
+ if (StringUtils.equals(
+ op1Conf.getTableMetadata().getDbName() + "." + op1Conf.getTableMetadata().getTableName(),
+ op2Conf.getTableMetadata().getDbName() + "." + op2Conf.getTableMetadata().getTableName()) &&
+ op1Conf.getNeededColumns().equals(op2Conf.getNeededColumns()) &&
+ StringUtils.equals(op1Conf.getFilterExprString(), op2Conf.getFilterExprString()) &&
+ pctx.getPrunedPartitions(tsOp1).getPartitions().equals(
+ pctx.getPrunedPartitions(tsOp2).getPartitions()) &&
+ op1Conf.getRowLimit() == op2Conf.getRowLimit()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ OperatorComparatorFactory.OperatorComparator operatorComparator =
+ OperatorComparatorFactory.getOperatorComparator(op1.getClass());
+ return operatorComparator.equals(op1, op2);
+ }
+
+ private static boolean validPreConditions(ParseContext pctx, SharedWorkOptimizerCache optimizerCache,
+ SharedResult sr) {
+
+ // We check whether merging the works would cause the size of
+ // the data in memory grow too large.
+ // TODO: Currently ignores GBY and PTF which may also buffer data in memory.
+ if (sr.dataSize > sr.maxDataSize) {
+ // Size surpasses limit, we cannot convert
+ LOG.debug("accumulated data size: {} / max size: {}", sr.dataSize, sr.maxDataSize);
+ return false;
+ }
+
+ TableScanOperator tsOp1 = (TableScanOperator) sr.retainableOps.get(0);
+ TableScanOperator tsOp2 = (TableScanOperator) sr.discardableOps.get(0);
+
+ // 1) The set of operators in the works of the TS operators need to meet
+ // some requirements. In particular:
+ // 1.1. None of the works that contain the TS operators can contain a Union
+ // operator. This is not supported yet as we might end up with cycles in
+ // the Tez DAG.
+ // 1.2. There cannot be more than one DummyStore operator in the new resulting
+ // work when the TS operators are merged. This is due to an assumption in
+ // MergeJoinProc that needs to be further explored.
+ // If any of these conditions are not met, we cannot merge.
+ // TODO: Extend rule so it can be applied for these cases.
+ final Set<Operator<?>> workOps1 = findWorkOperators(optimizerCache, tsOp1);
+ final Set<Operator<?>> workOps2 = findWorkOperators(optimizerCache, tsOp2);
+ boolean foundDummyStoreOp = false;
+ for (Operator<?> op : workOps1) {
+ if (op instanceof UnionOperator) {
+ // We cannot merge (1.1)
+ return false;
+ }
+ if (op instanceof DummyStoreOperator) {
+ foundDummyStoreOp = true;
+ }
+ }
+ for (Operator<?> op : workOps2) {
+ if (op instanceof UnionOperator) {
+ // We cannot merge (1.1)
+ return false;
+ }
+ if (foundDummyStoreOp && op instanceof DummyStoreOperator) {
+ // We cannot merge (1.2)
+ return false;
+ }
+ }
+ // 2) We check whether output works when we merge the operators will collide.
+ //
+ // Work1 Work2 (merge TS in W1 & W2) Work1
+ // \ / -> | | X
+ // Work3 Work3
+ //
+ // If we do, we cannot merge. The reason is that Tez currently does
+ // not support parallel edges, i.e., multiple edges from same work x
+ // into same work y.
+ final Set<Operator<?>> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, tsOp1);
+ final Set<Operator<?>> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, tsOp2);
+ if (!Collections.disjoint(outputWorksOps1, outputWorksOps2)) {
+ // We cannot merge
+ return false;
+ }
+ // 3) We check whether we will end up with same operators inputing on same work.
+ //
+ // Work1 (merge TS in W2 & W3) Work1
+ // / \ -> | | X
+ // Work2 Work3 Work2
+ //
+ // If we do, we cannot merge. The reason is the same as above, currently
+ // Tez currently does not support parallel edges.
+ final Set<Operator<?>> inputWorksOps1 = findParentWorkOperators(pctx, optimizerCache, tsOp1);
+ final Set<Operator<?>> inputWorksOps2 =
+ findParentWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps);
+ if (!Collections.disjoint(inputWorksOps1, inputWorksOps2)) {
+ // We cannot merge
+ return false;
+ }
+ // 4) We check whether one of the operators is part of a work that is an input for
+ // the work of the other operator.
+ //
+ // Work1 (merge TS in W1 & W3) Work1
+ // | -> | X
+ // Work2 Work2
+ // | |
+ // Work3 Work1
+ //
+ // If we do, we cannot merge, as we would end up with a cycle in the DAG.
+ final Set<Operator<?>> descendantWorksOps1 =
+ findDescendantWorkOperators(pctx, optimizerCache, tsOp1, sr.discardableInputOps);
+ final Set<Operator<?>> descendantWorksOps2 =
+ findDescendantWorkOperators(pctx, optimizerCache, tsOp2, sr.discardableInputOps);
+ if (!Collections.disjoint(descendantWorksOps1, workOps2)
+ || !Collections.disjoint(workOps1, descendantWorksOps2)) {
+ return false;
+ }
+ return true;
+ }
+
+ private static Set<Operator<?>> findParentWorkOperators(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache, Operator<?> start) {
+ return findParentWorkOperators(pctx, optimizerCache, start, ImmutableSet.of());
+ }
+
+ private static Set<Operator<?>> findParentWorkOperators(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache, Operator<?> start,
+ Set<Operator<?>> excludeOps) {
+ // Find operators in work
+ Set<Operator<?>> workOps = findWorkOperators(optimizerCache, start);
+ // Gather input works operators
+ Set<Operator<?>> set = new HashSet<Operator<?>>();
+ for (Operator<?> op : workOps) {
+ if (op.getParentOperators() != null) {
+ for (Operator<?> parent : op.getParentOperators()) {
+ if (parent instanceof ReduceSinkOperator && !excludeOps.contains(parent)) {
+ set.addAll(findWorkOperators(optimizerCache, parent));
+ }
+ }
+ } else if (op instanceof TableScanOperator) {
+ // Check for DPP and semijoin DPP
+ for (Operator<?> parent : optimizerCache.tableScanToDPPSource.get((TableScanOperator) op)) {
+ if (!excludeOps.contains(parent)) {
+ set.addAll(findWorkOperators(optimizerCache, parent));
+ }
+ }
+ }
+ }
+ return set;
+ }
+
+ private static Set<Operator<?>> findAscendantWorkOperators(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache, Operator<?> start) {
+ // Find operators in work
+ Set<Operator<?>> workOps = findWorkOperators(optimizerCache, start);
+ // Gather input works operators
+ Set<Operator<?>> result = new HashSet<Operator<?>>();
+ Set<Operator<?>> set;
+ while (!workOps.isEmpty()) {
+ set = new HashSet<Operator<?>>();
+ for (Operator<?> op : workOps) {
+ if (op.getParentOperators() != null) {
+ for (Operator<?> parent : op.getParentOperators()) {
+ if (parent instanceof ReduceSinkOperator) {
+ set.addAll(findWorkOperators(optimizerCache, parent));
+ }
+ }
+ } else if (op instanceof TableScanOperator) {
+ // Check for DPP and semijoin DPP
+ for (Operator<?> parent : optimizerCache.tableScanToDPPSource.get((TableScanOperator) op)) {
+ set.addAll(findWorkOperators(optimizerCache, parent));
+ }
+ }
+ }
+ workOps = set;
+ result.addAll(set);
+ }
+ return result;
+ }
+
+ private static Set<Operator<?>> findChildWorkOperators(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache, Operator<?> start) {
+ // Find operators in work
+ Set<Operator<?>> workOps = findWorkOperators(optimizerCache, start);
+ // Gather output works operators
+ Set<Operator<?>> set = new HashSet<Operator<?>>();
+ for (Operator<?> op : workOps) {
+ if (op instanceof ReduceSinkOperator) {
+ if (op.getChildOperators() != null) {
+ // All children of RS are descendants
+ for (Operator<?> child : op.getChildOperators()) {
+ set.addAll(findWorkOperators(optimizerCache, child));
+ }
+ }
+ // Semijoin DPP work is considered a child because work needs
+ // to finish for it to execute
+ SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op);
+ if (sjbi != null) {
+ set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp()));
+ }
+ } else if(op.getConf() instanceof DynamicPruningEventDesc) {
+ // DPP work is considered a child because work needs
+ // to finish for it to execute
+ set.addAll(findWorkOperators(
+ optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan()));
+ }
+ }
+ return set;
+ }
+
+ private static Set<Operator<?>> findDescendantWorkOperators(ParseContext pctx,
+ SharedWorkOptimizerCache optimizerCache, Operator<?> start,
+ Set<Operator<?>> excludeOps) {
+ // Find operators in work
+ Set<Operator<?>> workOps = findWorkOperators(optimizerCache, start);
+ // Gather output works operators
+ Set<Operator<?>> result = new HashSet<Operator<?>>();
+ Set<Operator<?>> set;
+ while (!workOps.isEmpty()) {
+ set = new HashSet<Operator<?>>();
+ for (Operator<?> op : workOps) {
+ if (excludeOps.contains(op)) {
+ continue;
+ }
+ if (op instanceof ReduceSinkOperator) {
+ if (op.getChildOperators() != null) {
+ // All children of RS are descendants
+ for (Operator<?> child : op.getChildOperators()) {
+ set.addAll(findWorkOperators(optimizerCache, child));
+ }
+ }
+ // Semijoin DPP work is considered a descendant because work needs
+ // to finish for it to execute
+ SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op);
+ if (sjbi != null) {
+ set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp()));
+ }
+ } else if(op.getConf() instanceof DynamicPruningEventDesc) {
+ // DPP work is considered a descendant because work needs
+ // to finish for it to execute
+ set.addAll(findWorkOperators(
+ optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan()));
+ }
+ }
+ workOps = set;
+ result.addAll(set);
+ }
+ return result;
+ }
+
+ // Stores result in cache
+ private static Set<Operator<?>> findWorkOperators(
+ SharedWorkOptimizerCache optimizerCache, Operator<?> start) {
+ Set<Operator<?>> c = optimizerCache.operatorToWorkOperators.get(start);
+ if (!c.isEmpty()) {
+ return c;
+ }
+ c = findWorkOperators(start, new HashSet<Operator<?>>());
+ for (Operator<?> op : c) {
+ optimizerCache.operatorToWorkOperators.putAll(op, c);
+ }
+ return c;
+ }
+
+ private static Set<Operator<?>> findWorkOperators(Operator<?> start, Set<Operator<?>> found) {
+ found.add(start);
+ if (start.getParentOperators() != null) {
+ for (Operator<?> parent : start.getParentOperators()) {
+ if (parent instanceof ReduceSinkOperator) {
+ continue;
+ }
+ if (!found.contains(parent)) {
+ findWorkOperators(parent, found);
+ }
+ }
+ }
+ if (start instanceof ReduceSinkOperator) {
+ return found;
+ }
+ if (start.getChildOperators() != null) {
+ for (Operator<?> child : start.getChildOperators()) {
+ if (!found.contains(child)) {
+ findWorkOperators(child, found);
+ }
+ }
+ }
+ return found;
+ }
+
+ private static void pushFilterToTopOfTableScan(
+ SharedWorkOptimizerCache optimizerCache, TableScanOperator tsOp)
+ throws UDFArgumentException {
+ ExprNodeGenericFuncDesc tableScanExprNode = tsOp.getConf().getFilterExpr();
+ List<Operator<? extends OperatorDesc>> allChildren =
+ Lists.newArrayList(tsOp.getChildOperators());
+ for (Operator<? extends OperatorDesc> op : allChildren) {
+ if (op instanceof FilterOperator) {
+ FilterOperator filterOp = (FilterOperator) op;
+ ExprNodeDesc filterExprNode = filterOp.getConf().getPredicate();
+ if (tableScanExprNode.isSame(filterExprNode)) {
+ // We do not need to do anything
+ return;
+ }
+ if (tableScanExprNode.getGenericUDF() instanceof GenericUDFOPOr) {
+ for (ExprNodeDesc childExprNode : tableScanExprNode.getChildren()) {
+ if (childExprNode.isSame(filterExprNode)) {
+ // We do not need to do anything, it is in the OR expression
+ // so probably we pushed previously
+ return;
+ }
+ }
+ }
+ ExprNodeGenericFuncDesc newPred = ExprNodeGenericFuncDesc.newInstance(
+ new GenericUDFOPAnd(),
+ Arrays.<ExprNodeDesc>asList(tableScanExprNode.clone(), filterExprNode));
+ filterOp.getConf().setPredicate(newPred);
+ } else {
+ Operator<FilterDesc> newOp = OperatorFactory.get(tsOp.getCompilationOpContext(),
+ new FilterDesc(tableScanExprNode.clone(), false),
+ new RowSchema(tsOp.getSchema().getSignature()));
+ tsOp.replaceChild(op, newOp);
+ newOp.getParentOperators().add(tsOp);
+ op.replaceParent(tsOp, newOp);
+ newOp.getChildOperators().add(op);
+ // Add to cache (same group as tsOp)
+ optimizerCache.putIfWorkExists(newOp, tsOp);
+ }
+ }
+ }
+
+ private static class SharedResult {
+ final List<Operator<?>> retainableOps;
+ final List<Operator<?>> discardableOps;
+ final Set<Operator<?>> discardableInputOps;
+ final long dataSize;
+ final long maxDataSize;
+
+ private SharedResult(Collection<Operator<?>> retainableOps, Collection<Operator<?>> discardableOps,
+ Set<Operator<?>> discardableInputOps, long dataSize, long maxDataSize) {
+ this.retainableOps = ImmutableList.copyOf(retainableOps);
+ this.discardableOps = ImmutableList.copyOf(discardableOps);
+ this.discardableInputOps = ImmutableSet.copyOf(discardableInputOps);
+ this.dataSize = dataSize;
+ this.maxDataSize = maxDataSize;
+ }
+ }
+
+ /** Cache to accelerate optimization */
+ private static class SharedWorkOptimizerCache {
+ // Operators that belong to each work
+ final HashMultimap<Operator<?>, Operator<?>> operatorToWorkOperators =
+ HashMultimap.<Operator<?>, Operator<?>>create();
+ // Table scan operators to DPP sources
+ final Multimap<TableScanOperator, Operator<?>> tableScanToDPPSource =
+ HashMultimap.<TableScanOperator, Operator<?>>create();
+
+ // Add new operator to cache work group of existing operator (if group exists)
+ void putIfWorkExists(Operator<?> opToAdd, Operator<?> existingOp) {
+ List<Operator<?>> c = ImmutableList.copyOf(operatorToWorkOperators.get(existingOp));
+ if (!c.isEmpty()) {
+ for (Operator<?> op : c) {
+ operatorToWorkOperators.get(op).add(opToAdd);
+ }
+ operatorToWorkOperators.putAll(opToAdd, c);
+ operatorToWorkOperators.put(opToAdd, opToAdd);
+ }
+ }
+
+ // Remove operator
+ void removeOp(Operator<?> opToRemove) {
+ Set<Operator<?>> s = operatorToWorkOperators.get(opToRemove);
+ s.remove(opToRemove);
+ List<Operator<?>> c1 = ImmutableList.copyOf(s);
+ if (!c1.isEmpty()) {
+ for (Operator<?> op1 : c1) {
+ operatorToWorkOperators.remove(op1, opToRemove); // Remove operator
+ }
+ operatorToWorkOperators.removeAll(opToRemove); // Remove entry for operator
+ }
+ }
+
+ // Remove operator and combine
+ void removeOpAndCombineWork(Operator<?> opToRemove, Operator<?> replacementOp) {
+ Set<Operator<?>> s = operatorToWorkOperators.get(opToRemove);
+ s.remove(opToRemove);
+ List<Operator<?>> c1 = ImmutableList.copyOf(s);
+ List<Operator<?>> c2 = ImmutableList.copyOf(operatorToWorkOperators.get(replacementOp));
+ if (!c1.isEmpty() && !c2.isEmpty()) {
+ for (Operator<?> op1 : c1) {
+ operatorToWorkOperators.remove(op1, opToRemove); // Remove operator
+ operatorToWorkOperators.putAll(op1, c2); // Add ops of new collection
+ }
+ operatorToWorkOperators.removeAll(opToRemove); // Remove entry for operator
+ for (Operator<?> op2 : c2) {
+ operatorToWorkOperators.putAll(op2, c1); // Add ops to existing collection
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index ca544b4..6423a6d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -275,7 +275,7 @@ public class GenTezUtils {
SemiJoinBranchInfo sjInfo = rsToSemiJoinBranchInfo.get(rs);
if (sjInfo.getTsOp() == orig) {
SemiJoinBranchInfo newSJInfo = new SemiJoinBranchInfo(
- (TableScanOperator)newRoot, sjInfo.getIsHint());
+ (TableScanOperator) newRoot, sjInfo.getIsHint());
rsToSemiJoinBranchInfo.put(rs, newSJInfo);
}
}
@@ -580,46 +580,135 @@ public class GenTezUtils {
LOG.debug("Removing ReduceSink " + rs + " and TableScan " + ts);
ExprNodeDesc constNode = new ExprNodeConstantDesc(
TypeInfoFactory.booleanTypeInfo, Boolean.TRUE);
+ // TS operator
DynamicValuePredicateContext filterDynamicValuePredicatesCollection =
new DynamicValuePredicateContext();
- FilterDesc filterDesc = ((FilterOperator)(ts.getChildOperators().get(0))).getConf();
- collectDynamicValuePredicates(filterDesc.getPredicate(),
- filterDynamicValuePredicatesCollection);
- for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection
- .childParentMapping.keySet()) {
- // Find out if this synthetic predicate belongs to the current cycle
- boolean skip = true;
- for (ExprNodeDesc expr : nodeToRemove.getChildren()) {
- if (expr instanceof ExprNodeDynamicValueDesc ) {
- String dynamicValueIdFromExpr = ((ExprNodeDynamicValueDesc) expr)
- .getDynamicValue().getId();
- List<String> dynamicValueIdsFromMap = context.
- getRsToRuntimeValuesInfoMap().get(rs).getDynamicValueIDs();
- for (String dynamicValueIdFromMap : dynamicValueIdsFromMap) {
- if (dynamicValueIdFromExpr.equals(dynamicValueIdFromMap)) {
- // Intended predicate to be removed
- skip = false;
- break;
- }
+ if (ts.getConf().getFilterExpr() != null) {
+ collectDynamicValuePredicates(ts.getConf().getFilterExpr(),
+ filterDynamicValuePredicatesCollection);
+ for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection
+ .childParentMapping.keySet()) {
+ // Find out if this synthetic predicate belongs to the current cycle
+ if (removeSemiJoinPredicate(context, rs, nodeToRemove)) {
+ ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection
+ .childParentMapping.get(nodeToRemove);
+ if (nodeParent == null) {
+ // This was the only predicate, set filter expression to null
+ ts.getConf().setFilterExpr(null);
+ } else {
+ int i = nodeParent.getChildren().indexOf(nodeToRemove);
+ nodeParent.getChildren().remove(i);
+ nodeParent.getChildren().add(i, constNode);
}
}
}
- if (!skip) {
- ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection
- .childParentMapping.get(nodeToRemove);
- if (nodeParent == null) {
+ }
+ // Filter operator
+ filterDynamicValuePredicatesCollection = new DynamicValuePredicateContext();
+ for (Operator<?> op : ts.getChildOperators()) {
+ if (!(op instanceof FilterOperator)) {
+ continue;
+ }
+ FilterDesc filterDesc = ((FilterOperator) op).getConf();
+ collectDynamicValuePredicates(filterDesc.getPredicate(),
+ filterDynamicValuePredicatesCollection);
+ for (ExprNodeDesc nodeToRemove : filterDynamicValuePredicatesCollection
+ .childParentMapping.keySet()) {
+ // Find out if this synthetic predicate belongs to the current cycle
+ if (removeSemiJoinPredicate(context, rs, nodeToRemove)) {
+ ExprNodeDesc nodeParent = filterDynamicValuePredicatesCollection
+ .childParentMapping.get(nodeToRemove);
+ if (nodeParent == null) {
+ // This was the only predicate, set filter expression to const
+ filterDesc.setPredicate(constNode);
+ } else {
+ int i = nodeParent.getChildren().indexOf(nodeToRemove);
+ nodeParent.getChildren().remove(i);
+ nodeParent.getChildren().add(i, constNode);
+ }
+ }
+ }
+ }
+ context.getRsToSemiJoinBranchInfo().remove(rs);
+ }
+
+ /** Find out if this predicate constains the synthetic predicate to be removed */
+ private static boolean removeSemiJoinPredicate(ParseContext context,
+ ReduceSinkOperator rs, ExprNodeDesc nodeToRemove) {
+ boolean remove = false;
+ for (ExprNodeDesc expr : nodeToRemove.getChildren()) {
+ if (expr instanceof ExprNodeDynamicValueDesc ) {
+ String dynamicValueIdFromExpr = ((ExprNodeDynamicValueDesc) expr)
+ .getDynamicValue().getId();
+ List<String> dynamicValueIdsFromMap = context.
+ getRsToRuntimeValuesInfoMap().get(rs).getDynamicValueIDs();
+ for (String dynamicValueIdFromMap : dynamicValueIdsFromMap) {
+ if (dynamicValueIdFromExpr.equals(dynamicValueIdFromMap)) {
+ // Intended predicate to be removed
+ remove = true;
+ break;
+ }
+ }
+ }
+ }
+ return remove;
+ }
+
+ // Functionality to remove semi-join optimization
+ public static void removeSemiJoinOperator(ParseContext context,
+ AppMasterEventOperator eventOp, TableScanOperator ts) throws SemanticException{
+ // Cleanup the synthetic predicate in the tablescan operator and filter by
+ // replacing it with "true"
+ LOG.debug("Removing AppMasterEventOperator " + eventOp + " and TableScan " + ts);
+ ExprNodeDesc constNode = new ExprNodeConstantDesc(
+ TypeInfoFactory.booleanTypeInfo, Boolean.TRUE);
+ // Retrieve generator
+ DynamicPruningEventDesc dped = (DynamicPruningEventDesc) eventOp.getConf();
+ // TS operator
+ DynamicPartitionPrunerContext filterDynamicListPredicatesCollection =
+ new DynamicPartitionPrunerContext();
+ if (ts.getConf().getFilterExpr() != null) {
+ collectDynamicPruningConditions(
+ ts.getConf().getFilterExpr(), filterDynamicListPredicatesCollection);
+ for (DynamicListContext ctx : filterDynamicListPredicatesCollection) {
+ if (ctx.generator != dped.getGenerator()) {
+ continue;
+ }
+ // remove the condition by replacing it with "true"
+ if (ctx.grandParent == null) {
+ // This was the only predicate, set filter expression to const
+ ts.getConf().setFilterExpr(null);
+ } else {
+ int i = ctx.grandParent.getChildren().indexOf(ctx.parent);
+ ctx.grandParent.getChildren().remove(i);
+ ctx.grandParent.getChildren().add(i, constNode);
+ }
+ }
+ }
+ // Filter operator
+ filterDynamicListPredicatesCollection.dynLists.clear();
+ for (Operator<?> op : ts.getChildOperators()) {
+ if (!(op instanceof FilterOperator)) {
+ continue;
+ }
+ FilterDesc filterDesc = ((FilterOperator) op).getConf();
+ collectDynamicPruningConditions(
+ filterDesc.getPredicate(), filterDynamicListPredicatesCollection);
+ for (DynamicListContext ctx : filterDynamicListPredicatesCollection) {
+ if (ctx.generator != dped.getGenerator()) {
+ continue;
+ }
+ // remove the condition by replacing it with "true"
+ if (ctx.grandParent == null) {
// This was the only predicate, set filter expression to const
filterDesc.setPredicate(constNode);
} else {
- int i = nodeParent.getChildren().indexOf(nodeToRemove);
- nodeParent.getChildren().remove(i);
- nodeParent.getChildren().add(i, constNode);
+ int i = ctx.grandParent.getChildren().indexOf(ctx.parent);
+ ctx.grandParent.getChildren().remove(i);
+ ctx.grandParent.getChildren().add(i, constNode);
}
- // skip the rest of the predicates
- skip = true;
}
}
- context.getRsToSemiJoinBranchInfo().remove(rs);
}
private static class DynamicValuePredicateContext implements NodeProcessorCtx {
@@ -647,7 +736,8 @@ public class GenTezUtils {
}
}
- private static void collectDynamicValuePredicates(ExprNodeDesc pred, NodeProcessorCtx ctx) throws SemanticException {
+ private static void collectDynamicValuePredicates(ExprNodeDesc pred, NodeProcessorCtx ctx)
+ throws SemanticException {
// create a walker which walks the tree in a DFS manner while maintaining
// the operator stack. The dispatcher
// generates the plan from the operator tree
@@ -660,4 +750,80 @@ public class GenTezUtils {
egw.startWalking(startNodes, null);
}
+
+ public static class DynamicListContext {
+ public ExprNodeDynamicListDesc desc;
+ public ExprNodeDesc parent;
+ public ExprNodeDesc grandParent;
+ public ReduceSinkOperator generator;
+
+ public DynamicListContext(ExprNodeDynamicListDesc desc, ExprNodeDesc parent,
+ ExprNodeDesc grandParent, ReduceSinkOperator generator) {
+ this.desc = desc;
+ this.parent = parent;
+ this.grandParent = grandParent;
+ this.generator = generator;
+ }
+ }
+
+ public static class DynamicPartitionPrunerContext implements NodeProcessorCtx,
+ Iterable<DynamicListContext> {
+ public List<DynamicListContext> dynLists = new ArrayList<DynamicListContext>();
+
+ public void addDynamicList(ExprNodeDynamicListDesc desc, ExprNodeDesc parent,
+ ExprNodeDesc grandParent, ReduceSinkOperator generator) {
+ dynLists.add(new DynamicListContext(desc, parent, grandParent, generator));
+ }
+
+ @Override
+ public Iterator<DynamicListContext> iterator() {
+ return dynLists.iterator();
+ }
+ }
+
+ public static class DynamicPartitionPrunerProc implements NodeProcessor {
+
+ /**
+ * process simply remembers all the dynamic partition pruning expressions
+ * found
+ */
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+ ExprNodeDynamicListDesc desc = (ExprNodeDynamicListDesc) nd;
+ DynamicPartitionPrunerContext context = (DynamicPartitionPrunerContext) procCtx;
+
+ // Rule is searching for dynamic pruning expr. There's at least an IN
+ // expression wrapping it.
+ ExprNodeDesc parent = (ExprNodeDesc) stack.get(stack.size() - 2);
+ ExprNodeDesc grandParent = stack.size() >= 3 ? (ExprNodeDesc) stack.get(stack.size() - 3) : null;
+
+ context.addDynamicList(desc, parent, grandParent, (ReduceSinkOperator) desc.getSource());
+
+ return context;
+ }
+ }
+
+ public static Map<Node, Object> collectDynamicPruningConditions(ExprNodeDesc pred, NodeProcessorCtx ctx)
+ throws SemanticException {
+
+ // create a walker which walks the tree in a DFS manner while maintaining
+ // the operator stack. The dispatcher
+ // generates the plan from the operator tree
+ Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
+ exprRules.put(new RuleRegExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"),
+ new DynamicPartitionPrunerProc());
+
+ // The dispatcher fires the processor corresponding to the closest matching
+ // rule and passes the context along
+ Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx);
+ GraphWalker egw = new DefaultGraphWalker(disp);
+
+ List<Node> startNodes = new ArrayList<Node>();
+ startNodes.add(pred);
+
+ HashMap<Node, Object> outputMap = new HashMap<Node, Object>();
+ egw.startWalking(startNodes, outputMap);
+ return outputMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index 20f16fb..888627a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -76,7 +76,7 @@ import org.apache.hadoop.hive.ql.optimizer.MergeJoinProc;
import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize;
import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
-import org.apache.hadoop.hive.ql.optimizer.SharedScanOptimizer;
+import org.apache.hadoop.hive.ql.optimizer.SharedWorkOptimizer;
import org.apache.hadoop.hive.ql.optimizer.metainfo.annotation.AnnotateWithOpTraits;
import org.apache.hadoop.hive.ql.optimizer.physical.AnnotateRunTimeStatsOptimizer;
import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck;
@@ -184,8 +184,8 @@ public class TezCompiler extends TaskCompiler {
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run cycle analysis for partition pruning");
perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
- if(procCtx.conf.getBoolVar(ConfVars.HIVE_SHARED_SCAN_OPTIMIZATION)) {
- new SharedScanOptimizer().transform(procCtx.parseContext);
+ if(procCtx.conf.getBoolVar(ConfVars.HIVE_SHARED_WORK_OPTIMIZATION)) {
+ new SharedWorkOptimizer().transform(procCtx.parseContext);
}
perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Shared scans optimization");
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
index 73bbebd..d88e110 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPruningEventDesc.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.IOException;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -38,6 +39,9 @@ public class DynamicPruningEventDesc extends AppMasterEventDesc {
// tableScan is only available during compile
private transient TableScanOperator tableScan;
+ // reduceSink is only available during compile
+ private transient ReduceSinkOperator generator;
+
// the partition column we're interested in
private ExprNodeDesc partKey;
@@ -49,6 +53,14 @@ public class DynamicPruningEventDesc extends AppMasterEventDesc {
this.tableScan = tableScan;
}
+ public ReduceSinkOperator getGenerator() {
+ return generator;
+ }
+
+ public void setGenerator(ReduceSinkOperator generator) {
+ this.generator = generator;
+ }
+
@Explain(displayName = "Target column")
public String displayTargetColumn() {
return targetColumnName + " (" + targetColumnType + ")";
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 04686f7..570bd6b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -176,6 +176,9 @@ public class TableScanDesc extends AbstractOperatorDesc {
@Explain(displayName = "filterExpr")
public String getFilterExprString() {
+ if (filterExpr == null) {
+ return null;
+ }
return PlanUtils.getExprListString(Arrays.asList(filterExpr));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/auto_join0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/auto_join0.q.out b/ql/src/test/results/clientpositive/llap/auto_join0.q.out
index 6d051ea..1141db2 100644
--- a/ql/src/test/results/clientpositive/llap/auto_join0.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_join0.q.out
@@ -52,13 +52,6 @@ STAGE PLANS:
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (key < 10) (type: boolean)
- Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/auto_join30.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/auto_join30.q.out b/ql/src/test/results/clientpositive/llap/auto_join30.q.out
index 90c4241..b1cb5fa 100644
--- a/ql/src/test/results/clientpositive/llap/auto_join30.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_join30.q.out
@@ -497,13 +497,6 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
@@ -678,10 +671,6 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
@@ -862,10 +851,6 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
@@ -1230,10 +1215,6 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
index bdb30d7..b53b2ed 100644
--- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
+++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_9.q.out
@@ -474,10 +474,9 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Map 1 <- Map 3 (BROADCAST_EDGE)
- Map 4 <- Map 3 (BROADCAST_EDGE)
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE)
- Reducer 5 <- Map 4 (SIMPLE_EDGE)
+ Map 1 <- Map 4 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (BROADCAST_EDGE)
+ Reducer 3 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -500,7 +499,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col0
input vertices:
- 1 Map 3
+ 1 Map 4
Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
@@ -514,9 +513,15 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Execution mode: llap
LLAP IO: no inputs
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: b
@@ -533,54 +538,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Execution mode: llap
- LLAP IO: no inputs
- Map 4
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0
- input vertices:
- 1 Map 3
- Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: _col0 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -600,7 +557,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col0, _col1, _col3
input vertices:
- 1 Reducer 5
+ 1 Reducer 3
Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint)
@@ -613,7 +570,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 5
+ Reducer 3
Execution mode: llap
Reduce Operator Tree:
Group By Operator
@@ -2298,10 +2255,9 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Map 1 <- Map 3 (BROADCAST_EDGE)
- Map 4 <- Map 3 (BROADCAST_EDGE)
- Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE)
- Reducer 5 <- Map 4 (SIMPLE_EDGE)
+ Map 1 <- Map 4 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 3 (BROADCAST_EDGE)
+ Reducer 3 <- Map 1 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -2324,7 +2280,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col0
input vertices:
- 1 Map 3
+ 1 Map 4
Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
@@ -2338,9 +2294,15 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Execution mode: llap
LLAP IO: no inputs
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: b
@@ -2357,54 +2319,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Execution mode: llap
- LLAP IO: no inputs
- Map 4
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0
- input vertices:
- 1 Map 3
- Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: _col0 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 11 Data size: 77 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
Execution mode: llap
LLAP IO: no inputs
Reducer 2
@@ -2424,7 +2338,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col0, _col1, _col3
input vertices:
- 1 Reducer 5
+ 1 Reducer 3
Statistics: Num rows: 5 Data size: 38 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: bigint), _col3 (type: bigint)
@@ -2437,7 +2351,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 5
+ Reducer 3
Execution mode: llap
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
index 042c60b..649afb5 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_map_join_tez1.q.out
@@ -818,13 +818,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: int), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 242 Data size: 4502 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out
index cdae4fb..8e85c02 100644
--- a/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out
+++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer2.q.out
@@ -1780,9 +1780,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(value)
keys: key (type: string)
@@ -1964,9 +1961,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(value)
keys: key (type: string)
[2/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query64.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query64.q.out b/ql/src/test/results/clientpositive/perf/query64.q.out
index f24b14d..ddd0614 100644
--- a/ql/src/test/results/clientpositive/perf/query64.q.out
+++ b/ql/src/test/results/clientpositive/perf/query64.q.out
@@ -317,9 +317,9 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_247]
PartitionCols:_col0
- Select Operator [SEL_227] (rows=1861800 width=385)
+ Select Operator [SEL_24] (rows=1861800 width=385)
Output:["_col0","_col1"]
- Filter Operator [FIL_574] (rows=1861800 width=385)
+ Filter Operator [FIL_543] (rows=1861800 width=385)
predicate:cd_demo_sk is not null
TableScan [TS_22] (rows=1861800 width=385)
default@customer_demographics,cd2,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_marital_status"]
@@ -336,11 +336,7 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_241]
PartitionCols:_col0
- Select Operator [SEL_152] (rows=1861800 width=385)
- Output:["_col0","_col1"]
- Filter Operator [FIL_562] (rows=1861800 width=385)
- predicate:cd_demo_sk is not null
- Please refer to the previous TableScan [TS_22]
+ Please refer to the previous Select Operator [SEL_24]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_240]
PartitionCols:_col1
@@ -349,9 +345,9 @@ Stage-0
<-Map 39 [SIMPLE_EDGE]
SHUFFLE [RS_238]
PartitionCols:_col0
- Select Operator [SEL_149] (rows=40000000 width=1014)
+ Select Operator [SEL_21] (rows=40000000 width=1014)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_561] (rows=40000000 width=1014)
+ Filter Operator [FIL_542] (rows=40000000 width=1014)
predicate:ca_address_sk is not null
TableScan [TS_19] (rows=40000000 width=1014)
default@customer_address,ad2,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_street_number","ca_street_name","ca_city","ca_zip"]
@@ -390,9 +386,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_228]
PartitionCols:_col5
- Select Operator [SEL_130] (rows=80000000 width=860)
+ Select Operator [SEL_2] (rows=80000000 width=860)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_556] (rows=80000000 width=860)
+ Filter Operator [FIL_537] (rows=80000000 width=860)
predicate:(c_customer_sk is not null and c_first_shipto_date_sk is not null and c_first_sales_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null)
TableScan [TS_0] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_hdemo_sk","c_current_addr_sk","c_first_shipto_date_sk","c_first_sales_date_sk"]
@@ -404,18 +400,18 @@ Stage-0
<-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_143]
PartitionCols:_col1
- Select Operator [SEL_139] (rows=7200 width=107)
+ Select Operator [SEL_11] (rows=7200 width=107)
Output:["_col0","_col1"]
- Filter Operator [FIL_559] (rows=7200 width=107)
+ Filter Operator [FIL_540] (rows=7200 width=107)
predicate:(hd_demo_sk is not null and hd_income_band_sk is not null)
TableScan [TS_9] (rows=7200 width=107)
default@household_demographics,hd2,Tbl:COMPLETE,Col:NONE,Output:["hd_demo_sk","hd_income_band_sk"]
<-Map 38 [SIMPLE_EDGE]
SHUFFLE [RS_144]
PartitionCols:_col0
- Select Operator [SEL_142] (rows=20 width=12)
+ Select Operator [SEL_14] (rows=20 width=12)
Output:["_col0"]
- Filter Operator [FIL_560] (rows=20 width=12)
+ Filter Operator [FIL_541] (rows=20 width=12)
predicate:ib_income_band_sk is not null
TableScan [TS_12] (rows=20 width=12)
default@income_band,ib2,Tbl:COMPLETE,Col:NONE,Output:["ib_income_band_sk"]
@@ -429,11 +425,7 @@ Stage-0
<-Map 39 [SIMPLE_EDGE]
SHUFFLE [RS_222]
PartitionCols:_col0
- Select Operator [SEL_208] (rows=40000000 width=1014)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_573] (rows=40000000 width=1014)
- predicate:ca_address_sk is not null
- Please refer to the previous TableScan [TS_19]
+ Please refer to the previous Select Operator [SEL_21]
<-Reducer 31 [SIMPLE_EDGE]
SHUFFLE [RS_221]
PartitionCols:_col13
@@ -442,9 +434,9 @@ Stage-0
<-Map 55 [SIMPLE_EDGE]
SHUFFLE [RS_219]
PartitionCols:_col0
- Select Operator [SEL_205] (rows=1704 width=1910)
+ Select Operator [SEL_77] (rows=1704 width=1910)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_572] (rows=1704 width=1910)
+ Filter Operator [FIL_553] (rows=1704 width=1910)
predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null)
TableScan [TS_75] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"]
@@ -468,9 +460,9 @@ Stage-0
<-Map 48 [SIMPLE_EDGE]
SHUFFLE [RS_184]
PartitionCols:_col0
- Select Operator [SEL_173] (rows=2300 width=1179)
+ Select Operator [SEL_45] (rows=2300 width=1179)
Output:["_col0"]
- Filter Operator [FIL_569] (rows=2300 width=1179)
+ Filter Operator [FIL_550] (rows=2300 width=1179)
predicate:p_promo_sk is not null
TableScan [TS_43] (rows=2300 width=1179)
default@promotion,promotion,Tbl:COMPLETE,Col:NONE,Output:["p_promo_sk"]
@@ -495,9 +487,9 @@ Stage-0
<-Map 47 [SIMPLE_EDGE]
SHUFFLE [RS_178]
PartitionCols:_col0, _col1
- Select Operator [SEL_167] (rows=57591150 width=77)
+ Select Operator [SEL_39] (rows=57591150 width=77)
Output:["_col0","_col1"]
- Filter Operator [FIL_567] (rows=57591150 width=77)
+ Filter Operator [FIL_548] (rows=57591150 width=77)
predicate:(sr_item_sk is not null and sr_ticket_number is not null)
TableScan [TS_37] (rows=57591150 width=77)
default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"]
@@ -509,18 +501,18 @@ Stage-0
<-Map 41 [SIMPLE_EDGE]
SHUFFLE [RS_174]
PartitionCols:_col1
- Select Operator [SEL_161] (rows=575995635 width=88)
+ Select Operator [SEL_33] (rows=575995635 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
- Filter Operator [FIL_565] (rows=575995635 width=88)
+ Filter Operator [FIL_546] (rows=575995635 width=88)
predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_customer_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null)
TableScan [TS_31] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_wholesale_cost","ss_list_price","ss_coupon_amt"]
<-Map 46 [SIMPLE_EDGE]
SHUFFLE [RS_175]
PartitionCols:_col0
- Select Operator [SEL_164] (rows=2851 width=1436)
+ Select Operator [SEL_36] (rows=2851 width=1436)
Output:["_col0","_col3"]
- Filter Operator [FIL_566] (rows=2851 width=1436)
+ Filter Operator [FIL_547] (rows=2851 width=1436)
predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null)
TableScan [TS_34] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_current_price","i_color","i_product_name"]
@@ -532,19 +524,11 @@ Stage-0
<-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_209]
PartitionCols:_col1
- Select Operator [SEL_155] (rows=7200 width=107)
- Output:["_col0","_col1"]
- Filter Operator [FIL_563] (rows=7200 width=107)
- predicate:(hd_demo_sk is not null and hd_income_band_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Map 38 [SIMPLE_EDGE]
SHUFFLE [RS_210]
PartitionCols:_col0
- Select Operator [SEL_158] (rows=20 width=12)
- Output:["_col0"]
- Filter Operator [FIL_564] (rows=20 width=12)
- predicate:ib_income_band_sk is not null
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 53 [SIMPLE_EDGE]
SHUFFLE [RS_216]
PartitionCols:_col0
@@ -566,18 +550,18 @@ Stage-0
<-Map 49 [SIMPLE_EDGE]
SHUFFLE [RS_193]
PartitionCols:_col0, _col1
- Select Operator [SEL_189] (rows=287989836 width=135)
+ Select Operator [SEL_61] (rows=287989836 width=135)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_570] (rows=287989836 width=135)
+ Filter Operator [FIL_551] (rows=287989836 width=135)
predicate:(cs_order_number is not null and cs_item_sk is not null)
TableScan [TS_59] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_item_sk","cs_order_number","cs_ext_list_price"]
<-Map 54 [SIMPLE_EDGE]
SHUFFLE [RS_194]
PartitionCols:_col0, _col1
- Select Operator [SEL_192] (rows=28798881 width=106)
+ Select Operator [SEL_64] (rows=28798881 width=106)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_571] (rows=28798881 width=106)
+ Filter Operator [FIL_552] (rows=28798881 width=106)
predicate:(cr_order_number is not null and cr_item_sk is not null)
TableScan [TS_62] (rows=28798881 width=106)
default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_refunded_cash","cr_reversed_charge","cr_store_credit"]
@@ -602,11 +586,7 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_119]
PartitionCols:_col0
- Select Operator [SEL_99] (rows=1861800 width=385)
- Output:["_col0","_col1"]
- Filter Operator [FIL_555] (rows=1861800 width=385)
- predicate:cd_demo_sk is not null
- Please refer to the previous TableScan [TS_22]
+ Please refer to the previous Select Operator [SEL_24]
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_118]
PartitionCols:_col39
@@ -622,11 +602,7 @@ Stage-0
<-Map 39 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
- Select Operator [SEL_80] (rows=40000000 width=1014)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_554] (rows=40000000 width=1014)
- predicate:ca_address_sk is not null
- Please refer to the previous TableScan [TS_19]
+ Please refer to the previous Select Operator [SEL_21]
<-Reducer 25 [SIMPLE_EDGE]
SHUFFLE [RS_93]
PartitionCols:_col13
@@ -635,11 +611,7 @@ Stage-0
<-Map 55 [SIMPLE_EDGE]
SHUFFLE [RS_91]
PartitionCols:_col0
- Select Operator [SEL_77] (rows=1704 width=1910)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_553] (rows=1704 width=1910)
- predicate:(s_store_sk is not null and s_store_name is not null and s_zip is not null)
- Please refer to the previous TableScan [TS_75]
+ Please refer to the previous Select Operator [SEL_77]
<-Reducer 24 [SIMPLE_EDGE]
SHUFFLE [RS_90]
PartitionCols:_col14
@@ -660,11 +632,7 @@ Stage-0
<-Map 48 [SIMPLE_EDGE]
SHUFFLE [RS_56]
PartitionCols:_col0
- Select Operator [SEL_45] (rows=2300 width=1179)
- Output:["_col0"]
- Filter Operator [FIL_550] (rows=2300 width=1179)
- predicate:p_promo_sk is not null
- Please refer to the previous TableScan [TS_43]
+ Please refer to the previous Select Operator [SEL_45]
<-Reducer 21 [SIMPLE_EDGE]
SHUFFLE [RS_55]
PartitionCols:_col7
@@ -686,11 +654,7 @@ Stage-0
<-Map 47 [SIMPLE_EDGE]
SHUFFLE [RS_50]
PartitionCols:_col0, _col1
- Select Operator [SEL_39] (rows=57591150 width=77)
- Output:["_col0","_col1"]
- Filter Operator [FIL_548] (rows=57591150 width=77)
- predicate:(sr_item_sk is not null and sr_ticket_number is not null)
- Please refer to the previous TableScan [TS_37]
+ Please refer to the previous Select Operator [SEL_39]
<-Reducer 42 [SIMPLE_EDGE]
SHUFFLE [RS_49]
PartitionCols:_col1, _col8
@@ -699,19 +663,11 @@ Stage-0
<-Map 41 [SIMPLE_EDGE]
SHUFFLE [RS_46]
PartitionCols:_col1
- Select Operator [SEL_33] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11"]
- Filter Operator [FIL_546] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_customer_sk is not null and ss_cdemo_sk is not null and ss_promo_sk is not null and ss_hdemo_sk is not null and ss_addr_sk is not null)
- Please refer to the previous TableScan [TS_31]
+ Please refer to the previous Select Operator [SEL_33]
<-Map 46 [SIMPLE_EDGE]
SHUFFLE [RS_47]
PartitionCols:_col0
- Select Operator [SEL_36] (rows=2851 width=1436)
- Output:["_col0","_col3"]
- Filter Operator [FIL_547] (rows=2851 width=1436)
- predicate:((i_color) IN ('maroon', 'burnished', 'dim', 'steel', 'navajo', 'chocolate') and i_current_price BETWEEN 35 AND 45 and i_current_price BETWEEN 36 AND 50 and i_item_sk is not null)
- Please refer to the previous TableScan [TS_34]
+ Please refer to the previous Select Operator [SEL_36]
<-Reducer 35 [SIMPLE_EDGE]
SHUFFLE [RS_84]
PartitionCols:_col0
@@ -720,19 +676,11 @@ Stage-0
<-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_81]
PartitionCols:_col1
- Select Operator [SEL_27] (rows=7200 width=107)
- Output:["_col0","_col1"]
- Filter Operator [FIL_544] (rows=7200 width=107)
- predicate:(hd_demo_sk is not null and hd_income_band_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Map 38 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col0
- Select Operator [SEL_30] (rows=20 width=12)
- Output:["_col0"]
- Filter Operator [FIL_545] (rows=20 width=12)
- predicate:ib_income_band_sk is not null
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 51 [SIMPLE_EDGE]
SHUFFLE [RS_88]
PartitionCols:_col0
@@ -754,19 +702,11 @@ Stage-0
<-Map 49 [SIMPLE_EDGE]
SHUFFLE [RS_65]
PartitionCols:_col0, _col1
- Select Operator [SEL_61] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_551] (rows=287989836 width=135)
- predicate:(cs_order_number is not null and cs_item_sk is not null)
- Please refer to the previous TableScan [TS_59]
+ Please refer to the previous Select Operator [SEL_61]
<-Map 54 [SIMPLE_EDGE]
SHUFFLE [RS_66]
PartitionCols:_col0, _col1
- Select Operator [SEL_64] (rows=28798881 width=106)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_552] (rows=28798881 width=106)
- predicate:(cr_order_number is not null and cr_item_sk is not null)
- Please refer to the previous TableScan [TS_62]
+ Please refer to the previous Select Operator [SEL_64]
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_115]
PartitionCols:_col0
@@ -775,11 +715,7 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_113]
PartitionCols:_col0
- Select Operator [SEL_24] (rows=1861800 width=385)
- Output:["_col0","_col1"]
- Filter Operator [FIL_543] (rows=1861800 width=385)
- predicate:cd_demo_sk is not null
- Please refer to the previous TableScan [TS_22]
+ Please refer to the previous Select Operator [SEL_24]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_112]
PartitionCols:_col1
@@ -788,11 +724,7 @@ Stage-0
<-Map 39 [SIMPLE_EDGE]
SHUFFLE [RS_110]
PartitionCols:_col0
- Select Operator [SEL_21] (rows=40000000 width=1014)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_542] (rows=40000000 width=1014)
- predicate:ca_address_sk is not null
- Please refer to the previous TableScan [TS_19]
+ Please refer to the previous Select Operator [SEL_21]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_109]
PartitionCols:_col3
@@ -806,9 +738,9 @@ Stage-0
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_104]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=73049 width=1119)
+ Select Operator [SEL_5] (rows=73049 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_539] (rows=73049 width=1119)
+ Filter Operator [FIL_538] (rows=73049 width=1119)
predicate:d_date_sk is not null
Please refer to the previous TableScan [TS_3]
<-Reducer 2 [SIMPLE_EDGE]
@@ -819,19 +751,11 @@ Stage-0
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_101]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_538] (rows=73049 width=1119)
- predicate:d_date_sk is not null
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_100]
PartitionCols:_col5
- Select Operator [SEL_2] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_537] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_first_shipto_date_sk is not null and c_first_sales_date_sk is not null and c_current_cdemo_sk is not null and c_current_hdemo_sk is not null and c_current_addr_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 34 [SIMPLE_EDGE]
SHUFFLE [RS_107]
PartitionCols:_col0
@@ -840,17 +764,9 @@ Stage-0
<-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col1
- Select Operator [SEL_11] (rows=7200 width=107)
- Output:["_col0","_col1"]
- Filter Operator [FIL_540] (rows=7200 width=107)
- predicate:(hd_demo_sk is not null and hd_income_band_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Map 38 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=20 width=12)
- Output:["_col0"]
- Filter Operator [FIL_541] (rows=20 width=12)
- predicate:ib_income_band_sk is not null
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query65.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query65.q.out b/ql/src/test/results/clientpositive/perf/query65.q.out
index b2035c2..1b32ed6 100644
--- a/ql/src/test/results/clientpositive/perf/query65.q.out
+++ b/ql/src/test/results/clientpositive/perf/query65.q.out
@@ -165,9 +165,5 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_21]
PartitionCols:_col0
- Select Operator [SEL_19] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_75] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query66.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query66.q.out b/ql/src/test/results/clientpositive/perf/query66.q.out
index 2c74815..9a37cf3 100644
--- a/ql/src/test/results/clientpositive/perf/query66.q.out
+++ b/ql/src/test/results/clientpositive/perf/query66.q.out
@@ -494,9 +494,9 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_58]
PartitionCols:_col0
- Select Operator [SEL_47] (rows=27 width=1029)
+ Select Operator [SEL_14] (rows=27 width=1029)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Filter Operator [FIL_115] (rows=27 width=1029)
+ Filter Operator [FIL_110] (rows=27 width=1029)
predicate:w_warehouse_sk is not null
TableScan [TS_12] (rows=27 width=1029)
default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name","w_warehouse_sq_ft","w_city","w_county","w_state","w_country"]
@@ -508,9 +508,9 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_55]
PartitionCols:_col0
- Select Operator [SEL_44] (rows=1 width=0)
+ Select Operator [SEL_11] (rows=1 width=0)
Output:["_col0"]
- Filter Operator [FIL_114] (rows=1 width=0)
+ Filter Operator [FIL_109] (rows=1 width=0)
predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null)
TableScan [TS_9] (rows=1 width=0)
default@ship_mode,ship_mode,Tbl:PARTIAL,Col:NONE,Output:["sm_ship_mode_sk","sm_carrier"]
@@ -522,9 +522,9 @@ Stage-0
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_52]
PartitionCols:_col0
- Select Operator [SEL_41] (rows=36524 width=1119)
+ Select Operator [SEL_8] (rows=36524 width=1119)
Output:["_col0","_col2"]
- Filter Operator [FIL_113] (rows=36524 width=1119)
+ Filter Operator [FIL_108] (rows=36524 width=1119)
predicate:((d_year = 2002) and d_date_sk is not null)
TableScan [TS_6] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -536,9 +536,9 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_49]
PartitionCols:_col0
- Select Operator [SEL_38] (rows=9600 width=471)
+ Select Operator [SEL_5] (rows=9600 width=471)
Output:["_col0"]
- Filter Operator [FIL_112] (rows=9600 width=471)
+ Filter Operator [FIL_107] (rows=9600 width=471)
predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null)
TableScan [TS_3] (rows=86400 width=471)
default@time_dim,time_dim,Tbl:COMPLETE,Col:NONE,Output:["t_time_sk","t_time"]
@@ -572,11 +572,7 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_25]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=27 width=1029)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Filter Operator [FIL_110] (rows=27 width=1029)
- predicate:w_warehouse_sk is not null
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_24]
PartitionCols:_col3
@@ -585,11 +581,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=1 width=0)
- Output:["_col0"]
- Filter Operator [FIL_109] (rows=1 width=0)
- predicate:((sm_carrier) IN ('DIAMOND', 'AIRBORNE') and sm_ship_mode_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_21]
PartitionCols:_col2
@@ -598,11 +590,7 @@ Stage-0
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=36524 width=1119)
- Output:["_col0","_col2"]
- Filter Operator [FIL_108] (rows=36524 width=1119)
- predicate:((d_year = 2002) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col0
@@ -611,11 +599,7 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=9600 width=471)
- Output:["_col0"]
- Filter Operator [FIL_107] (rows=9600 width=471)
- predicate:(t_time BETWEEN 49530 AND 78330 and t_time_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col1
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query68.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query68.q.out b/ql/src/test/results/clientpositive/perf/query68.q.out
index bd9b5ec..5f763d5 100644
--- a/ql/src/test/results/clientpositive/perf/query68.q.out
+++ b/ql/src/test/results/clientpositive/perf/query68.q.out
@@ -150,11 +150,7 @@ Stage-0
<-Map 5 [SIMPLE_EDGE]
SHUFFLE [RS_31]
PartitionCols:_col0
- Select Operator [SEL_20] (rows=40000000 width=1014)
- Output:["_col0","_col1"]
- Filter Operator [FIL_80] (rows=40000000 width=1014)
- predicate:ca_address_sk is not null
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query69.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query69.q.out b/ql/src/test/results/clientpositive/perf/query69.q.out
index a55c368..2769bc5 100644
--- a/ql/src/test/results/clientpositive/perf/query69.q.out
+++ b/ql/src/test/results/clientpositive/perf/query69.q.out
@@ -150,9 +150,9 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_57]
PartitionCols:_col0
- Select Operator [SEL_55] (rows=4058 width=1119)
+ Select Operator [SEL_14] (rows=4058 width=1119)
Output:["_col0"]
- Filter Operator [FIL_107] (rows=4058 width=1119)
+ Filter Operator [FIL_103] (rows=4058 width=1119)
predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null)
TableScan [TS_12] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -193,11 +193,7 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=4058 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_103] (rows=4058 width=1119)
- predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
@@ -222,11 +218,7 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=4058 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_105] (rows=4058 width=1119)
- predicate:((d_year = 1999) and d_moy BETWEEN 1 AND 3 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query70.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query70.q.out b/ql/src/test/results/clientpositive/perf/query70.q.out
index ee1fe86..9f9a8e1 100644
--- a/ql/src/test/results/clientpositive/perf/query70.q.out
+++ b/ql/src/test/results/clientpositive/perf/query70.q.out
@@ -160,18 +160,18 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=575995635 width=88)
+ Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_82] (rows=575995635 width=88)
- predicate:(ss_store_sk is not null and ss_sold_date_sk is not null)
+ Filter Operator [FIL_78] (rows=575995635 width=88)
+ predicate:(ss_sold_date_sk is not null and ss_store_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_net_profit"]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=8116 width=1119)
+ Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_83] (rows=8116 width=1119)
+ Filter Operator [FIL_79] (rows=8116 width=1119)
predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_month_seq"]
@@ -197,17 +197,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_37]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_78] (rows=575995635 width=88)
- predicate:(ss_sold_date_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_38]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_79] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query74.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query74.q.out b/ql/src/test/results/clientpositive/perf/query74.q.out
index 50d97e7..9dddd6a 100644
--- a/ql/src/test/results/clientpositive/perf/query74.q.out
+++ b/ql/src/test/results/clientpositive/perf/query74.q.out
@@ -173,9 +173,9 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_33]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=80000000 width=860)
+ Select Operator [SEL_8] (rows=80000000 width=860)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_154] (rows=80000000 width=860)
+ Filter Operator [FIL_151] (rows=80000000 width=860)
predicate:(c_customer_sk is not null and c_customer_id is not null)
TableScan [TS_6] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name"]
@@ -219,11 +219,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_75]
PartitionCols:_col0
- Select Operator [SEL_70] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_160] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_74]
PartitionCols:_col1
@@ -240,11 +236,7 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_71]
PartitionCols:_col0
- Select Operator [SEL_64] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_158] (rows=575995635 width=88)
- predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_20]
+ Please refer to the previous Select Operator [SEL_22]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col0
@@ -262,11 +254,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_151] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col1
@@ -308,11 +296,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_54]
PartitionCols:_col0
- Select Operator [SEL_49] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_157] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_53]
PartitionCols:_col1
@@ -329,9 +313,5 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_50]
PartitionCols:_col0
- Select Operator [SEL_43] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_155] (rows=144002668 width=135)
- predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query75.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query75.q.out b/ql/src/test/results/clientpositive/perf/query75.q.out
index d399567..c3dfff5 100644
--- a/ql/src/test/results/clientpositive/perf/query75.q.out
+++ b/ql/src/test/results/clientpositive/perf/query75.q.out
@@ -212,9 +212,9 @@ Stage-0
<-Map 32 [SIMPLE_EDGE]
SHUFFLE [RS_104]
PartitionCols:_col0, _col1
- Select Operator [SEL_96] (rows=28798881 width=106)
+ Select Operator [SEL_11] (rows=28798881 width=106)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_255] (rows=28798881 width=106)
+ Filter Operator [FIL_243] (rows=28798881 width=106)
predicate:cr_item_sk is not null
TableScan [TS_9] (rows=28798881 width=106)
default@catalog_returns,catalog_returns,Tbl:COMPLETE,Col:NONE,Output:["cr_item_sk","cr_order_number","cr_return_quantity","cr_return_amount"]
@@ -226,9 +226,9 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_101]
PartitionCols:_col0
- Select Operator [SEL_93] (rows=231000 width=1436)
+ Select Operator [SEL_8] (rows=231000 width=1436)
Output:["_col0","_col1","_col2","_col3","_col5"]
- Filter Operator [FIL_254] (rows=231000 width=1436)
+ Filter Operator [FIL_242] (rows=231000 width=1436)
predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null)
TableScan [TS_6] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand_id","i_class_id","i_category_id","i_category","i_manufact_id"]
@@ -249,9 +249,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col0
- Select Operator [SEL_87] (rows=287989836 width=135)
+ Select Operator [SEL_2] (rows=287989836 width=135)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_252] (rows=287989836 width=135)
+ Filter Operator [FIL_240] (rows=287989836 width=135)
predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
TableScan [TS_0] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk","cs_order_number","cs_quantity","cs_ext_sales_price"]
@@ -267,9 +267,9 @@ Stage-0
<-Map 34 [SIMPLE_EDGE]
SHUFFLE [RS_126]
PartitionCols:_col0, _col1
- Select Operator [SEL_118] (rows=57591150 width=77)
+ Select Operator [SEL_33] (rows=57591150 width=77)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_259] (rows=57591150 width=77)
+ Filter Operator [FIL_247] (rows=57591150 width=77)
predicate:sr_item_sk is not null
TableScan [TS_31] (rows=57591150 width=77)
default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number","sr_return_quantity","sr_return_amt"]
@@ -281,11 +281,7 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_123]
PartitionCols:_col0
- Select Operator [SEL_115] (rows=231000 width=1436)
- Output:["_col0","_col1","_col2","_col3","_col5"]
- Filter Operator [FIL_258] (rows=231000 width=1436)
- predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 25 [SIMPLE_EDGE]
SHUFFLE [RS_122]
PartitionCols:_col1
@@ -302,9 +298,9 @@ Stage-0
<-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_119]
PartitionCols:_col0
- Select Operator [SEL_109] (rows=575995635 width=88)
+ Select Operator [SEL_24] (rows=575995635 width=88)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_256] (rows=575995635 width=88)
+ Filter Operator [FIL_244] (rows=575995635 width=88)
predicate:(ss_item_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_22] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_ticket_number","ss_quantity","ss_ext_sales_price"]
@@ -320,9 +316,9 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_155]
PartitionCols:_col0, _col1
- Select Operator [SEL_147] (rows=14398467 width=92)
+ Select Operator [SEL_62] (rows=14398467 width=92)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_263] (rows=14398467 width=92)
+ Filter Operator [FIL_251] (rows=14398467 width=92)
predicate:wr_item_sk is not null
TableScan [TS_60] (rows=14398467 width=92)
default@web_returns,web_returns,Tbl:COMPLETE,Col:NONE,Output:["wr_item_sk","wr_order_number","wr_return_quantity","wr_return_amt"]
@@ -334,11 +330,7 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_152]
PartitionCols:_col0
- Select Operator [SEL_144] (rows=231000 width=1436)
- Output:["_col0","_col1","_col2","_col3","_col5"]
- Filter Operator [FIL_262] (rows=231000 width=1436)
- predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 28 [SIMPLE_EDGE]
SHUFFLE [RS_151]
PartitionCols:_col1
@@ -355,9 +347,9 @@ Stage-0
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_148]
PartitionCols:_col0
- Select Operator [SEL_138] (rows=144002668 width=135)
+ Select Operator [SEL_53] (rows=144002668 width=135)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_260] (rows=144002668 width=135)
+ Filter Operator [FIL_248] (rows=144002668 width=135)
predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
TableScan [TS_51] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk","ws_order_number","ws_quantity","ws_ext_sales_price"]
@@ -381,11 +373,7 @@ Stage-0
<-Map 36 [SIMPLE_EDGE]
SHUFFLE [RS_70]
PartitionCols:_col0, _col1
- Select Operator [SEL_62] (rows=14398467 width=92)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_251] (rows=14398467 width=92)
- predicate:wr_item_sk is not null
- Please refer to the previous TableScan [TS_60]
+ Please refer to the previous Select Operator [SEL_62]
<-Reducer 23 [SIMPLE_EDGE]
SHUFFLE [RS_69]
PartitionCols:_col1, _col2
@@ -394,11 +382,7 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_67]
PartitionCols:_col0
- Select Operator [SEL_59] (rows=231000 width=1436)
- Output:["_col0","_col1","_col2","_col3","_col5"]
- Filter Operator [FIL_250] (rows=231000 width=1436)
- predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 22 [SIMPLE_EDGE]
SHUFFLE [RS_66]
PartitionCols:_col1
@@ -407,19 +391,15 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_64]
PartitionCols:_col0
- Select Operator [SEL_56] (rows=36524 width=1119)
+ Select Operator [SEL_5] (rows=36524 width=1119)
Output:["_col0"]
- Filter Operator [FIL_249] (rows=36524 width=1119)
+ Filter Operator [FIL_241] (rows=36524 width=1119)
predicate:((d_year = 2001) and d_date_sk is not null)
Please refer to the previous TableScan [TS_3]
<-Map 35 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col0
- Select Operator [SEL_53] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_248] (rows=144002668 width=135)
- predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_51]
+ Please refer to the previous Select Operator [SEL_53]
<-Reducer 6 [CONTAINS]
Reduce Output Operator [RS_77]
PartitionCols:_col0, _col1, _col2, _col3
@@ -440,11 +420,7 @@ Stage-0
<-Map 34 [SIMPLE_EDGE]
SHUFFLE [RS_41]
PartitionCols:_col0, _col1
- Select Operator [SEL_33] (rows=57591150 width=77)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_247] (rows=57591150 width=77)
- predicate:sr_item_sk is not null
- Please refer to the previous TableScan [TS_31]
+ Please refer to the previous Select Operator [SEL_33]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_40]
PartitionCols:_col1, _col2
@@ -453,11 +429,7 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_38]
PartitionCols:_col0
- Select Operator [SEL_30] (rows=231000 width=1436)
- Output:["_col0","_col1","_col2","_col3","_col5"]
- Filter Operator [FIL_246] (rows=231000 width=1436)
- predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 19 [SIMPLE_EDGE]
SHUFFLE [RS_37]
PartitionCols:_col1
@@ -466,19 +438,11 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_35]
PartitionCols:_col0
- Select Operator [SEL_27] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_245] (rows=36524 width=1119)
- predicate:((d_year = 2001) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_34]
PartitionCols:_col0
- Select Operator [SEL_24] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_244] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_22]
+ Please refer to the previous Select Operator [SEL_24]
<-Reducer 4 [CONTAINS]
Reduce Output Operator [RS_48]
PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5
@@ -491,11 +455,7 @@ Stage-0
<-Map 32 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0, _col1
- Select Operator [SEL_11] (rows=28798881 width=106)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_243] (rows=28798881 width=106)
- predicate:cr_item_sk is not null
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col1, _col2
@@ -504,11 +464,7 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=231000 width=1436)
- Output:["_col0","_col1","_col2","_col3","_col5"]
- Filter Operator [FIL_242] (rows=231000 width=1436)
- predicate:((i_category = 'Sports') and i_item_sk is not null and i_brand_id is not null and i_class_id is not null and i_category_id is not null and i_manufact_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col1
@@ -517,17 +473,9 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_241] (rows=36524 width=1119)
- predicate:((d_year = 2001) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_240] (rows=287989836 width=135)
- predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
[6/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/subquery_views.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_views.q.out b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
index 1a21a02..5905ef0 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_views.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_views.q.out
@@ -243,11 +243,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 205 Data size: 17835 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
index b4b6019..ead0909 100644
--- a/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/unionDistinct_1.q.out
@@ -56,13 +56,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Select Operator
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -289,13 +282,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Select Operator
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(1)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -5395,11 +5381,6 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: bigint)
- Group By Operator
- aggregations: count(1)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
@@ -11369,19 +11350,9 @@ STAGE PLANS:
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/union_top_level.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/union_top_level.q.out b/ql/src/test/results/clientpositive/llap/union_top_level.q.out
index 2fac8cc..268e041 100644
--- a/ql/src/test/results/clientpositive/llap/union_top_level.q.out
+++ b/ql/src/test/results/clientpositive/llap/union_top_level.q.out
@@ -232,13 +232,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
@@ -264,13 +257,6 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
index e1ad06c..4f914c8 100644
--- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets4.q.out
@@ -77,15 +77,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint)
- Filter Operator
- predicate: (UDFToDouble(a) < 3.0) (type: boolean)
- Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: a (type: string), b (type: string), 0 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
sort order: +++
@@ -214,15 +205,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint)
- Filter Operator
- predicate: (UDFToDouble(a) < 3.0) (type: boolean)
- Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: a (type: string), b (type: string), 0 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 2040 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
sort order: +++
@@ -383,15 +365,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint)
- Filter Operator
- predicate: (UDFToDouble(a) < 3.0) (type: boolean)
- Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: a (type: string), b (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 510 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/vector_join30.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_join30.q.out b/ql/src/test/results/clientpositive/llap/vector_join30.q.out
index ec76750..6b544f3 100644
--- a/ql/src/test/results/clientpositive/llap/vector_join30.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_join30.q.out
@@ -854,21 +854,6 @@ STAGE PLANS:
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Filter Operator
- Filter Vectorization:
- className: VectorFilterOperator
- native: true
- predicateExpression: SelectColumnIsNotNull(col 0) -> boolean
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumns: [0, 1]
- Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
@@ -1162,14 +1147,6 @@ STAGE PLANS:
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumns: [0, 1]
- Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
@@ -1461,14 +1438,6 @@ STAGE PLANS:
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumns: [0, 1]
- Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
@@ -2059,14 +2028,6 @@ STAGE PLANS:
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Select Vectorization:
- className: VectorSelectOperator
- native: true
- projectedOutputColumns: [0, 1]
- Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string)
sort order: +
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
index 80cfa95..9172136 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out
@@ -2386,27 +2386,21 @@ STAGE PLANS:
alias: srcpart
filterExpr: (ds = '2008-04-08') (type: boolean)
Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (ds = '2008-04-08') (type: boolean)
- Select Operator
+ Select Operator
+ Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ sort order:
Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ keys: '2008-04-08' (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
- sort order:
- Statistics: Num rows: 1000 Data size: 94000 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (ds = '2008-04-08') (type: boolean)
- Select Operator
- Statistics: Num rows: 1000 Data size: 18624 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- keys: '2008-04-08' (type: string)
- mode: hash
- outputColumnNames: _col0
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: no inputs
Map Vectorization:
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query1.q.out b/ql/src/test/results/clientpositive/perf/query1.q.out
index da4a65c..9c5d440 100644
--- a/ql/src/test/results/clientpositive/perf/query1.q.out
+++ b/ql/src/test/results/clientpositive/perf/query1.q.out
@@ -165,9 +165,5 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
- Select Operator [SEL_25] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_73] (rows=36524 width=1119)
- predicate:((d_year = 2000) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query10.q.out b/ql/src/test/results/clientpositive/perf/query10.q.out
index 9b6621c..73c474a 100644
--- a/ql/src/test/results/clientpositive/perf/query10.q.out
+++ b/ql/src/test/results/clientpositive/perf/query10.q.out
@@ -174,9 +174,9 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_55]
PartitionCols:_col0
- Select Operator [SEL_53] (rows=4058 width=1119)
+ Select Operator [SEL_14] (rows=4058 width=1119)
Output:["_col0"]
- Filter Operator [FIL_105] (rows=4058 width=1119)
+ Filter Operator [FIL_101] (rows=4058 width=1119)
predicate:((d_year = 2002) and d_moy BETWEEN 4 AND 7 and d_date_sk is not null)
TableScan [TS_12] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -213,11 +213,7 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=4058 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_101] (rows=4058 width=1119)
- predicate:((d_year = 2002) and d_moy BETWEEN 4 AND 7 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
@@ -242,11 +238,7 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=4058 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_103] (rows=4058 width=1119)
- predicate:((d_year = 2002) and d_moy BETWEEN 4 AND 7 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query11.q.out b/ql/src/test/results/clientpositive/perf/query11.q.out
index cce6350..11a4a6b 100644
--- a/ql/src/test/results/clientpositive/perf/query11.q.out
+++ b/ql/src/test/results/clientpositive/perf/query11.q.out
@@ -205,9 +205,9 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_34]
PartitionCols:_col0
- Select Operator [SEL_29] (rows=80000000 width=860)
+ Select Operator [SEL_8] (rows=80000000 width=860)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_158] (rows=80000000 width=860)
+ Filter Operator [FIL_155] (rows=80000000 width=860)
predicate:(c_customer_sk is not null and c_customer_id is not null)
TableScan [TS_6] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name","c_preferred_cust_flag","c_birth_country","c_login","c_email_address"]
@@ -253,11 +253,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_78]
PartitionCols:_col0
- Select Operator [SEL_73] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_164] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_77]
PartitionCols:_col1
@@ -274,11 +270,7 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_74]
PartitionCols:_col0
- Select Operator [SEL_67] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_162] (rows=575995635 width=88)
- predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_21]
+ Please refer to the previous Select Operator [SEL_23]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_86]
PartitionCols:_col0
@@ -298,11 +290,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_155] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col1
@@ -346,11 +334,7 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_56]
PartitionCols:_col0
- Select Operator [SEL_51] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_161] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_55]
PartitionCols:_col1
@@ -367,9 +351,5 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_52]
PartitionCols:_col0
- Select Operator [SEL_45] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_159] (rows=144002668 width=135)
- predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
[5/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query14.q.out b/ql/src/test/results/clientpositive/perf/query14.q.out
index 64c5921..eab635e 100644
--- a/ql/src/test/results/clientpositive/perf/query14.q.out
+++ b/ql/src/test/results/clientpositive/perf/query14.q.out
@@ -1,6 +1,6 @@
+Warning: Shuffle Join MERGEJOIN[899][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product
Warning: Shuffle Join MERGEJOIN[900][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product
Warning: Shuffle Join MERGEJOIN[901][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product
-Warning: Shuffle Join MERGEJOIN[899][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product
PREHOOK: query: explain
with cross_items as
(select i_item_sk ss_item_sk
@@ -337,18 +337,18 @@ Stage-0
<-Map 76 [SIMPLE_EDGE]
SHUFFLE [RS_208]
PartitionCols:_col0
- Select Operator [SEL_206] (rows=8116 width=1119)
+ Select Operator [SEL_15] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_807] (rows=8116 width=1119)
+ Filter Operator [FIL_782] (rows=8116 width=1119)
predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
TableScan [TS_13] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
<-Map 72 [SIMPLE_EDGE]
SHUFFLE [RS_207]
PartitionCols:_col0
- Select Operator [SEL_203] (rows=287989836 width=135)
+ Select Operator [SEL_12] (rows=287989836 width=135)
Output:["_col0","_col1"]
- Filter Operator [FIL_806] (rows=287989836 width=135)
+ Filter Operator [FIL_781] (rows=287989836 width=135)
predicate:cs_sold_date_sk is not null
TableScan [TS_10] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity"]
@@ -365,18 +365,18 @@ Stage-0
<-Map 84 [SIMPLE_EDGE]
SHUFFLE [RS_219]
PartitionCols:_col0
- Select Operator [SEL_217] (rows=8116 width=1119)
+ Select Operator [SEL_26] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_809] (rows=8116 width=1119)
+ Filter Operator [FIL_784] (rows=8116 width=1119)
predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
TableScan [TS_24] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
<-Map 80 [SIMPLE_EDGE]
SHUFFLE [RS_218]
PartitionCols:_col0
- Select Operator [SEL_214] (rows=144002668 width=135)
+ Select Operator [SEL_23] (rows=144002668 width=135)
Output:["_col0","_col1"]
- Filter Operator [FIL_808] (rows=144002668 width=135)
+ Filter Operator [FIL_783] (rows=144002668 width=135)
predicate:ws_sold_date_sk is not null
TableScan [TS_21] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity"]
@@ -393,18 +393,18 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_198]
PartitionCols:_col0
- Select Operator [SEL_196] (rows=8116 width=1119)
+ Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_805] (rows=8116 width=1119)
+ Filter Operator [FIL_780] (rows=8116 width=1119)
predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_197]
PartitionCols:_col0
- Select Operator [SEL_193] (rows=575995635 width=88)
+ Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1"]
- Filter Operator [FIL_804] (rows=575995635 width=88)
+ Filter Operator [FIL_779] (rows=575995635 width=88)
predicate:ss_sold_date_sk is not null
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity"]
@@ -426,17 +426,13 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_242]
PartitionCols:_col0
- Select Operator [SEL_240] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_811] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 88 [SIMPLE_EDGE]
SHUFFLE [RS_241]
PartitionCols:_col0
- Select Operator [SEL_237] (rows=575995635 width=88)
+ Select Operator [SEL_46] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_810] (rows=575995635 width=88)
+ Filter Operator [FIL_785] (rows=575995635 width=88)
predicate:ss_sold_date_sk is not null
TableScan [TS_44] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_quantity","ss_list_price"]
@@ -453,17 +449,13 @@ Stage-0
<-Map 76 [SIMPLE_EDGE]
SHUFFLE [RS_252]
PartitionCols:_col0
- Select Operator [SEL_250] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_813] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 89 [SIMPLE_EDGE]
SHUFFLE [RS_251]
PartitionCols:_col0
- Select Operator [SEL_247] (rows=287989836 width=135)
+ Select Operator [SEL_56] (rows=287989836 width=135)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_812] (rows=287989836 width=135)
+ Filter Operator [FIL_787] (rows=287989836 width=135)
predicate:cs_sold_date_sk is not null
TableScan [TS_54] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_quantity","cs_list_price"]
@@ -480,17 +472,13 @@ Stage-0
<-Map 84 [SIMPLE_EDGE]
SHUFFLE [RS_263]
PartitionCols:_col0
- Select Operator [SEL_261] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_815] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_24]
+ Please refer to the previous Select Operator [SEL_26]
<-Map 90 [SIMPLE_EDGE]
SHUFFLE [RS_262]
PartitionCols:_col0
- Select Operator [SEL_258] (rows=144002668 width=135)
+ Select Operator [SEL_67] (rows=144002668 width=135)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_814] (rows=144002668 width=135)
+ Filter Operator [FIL_789] (rows=144002668 width=135)
predicate:ws_sold_date_sk is not null
TableScan [TS_65] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_quantity","ws_list_price"]
@@ -578,17 +566,13 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_294]
PartitionCols:_col0
- Select Operator [SEL_289] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_821] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 97 [SIMPLE_EDGE]
SHUFFLE [RS_293]
PartitionCols:_col0
- Select Operator [SEL_286] (rows=575995635 width=88)
+ Select Operator [SEL_95] (rows=575995635 width=88)
Output:["_col0","_col1"]
- Filter Operator [FIL_820] (rows=575995635 width=88)
+ Filter Operator [FIL_795] (rows=575995635 width=88)
predicate:(ss_item_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_93] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"]
@@ -624,17 +608,13 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_315]
PartitionCols:_col0
- Select Operator [SEL_310] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_824] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 98 [SIMPLE_EDGE]
SHUFFLE [RS_314]
PartitionCols:_col0
- Select Operator [SEL_307] (rows=287989836 width=135)
+ Select Operator [SEL_116] (rows=287989836 width=135)
Output:["_col0","_col1"]
- Filter Operator [FIL_823] (rows=287989836 width=135)
+ Filter Operator [FIL_798] (rows=287989836 width=135)
predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
TableScan [TS_114] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_item_sk"]
@@ -670,17 +650,13 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_337]
PartitionCols:_col0
- Select Operator [SEL_332] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_827] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 99 [SIMPLE_EDGE]
SHUFFLE [RS_336]
PartitionCols:_col0
- Select Operator [SEL_329] (rows=144002668 width=135)
+ Select Operator [SEL_138] (rows=144002668 width=135)
Output:["_col0","_col1"]
- Filter Operator [FIL_826] (rows=144002668 width=135)
+ Filter Operator [FIL_801] (rows=144002668 width=135)
predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
TableScan [TS_136] (rows=144002668 width=135)
default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_item_sk"]
@@ -692,9 +668,9 @@ Stage-0
<-Map 93 [SIMPLE_EDGE]
SHUFFLE [RS_364]
PartitionCols:_col0
- Select Operator [SEL_277] (rows=18262 width=1119)
+ Select Operator [SEL_86] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_817] (rows=18262 width=1119)
+ Filter Operator [FIL_792] (rows=18262 width=1119)
predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null)
TableScan [TS_84] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -742,19 +718,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_390]
PartitionCols:_col0
- Select Operator [SEL_388] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_830] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_389]
PartitionCols:_col0
- Select Operator [SEL_385] (rows=575995635 width=88)
- Output:["_col0","_col1"]
- Filter Operator [FIL_829] (rows=575995635 width=88)
- predicate:ss_sold_date_sk is not null
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 75 [CONTAINS]
Reduce Output Operator [RS_416]
Group By Operator [GBY_415] (rows=1 width=8)
@@ -768,19 +736,11 @@ Stage-0
<-Map 76 [SIMPLE_EDGE]
SHUFFLE [RS_400]
PartitionCols:_col0
- Select Operator [SEL_398] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_832] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 72 [SIMPLE_EDGE]
SHUFFLE [RS_399]
PartitionCols:_col0
- Select Operator [SEL_395] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_831] (rows=287989836 width=135)
- predicate:cs_sold_date_sk is not null
- Please refer to the previous TableScan [TS_10]
+ Please refer to the previous Select Operator [SEL_12]
<-Reducer 83 [CONTAINS]
Reduce Output Operator [RS_416]
Group By Operator [GBY_415] (rows=1 width=8)
@@ -794,19 +754,11 @@ Stage-0
<-Map 84 [SIMPLE_EDGE]
SHUFFLE [RS_411]
PartitionCols:_col0
- Select Operator [SEL_409] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_834] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_24]
+ Please refer to the previous Select Operator [SEL_26]
<-Map 80 [SIMPLE_EDGE]
SHUFFLE [RS_410]
PartitionCols:_col0
- Select Operator [SEL_406] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_833] (rows=144002668 width=135)
- predicate:ws_sold_date_sk is not null
- Please refer to the previous TableScan [TS_21]
+ Please refer to the previous Select Operator [SEL_23]
<-Reducer 56 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_569]
Group By Operator [GBY_462] (rows=1 width=288)
@@ -825,19 +777,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_434]
PartitionCols:_col0
- Select Operator [SEL_432] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_836] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 88 [SIMPLE_EDGE]
SHUFFLE [RS_433]
PartitionCols:_col0
- Select Operator [SEL_429] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_835] (rows=575995635 width=88)
- predicate:ss_sold_date_sk is not null
- Please refer to the previous TableScan [TS_44]
+ Please refer to the previous Select Operator [SEL_46]
<-Reducer 79 [CONTAINS]
Reduce Output Operator [RS_461]
Group By Operator [GBY_460] (rows=1 width=288)
@@ -851,19 +795,11 @@ Stage-0
<-Map 76 [SIMPLE_EDGE]
SHUFFLE [RS_444]
PartitionCols:_col0
- Select Operator [SEL_442] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_838] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 89 [SIMPLE_EDGE]
SHUFFLE [RS_443]
PartitionCols:_col0
- Select Operator [SEL_439] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_837] (rows=287989836 width=135)
- predicate:cs_sold_date_sk is not null
- Please refer to the previous TableScan [TS_54]
+ Please refer to the previous Select Operator [SEL_56]
<-Reducer 87 [CONTAINS]
Reduce Output Operator [RS_461]
Group By Operator [GBY_460] (rows=1 width=288)
@@ -877,19 +813,11 @@ Stage-0
<-Map 84 [SIMPLE_EDGE]
SHUFFLE [RS_455]
PartitionCols:_col0
- Select Operator [SEL_453] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_840] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_24]
+ Please refer to the previous Select Operator [SEL_26]
<-Map 90 [SIMPLE_EDGE]
SHUFFLE [RS_454]
PartitionCols:_col0
- Select Operator [SEL_450] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_839] (rows=144002668 width=135)
- predicate:ws_sold_date_sk is not null
- Please refer to the previous TableScan [TS_65]
+ Please refer to the previous Select Operator [SEL_67]
<-Reducer 65 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_570]
Group By Operator [GBY_566] (rows=174243235 width=135)
@@ -973,19 +901,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_486]
PartitionCols:_col0
- Select Operator [SEL_481] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_846] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 97 [SIMPLE_EDGE]
SHUFFLE [RS_485]
PartitionCols:_col0
- Select Operator [SEL_478] (rows=575995635 width=88)
- Output:["_col0","_col1"]
- Filter Operator [FIL_845] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_93]
+ Please refer to the previous Select Operator [SEL_95]
<-Reducer 68 [CONTAINS]
Reduce Output Operator [RS_543]
PartitionCols:_col0, _col1, _col2
@@ -1018,19 +938,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_507]
PartitionCols:_col0
- Select Operator [SEL_502] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_849] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 98 [SIMPLE_EDGE]
SHUFFLE [RS_506]
PartitionCols:_col0
- Select Operator [SEL_499] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_848] (rows=287989836 width=135)
- predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_114]
+ Please refer to the previous Select Operator [SEL_116]
<-Reducer 71 [CONTAINS]
Reduce Output Operator [RS_543]
PartitionCols:_col0, _col1, _col2
@@ -1063,19 +975,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_529]
PartitionCols:_col0
- Select Operator [SEL_524] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_852] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 99 [SIMPLE_EDGE]
SHUFFLE [RS_528]
PartitionCols:_col0
- Select Operator [SEL_521] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_851] (rows=144002668 width=135)
- predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_136]
+ Please refer to the previous Select Operator [SEL_138]
<-Reducer 95 [SIMPLE_EDGE]
SHUFFLE [RS_558]
PartitionCols:_col1
@@ -1084,11 +988,7 @@ Stage-0
<-Map 93 [SIMPLE_EDGE]
SHUFFLE [RS_556]
PartitionCols:_col0
- Select Operator [SEL_469] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_842] (rows=18262 width=1119)
- predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_84]
+ Please refer to the previous Select Operator [SEL_86]
<-Map 101 [SIMPLE_EDGE]
SHUFFLE [RS_555]
PartitionCols:_col0
@@ -1127,19 +1027,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_51]
PartitionCols:_col0
- Select Operator [SEL_49] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_786] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 88 [SIMPLE_EDGE]
SHUFFLE [RS_50]
PartitionCols:_col0
- Select Operator [SEL_46] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_785] (rows=575995635 width=88)
- predicate:ss_sold_date_sk is not null
- Please refer to the previous TableScan [TS_44]
+ Please refer to the previous Select Operator [SEL_46]
<-Reducer 77 [CONTAINS]
Reduce Output Operator [RS_78]
Group By Operator [GBY_77] (rows=1 width=288)
@@ -1153,19 +1045,11 @@ Stage-0
<-Map 76 [SIMPLE_EDGE]
SHUFFLE [RS_61]
PartitionCols:_col0
- Select Operator [SEL_59] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_788] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 89 [SIMPLE_EDGE]
SHUFFLE [RS_60]
PartitionCols:_col0
- Select Operator [SEL_56] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_787] (rows=287989836 width=135)
- predicate:cs_sold_date_sk is not null
- Please refer to the previous TableScan [TS_54]
+ Please refer to the previous Select Operator [SEL_56]
<-Reducer 85 [CONTAINS]
Reduce Output Operator [RS_78]
Group By Operator [GBY_77] (rows=1 width=288)
@@ -1179,19 +1063,11 @@ Stage-0
<-Map 84 [SIMPLE_EDGE]
SHUFFLE [RS_72]
PartitionCols:_col0
- Select Operator [SEL_70] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_790] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_24]
+ Please refer to the previous Select Operator [SEL_26]
<-Map 90 [SIMPLE_EDGE]
SHUFFLE [RS_71]
PartitionCols:_col0
- Select Operator [SEL_67] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_789] (rows=144002668 width=135)
- predicate:ws_sold_date_sk is not null
- Please refer to the previous TableScan [TS_65]
+ Please refer to the previous Select Operator [SEL_67]
<-Reducer 29 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_187]
Group By Operator [GBY_183] (rows=696954748 width=88)
@@ -1275,19 +1151,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_103]
PartitionCols:_col0
- Select Operator [SEL_98] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_796] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 97 [SIMPLE_EDGE]
SHUFFLE [RS_102]
PartitionCols:_col0
- Select Operator [SEL_95] (rows=575995635 width=88)
- Output:["_col0","_col1"]
- Filter Operator [FIL_795] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_93]
+ Please refer to the previous Select Operator [SEL_95]
<-Reducer 32 [CONTAINS]
Reduce Output Operator [RS_160]
PartitionCols:_col0, _col1, _col2
@@ -1320,19 +1188,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_124]
PartitionCols:_col0
- Select Operator [SEL_119] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_799] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 98 [SIMPLE_EDGE]
SHUFFLE [RS_123]
PartitionCols:_col0
- Select Operator [SEL_116] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_798] (rows=287989836 width=135)
- predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_114]
+ Please refer to the previous Select Operator [SEL_116]
<-Reducer 35 [CONTAINS]
Reduce Output Operator [RS_160]
PartitionCols:_col0, _col1, _col2
@@ -1365,19 +1225,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_146]
PartitionCols:_col0
- Select Operator [SEL_141] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_802] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 99 [SIMPLE_EDGE]
SHUFFLE [RS_145]
PartitionCols:_col0
- Select Operator [SEL_138] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_801] (rows=144002668 width=135)
- predicate:(ws_item_sk is not null and ws_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_136]
+ Please refer to the previous Select Operator [SEL_138]
<-Reducer 92 [SIMPLE_EDGE]
SHUFFLE [RS_175]
PartitionCols:_col1
@@ -1386,11 +1238,7 @@ Stage-0
<-Map 93 [SIMPLE_EDGE]
SHUFFLE [RS_173]
PartitionCols:_col0
- Select Operator [SEL_86] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_792] (rows=18262 width=1119)
- predicate:((d_year = 2000) and (d_moy = 11) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_84]
+ Please refer to the previous Select Operator [SEL_86]
<-Map 91 [SIMPLE_EDGE]
SHUFFLE [RS_172]
PartitionCols:_col0
@@ -1424,19 +1272,11 @@ Stage-0
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_780] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1999 AND 2001 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_6]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=575995635 width=88)
- Output:["_col0","_col1"]
- Filter Operator [FIL_779] (rows=575995635 width=88)
- predicate:ss_sold_date_sk is not null
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 73 [CONTAINS]
Reduce Output Operator [RS_33]
Group By Operator [GBY_32] (rows=1 width=8)
@@ -1450,19 +1290,11 @@ Stage-0
<-Map 76 [SIMPLE_EDGE]
SHUFFLE [RS_17]
PartitionCols:_col0
- Select Operator [SEL_15] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_782] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 72 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_12] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_781] (rows=287989836 width=135)
- predicate:cs_sold_date_sk is not null
- Please refer to the previous TableScan [TS_10]
+ Please refer to the previous Select Operator [SEL_12]
<-Reducer 81 [CONTAINS]
Reduce Output Operator [RS_33]
Group By Operator [GBY_32] (rows=1 width=8)
@@ -1476,17 +1308,9 @@ Stage-0
<-Map 84 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
- Select Operator [SEL_26] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_784] (rows=8116 width=1119)
- predicate:(d_year BETWEEN 1998 AND 2000 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_24]
+ Please refer to the previous Select Operator [SEL_26]
<-Map 80 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
- Select Operator [SEL_23] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_783] (rows=144002668 width=135)
- predicate:ws_sold_date_sk is not null
- Please refer to the previous TableScan [TS_21]
+ Please refer to the previous Select Operator [SEL_23]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query16.q.out b/ql/src/test/results/clientpositive/perf/query16.q.out
index a7f93f9..239f2b7 100644
--- a/ql/src/test/results/clientpositive/perf/query16.q.out
+++ b/ql/src/test/results/clientpositive/perf/query16.q.out
@@ -150,9 +150,7 @@ Stage-0
Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
<-Map 15 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_32]
- Select Operator [SEL_28] (rows=287989836 width=135)
- Output:["_col0","_col1"]
- Please refer to the previous TableScan [TS_19]
+ Please refer to the previous Select Operator [SEL_20]
<-Map 20 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_29]
Select Operator [SEL_22] (rows=73049 width=4)
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query2.q.out b/ql/src/test/results/clientpositive/perf/query2.q.out
index 50d7f7b..1686f57 100644
--- a/ql/src/test/results/clientpositive/perf/query2.q.out
+++ b/ql/src/test/results/clientpositive/perf/query2.q.out
@@ -177,9 +177,9 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_39]
PartitionCols:_col0
- Select Operator [SEL_37] (rows=73049 width=1119)
+ Select Operator [SEL_10] (rows=73049 width=1119)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_89] (rows=73049 width=1119)
+ Filter Operator [FIL_85] (rows=73049 width=1119)
predicate:(d_date_sk is not null and d_week_seq is not null)
TableScan [TS_8] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"]
@@ -232,11 +232,7 @@ Stage-0
<-Map 9 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_10] (rows=73049 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_85] (rows=73049 width=1119)
- predicate:(d_date_sk is not null and d_week_seq is not null)
- Please refer to the previous TableScan [TS_8]
+ Please refer to the previous Select Operator [SEL_10]
<-Union 2 [SIMPLE_EDGE]
<-Map 1 [CONTAINS]
Reduce Output Operator [RS_11]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query23.q.out b/ql/src/test/results/clientpositive/perf/query23.q.out
index b8cdad3..ebd2271 100644
--- a/ql/src/test/results/clientpositive/perf/query23.q.out
+++ b/ql/src/test/results/clientpositive/perf/query23.q.out
@@ -195,9 +195,9 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_165]
PartitionCols:_col0
- Select Operator [SEL_160] (rows=80000000 width=860)
+ Select Operator [SEL_41] (rows=80000000 width=860)
Output:["_col0"]
- Filter Operator [FIL_341] (rows=80000000 width=860)
+ Filter Operator [FIL_328] (rows=80000000 width=860)
predicate:c_customer_sk is not null
TableScan [TS_39] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk"]
@@ -209,18 +209,18 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_162]
PartitionCols:_col0
- Select Operator [SEL_157] (rows=36525 width=1119)
+ Select Operator [SEL_38] (rows=36525 width=1119)
Output:["_col0"]
- Filter Operator [FIL_340] (rows=36525 width=1119)
+ Filter Operator [FIL_327] (rows=36525 width=1119)
predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null)
TableScan [TS_36] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_161]
PartitionCols:_col0
- Select Operator [SEL_154] (rows=575995635 width=88)
+ Select Operator [SEL_35] (rows=575995635 width=88)
Output:["_col0","_col1"]
- Filter Operator [FIL_339] (rows=575995635 width=88)
+ Filter Operator [FIL_326] (rows=575995635 width=88)
predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_33] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"]
@@ -248,11 +248,7 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_194]
PartitionCols:_col0
- Select Operator [SEL_189] (rows=80000000 width=860)
- Output:["_col0"]
- Filter Operator [FIL_344] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_39]
+ Please refer to the previous Select Operator [SEL_41]
<-Reducer 36 [SIMPLE_EDGE]
SHUFFLE [RS_193]
PartitionCols:_col1
@@ -261,17 +257,13 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_191]
PartitionCols:_col0
- Select Operator [SEL_186] (rows=36525 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_343] (rows=36525 width=1119)
- predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_36]
+ Please refer to the previous Select Operator [SEL_38]
<-Map 45 [SIMPLE_EDGE]
SHUFFLE [RS_190]
PartitionCols:_col0
- Select Operator [SEL_183] (rows=575995635 width=88)
+ Select Operator [SEL_64] (rows=575995635 width=88)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_342] (rows=575995635 width=88)
+ Filter Operator [FIL_329] (rows=575995635 width=88)
predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
TableScan [TS_62] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk","ss_quantity","ss_sales_price"]
@@ -291,17 +283,13 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_214]
PartitionCols:_col0
- Select Operator [SEL_212] (rows=80000000 width=860)
- Output:["_col0"]
- Filter Operator [FIL_346] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_39]
+ Please refer to the previous Select Operator [SEL_41]
<-Map 46 [SIMPLE_EDGE]
SHUFFLE [RS_213]
PartitionCols:_col0
- Select Operator [SEL_209] (rows=575995635 width=88)
+ Select Operator [SEL_90] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_345] (rows=575995635 width=88)
+ Filter Operator [FIL_332] (rows=575995635 width=88)
predicate:ss_customer_sk is not null
TableScan [TS_88] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_customer_sk","ss_quantity","ss_sales_price"]
@@ -335,9 +323,9 @@ Stage-0
<-Map 19 [SIMPLE_EDGE]
SHUFFLE [RS_138]
PartitionCols:_col0
- Select Operator [SEL_133] (rows=462000 width=1436)
+ Select Operator [SEL_14] (rows=462000 width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_338] (rows=462000 width=1436)
+ Filter Operator [FIL_325] (rows=462000 width=1436)
predicate:i_item_sk is not null
TableScan [TS_12] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_desc"]
@@ -349,18 +337,18 @@ Stage-0
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_134]
PartitionCols:_col0
- Select Operator [SEL_127] (rows=575995635 width=88)
+ Select Operator [SEL_8] (rows=575995635 width=88)
Output:["_col0","_col1"]
- Filter Operator [FIL_336] (rows=575995635 width=88)
+ Filter Operator [FIL_323] (rows=575995635 width=88)
predicate:(ss_sold_date_sk is not null and ss_item_sk is not null)
TableScan [TS_6] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk"]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_135]
PartitionCols:_col0
- Select Operator [SEL_130] (rows=36525 width=1119)
+ Select Operator [SEL_11] (rows=36525 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_337] (rows=36525 width=1119)
+ Filter Operator [FIL_324] (rows=36525 width=1119)
predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null)
TableScan [TS_9] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_year"]
@@ -372,9 +360,9 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_229]
PartitionCols:_col0
- Select Operator [SEL_124] (rows=18262 width=1119)
+ Select Operator [SEL_5] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_335] (rows=18262 width=1119)
+ Filter Operator [FIL_322] (rows=18262 width=1119)
predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -430,11 +418,7 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_46]
PartitionCols:_col0
- Select Operator [SEL_41] (rows=80000000 width=860)
- Output:["_col0"]
- Filter Operator [FIL_328] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_39]
+ Please refer to the previous Select Operator [SEL_41]
<-Reducer 21 [SIMPLE_EDGE]
SHUFFLE [RS_45]
PartitionCols:_col1
@@ -443,19 +427,11 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_43]
PartitionCols:_col0
- Select Operator [SEL_38] (rows=36525 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_327] (rows=36525 width=1119)
- predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_36]
+ Please refer to the previous Select Operator [SEL_38]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_42]
PartitionCols:_col0
- Select Operator [SEL_35] (rows=575995635 width=88)
- Output:["_col0","_col1"]
- Filter Operator [FIL_326] (rows=575995635 width=88)
- predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_33]
+ Please refer to the previous Select Operator [SEL_35]
<-Reducer 35 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_104]
Group By Operator [GBY_86] (rows=1 width=224)
@@ -480,11 +456,7 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_75]
PartitionCols:_col0
- Select Operator [SEL_70] (rows=80000000 width=860)
- Output:["_col0"]
- Filter Operator [FIL_331] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_39]
+ Please refer to the previous Select Operator [SEL_41]
<-Reducer 32 [SIMPLE_EDGE]
SHUFFLE [RS_74]
PartitionCols:_col1
@@ -493,19 +465,11 @@ Stage-0
<-Map 31 [SIMPLE_EDGE]
SHUFFLE [RS_72]
PartitionCols:_col0
- Select Operator [SEL_67] (rows=36525 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_330] (rows=36525 width=1119)
- predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_36]
+ Please refer to the previous Select Operator [SEL_38]
<-Map 45 [SIMPLE_EDGE]
SHUFFLE [RS_71]
PartitionCols:_col0
- Select Operator [SEL_64] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_329] (rows=575995635 width=88)
- predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_62]
+ Please refer to the previous Select Operator [SEL_64]
<-Reducer 42 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_105]
Group By Operator [GBY_101] (rows=316797606 width=88)
@@ -522,19 +486,11 @@ Stage-0
<-Map 40 [SIMPLE_EDGE]
SHUFFLE [RS_95]
PartitionCols:_col0
- Select Operator [SEL_93] (rows=80000000 width=860)
- Output:["_col0"]
- Filter Operator [FIL_333] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_39]
+ Please refer to the previous Select Operator [SEL_41]
<-Map 46 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
- Select Operator [SEL_90] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_332] (rows=575995635 width=88)
- predicate:ss_customer_sk is not null
- Please refer to the previous TableScan [TS_88]
+ Please refer to the previous Select Operator [SEL_90]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_115]
PartitionCols:_col1
@@ -565,11 +521,7 @@ Stage-0
<-Map 19 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_325] (rows=462000 width=1436)
- predicate:i_item_sk is not null
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 12 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col1
@@ -578,19 +530,11 @@ Stage-0
<-Map 11 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=575995635 width=88)
- Output:["_col0","_col1"]
- Filter Operator [FIL_323] (rows=575995635 width=88)
- predicate:(ss_sold_date_sk is not null and ss_item_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=36525 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_324] (rows=36525 width=1119)
- predicate:((d_year) IN (1999, 2000, 2001, 2002) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_112]
PartitionCols:_col2
@@ -599,11 +543,7 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_110]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_322] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 1) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_109]
PartitionCols:_col0
[3/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query54.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query54.q.out b/ql/src/test/results/clientpositive/perf/query54.q.out
index 3cbcbe3..b9d0b8b 100644
--- a/ql/src/test/results/clientpositive/perf/query54.q.out
+++ b/ql/src/test/results/clientpositive/perf/query54.q.out
@@ -222,7 +222,7 @@ Stage-0
Output:["_col0"],keys:_col0
Select Operator [SEL_65] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_179] (rows=18262 width=1119)
+ Filter Operator [FIL_169] (rows=18262 width=1119)
predicate:((d_year = 1999) and (d_moy = 3))
TableScan [TS_0] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_month_seq","d_year","d_moy"]
@@ -348,9 +348,7 @@ Stage-0
Output:["_col0"],keys:_col0
Select Operator [SEL_83] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_181] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 3))
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Filter Operator [FIL_169]
<-Reducer 3 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_111]
Merge Join Operator [MERGEJOIN_191] (rows=9131 width=1128)
@@ -366,9 +364,7 @@ Stage-0
Output:["_col0"],keys:_col0
Select Operator [SEL_2] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_169] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 3))
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Filter Operator [FIL_169]
<-Reducer 9 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_109]
Select Operator [SEL_22] (rows=1 width=8)
@@ -386,11 +382,5 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Group By Operator [GBY_12] (rows=18262 width=1119)
- Output:["_col0"],keys:_col0
- Select Operator [SEL_10] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_170] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 3))
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Group By Operator [GBY_4]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query56.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query56.q.out b/ql/src/test/results/clientpositive/perf/query56.q.out
index 4c11d01..8f4606a 100644
--- a/ql/src/test/results/clientpositive/perf/query56.q.out
+++ b/ql/src/test/results/clientpositive/perf/query56.q.out
@@ -195,9 +195,9 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_60]
PartitionCols:_col0
- Select Operator [SEL_55] (rows=20000000 width=1014)
+ Select Operator [SEL_18] (rows=20000000 width=1014)
Output:["_col0"]
- Filter Operator [FIL_167] (rows=20000000 width=1014)
+ Filter Operator [FIL_162] (rows=20000000 width=1014)
predicate:((ca_gmt_offset = -8) and ca_address_sk is not null)
TableScan [TS_16] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
@@ -209,9 +209,9 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_57]
PartitionCols:_col0
- Select Operator [SEL_52] (rows=18262 width=1119)
+ Select Operator [SEL_15] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_166] (rows=18262 width=1119)
+ Filter Operator [FIL_161] (rows=18262 width=1119)
predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null)
TableScan [TS_13] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -232,9 +232,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col1
- Select Operator [SEL_39] (rows=462000 width=1436)
+ Select Operator [SEL_2] (rows=462000 width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_163] (rows=462000 width=1436)
+ Filter Operator [FIL_158] (rows=462000 width=1436)
predicate:(i_item_id is not null and i_item_sk is not null)
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
@@ -246,11 +246,11 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_44]
PartitionCols:_col0
- Group By Operator [GBY_43] (rows=231000 width=1436)
+ Group By Operator [GBY_6] (rows=231000 width=1436)
Output:["_col0"],keys:i_item_id
- Select Operator [SEL_42] (rows=231000 width=1436)
+ Select Operator [SEL_5] (rows=231000 width=1436)
Output:["i_item_id"]
- Filter Operator [FIL_164] (rows=231000 width=1436)
+ Filter Operator [FIL_159] (rows=231000 width=1436)
predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_color"]
@@ -276,11 +276,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_101]
PartitionCols:_col1
- Select Operator [SEL_77] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_168] (rows=462000 width=1436)
- predicate:(i_item_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_102]
PartitionCols:_col0
@@ -289,13 +285,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col0
- Group By Operator [GBY_81] (rows=231000 width=1436)
- Output:["_col0"],keys:i_item_id
- Select Operator [SEL_80] (rows=231000 width=1436)
- Output:["i_item_id"]
- Filter Operator [FIL_169] (rows=231000 width=1436)
- predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Group By Operator [GBY_6]
<-Reducer 25 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col3
@@ -306,11 +296,7 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_98]
PartitionCols:_col0
- Select Operator [SEL_93] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_172] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -8) and ca_address_sk is not null)
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Reducer 24 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col2
@@ -319,11 +305,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_95]
PartitionCols:_col0
- Select Operator [SEL_90] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_171] (rows=18262 width=1119)
- predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 28 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
@@ -355,11 +337,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_26]
PartitionCols:_col1
- Select Operator [SEL_2] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_158] (rows=462000 width=1436)
- predicate:(i_item_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
@@ -368,13 +346,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Group By Operator [GBY_6] (rows=231000 width=1436)
- Output:["_col0"],keys:i_item_id
- Select Operator [SEL_5] (rows=231000 width=1436)
- Output:["i_item_id"]
- Filter Operator [FIL_159] (rows=231000 width=1436)
- predicate:((i_color) IN ('orchid', 'chiffon', 'lace') and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Group By Operator [GBY_6]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
@@ -385,11 +357,7 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
- Select Operator [SEL_18] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_162] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -8) and ca_address_sk is not null)
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Reducer 19 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col2
@@ -398,11 +366,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_20]
PartitionCols:_col0
- Select Operator [SEL_15] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_161] (rows=18262 width=1119)
- predicate:((d_year = 2000) and (d_moy = 1) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query57.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query57.q.out b/ql/src/test/results/clientpositive/perf/query57.q.out
index 6c237bf..372e760 100644
--- a/ql/src/test/results/clientpositive/perf/query57.q.out
+++ b/ql/src/test/results/clientpositive/perf/query57.q.out
@@ -171,9 +171,9 @@ Stage-0
<-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_52]
PartitionCols:_col0
- Select Operator [SEL_44] (rows=462000 width=1436)
+ Select Operator [SEL_11] (rows=462000 width=1436)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_174] (rows=462000 width=1436)
+ Filter Operator [FIL_168] (rows=462000 width=1436)
predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
TableScan [TS_9] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
@@ -185,9 +185,9 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_49]
PartitionCols:_col0
- Select Operator [SEL_41] (rows=60 width=2045)
+ Select Operator [SEL_8] (rows=60 width=2045)
Output:["_col0","_col1"]
- Filter Operator [FIL_173] (rows=60 width=2045)
+ Filter Operator [FIL_167] (rows=60 width=2045)
predicate:(cc_call_center_sk is not null and cc_name is not null)
TableScan [TS_6] (rows=60 width=2045)
default@call_center,call_center,Tbl:COMPLETE,Col:NONE,Output:["cc_call_center_sk","cc_name"]
@@ -199,18 +199,18 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_45]
PartitionCols:_col0
- Select Operator [SEL_35] (rows=287989836 width=135)
+ Select Operator [SEL_2] (rows=287989836 width=135)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_171] (rows=287989836 width=135)
+ Filter Operator [FIL_165] (rows=287989836 width=135)
predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null)
TableScan [TS_0] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_call_center_sk","cs_item_sk","cs_sales_price"]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_46]
PartitionCols:_col0
- Select Operator [SEL_38] (rows=73048 width=1119)
+ Select Operator [SEL_5] (rows=73048 width=1119)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_172] (rows=73048 width=1119)
+ Filter Operator [FIL_166] (rows=73048 width=1119)
predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -242,11 +242,7 @@ Stage-0
<-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_89]
PartitionCols:_col0
- Select Operator [SEL_81] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_179] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_88]
PartitionCols:_col2
@@ -255,11 +251,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_86]
PartitionCols:_col0
- Select Operator [SEL_78] (rows=60 width=2045)
- Output:["_col0","_col1"]
- Filter Operator [FIL_178] (rows=60 width=2045)
- predicate:(cc_call_center_sk is not null and cc_name is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_85]
PartitionCols:_col1
@@ -268,19 +260,11 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col0
- Select Operator [SEL_72] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_176] (rows=287989836 width=135)
- predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_83]
PartitionCols:_col0
- Select Operator [SEL_75] (rows=73048 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_177] (rows=73048 width=1119)
- predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_103]
PartitionCols:_col0, _col2, _col1, (_col6 + 1)
@@ -309,11 +293,7 @@ Stage-0
<-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_168] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col2
@@ -322,11 +302,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=60 width=2045)
- Output:["_col0","_col1"]
- Filter Operator [FIL_167] (rows=60 width=2045)
- predicate:(cc_call_center_sk is not null and cc_name is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col1
@@ -335,17 +311,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_165] (rows=287989836 width=135)
- predicate:(cs_item_sk is not null and cs_sold_date_sk is not null and cs_call_center_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=73048 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_166] (rows=73048 width=1119)
- predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query58.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query58.q.out b/ql/src/test/results/clientpositive/perf/query58.q.out
index acdfc07..5b37f36 100644
--- a/ql/src/test/results/clientpositive/perf/query58.q.out
+++ b/ql/src/test/results/clientpositive/perf/query58.q.out
@@ -197,9 +197,9 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_85]
PartitionCols:_col1
- Select Operator [SEL_58] (rows=73049 width=1119)
+ Select Operator [SEL_8] (rows=73049 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_254] (rows=73049 width=1119)
+ Filter Operator [FIL_248] (rows=73049 width=1119)
predicate:(d_date is not null and d_date_sk is not null)
TableScan [TS_6] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
@@ -245,10 +245,10 @@ Stage-0
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 18 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_63]
- Group By Operator [GBY_62] (rows=1 width=8)
+ Group By Operator [GBY_12] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_61] (rows=36524 width=1119)
- Filter Operator [FIL_255] (rows=36524 width=1119)
+ Select Operator [SEL_11] (rows=36524 width=1119)
+ Filter Operator [FIL_249] (rows=36524 width=1119)
predicate:(d_date = '1998-02-19')
TableScan [TS_9] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date"]
@@ -260,9 +260,9 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_90]
PartitionCols:_col0
- Select Operator [SEL_55] (rows=462000 width=1436)
+ Select Operator [SEL_5] (rows=462000 width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_253] (rows=462000 width=1436)
+ Filter Operator [FIL_247] (rows=462000 width=1436)
predicate:(i_item_sk is not null and i_item_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
@@ -295,11 +295,7 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_140]
PartitionCols:_col0
- Select Operator [SEL_105] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_259] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 33 [SIMPLE_EDGE]
SHUFFLE [RS_139]
PartitionCols:_col1
@@ -317,11 +313,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_135]
PartitionCols:_col1
- Select Operator [SEL_108] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_260] (rows=73049 width=1119)
- predicate:(d_date is not null and d_date_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 30 [SIMPLE_EDGE]
SHUFFLE [RS_136]
PartitionCols:_col0
@@ -363,12 +355,7 @@ Stage-0
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 18 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_113]
- Group By Operator [GBY_112] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_111] (rows=36524 width=1119)
- Filter Operator [FIL_261] (rows=36524 width=1119)
- predicate:(d_date = '1998-02-19')
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Group By Operator [GBY_12]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_150]
PartitionCols:_col0
@@ -389,11 +376,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_35]
PartitionCols:_col1
- Select Operator [SEL_8] (rows=73049 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_248] (rows=73049 width=1119)
- predicate:(d_date is not null and d_date_sk is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 22 [SIMPLE_EDGE]
SHUFFLE [RS_36]
PartitionCols:_col0
@@ -435,12 +418,7 @@ Stage-0
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 18 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_13]
- Group By Operator [GBY_12] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_11] (rows=36524 width=1119)
- Filter Operator [FIL_249] (rows=36524 width=1119)
- predicate:(d_date = '1998-02-19')
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Group By Operator [GBY_12]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_42]
PartitionCols:_col0
@@ -449,11 +427,7 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_40]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_247] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_39]
PartitionCols:_col1
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query59.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query59.q.out b/ql/src/test/results/clientpositive/perf/query59.q.out
index b570b96..baa9962 100644
--- a/ql/src/test/results/clientpositive/perf/query59.q.out
+++ b/ql/src/test/results/clientpositive/perf/query59.q.out
@@ -161,18 +161,18 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_34]
PartitionCols:_col0
- Select Operator [SEL_30] (rows=575995635 width=88)
+ Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_94] (rows=575995635 width=88)
+ Filter Operator [FIL_90] (rows=575995635 width=88)
predicate:(ss_sold_date_sk is not null and ss_store_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_store_sk","ss_sales_price"]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_35]
PartitionCols:_col0
- Select Operator [SEL_33] (rows=73049 width=1119)
+ Select Operator [SEL_5] (rows=73049 width=1119)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_95] (rows=73049 width=1119)
+ Filter Operator [FIL_91] (rows=73049 width=1119)
predicate:(d_date_sk is not null and d_week_seq is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"]
@@ -222,17 +222,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_6]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_90] (rows=575995635 width=88)
- predicate:(ss_sold_date_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=73049 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_91] (rows=73049 width=1119)
- predicate:(d_date_sk is not null and d_week_seq is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query60.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query60.q.out b/ql/src/test/results/clientpositive/perf/query60.q.out
index 4b7f8b3..7d63986 100644
--- a/ql/src/test/results/clientpositive/perf/query60.q.out
+++ b/ql/src/test/results/clientpositive/perf/query60.q.out
@@ -215,9 +215,9 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_60]
PartitionCols:_col0
- Select Operator [SEL_55] (rows=20000000 width=1014)
+ Select Operator [SEL_18] (rows=20000000 width=1014)
Output:["_col0"]
- Filter Operator [FIL_167] (rows=20000000 width=1014)
+ Filter Operator [FIL_162] (rows=20000000 width=1014)
predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
TableScan [TS_16] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
@@ -229,9 +229,9 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_57]
PartitionCols:_col0
- Select Operator [SEL_52] (rows=18262 width=1119)
+ Select Operator [SEL_15] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_166] (rows=18262 width=1119)
+ Filter Operator [FIL_161] (rows=18262 width=1119)
predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null)
TableScan [TS_13] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -252,9 +252,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col1
- Select Operator [SEL_39] (rows=462000 width=1436)
+ Select Operator [SEL_2] (rows=462000 width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_163] (rows=462000 width=1436)
+ Filter Operator [FIL_158] (rows=462000 width=1436)
predicate:(i_item_id is not null and i_item_sk is not null)
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
@@ -266,11 +266,11 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_44]
PartitionCols:_col0
- Group By Operator [GBY_43] (rows=231000 width=1436)
+ Group By Operator [GBY_6] (rows=231000 width=1436)
Output:["_col0"],keys:i_item_id
- Select Operator [SEL_42] (rows=231000 width=1436)
+ Select Operator [SEL_5] (rows=231000 width=1436)
Output:["i_item_id"]
- Filter Operator [FIL_164] (rows=231000 width=1436)
+ Filter Operator [FIL_159] (rows=231000 width=1436)
predicate:((i_category) IN ('Children') and i_item_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_id","i_category"]
@@ -296,11 +296,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_101]
PartitionCols:_col1
- Select Operator [SEL_77] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_168] (rows=462000 width=1436)
- predicate:(i_item_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_102]
PartitionCols:_col0
@@ -309,13 +305,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col0
- Group By Operator [GBY_81] (rows=231000 width=1436)
- Output:["_col0"],keys:i_item_id
- Select Operator [SEL_80] (rows=231000 width=1436)
- Output:["i_item_id"]
- Filter Operator [FIL_169] (rows=231000 width=1436)
- predicate:((i_category) IN ('Children') and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Group By Operator [GBY_6]
<-Reducer 25 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col3
@@ -326,11 +316,7 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_98]
PartitionCols:_col0
- Select Operator [SEL_93] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_172] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Reducer 24 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col2
@@ -339,11 +325,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_95]
PartitionCols:_col0
- Select Operator [SEL_90] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_171] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 28 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
@@ -375,11 +357,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_26]
PartitionCols:_col1
- Select Operator [SEL_2] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_158] (rows=462000 width=1436)
- predicate:(i_item_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
@@ -388,13 +366,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Group By Operator [GBY_6] (rows=231000 width=1436)
- Output:["_col0"],keys:i_item_id
- Select Operator [SEL_5] (rows=231000 width=1436)
- Output:["i_item_id"]
- Filter Operator [FIL_159] (rows=231000 width=1436)
- predicate:((i_category) IN ('Children') and i_item_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Group By Operator [GBY_6]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
@@ -405,11 +377,7 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
- Select Operator [SEL_18] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_162] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Reducer 19 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col2
@@ -418,11 +386,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_20]
PartitionCols:_col0
- Select Operator [SEL_15] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_161] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 9) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query61.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query61.q.out b/ql/src/test/results/clientpositive/perf/query61.q.out
index 6838332..93d4d42 100644
--- a/ql/src/test/results/clientpositive/perf/query61.q.out
+++ b/ql/src/test/results/clientpositive/perf/query61.q.out
@@ -236,11 +236,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_70]
PartitionCols:_col0
- Select Operator [SEL_62] (rows=852 width=1910)
- Output:["_col0"]
- Filter Operator [FIL_144] (rows=852 width=1910)
- predicate:((s_gmt_offset = -7) and s_store_sk is not null)
- Please refer to the previous TableScan [TS_15]
+ Please refer to the previous Select Operator [SEL_17]
<-Reducer 18 [SIMPLE_EDGE]
SHUFFLE [RS_69]
PartitionCols:_col3
@@ -249,11 +245,7 @@ Stage-0
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_67]
PartitionCols:_col0
- Select Operator [SEL_59] (rows=231000 width=1436)
- Output:["_col0"]
- Filter Operator [FIL_143] (rows=231000 width=1436)
- predicate:((i_category = 'Electronics') and i_item_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_66]
PartitionCols:_col1
@@ -262,11 +254,7 @@ Stage-0
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_64]
PartitionCols:_col0
- Select Operator [SEL_56] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_142] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 11) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col0
@@ -284,17 +272,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_73]
PartitionCols:_col1
- Select Operator [SEL_47] (rows=80000000 width=860)
- Output:["_col0","_col1"]
- Filter Operator [FIL_139] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_current_addr_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_74]
PartitionCols:_col0
- Select Operator [SEL_50] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_140] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -7) and ca_address_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
[9/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
HIVE-16867: Extend shared scan optimizer to reuse computation from other operators (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7819cd36
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7819cd36
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7819cd36
Branch: refs/heads/master
Commit: 7819cd3662f009e9416254ce7b63457285af8cd3
Parents: 71f52d8
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed Jun 21 10:03:31 2017 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Thu Jun 22 16:12:42 2017 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 +-
.../hadoop/hive/ql/exec/OperatorUtils.java | 22 +
.../DynamicPartitionPruningOptimization.java | 137 +-
.../ql/optimizer/OperatorComparatorFactory.java | 64 +-
.../hive/ql/optimizer/SharedScanOptimizer.java | 626 ----------
.../hive/ql/optimizer/SharedWorkOptimizer.java | 1176 ++++++++++++++++++
.../hadoop/hive/ql/parse/GenTezUtils.java | 228 +++-
.../hadoop/hive/ql/parse/TezCompiler.java | 6 +-
.../hive/ql/plan/DynamicPruningEventDesc.java | 12 +
.../hadoop/hive/ql/plan/TableScanDesc.java | 3 +
.../clientpositive/llap/auto_join0.q.out | 7 -
.../clientpositive/llap/auto_join30.q.out | 19 -
.../llap/auto_sortmerge_join_9.q.out | 138 +-
.../llap/bucket_map_join_tez1.q.out | 7 -
.../llap/correlationoptimizer2.q.out | 6 -
.../llap/correlationoptimizer3.q.out | 28 -
.../llap/correlationoptimizer6.q.out | 28 -
.../llap/dynamic_partition_pruning.q.out | 30 +-
.../clientpositive/llap/except_distinct.q.out | 8 -
.../clientpositive/llap/explainuser_1.q.out | 10 +-
.../clientpositive/llap/explainuser_2.q.out | 104 +-
.../clientpositive/llap/intersect_merge.q.out | 10 -
.../clientpositive/llap/limit_pushdown.q.out | 10 -
.../test/results/clientpositive/llap/mrr.q.out | 18 -
.../clientpositive/llap/multiMapJoin2.q.out | 99 +-
.../llap/offset_limit_ppd_optimizer.q.out | 4 -
.../llap/partition_shared_scan.q.out | 7 -
.../clientpositive/llap/subquery_in.q.out | 18 -
.../clientpositive/llap/subquery_multi.q.out | 42 -
.../clientpositive/llap/subquery_notin.q.out | 116 --
.../clientpositive/llap/subquery_null_agg.q.out | 4 -
.../clientpositive/llap/subquery_scalar.q.out | 32 -
.../clientpositive/llap/subquery_select.q.out | 42 -
.../clientpositive/llap/subquery_views.q.out | 5 -
.../clientpositive/llap/unionDistinct_1.q.out | 29 -
.../clientpositive/llap/union_top_level.q.out | 14 -
.../llap/vector_groupby_grouping_sets4.q.out | 27 -
.../clientpositive/llap/vector_join30.q.out | 39 -
.../vectorized_dynamic_partition_pruning.q.out | 30 +-
.../results/clientpositive/perf/query1.q.out | 6 +-
.../results/clientpositive/perf/query10.q.out | 16 +-
.../results/clientpositive/perf/query11.q.out | 34 +-
.../results/clientpositive/perf/query14.q.out | 318 ++---
.../results/clientpositive/perf/query16.q.out | 4 +-
.../results/clientpositive/perf/query2.q.out | 10 +-
.../results/clientpositive/perf/query23.q.out | 126 +-
.../results/clientpositive/perf/query24.q.out | 50 +-
.../results/clientpositive/perf/query30.q.out | 10 +-
.../results/clientpositive/perf/query31.q.out | 62 +-
.../results/clientpositive/perf/query32.q.out | 12 +-
.../results/clientpositive/perf/query33.q.out | 70 +-
.../results/clientpositive/perf/query35.q.out | 16 +-
.../results/clientpositive/perf/query38.q.out | 32 +-
.../results/clientpositive/perf/query39.q.out | 30 +-
.../results/clientpositive/perf/query4.q.out | 52 +-
.../results/clientpositive/perf/query44.q.out | 29 +-
.../results/clientpositive/perf/query46.q.out | 6 +-
.../results/clientpositive/perf/query47.q.out | 64 +-
.../results/clientpositive/perf/query49.q.out | 16 +-
.../results/clientpositive/perf/query5.q.out | 16 +-
.../results/clientpositive/perf/query51.q.out | 6 +-
.../results/clientpositive/perf/query54.q.out | 18 +-
.../results/clientpositive/perf/query56.q.out | 70 +-
.../results/clientpositive/perf/query57.q.out | 64 +-
.../results/clientpositive/perf/query58.q.out | 52 +-
.../results/clientpositive/perf/query59.q.out | 20 +-
.../results/clientpositive/perf/query60.q.out | 70 +-
.../results/clientpositive/perf/query61.q.out | 30 +-
.../results/clientpositive/perf/query64.q.out | 178 +--
.../results/clientpositive/perf/query65.q.out | 6 +-
.../results/clientpositive/perf/query66.q.out | 40 +-
.../results/clientpositive/perf/query68.q.out | 6 +-
.../results/clientpositive/perf/query69.q.out | 16 +-
.../results/clientpositive/perf/query70.q.out | 22 +-
.../results/clientpositive/perf/query74.q.out | 34 +-
.../results/clientpositive/perf/query75.q.out | 110 +-
.../results/clientpositive/perf/query76.q.out | 16 +-
.../results/clientpositive/perf/query77.q.out | 50 +-
.../results/clientpositive/perf/query78.q.out | 16 +-
.../results/clientpositive/perf/query80.q.out | 48 +-
.../results/clientpositive/perf/query81.q.out | 10 +-
.../results/clientpositive/perf/query83.q.out | 32 +-
.../results/clientpositive/perf/query85.q.out | 10 +-
.../results/clientpositive/perf/query87.q.out | 32 +-
.../results/clientpositive/perf/query88.q.out | 138 +-
.../results/clientpositive/perf/query90.q.out | 30 +-
.../results/clientpositive/perf/query92.q.out | 12 +-
.../results/clientpositive/perf/query94.q.out | 4 +-
.../results/clientpositive/perf/query95.q.out | 12 +-
.../results/clientpositive/perf/query97.q.out | 6 +-
.../clientpositive/tez/explainanalyze_2.q.out | 52 +-
91 files changed, 2134 insertions(+), 3266 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 1529788..da48a7c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1661,9 +1661,9 @@ public class HiveConf extends Configuration {
"If the skew information is correctly stored in the metadata, hive.optimize.skewjoin.compiletime\n" +
"would change the query plan to take care of it, and hive.optimize.skewjoin will be a no-op."),
- HIVE_SHARED_SCAN_OPTIMIZATION("hive.optimize.shared.scan", true,
- "Whether to enable shared scan optimizer. The optimizer finds scan operator over the same table\n" +
- "in the query plan and merges them if they meet some preconditions."),
+ HIVE_SHARED_WORK_OPTIMIZATION("hive.optimize.shared.work", true,
+ "Whether to enable shared work optimizer. The optimizer finds scan operator over the same table\n" +
+ "and follow-up operators in the query plan and merges them if they meet some preconditions."),
// CTE
HIVE_CTE_MATERIALIZE_THRESHOLD("hive.optimize.cte.materialize.threshold", -1,
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
index d5006bd..be0795d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
@@ -33,6 +33,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
public class OperatorUtils {
@@ -381,6 +382,27 @@ public class OperatorUtils {
curr.removeChild(child);
}
+ /**
+ * Remove operator from the tree, disconnecting it from its
+ * parents and children.
+ */
+ public static void removeOperator(Operator<?> op) {
+ if (op.getNumParent() != 0) {
+ List<Operator<? extends OperatorDesc>> allParent =
+ Lists.newArrayList(op.getParentOperators());
+ for (Operator<?> parentOp : allParent) {
+ parentOp.removeChild(op);
+ }
+ }
+ if (op.getNumChild() != 0) {
+ List<Operator<? extends OperatorDesc>> allChildren =
+ Lists.newArrayList(op.getChildOperators());
+ for (Operator<?> childOp : allChildren) {
+ childOp.removeParent(op);
+ }
+ }
+ }
+
public static String getOpNamePretty(Operator<?> op) {
if (op instanceof TableScanOperator) {
return op.toString() + " (" + ((TableScanOperator) op).getConf().getAlias() + ")";
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index a9099b8..bd2d73e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -21,43 +21,67 @@ package org.apache.hadoop.hive.ql.optimizer;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Stack;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.exec.*;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.FilterOperator;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorFactory;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
-import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
-import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
-import org.apache.hadoop.hive.ql.lib.Dispatcher;
-import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.lib.Rule;
-import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc;
-import org.apache.hadoop.hive.ql.parse.*;
+import org.apache.hadoop.hive.ql.parse.GenTezUtils;
+import org.apache.hadoop.hive.ql.parse.GenTezUtils.DynamicListContext;
+import org.apache.hadoop.hive.ql.parse.GenTezUtils.DynamicPartitionPrunerContext;
+import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
+import org.apache.hadoop.hive.ql.parse.RuntimeValuesInfo;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo;
+import org.apache.hadoop.hive.ql.parse.SemiJoinHint;
import org.apache.hadoop.hive.ql.parse.spark.OptimizeSparkProcContext;
-import org.apache.hadoop.hive.ql.plan.*;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
+import org.apache.hadoop.hive.ql.plan.DynamicValue;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
+import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
+import org.apache.hadoop.hive.ql.plan.SelectDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBloomFilter.GenericUDAFBloomFilterEvaluator;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
/**
* This optimization looks for expressions of the kind "x IN (RS[n])". If such
* an expression made it to a table scan operator and x is a partition column we
@@ -69,59 +93,6 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
static final private Logger LOG = LoggerFactory.getLogger(DynamicPartitionPruningOptimization.class
.getName());
- private static class DynamicPartitionPrunerProc implements NodeProcessor {
-
- /**
- * process simply remembers all the dynamic partition pruning expressions
- * found
- */
- @Override
- public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
- Object... nodeOutputs) throws SemanticException {
- ExprNodeDynamicListDesc desc = (ExprNodeDynamicListDesc) nd;
- DynamicPartitionPrunerContext context = (DynamicPartitionPrunerContext) procCtx;
-
- // Rule is searching for dynamic pruning expr. There's at least an IN
- // expression wrapping it.
- ExprNodeDesc parent = (ExprNodeDesc) stack.get(stack.size() - 2);
- ExprNodeDesc grandParent = stack.size() >= 3 ? (ExprNodeDesc) stack.get(stack.size() - 3) : null;
-
- context.addDynamicList(desc, parent, grandParent, (ReduceSinkOperator) desc.getSource());
-
- return context;
- }
- }
-
- private static class DynamicListContext {
- public ExprNodeDynamicListDesc desc;
- public ExprNodeDesc parent;
- public ExprNodeDesc grandParent;
- public ReduceSinkOperator generator;
-
- public DynamicListContext(ExprNodeDynamicListDesc desc, ExprNodeDesc parent,
- ExprNodeDesc grandParent, ReduceSinkOperator generator) {
- this.desc = desc;
- this.parent = parent;
- this.grandParent = grandParent;
- this.generator = generator;
- }
- }
-
- private static class DynamicPartitionPrunerContext implements NodeProcessorCtx,
- Iterable<DynamicListContext> {
- public List<DynamicListContext> dynLists = new ArrayList<DynamicListContext>();
-
- public void addDynamicList(ExprNodeDynamicListDesc desc, ExprNodeDesc parent,
- ExprNodeDesc grandParent, ReduceSinkOperator generator) {
- dynLists.add(new DynamicListContext(desc, parent, grandParent, generator));
- }
-
- @Override
- public Iterator<DynamicListContext> iterator() {
- return dynLists.iterator();
- }
- }
-
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs)
throws SemanticException {
@@ -162,7 +133,7 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
// collect the dynamic pruning conditions
removerContext.dynLists.clear();
- collectDynamicPruningConditions(desc.getPredicate(), removerContext);
+ GenTezUtils.collectDynamicPruningConditions(desc.getPredicate(), removerContext);
if (ts == null) {
// Replace the synthetic predicate with true and bail out
@@ -402,7 +373,7 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
// collect the dynamic pruning conditions
removerContext.dynLists.clear();
- collectDynamicPruningConditions(ts.getConf().getFilterExpr(), removerContext);
+ GenTezUtils.collectDynamicPruningConditions(ts.getConf().getFilterExpr(), removerContext);
for (DynamicListContext ctx : removerContext) {
// remove the condition by replacing it with "true"
@@ -476,6 +447,7 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
DynamicPruningEventDesc eventDesc = new DynamicPruningEventDesc();
eventDesc.setTableScan(ts);
+ eventDesc.setGenerator(ctx.generator);
eventDesc.setTable(PlanUtils.getReduceValueTableDesc(PlanUtils
.getFieldSchemasFromColumnList(keyExprs, "key")));
eventDesc.setTargetColumnName(column);
@@ -784,27 +756,4 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor {
parseContext.getColExprToGBMap().put(key, gb);
}
- private Map<Node, Object> collectDynamicPruningConditions(ExprNodeDesc pred, NodeProcessorCtx ctx)
- throws SemanticException {
-
- // create a walker which walks the tree in a DFS manner while maintaining
- // the operator stack. The dispatcher
- // generates the plan from the operator tree
- Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>();
- exprRules.put(new RuleRegExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"),
- new DynamicPartitionPrunerProc());
-
- // The dispatcher fires the processor corresponding to the closest matching
- // rule and passes the context along
- Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx);
- GraphWalker egw = new DefaultGraphWalker(disp);
-
- List<Node> startNodes = new ArrayList<Node>();
- startNodes.add(pred);
-
- HashMap<Node, Object> outputMap = new HashMap<Node, Object>();
- egw.startWalking(startNodes, outputMap);
- return outputMap;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java
index 1da9164..0373e53 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorComparatorFactory.java
@@ -25,6 +25,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
import org.apache.hadoop.hive.ql.exec.CollectOperator;
import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
import org.apache.hadoop.hive.ql.exec.DemuxOperator;
@@ -56,6 +57,9 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorLimitOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorSparkHashTableSinkOperator;
+import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc;
+import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
+import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
@@ -93,13 +97,14 @@ public class OperatorComparatorFactory {
new SparkHashTableSinkOperatorComparator());
comparatorMapping.put(LateralViewJoinOperator.class, new LateralViewJoinOperatorComparator());
comparatorMapping.put(VectorGroupByOperator.class, new VectorGroupByOperatorComparator());
- comparatorMapping.put(CommonMergeJoinOperator.class, new MapJoinOperatorComparator());
+ comparatorMapping.put(CommonMergeJoinOperator.class, new CommonMergeJoinOperatorComparator());
comparatorMapping.put(VectorFilterOperator.class, new FilterOperatorComparator());
comparatorMapping.put(UDTFOperator.class, new UDTFOperatorComparator());
comparatorMapping.put(VectorSelectOperator.class, new VectorSelectOperatorComparator());
comparatorMapping.put(VectorLimitOperator.class, new LimitOperatorComparator());
comparatorMapping.put(ScriptOperator.class, new ScriptOperatorComparator());
comparatorMapping.put(TemporaryHashSinkOperator.class, new HashTableSinkOperatorComparator());
+ comparatorMapping.put(AppMasterEventOperator.class, new AppMasterEventOperatorComparator());
// these operators does not have state, so they always equal with the same kind.
comparatorMapping.put(UnionOperator.class, new AlwaysTrueOperatorComparator());
comparatorMapping.put(ForwardOperator.class, new AlwaysTrueOperatorComparator());
@@ -364,7 +369,33 @@ public class OperatorComparatorFactory {
compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
desc1.getPosBigTable() == desc2.getPosBigTable() &&
desc1.isBucketMapJoin() == desc2.isBucketMapJoin() &&
+ compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
+ compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
+ compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
+ desc1.getHandleSkewJoin() == desc2.getHandleSkewJoin() &&
+ compareString(desc1.getNullSafeString(), desc2.getNullSafeString())) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ static class CommonMergeJoinOperatorComparator implements OperatorComparator<CommonMergeJoinOperator> {
+
+ @Override
+ public boolean equals(CommonMergeJoinOperator op1, CommonMergeJoinOperator op2) {
+ Preconditions.checkNotNull(op1);
+ Preconditions.checkNotNull(op2);
+ CommonMergeJoinDesc desc1 = op1.getConf();
+ CommonMergeJoinDesc desc2 = op2.getConf();
+
+ if (compareObject(desc1.getParentToInput(), desc2.getParentToInput()) &&
+ compareString(desc1.getKeyCountsExplainDesc(), desc2.getKeyCountsExplainDesc()) &&
compareObject(desc1.getKeysString(), desc2.getKeysString()) &&
+ desc1.getPosBigTable() == desc2.getPosBigTable() &&
+ desc1.isBucketMapJoin() == desc2.isBucketMapJoin() &&
+ desc1.getNumBuckets() == desc2.getNumBuckets() &&
compareObject(desc1.getFiltersStringMap(), desc2.getFiltersStringMap()) &&
compareObject(desc1.getOutputColumnNames(), desc2.getOutputColumnNames()) &&
compareObject(desc1.getCondsList(), desc2.getCondsList()) &&
@@ -516,6 +547,37 @@ public class OperatorComparatorFactory {
}
}
+ static class AppMasterEventOperatorComparator implements OperatorComparator<AppMasterEventOperator> {
+
+ @Override
+ public boolean equals(AppMasterEventOperator op1, AppMasterEventOperator op2) {
+ Preconditions.checkNotNull(op1);
+ Preconditions.checkNotNull(op2);
+ AppMasterEventDesc op1Conf = op1.getConf();
+ AppMasterEventDesc op2Conf = op2.getConf();
+
+ if (compareString(op1Conf.getInputName(), op2Conf.getInputName()) &&
+ compareString(op1Conf.getVertexName(), op2Conf.getVertexName()) &&
+ compareObject(op1Conf.getTable(), op2Conf.getTable())) {
+ if (op1Conf instanceof DynamicPruningEventDesc && op2Conf instanceof DynamicPruningEventDesc) {
+ DynamicPruningEventDesc op1DPPConf = (DynamicPruningEventDesc) op1Conf;
+ DynamicPruningEventDesc op2DPPConf = (DynamicPruningEventDesc) op2Conf;
+ if (compareString(op1DPPConf.getTargetColumnName(), op2DPPConf.getTargetColumnName()) &&
+ compareString(op1DPPConf.getTargetColumnType(), op2DPPConf.getTargetColumnType()) &&
+ compareString(op1DPPConf.getPartKeyString(), op2DPPConf.getPartKeyString())) {
+ return true;
+ }
+ return false;
+ } else if (op1Conf instanceof DynamicPruningEventDesc || op2Conf instanceof DynamicPruningEventDesc) {
+ return false;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
static boolean compareString(String first, String second) {
return compareObject(first, second);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedScanOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedScanOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedScanOptimizer.java
deleted file mode 100644
index e31119f..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedScanOptimizer.java
+++ /dev/null
@@ -1,626 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
-import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.OperatorFactory;
-import org.apache.hadoop.hive.ql.exec.OperatorUtils;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.RowSchema;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
-import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo;
-import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.FilterDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.stats.StatsUtils;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-
-/**
- * Shared scan optimizer. This rule finds scan operator over the same table
- * in the query plan and merges them if they meet some preconditions.
- *
- * TS TS TS
- * | | -> / \
- * Op Op Op Op
- *
- * <p>Currently it only works with the Tez execution engine.
- */
-public class SharedScanOptimizer extends Transform {
-
- private final static Logger LOG = LoggerFactory.getLogger(SharedScanOptimizer.class);
-
- @Override
- public ParseContext transform(ParseContext pctx) throws SemanticException {
-
- final Map<String, TableScanOperator> topOps = pctx.getTopOps();
- if (topOps.size() < 2) {
- // Nothing to do, bail out
- return pctx;
- }
-
- // Cache to use during optimization
- SharedScanOptimizerCache optimizerCache = new SharedScanOptimizerCache();
-
- // We will not apply this optimization on some table scan operators.
- Set<TableScanOperator> excludeTableScanOps = gatherNotValidTableScanOps(pctx, optimizerCache);
- LOG.debug("Exclude TableScan ops: {}", excludeTableScanOps);
-
- // Map of dbName.TblName -> Pair(tableAlias, TSOperator)
- Multimap<String, Entry<String, TableScanOperator>> tableNameToOps = splitTableScanOpsByTable(pctx);
-
- // We enforce a certain order when we do the reutilization.
- // In particular, we use size of table x number of reads to
- // rank the tables.
- List<Entry<String, Long>> sortedTables = rankTablesByAccumulatedSize(pctx, excludeTableScanOps);
- LOG.debug("Sorted tables by size: {}", sortedTables);
-
- // Execute optimization
- Multimap<String, TableScanOperator> existingOps = ArrayListMultimap.create();
- Set<String> entriesToRemove = new HashSet<>();
- for (Entry<String, Long> tablePair : sortedTables) {
- for (Entry<String, TableScanOperator> tableScanOpPair : tableNameToOps.get(tablePair.getKey())) {
- TableScanOperator tsOp = tableScanOpPair.getValue();
- if (excludeTableScanOps.contains(tsOp)) {
- // Skip operator, currently we do not merge
- continue;
- }
- String tableName = tablePair.getKey();
- Collection<TableScanOperator> prevTsOps = existingOps.get(tableName);
- if (!prevTsOps.isEmpty()) {
- for (TableScanOperator prevTsOp : prevTsOps) {
-
- // First we check if the two table scan operators can actually be merged
- // If schemas do not match, we currently do not merge
- List<String> prevTsOpNeededColumns = prevTsOp.getNeededColumns();
- List<String> tsOpNeededColumns = tsOp.getNeededColumns();
- if (prevTsOpNeededColumns.size() != tsOpNeededColumns.size()) {
- // Skip
- continue;
- }
- boolean notEqual = false;
- for (int i = 0; i < prevTsOpNeededColumns.size(); i++) {
- if (!prevTsOpNeededColumns.get(i).equals(tsOpNeededColumns.get(i))) {
- notEqual = true;
- break;
- }
- }
- if (notEqual) {
- // Skip
- continue;
- }
- // If row limit does not match, we currently do not merge
- if (prevTsOp.getConf().getRowLimit() != tsOp.getConf().getRowLimit()) {
- // Skip
- continue;
- }
- // If partitions do not match, we currently do not merge
- PrunedPartitionList prevTsOpPPList = pctx.getPrunedPartitions(prevTsOp);
- PrunedPartitionList tsOpPPList = pctx.getPrunedPartitions(tsOp);
- if (prevTsOpPPList.hasUnknownPartitions()
- || tsOpPPList.hasUnknownPartitions()
- || !prevTsOpPPList.getPartitions().equals(tsOpPPList.getPartitions())) {
- // Skip
- continue;
- }
-
- // It seems these two operators can be merged.
- // Check that plan meets some preconditions before doing it.
- // In particular, in the presence of map joins in the upstream plan:
- // - we cannot exceed the noconditional task size, and
- // - if we already merged the big table, we cannot merge the broadcast
- // tables.
- if (!validPreConditions(pctx, optimizerCache, prevTsOp, tsOp)) {
- // Skip
- LOG.debug("{} does not meet preconditions", tsOp);
- continue;
- }
-
- // We can merge
- ExprNodeGenericFuncDesc exprNode = null;
- if (prevTsOp.getConf().getFilterExpr() != null) {
- // Push filter on top of children
- pushFilterToTopOfTableScan(optimizerCache, prevTsOp);
- // Clone to push to table scan
- exprNode = (ExprNodeGenericFuncDesc) prevTsOp.getConf().getFilterExpr();
- }
- if (tsOp.getConf().getFilterExpr() != null) {
- // Push filter on top
- pushFilterToTopOfTableScan(optimizerCache, tsOp);
- ExprNodeGenericFuncDesc tsExprNode = tsOp.getConf().getFilterExpr();
- if (exprNode != null && !exprNode.isSame(tsExprNode)) {
- // We merge filters from previous scan by ORing with filters from current scan
- if (exprNode.getGenericUDF() instanceof GenericUDFOPOr) {
- List<ExprNodeDesc> newChildren = new ArrayList<>(exprNode.getChildren().size() + 1);
- for (ExprNodeDesc childExprNode : exprNode.getChildren()) {
- if (childExprNode.isSame(tsExprNode)) {
- // We do not need to do anything, it is in the OR expression
- break;
- }
- newChildren.add(childExprNode);
- }
- if (exprNode.getChildren().size() == newChildren.size()) {
- newChildren.add(tsExprNode);
- exprNode = ExprNodeGenericFuncDesc.newInstance(
- new GenericUDFOPOr(),
- newChildren);
- }
- } else {
- exprNode = ExprNodeGenericFuncDesc.newInstance(
- new GenericUDFOPOr(),
- Arrays.<ExprNodeDesc>asList(exprNode, tsExprNode));
- }
- }
- }
- // Replace filter
- prevTsOp.getConf().setFilterExpr(exprNode);
- // Replace table scan operator
- List<Operator<? extends OperatorDesc>> allChildren =
- Lists.newArrayList(tsOp.getChildOperators());
- for (Operator<? extends OperatorDesc> op : allChildren) {
- tsOp.getChildOperators().remove(op);
- op.replaceParent(tsOp, prevTsOp);
- prevTsOp.getChildOperators().add(op);
- }
- entriesToRemove.add(tableScanOpPair.getKey());
- // Remove and combine
- optimizerCache.removeOpAndCombineWork(tsOp, prevTsOp);
-
- LOG.debug("Merged {} into {}", tsOp, prevTsOp);
-
- break;
- }
- if (!entriesToRemove.contains(tableScanOpPair.getKey())) {
- existingOps.put(tableName, tsOp);
- }
- } else {
- // Add to existing ops
- existingOps.put(tableName, tsOp);
- }
- }
- }
- // Remove unused operators
- for (String key : entriesToRemove) {
- topOps.remove(key);
- }
-
- return pctx;
- }
-
- private static Set<TableScanOperator> gatherNotValidTableScanOps(
- ParseContext pctx, SharedScanOptimizerCache optimizerCache) throws SemanticException {
- // Find TS operators with partition pruning enabled in plan
- // because these TS may potentially read different data for
- // different pipeline.
- // These can be:
- // 1) TS with DPP.
- // TODO: Check if dynamic filters are identical and do not add.
- // 2) TS with semijoin DPP.
- // TODO: Check for dynamic filters.
- Set<TableScanOperator> notValidTableScanOps = new HashSet<>();
- // 1) TS with DPP.
- Map<String, TableScanOperator> topOps = pctx.getTopOps();
- Collection<Operator<? extends OperatorDesc>> tableScanOps =
- Lists.<Operator<?>>newArrayList(topOps.values());
- Set<AppMasterEventOperator> s =
- OperatorUtils.findOperators(tableScanOps, AppMasterEventOperator.class);
- for (AppMasterEventOperator a : s) {
- if (a.getConf() instanceof DynamicPruningEventDesc) {
- DynamicPruningEventDesc dped = (DynamicPruningEventDesc) a.getConf();
- notValidTableScanOps.add(dped.getTableScan());
- optimizerCache.tableScanToDPPSource.put(dped.getTableScan(), a);
- }
- }
- // 2) TS with semijoin DPP.
- for (Entry<ReduceSinkOperator, SemiJoinBranchInfo> e
- : pctx.getRsToSemiJoinBranchInfo().entrySet()) {
- notValidTableScanOps.add(e.getValue().getTsOp());
- optimizerCache.tableScanToDPPSource.put(e.getValue().getTsOp(), e.getKey());
- }
- return notValidTableScanOps;
- }
-
- private static Multimap<String, Entry<String, TableScanOperator>> splitTableScanOpsByTable(
- ParseContext pctx) {
- Multimap<String, Entry<String, TableScanOperator>> tableNameToOps = ArrayListMultimap.create();
- for (Entry<String, TableScanOperator> e : pctx.getTopOps().entrySet()) {
- TableScanOperator tsOp = e.getValue();
- tableNameToOps.put(
- tsOp.getConf().getTableMetadata().getDbName() + "."
- + tsOp.getConf().getTableMetadata().getTableName(), e);
- }
- return tableNameToOps;
- }
-
- private static List<Entry<String, Long>> rankTablesByAccumulatedSize(ParseContext pctx,
- Set<TableScanOperator> excludeTables) {
- Map<String, Long> tableToTotalSize = new HashMap<>();
- for (Entry<String, TableScanOperator> e : pctx.getTopOps().entrySet()) {
- TableScanOperator tsOp = e.getValue();
- if (excludeTables.contains(tsOp)) {
- // Skip operator, currently we do not merge
- continue;
- }
- String tableName = tsOp.getConf().getTableMetadata().getDbName() + "."
- + tsOp.getConf().getTableMetadata().getTableName();
- long tableSize = tsOp.getStatistics() != null ?
- tsOp.getStatistics().getDataSize() : 0L;
- Long totalSize = tableToTotalSize.get(tableName);
- if (totalSize != null) {
- tableToTotalSize.put(tableName,
- StatsUtils.safeAdd(totalSize, tableSize));
- } else {
- tableToTotalSize.put(tableName, tableSize);
- }
- }
- List<Entry<String, Long>> sortedTables =
- new LinkedList<>(tableToTotalSize.entrySet());
- Collections.sort(sortedTables, Collections.reverseOrder(
- new Comparator<Map.Entry<String, Long>>() {
- public int compare(Map.Entry<String, Long> o1, Map.Entry<String, Long> o2) {
- return (o1.getValue()).compareTo(o2.getValue());
- }
- }));
- return sortedTables;
- }
-
- private static boolean validPreConditions(ParseContext pctx, SharedScanOptimizerCache optimizerCache,
- TableScanOperator prevTsOp, TableScanOperator tsOp) {
- // 1) The set of operators in the works of the TS operators need to meet
- // some requirements. In particular:
- // 1.1. None of the works that contain the TS operators can contain a Union
- // operator. This is not supported yet as we might end up with cycles in
- // the Tez DAG.
- // 1.2. There cannot be more than one DummyStore operator in the new resulting
- // work when the TS operators are merged. This is due to an assumption in
- // MergeJoinProc that needs to be further explored.
- // If any of these conditions are not met, we cannot merge.
- // TODO: Extend rule so it can be applied for these cases.
- final Set<Operator<?>> workOps1 = findWorkOperators(optimizerCache, prevTsOp);
- final Set<Operator<?>> workOps2 = findWorkOperators(optimizerCache, tsOp);
- boolean foundDummyStoreOp = false;
- for (Operator<?> op : workOps1) {
- if (op instanceof UnionOperator) {
- // We cannot merge (1.1)
- return false;
- }
- if (op instanceof DummyStoreOperator) {
- foundDummyStoreOp = true;
- }
- }
- for (Operator<?> op : workOps2) {
- if (op instanceof UnionOperator) {
- // We cannot merge (1.1)
- return false;
- }
- if (foundDummyStoreOp && op instanceof DummyStoreOperator) {
- // We cannot merge (1.2)
- return false;
- }
- }
- // 2) We check whether output works when we merge the operators will collide.
- //
- // Work1 Work2 (merge TS in W1 & W2) Work1
- // \ / -> | | X
- // Work3 Work3
- //
- // If we do, we cannot merge. The reason is that Tez currently does
- // not support parallel edges, i.e., multiple edges from same work x
- // into same work y.
- final Set<Operator<?>> outputWorksOps1 = findChildWorkOperators(pctx, optimizerCache, prevTsOp);
- final Set<Operator<?>> outputWorksOps2 = findChildWorkOperators(pctx, optimizerCache, tsOp);
- if (!Collections.disjoint(outputWorksOps1, outputWorksOps2)) {
- // We cannot merge
- return false;
- }
- // 3) We check whether we will end up with same operators inputing on same work.
- //
- // Work1 (merge TS in W2 & W3) Work1
- // / \ -> | | X
- // Work2 Work3 Work2
- //
- // If we do, we cannot merge. The reason is the same as above, currently
- // Tez currently does not support parallel edges.
- final Set<Operator<?>> inputWorksOps1 = findParentWorkOperators(pctx, optimizerCache, prevTsOp);
- final Set<Operator<?>> inputWorksOps2 = findParentWorkOperators(pctx, optimizerCache, tsOp);
- if (!Collections.disjoint(inputWorksOps1, inputWorksOps2)) {
- // We cannot merge
- return false;
- }
- // 4) We check whether one of the operators is part of a work that is an input for
- // the work of the other operator.
- //
- // Work1 (merge TS in W1 & W3) Work1
- // | -> | X
- // Work2 Work2
- // | |
- // Work3 Work1
- //
- // If we do, we cannot merge, as we would end up with a cycle in the DAG.
- final Set<Operator<?>> descendantWorksOps1 =
- findDescendantWorkOperators(pctx, optimizerCache, prevTsOp);
- final Set<Operator<?>> descendantWorksOps2 =
- findDescendantWorkOperators(pctx, optimizerCache, tsOp);
- if (!Collections.disjoint(descendantWorksOps1, workOps2)
- || !Collections.disjoint(workOps1, descendantWorksOps2)) {
- return false;
- }
- // 5) We check whether merging the works would cause the size of
- // the data in memory grow too large.
- // TODO: Currently ignores GBY and PTF which may also buffer data in memory.
- final Set<Operator<?>> newWorkOps = workOps1;
- newWorkOps.addAll(workOps2);
- long dataSize = 0L;
- for (Operator<?> op : newWorkOps) {
- if (op instanceof MapJoinOperator) {
- MapJoinOperator mop = (MapJoinOperator) op;
- dataSize = StatsUtils.safeAdd(dataSize, mop.getConf().getInMemoryDataSize());
- if (dataSize > mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize()) {
- // Size surpasses limit, we cannot convert
- LOG.debug("accumulated data size: {} / max size: {}",
- dataSize, mop.getConf().getMemoryMonitorInfo().getAdjustedNoConditionalTaskSize());
- return false;
- }
- }
- }
- return true;
- }
-
- private static Set<Operator<?>> findParentWorkOperators(ParseContext pctx,
- SharedScanOptimizerCache optimizerCache, Operator<?> start) {
- // Find operators in work
- Set<Operator<?>> workOps = findWorkOperators(optimizerCache, start);
- // Gather input works operators
- Set<Operator<?>> set = new HashSet<Operator<?>>();
- for (Operator<?> op : workOps) {
- if (op.getParentOperators() != null) {
- for (Operator<?> parent : op.getParentOperators()) {
- if (parent instanceof ReduceSinkOperator) {
- set.addAll(findWorkOperators(optimizerCache, parent));
- }
- }
- } else if (op instanceof TableScanOperator) {
- // Check for DPP and semijoin DPP
- for (Operator<?> parent : optimizerCache.tableScanToDPPSource.get((TableScanOperator) op)) {
- set.addAll(findWorkOperators(optimizerCache, parent));
- }
- }
- }
- return set;
- }
-
- private static Set<Operator<?>> findChildWorkOperators(ParseContext pctx,
- SharedScanOptimizerCache optimizerCache, Operator<?> start) {
- // Find operators in work
- Set<Operator<?>> workOps = findWorkOperators(optimizerCache, start);
- // Gather output works operators
- Set<Operator<?>> set = new HashSet<Operator<?>>();
- for (Operator<?> op : workOps) {
- if (op instanceof ReduceSinkOperator) {
- if (op.getChildOperators() != null) {
- // All children of RS are descendants
- for (Operator<?> child : op.getChildOperators()) {
- set.addAll(findWorkOperators(optimizerCache, child));
- }
- }
- // Semijoin DPP work is considered a child because work needs
- // to finish for it to execute
- SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op);
- if (sjbi != null) {
- set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp()));
- }
- } else if(op.getConf() instanceof DynamicPruningEventDesc) {
- // DPP work is considered a child because work needs
- // to finish for it to execute
- set.addAll(findWorkOperators(
- optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan()));
- }
- }
- return set;
- }
-
- private static Set<Operator<?>> findDescendantWorkOperators(ParseContext pctx,
- SharedScanOptimizerCache optimizerCache, Operator<?> start) {
- // Find operators in work
- Set<Operator<?>> workOps = findWorkOperators(optimizerCache, start);
- // Gather output works operators
- Set<Operator<?>> result = new HashSet<Operator<?>>();
- Set<Operator<?>> set;
- while (!workOps.isEmpty()) {
- set = new HashSet<Operator<?>>();
- for (Operator<?> op : workOps) {
- if (op instanceof ReduceSinkOperator) {
- if (op.getChildOperators() != null) {
- // All children of RS are descendants
- for (Operator<?> child : op.getChildOperators()) {
- set.addAll(findWorkOperators(optimizerCache, child));
- }
- }
- // Semijoin DPP work is considered a descendant because work needs
- // to finish for it to execute
- SemiJoinBranchInfo sjbi = pctx.getRsToSemiJoinBranchInfo().get(op);
- if (sjbi != null) {
- set.addAll(findWorkOperators(optimizerCache, sjbi.getTsOp()));
- }
- } else if(op.getConf() instanceof DynamicPruningEventDesc) {
- // DPP work is considered a descendant because work needs
- // to finish for it to execute
- set.addAll(findWorkOperators(
- optimizerCache, ((DynamicPruningEventDesc) op.getConf()).getTableScan()));
- }
- }
- workOps = set;
- result.addAll(set);
- }
- return result;
- }
-
- // Stores result in cache
- private static Set<Operator<?>> findWorkOperators(
- SharedScanOptimizerCache optimizerCache, Operator<?> start) {
- Set<Operator<?>> c = optimizerCache.operatorToWorkOperators.get(start);
- if (!c.isEmpty()) {
- return c;
- }
- c = findWorkOperators(start, new HashSet<Operator<?>>());
- for (Operator<?> op : c) {
- optimizerCache.operatorToWorkOperators.putAll(op, c);
- }
- return c;
- }
-
- private static Set<Operator<?>> findWorkOperators(Operator<?> start, Set<Operator<?>> found) {
- found.add(start);
- if (start.getParentOperators() != null) {
- for (Operator<?> parent : start.getParentOperators()) {
- if (parent instanceof ReduceSinkOperator) {
- continue;
- }
- if (!found.contains(parent)) {
- findWorkOperators(parent, found);
- }
- }
- }
- if (start instanceof ReduceSinkOperator) {
- return found;
- }
- if (start.getChildOperators() != null) {
- for (Operator<?> child : start.getChildOperators()) {
- if (!found.contains(child)) {
- findWorkOperators(child, found);
- }
- }
- }
- return found;
- }
-
- private static void pushFilterToTopOfTableScan(
- SharedScanOptimizerCache optimizerCache, TableScanOperator tsOp)
- throws UDFArgumentException {
- ExprNodeGenericFuncDesc tableScanExprNode = tsOp.getConf().getFilterExpr();
- List<Operator<? extends OperatorDesc>> allChildren =
- Lists.newArrayList(tsOp.getChildOperators());
- for (Operator<? extends OperatorDesc> op : allChildren) {
- if (op instanceof FilterOperator) {
- FilterOperator filterOp = (FilterOperator) op;
- ExprNodeDesc filterExprNode = filterOp.getConf().getPredicate();
- if (tableScanExprNode.isSame(filterExprNode)) {
- // We do not need to do anything
- return;
- }
- if (tableScanExprNode.getGenericUDF() instanceof GenericUDFOPOr) {
- for (ExprNodeDesc childExprNode : tableScanExprNode.getChildren()) {
- if (childExprNode.isSame(filterExprNode)) {
- // We do not need to do anything, it is in the OR expression
- // so probably we pushed previously
- return;
- }
- }
- }
- ExprNodeGenericFuncDesc newPred = ExprNodeGenericFuncDesc.newInstance(
- new GenericUDFOPAnd(),
- Arrays.<ExprNodeDesc>asList(tableScanExprNode.clone(), filterExprNode));
- filterOp.getConf().setPredicate(newPred);
- } else {
- Operator<FilterDesc> newOp = OperatorFactory.get(tsOp.getCompilationOpContext(),
- new FilterDesc(tableScanExprNode.clone(), false),
- new RowSchema(tsOp.getSchema().getSignature()));
- tsOp.replaceChild(op, newOp);
- newOp.getParentOperators().add(tsOp);
- op.replaceParent(tsOp, newOp);
- newOp.getChildOperators().add(op);
- // Add to cache (same group as tsOp)
- optimizerCache.putIfWorkExists(newOp, tsOp);
- }
- }
- }
-
- /** Cache to accelerate optimization */
- private static class SharedScanOptimizerCache {
- // Operators that belong to each work
- final HashMultimap<Operator<?>, Operator<?>> operatorToWorkOperators =
- HashMultimap.<Operator<?>, Operator<?>>create();
- // Table scan operators to DPP sources
- final Multimap<TableScanOperator, Operator<?>> tableScanToDPPSource =
- HashMultimap.<TableScanOperator, Operator<?>>create();
-
- // Add new operator to cache work group of existing operator (if group exists)
- void putIfWorkExists(Operator<?> opToAdd, Operator<?> existingOp) {
- List<Operator<?>> c = ImmutableList.copyOf(operatorToWorkOperators.get(existingOp));
- if (!c.isEmpty()) {
- for (Operator<?> op : c) {
- operatorToWorkOperators.get(op).add(opToAdd);
- }
- operatorToWorkOperators.putAll(opToAdd, c);
- operatorToWorkOperators.put(opToAdd, opToAdd);
- }
- }
-
- // Remove operator and combine
- void removeOpAndCombineWork(Operator<?> opToRemove, Operator<?> replacementOp) {
- Set<Operator<?>> s = operatorToWorkOperators.get(opToRemove);
- s.remove(opToRemove);
- List<Operator<?>> c1 = ImmutableList.copyOf(s);
- List<Operator<?>> c2 = ImmutableList.copyOf(operatorToWorkOperators.get(replacementOp));
- if (!c1.isEmpty() && !c2.isEmpty()) {
- for (Operator<?> op1 : c1) {
- operatorToWorkOperators.remove(op1, opToRemove); // Remove operator
- operatorToWorkOperators.putAll(op1, c2); // Add ops of new collection
- }
- operatorToWorkOperators.removeAll(opToRemove); // Remove entry for operator
- for (Operator<?> op2 : c2) {
- operatorToWorkOperators.putAll(op2, c1); // Add ops to existing collection
- }
- }
- }
- }
-
-}
[4/9] hive git commit: HIVE-16867: Extend shared scan optimizer to
reuse computation from other operators (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query24.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query24.q.out b/ql/src/test/results/clientpositive/perf/query24.q.out
index 2aa0c19..bfc89e2 100644
--- a/ql/src/test/results/clientpositive/perf/query24.q.out
+++ b/ql/src/test/results/clientpositive/perf/query24.q.out
@@ -151,9 +151,9 @@ Stage-0
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_74]
PartitionCols:_col1, upper(_col2)
- Select Operator [SEL_60] (rows=40000000 width=1014)
+ Select Operator [SEL_17] (rows=40000000 width=1014)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_143] (rows=40000000 width=1014)
+ Filter Operator [FIL_137] (rows=40000000 width=1014)
predicate:(ca_zip is not null and ca_country is not null)
TableScan [TS_15] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_state","ca_zip","ca_country"]
@@ -179,9 +179,9 @@ Stage-0
<-Map 19 [SIMPLE_EDGE]
SHUFFLE [RS_68]
PartitionCols:_col0
- Select Operator [SEL_54] (rows=80000000 width=860)
+ Select Operator [SEL_14] (rows=80000000 width=860)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_141] (rows=80000000 width=860)
+ Filter Operator [FIL_136] (rows=80000000 width=860)
predicate:(c_customer_sk is not null and c_birth_country is not null)
TableScan [TS_12] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name","c_birth_country"]
@@ -193,9 +193,9 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_65]
PartitionCols:_col0
- Select Operator [SEL_51] (rows=852 width=1910)
+ Select Operator [SEL_11] (rows=852 width=1910)
Output:["_col0","_col1","_col3","_col4"]
- Filter Operator [FIL_140] (rows=852 width=1910)
+ Filter Operator [FIL_135] (rows=852 width=1910)
predicate:((s_market_id = 7) and s_store_sk is not null and s_zip is not null)
TableScan [TS_9] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_market_id","s_state","s_zip"]
@@ -207,18 +207,18 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_61]
PartitionCols:_col0, _col3
- Select Operator [SEL_45] (rows=575995635 width=88)
+ Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_138] (rows=575995635 width=88)
+ Filter Operator [FIL_132] (rows=575995635 width=88)
predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_store_sk is not null and ss_customer_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_sales_price"]
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_62]
PartitionCols:_col0, _col1
- Select Operator [SEL_48] (rows=57591150 width=77)
+ Select Operator [SEL_5] (rows=57591150 width=77)
Output:["_col0","_col1"]
- Filter Operator [FIL_139] (rows=57591150 width=77)
+ Filter Operator [FIL_133] (rows=57591150 width=77)
predicate:(sr_item_sk is not null and sr_ticket_number is not null)
TableScan [TS_3] (rows=57591150 width=77)
default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_item_sk","sr_ticket_number"]
@@ -242,11 +242,7 @@ Stage-0
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_31]
PartitionCols:_col1, upper(_col2)
- Select Operator [SEL_17] (rows=40000000 width=1014)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_137] (rows=40000000 width=1014)
- predicate:(ca_zip is not null and ca_country is not null)
- Please refer to the previous TableScan [TS_15]
+ Please refer to the previous Select Operator [SEL_17]
<-Reducer 5 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col17, _col21
@@ -255,11 +251,7 @@ Stage-0
<-Map 19 [SIMPLE_EDGE]
SHUFFLE [RS_28]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_136] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_birth_country is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col1
@@ -268,11 +260,7 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_25]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=852 width=1910)
- Output:["_col0","_col1","_col3","_col4"]
- Filter Operator [FIL_135] (rows=852 width=1910)
- predicate:((s_market_id = 7) and s_store_sk is not null and s_zip is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_24]
PartitionCols:_col2
@@ -294,17 +282,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col0, _col3
- Select Operator [SEL_2] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_132] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_ticket_number is not null and ss_store_sk is not null and ss_customer_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 16 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0, _col1
- Select Operator [SEL_5] (rows=57591150 width=77)
- Output:["_col0","_col1"]
- Filter Operator [FIL_133] (rows=57591150 width=77)
- predicate:(sr_item_sk is not null and sr_ticket_number is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query30.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query30.q.out b/ql/src/test/results/clientpositive/perf/query30.q.out
index a86f11d..29f2061 100644
--- a/ql/src/test/results/clientpositive/perf/query30.q.out
+++ b/ql/src/test/results/clientpositive/perf/query30.q.out
@@ -163,9 +163,9 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_36]
PartitionCols:_col0
- Select Operator [SEL_31] (rows=36524 width=1119)
+ Select Operator [SEL_11] (rows=36524 width=1119)
Output:["_col0"]
- Filter Operator [FIL_97] (rows=36524 width=1119)
+ Filter Operator [FIL_94] (rows=36524 width=1119)
predicate:((d_year = 2002) and d_date_sk is not null)
TableScan [TS_9] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year"]
@@ -207,9 +207,5 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=36524 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_94] (rows=36524 width=1119)
- predicate:((d_year = 2002) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query31.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query31.q.out b/ql/src/test/results/clientpositive/perf/query31.q.out
index b3a74f3..79b9e52 100644
--- a/ql/src/test/results/clientpositive/perf/query31.q.out
+++ b/ql/src/test/results/clientpositive/perf/query31.q.out
@@ -151,9 +151,9 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_53]
PartitionCols:_col0
- Select Operator [SEL_48] (rows=40000000 width=1014)
+ Select Operator [SEL_8] (rows=40000000 width=1014)
Output:["_col0","_col1"]
- Filter Operator [FIL_244] (rows=40000000 width=1014)
+ Filter Operator [FIL_238] (rows=40000000 width=1014)
predicate:(ca_address_sk is not null and ca_county is not null)
TableScan [TS_6] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_county"]
@@ -174,9 +174,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_49]
PartitionCols:_col0
- Select Operator [SEL_42] (rows=575995635 width=88)
+ Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_242] (rows=575995635 width=88)
+ Filter Operator [FIL_236] (rows=575995635 width=88)
predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_addr_sk","ss_ext_sales_price"]
@@ -200,11 +200,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_73]
PartitionCols:_col0
- Select Operator [SEL_68] (rows=40000000 width=1014)
- Output:["_col0","_col1"]
- Filter Operator [FIL_247] (rows=40000000 width=1014)
- predicate:(ca_address_sk is not null and ca_county is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 13 [SIMPLE_EDGE]
SHUFFLE [RS_72]
PartitionCols:_col1
@@ -242,11 +238,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_93]
PartitionCols:_col0
- Select Operator [SEL_88] (rows=40000000 width=1014)
- Output:["_col0","_col1"]
- Filter Operator [FIL_250] (rows=40000000 width=1014)
- predicate:(ca_address_sk is not null and ca_county is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_92]
PartitionCols:_col1
@@ -263,11 +255,7 @@ Stage-0
<-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_89]
PartitionCols:_col0
- Select Operator [SEL_82] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_248] (rows=144002668 width=135)
- predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null)
- Please refer to the previous TableScan [TS_60]
+ Please refer to the previous Select Operator [SEL_62]
<-Reducer 22 [SIMPLE_EDGE]
SHUFFLE [RS_122]
PartitionCols:_col0
@@ -283,11 +271,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_113]
PartitionCols:_col0
- Select Operator [SEL_108] (rows=40000000 width=1014)
- Output:["_col0","_col1"]
- Filter Operator [FIL_253] (rows=40000000 width=1014)
- predicate:(ca_address_sk is not null and ca_county is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_112]
PartitionCols:_col1
@@ -304,11 +288,7 @@ Stage-0
<-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_109]
PartitionCols:_col0
- Select Operator [SEL_102] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_251] (rows=144002668 width=135)
- predicate:(ws_sold_date_sk is not null and ws_bill_addr_sk is not null)
- Please refer to the previous TableScan [TS_60]
+ Please refer to the previous Select Operator [SEL_62]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_125]
PartitionCols:_col0
@@ -324,11 +304,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=40000000 width=1014)
- Output:["_col0","_col1"]
- Filter Operator [FIL_238] (rows=40000000 width=1014)
- predicate:(ca_address_sk is not null and ca_county is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col1
@@ -345,11 +321,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_236] (rows=575995635 width=88)
- predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 8 [SIMPLE_EDGE]
SHUFFLE [RS_126]
PartitionCols:_col0
@@ -365,11 +337,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_33]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=40000000 width=1014)
- Output:["_col0","_col1"]
- Filter Operator [FIL_241] (rows=40000000 width=1014)
- predicate:(ca_address_sk is not null and ca_county is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_32]
PartitionCols:_col1
@@ -386,9 +354,5 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
- Select Operator [SEL_22] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_239] (rows=575995635 width=88)
- predicate:(ss_sold_date_sk is not null and ss_addr_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query32.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query32.q.out b/ql/src/test/results/clientpositive/perf/query32.q.out
index 66b8485..10ad060 100644
--- a/ql/src/test/results/clientpositive/perf/query32.q.out
+++ b/ql/src/test/results/clientpositive/perf/query32.q.out
@@ -136,17 +136,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_55] (rows=287989836 width=135)
- predicate:(cs_item_sk is not null and cs_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_56] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-03-18 00:00:00.0 AND 1998-06-16 01:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query33.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query33.q.out b/ql/src/test/results/clientpositive/perf/query33.q.out
index 2058830..aab3a19 100644
--- a/ql/src/test/results/clientpositive/perf/query33.q.out
+++ b/ql/src/test/results/clientpositive/perf/query33.q.out
@@ -209,9 +209,9 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_60]
PartitionCols:_col0
- Select Operator [SEL_55] (rows=20000000 width=1014)
+ Select Operator [SEL_18] (rows=20000000 width=1014)
Output:["_col0"]
- Filter Operator [FIL_167] (rows=20000000 width=1014)
+ Filter Operator [FIL_162] (rows=20000000 width=1014)
predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
TableScan [TS_16] (rows=40000000 width=1014)
default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_gmt_offset"]
@@ -223,9 +223,9 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_57]
PartitionCols:_col0
- Select Operator [SEL_52] (rows=18262 width=1119)
+ Select Operator [SEL_15] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_166] (rows=18262 width=1119)
+ Filter Operator [FIL_161] (rows=18262 width=1119)
predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null)
TableScan [TS_13] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -246,9 +246,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col1
- Select Operator [SEL_39] (rows=462000 width=1436)
+ Select Operator [SEL_2] (rows=462000 width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_163] (rows=462000 width=1436)
+ Filter Operator [FIL_158] (rows=462000 width=1436)
predicate:(i_manufact_id is not null and i_item_sk is not null)
TableScan [TS_0] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_manufact_id"]
@@ -260,11 +260,11 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_44]
PartitionCols:_col0
- Group By Operator [GBY_43] (rows=231000 width=1436)
+ Group By Operator [GBY_6] (rows=231000 width=1436)
Output:["_col0"],keys:i_manufact_id
- Select Operator [SEL_42] (rows=231000 width=1436)
+ Select Operator [SEL_5] (rows=231000 width=1436)
Output:["i_manufact_id"]
- Filter Operator [FIL_164] (rows=231000 width=1436)
+ Filter Operator [FIL_159] (rows=231000 width=1436)
predicate:((i_category) IN ('Books') and i_manufact_id is not null)
TableScan [TS_3] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_category","i_manufact_id"]
@@ -290,11 +290,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_101]
PartitionCols:_col1
- Select Operator [SEL_77] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_168] (rows=462000 width=1436)
- predicate:(i_manufact_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_102]
PartitionCols:_col0
@@ -303,13 +299,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col0
- Group By Operator [GBY_81] (rows=231000 width=1436)
- Output:["_col0"],keys:i_manufact_id
- Select Operator [SEL_80] (rows=231000 width=1436)
- Output:["i_manufact_id"]
- Filter Operator [FIL_169] (rows=231000 width=1436)
- predicate:((i_category) IN ('Books') and i_manufact_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Group By Operator [GBY_6]
<-Reducer 25 [SIMPLE_EDGE]
SHUFFLE [RS_105]
PartitionCols:_col3
@@ -320,11 +310,7 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_98]
PartitionCols:_col0
- Select Operator [SEL_93] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_172] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Reducer 24 [SIMPLE_EDGE]
SHUFFLE [RS_97]
PartitionCols:_col2
@@ -333,11 +319,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_95]
PartitionCols:_col0
- Select Operator [SEL_90] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_171] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 28 [SIMPLE_EDGE]
SHUFFLE [RS_94]
PartitionCols:_col0
@@ -369,11 +351,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_26]
PartitionCols:_col1
- Select Operator [SEL_2] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_158] (rows=462000 width=1436)
- predicate:(i_manufact_id is not null and i_item_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
@@ -382,13 +360,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_7]
PartitionCols:_col0
- Group By Operator [GBY_6] (rows=231000 width=1436)
- Output:["_col0"],keys:i_manufact_id
- Select Operator [SEL_5] (rows=231000 width=1436)
- Output:["i_manufact_id"]
- Filter Operator [FIL_159] (rows=231000 width=1436)
- predicate:((i_category) IN ('Books') and i_manufact_id is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Group By Operator [GBY_6]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
@@ -399,11 +371,7 @@ Stage-0
<-Map 26 [SIMPLE_EDGE]
SHUFFLE [RS_23]
PartitionCols:_col0
- Select Operator [SEL_18] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_162] (rows=20000000 width=1014)
- predicate:((ca_gmt_offset = -6) and ca_address_sk is not null)
- Please refer to the previous TableScan [TS_16]
+ Please refer to the previous Select Operator [SEL_18]
<-Reducer 19 [SIMPLE_EDGE]
SHUFFLE [RS_22]
PartitionCols:_col2
@@ -412,11 +380,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_20]
PartitionCols:_col0
- Select Operator [SEL_15] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_161] (rows=18262 width=1119)
- predicate:((d_year = 1999) and (d_moy = 3) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_13]
+ Please refer to the previous Select Operator [SEL_15]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query35.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query35.q.out b/ql/src/test/results/clientpositive/perf/query35.q.out
index b286a07..2815257 100644
--- a/ql/src/test/results/clientpositive/perf/query35.q.out
+++ b/ql/src/test/results/clientpositive/perf/query35.q.out
@@ -170,9 +170,9 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_55]
PartitionCols:_col0
- Select Operator [SEL_53] (rows=12174 width=1119)
+ Select Operator [SEL_14] (rows=12174 width=1119)
Output:["_col0"]
- Filter Operator [FIL_106] (rows=12174 width=1119)
+ Filter Operator [FIL_102] (rows=12174 width=1119)
predicate:((d_year = 1999) and (d_qoy < 4) and d_date_sk is not null)
TableScan [TS_12] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_qoy"]
@@ -209,11 +209,7 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_14] (rows=12174 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_102] (rows=12174 width=1119)
- predicate:((d_year = 1999) and (d_qoy < 4) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
@@ -238,11 +234,7 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col0
- Select Operator [SEL_28] (rows=12174 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_104] (rows=12174 width=1119)
- predicate:((d_year = 1999) and (d_qoy < 4) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_12]
+ Please refer to the previous Select Operator [SEL_14]
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_29]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query38.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query38.q.out b/ql/src/test/results/clientpositive/perf/query38.q.out
index ae9ada5..1fae4e2 100644
--- a/ql/src/test/results/clientpositive/perf/query38.q.out
+++ b/ql/src/test/results/clientpositive/perf/query38.q.out
@@ -98,9 +98,9 @@ Stage-0
<-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_39]
PartitionCols:_col0
- Select Operator [SEL_34] (rows=80000000 width=860)
+ Select Operator [SEL_8] (rows=80000000 width=860)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_113] (rows=80000000 width=860)
+ Filter Operator [FIL_110] (rows=80000000 width=860)
predicate:c_customer_sk is not null
TableScan [TS_6] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_first_name","c_last_name"]
@@ -112,9 +112,9 @@ Stage-0
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_36]
PartitionCols:_col0
- Select Operator [SEL_31] (rows=8116 width=1119)
+ Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0","_col1"]
- Filter Operator [FIL_112] (rows=8116 width=1119)
+ Filter Operator [FIL_109] (rows=8116 width=1119)
predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date","d_month_seq"]
@@ -146,11 +146,7 @@ Stage-0
<-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_66]
PartitionCols:_col0
- Select Operator [SEL_61] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_116] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 12 [SIMPLE_EDGE]
SHUFFLE [RS_65]
PartitionCols:_col1
@@ -159,11 +155,7 @@ Stage-0
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_63]
PartitionCols:_col0
- Select Operator [SEL_58] (rows=8116 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_115] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 17 [SIMPLE_EDGE]
SHUFFLE [RS_62]
PartitionCols:_col0
@@ -192,11 +184,7 @@ Stage-0
<-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_110] (rows=80000000 width=860)
- predicate:c_customer_sk is not null
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col1
@@ -205,11 +193,7 @@ Stage-0
<-Map 8 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_109] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query39.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query39.q.out b/ql/src/test/results/clientpositive/perf/query39.q.out
index cf139f2..7607ba0 100644
--- a/ql/src/test/results/clientpositive/perf/query39.q.out
+++ b/ql/src/test/results/clientpositive/perf/query39.q.out
@@ -99,9 +99,9 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_47]
PartitionCols:_col0
- Select Operator [SEL_39] (rows=27 width=1029)
+ Select Operator [SEL_11] (rows=27 width=1029)
Output:["_col0","_col1"]
- Filter Operator [FIL_97] (rows=27 width=1029)
+ Filter Operator [FIL_93] (rows=27 width=1029)
predicate:w_warehouse_sk is not null
TableScan [TS_9] (rows=27 width=1029)
default@warehouse,warehouse,Tbl:COMPLETE,Col:NONE,Output:["w_warehouse_sk","w_warehouse_name"]
@@ -113,9 +113,9 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_44]
PartitionCols:_col0
- Select Operator [SEL_36] (rows=462000 width=1436)
+ Select Operator [SEL_8] (rows=462000 width=1436)
Output:["_col0"]
- Filter Operator [FIL_96] (rows=462000 width=1436)
+ Filter Operator [FIL_92] (rows=462000 width=1436)
predicate:i_item_sk is not null
TableScan [TS_6] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk"]
@@ -127,9 +127,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_40]
PartitionCols:_col0
- Select Operator [SEL_30] (rows=37584000 width=15)
+ Select Operator [SEL_2] (rows=37584000 width=15)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_94] (rows=37584000 width=15)
+ Filter Operator [FIL_90] (rows=37584000 width=15)
predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null)
TableScan [TS_0] (rows=37584000 width=15)
default@inventory,inventory,Tbl:COMPLETE,Col:NONE,Output:["inv_date_sk","inv_item_sk","inv_warehouse_sk","inv_quantity_on_hand"]
@@ -163,11 +163,7 @@ Stage-0
<-Map 14 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=27 width=1029)
- Output:["_col0","_col1"]
- Filter Operator [FIL_93] (rows=27 width=1029)
- predicate:w_warehouse_sk is not null
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col2
@@ -176,11 +172,7 @@ Stage-0
<-Map 13 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=462000 width=1436)
- Output:["_col0"]
- Filter Operator [FIL_92] (rows=462000 width=1436)
- predicate:i_item_sk is not null
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col1
@@ -189,11 +181,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=37584000 width=15)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_90] (rows=37584000 width=15)
- predicate:(inv_item_sk is not null and inv_warehouse_sk is not null and inv_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query4.q.out b/ql/src/test/results/clientpositive/perf/query4.q.out
index 5f41b4a..b03a91e 100644
--- a/ql/src/test/results/clientpositive/perf/query4.q.out
+++ b/ql/src/test/results/clientpositive/perf/query4.q.out
@@ -277,9 +277,9 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_34]
PartitionCols:_col0
- Select Operator [SEL_29] (rows=80000000 width=860)
+ Select Operator [SEL_8] (rows=80000000 width=860)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_268] (rows=80000000 width=860)
+ Filter Operator [FIL_265] (rows=80000000 width=860)
predicate:(c_customer_sk is not null and c_customer_id is not null)
TableScan [TS_6] (rows=80000000 width=860)
default@customer,customer,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_customer_id","c_first_name","c_last_name","c_preferred_cust_flag","c_birth_country","c_login","c_email_address"]
@@ -327,11 +327,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_56]
PartitionCols:_col0
- Select Operator [SEL_51] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_271] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_55]
PartitionCols:_col1
@@ -373,11 +369,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_100]
PartitionCols:_col0
- Select Operator [SEL_95] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_277] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 17 [SIMPLE_EDGE]
SHUFFLE [RS_99]
PartitionCols:_col1
@@ -394,11 +386,7 @@ Stage-0
<-Map 25 [SIMPLE_EDGE]
SHUFFLE [RS_96]
PartitionCols:_col0
- Select Operator [SEL_89] (rows=287989836 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_275] (rows=287989836 width=135)
- predicate:(cs_bill_customer_sk is not null and cs_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_43]
+ Please refer to the previous Select Operator [SEL_45]
<-Reducer 22 [SIMPLE_EDGE]
SHUFFLE [RS_134]
PartitionCols:_col0
@@ -418,11 +406,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_121]
PartitionCols:_col0
- Select Operator [SEL_116] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_280] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 20 [SIMPLE_EDGE]
SHUFFLE [RS_120]
PartitionCols:_col1
@@ -439,11 +423,7 @@ Stage-0
<-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_117]
PartitionCols:_col0
- Select Operator [SEL_110] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_278] (rows=575995635 width=88)
- predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_21]
+ Please refer to the previous Select Operator [SEL_23]
<-Reducer 4 [SIMPLE_EDGE]
SHUFFLE [RS_129]
PartitionCols:_col0
@@ -463,11 +443,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_265] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col1
@@ -511,11 +487,7 @@ Stage-0
<-Map 23 [SIMPLE_EDGE]
SHUFFLE [RS_78]
PartitionCols:_col0
- Select Operator [SEL_73] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
- Filter Operator [FIL_274] (rows=80000000 width=860)
- predicate:(c_customer_sk is not null and c_customer_id is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 7 [SIMPLE_EDGE]
SHUFFLE [RS_77]
PartitionCols:_col1
@@ -532,9 +504,5 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_74]
PartitionCols:_col0
- Select Operator [SEL_67] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_272] (rows=144002668 width=135)
- predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query44.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query44.q.out b/ql/src/test/results/clientpositive/perf/query44.q.out
index 5665480..07187b1 100644
--- a/ql/src/test/results/clientpositive/perf/query44.q.out
+++ b/ql/src/test/results/clientpositive/perf/query44.q.out
@@ -200,11 +200,7 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_90]
PartitionCols:_col0
- Select Operator [SEL_47] (rows=462000 width=1436)
- Output:["_col0","_col1"]
- Filter Operator [FIL_126] (rows=462000 width=1436)
- predicate:i_item_sk is not null
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Reducer 14 [SIMPLE_EDGE]
SHUFFLE [RS_91]
PartitionCols:_col0
@@ -240,12 +236,7 @@ Stage-0
<-Map 6 [SIMPLE_EDGE]
SHUFFLE [RS_53]
PartitionCols:_col0
- Group By Operator [GBY_52] (rows=143998908 width=88)
- Output:["_col0"],keys:410
- Select Operator [SEL_50] (rows=143998908 width=88)
- Filter Operator [FIL_128] (rows=143998908 width=88)
- predicate:((ss_store_sk = 410) and ss_hdemo_sk is null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Group By Operator [GBY_7]
<-Reducer 17 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_79]
Select Operator [SEL_70] (rows=71999454 width=88)
@@ -255,13 +246,7 @@ Stage-0
<-Map 15 [SIMPLE_EDGE]
SHUFFLE [RS_68]
PartitionCols:_col0
- Group By Operator [GBY_67] (rows=143998908 width=88)
- Output:["_col0","_col1"],aggregations:["avg(_col1)"],keys:410
- Select Operator [SEL_65] (rows=143998908 width=88)
- Output:["_col1"]
- Filter Operator [FIL_129] (rows=143998908 width=88)
- predicate:((ss_store_sk = 410) and ss_hdemo_sk is null)
- Please refer to the previous TableScan [TS_18]
+ Please refer to the previous Group By Operator [GBY_22]
<-Reducer 20 [CUSTOM_SIMPLE_EDGE]
PARTITION_ONLY_SHUFFLE [RS_80]
Group By Operator [GBY_76] (rows=143998908 width=88)
@@ -269,11 +254,5 @@ Stage-0
<-Map 18 [SIMPLE_EDGE]
SHUFFLE [RS_75]
PartitionCols:_col0
- Group By Operator [GBY_74] (rows=287997817 width=88)
- Output:["_col0","_col1"],aggregations:["avg(ss_net_profit)"],keys:ss_item_sk
- Select Operator [SEL_73] (rows=287997817 width=88)
- Output:["ss_item_sk","ss_net_profit"]
- Filter Operator [FIL_130] (rows=287997817 width=88)
- predicate:(ss_store_sk = 410)
- Please refer to the previous TableScan [TS_26]
+ Please refer to the previous Group By Operator [GBY_29]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query46.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query46.q.out b/ql/src/test/results/clientpositive/perf/query46.q.out
index 6806703..380944a 100644
--- a/ql/src/test/results/clientpositive/perf/query46.q.out
+++ b/ql/src/test/results/clientpositive/perf/query46.q.out
@@ -136,11 +136,7 @@ Stage-0
<-Map 5 [SIMPLE_EDGE]
SHUFFLE [RS_31]
PartitionCols:_col0
- Select Operator [SEL_20] (rows=40000000 width=1014)
- Output:["_col0","_col1"]
- Filter Operator [FIL_80] (rows=40000000 width=1014)
- predicate:ca_address_sk is not null
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Reducer 11 [SIMPLE_EDGE]
SHUFFLE [RS_30]
PartitionCols:_col3
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query47.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query47.q.out b/ql/src/test/results/clientpositive/perf/query47.q.out
index d5e1922..efbd89c 100644
--- a/ql/src/test/results/clientpositive/perf/query47.q.out
+++ b/ql/src/test/results/clientpositive/perf/query47.q.out
@@ -177,9 +177,9 @@ Stage-0
<-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_52]
PartitionCols:_col0
- Select Operator [SEL_44] (rows=1704 width=1910)
+ Select Operator [SEL_11] (rows=1704 width=1910)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_174] (rows=1704 width=1910)
+ Filter Operator [FIL_168] (rows=1704 width=1910)
predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null)
TableScan [TS_9] (rows=1704 width=1910)
default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_company_name"]
@@ -191,9 +191,9 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_49]
PartitionCols:_col0
- Select Operator [SEL_41] (rows=462000 width=1436)
+ Select Operator [SEL_8] (rows=462000 width=1436)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_173] (rows=462000 width=1436)
+ Filter Operator [FIL_167] (rows=462000 width=1436)
predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
TableScan [TS_6] (rows=462000 width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_brand","i_category"]
@@ -205,18 +205,18 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_45]
PartitionCols:_col0
- Select Operator [SEL_35] (rows=575995635 width=88)
+ Select Operator [SEL_2] (rows=575995635 width=88)
Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_171] (rows=575995635 width=88)
+ Filter Operator [FIL_165] (rows=575995635 width=88)
predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
TableScan [TS_0] (rows=575995635 width=88)
default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_store_sk","ss_sales_price"]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_46]
PartitionCols:_col0
- Select Operator [SEL_38] (rows=73048 width=1119)
+ Select Operator [SEL_5] (rows=73048 width=1119)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_172] (rows=73048 width=1119)
+ Filter Operator [FIL_166] (rows=73048 width=1119)
predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -248,11 +248,7 @@ Stage-0
<-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_89]
PartitionCols:_col0
- Select Operator [SEL_81] (rows=1704 width=1910)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_179] (rows=1704 width=1910)
- predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 16 [SIMPLE_EDGE]
SHUFFLE [RS_88]
PartitionCols:_col2
@@ -261,11 +257,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_86]
PartitionCols:_col0
- Select Operator [SEL_78] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_178] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 15 [SIMPLE_EDGE]
SHUFFLE [RS_85]
PartitionCols:_col1
@@ -274,19 +266,11 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_82]
PartitionCols:_col0
- Select Operator [SEL_72] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_176] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_83]
PartitionCols:_col0
- Select Operator [SEL_75] (rows=73048 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_177] (rows=73048 width=1119)
- predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Reducer 6 [SIMPLE_EDGE]
SHUFFLE [RS_103]
PartitionCols:_col0, _col1, _col2, _col3, (_col7 + 1)
@@ -315,11 +299,7 @@ Stage-0
<-Map 22 [SIMPLE_EDGE]
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=1704 width=1910)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_168] (rows=1704 width=1910)
- predicate:(s_store_sk is not null and s_store_name is not null and s_company_name is not null)
- Please refer to the previous TableScan [TS_9]
+ Please refer to the previous Select Operator [SEL_11]
<-Reducer 3 [SIMPLE_EDGE]
SHUFFLE [RS_18]
PartitionCols:_col2
@@ -328,11 +308,7 @@ Stage-0
<-Map 21 [SIMPLE_EDGE]
SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=462000 width=1436)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_167] (rows=462000 width=1436)
- predicate:(i_item_sk is not null and i_category is not null and i_brand is not null)
- Please refer to the previous TableScan [TS_6]
+ Please refer to the previous Select Operator [SEL_8]
<-Reducer 2 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col1
@@ -341,17 +317,9 @@ Stage-0
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=575995635 width=88)
- Output:["_col0","_col1","_col2","_col3"]
- Filter Operator [FIL_165] (rows=575995635 width=88)
- predicate:(ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null)
- Please refer to the previous TableScan [TS_0]
+ Please refer to the previous Select Operator [SEL_2]
<-Map 20 [SIMPLE_EDGE]
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=73048 width=1119)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_166] (rows=73048 width=1119)
- predicate:(((d_year = 2000) or ((d_year = 1999) and (d_moy = 12)) or ((d_year = 2001) and (d_moy = 1))) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query49.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query49.q.out b/ql/src/test/results/clientpositive/perf/query49.q.out
index 8b8ad8b..f8c0649 100644
--- a/ql/src/test/results/clientpositive/perf/query49.q.out
+++ b/ql/src/test/results/clientpositive/perf/query49.q.out
@@ -342,9 +342,9 @@ Stage-0
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_81]
PartitionCols:_col0
- Select Operator [SEL_76] (rows=18262 width=1119)
+ Select Operator [SEL_5] (rows=18262 width=1119)
Output:["_col0"]
- Filter Operator [FIL_139] (rows=18262 width=1119)
+ Filter Operator [FIL_131] (rows=18262 width=1119)
predicate:((d_year = 2000) and (d_moy = 12) and d_date_sk is not null)
TableScan [TS_3] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
@@ -420,11 +420,7 @@ Stage-0
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_42]
PartitionCols:_col0
- Select Operator [SEL_37] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_135] (rows=18262 width=1119)
- predicate:((d_year = 2000) and (d_moy = 12) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 24 [SIMPLE_EDGE]
SHUFFLE [RS_41]
PartitionCols:_col0
@@ -487,11 +483,7 @@ Stage-0
<-Map 12 [SIMPLE_EDGE]
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=18262 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_131] (rows=18262 width=1119)
- predicate:((d_year = 2000) and (d_moy = 12) and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_9]
PartitionCols:_col0
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query5.q.out b/ql/src/test/results/clientpositive/perf/query5.q.out
index f3f122c..ea072d9 100644
--- a/ql/src/test/results/clientpositive/perf/query5.q.out
+++ b/ql/src/test/results/clientpositive/perf/query5.q.out
@@ -323,9 +323,9 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_40]
PartitionCols:_col0
- Select Operator [SEL_35] (rows=8116 width=1119)
+ Select Operator [SEL_10] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_124] (rows=8116 width=1119)
+ Filter Operator [FIL_120] (rows=8116 width=1119)
predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-08-18 00:00:00.0 and d_date_sk is not null)
TableScan [TS_8] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
@@ -381,11 +381,7 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_73]
PartitionCols:_col0
- Select Operator [SEL_68] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_129] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-08-18 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_8]
+ Please refer to the previous Select Operator [SEL_10]
<-Union 23 [SIMPLE_EDGE]
<-Map 22 [CONTAINS]
Reduce Output Operator [RS_72]
@@ -454,11 +450,7 @@ Stage-0
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_15]
PartitionCols:_col0
- Select Operator [SEL_10] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_120] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1998-08-04 00:00:00.0 AND 1998-08-18 00:00:00.0 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_8]
+ Please refer to the previous Select Operator [SEL_10]
<-Union 2 [SIMPLE_EDGE]
<-Map 1 [CONTAINS]
Reduce Output Operator [RS_14]
http://git-wip-us.apache.org/repos/asf/hive/blob/7819cd36/ql/src/test/results/clientpositive/perf/query51.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/query51.q.out b/ql/src/test/results/clientpositive/perf/query51.q.out
index 0ce3e9f..5f549c5 100644
--- a/ql/src/test/results/clientpositive/perf/query51.q.out
+++ b/ql/src/test/results/clientpositive/perf/query51.q.out
@@ -175,11 +175,7 @@ Stage-0
<-Map 7 [SIMPLE_EDGE]
SHUFFLE [RS_27]
PartitionCols:_col0
- Select Operator [SEL_25] (rows=8116 width=1119)
- Output:["_col0","_col1"]
- Filter Operator [FIL_62] (rows=8116 width=1119)
- predicate:(d_month_seq BETWEEN 1212 AND 1223 and d_date_sk is not null)
- Please refer to the previous TableScan [TS_3]
+ Please refer to the previous Select Operator [SEL_5]
<-Map 10 [SIMPLE_EDGE]
SHUFFLE [RS_26]
PartitionCols:_col0