You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2017/11/07 06:27:27 UTC
[08/17] hive git commit: HIVE-17767 Rewrite correlated EXISTS/IN
subqueries into LEFT SEMI JOIN (Vineet Garg, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/aee0eaa0/ql/src/test/results/clientpositive/perf/tez/query69.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query69.q.out b/ql/src/test/results/clientpositive/perf/tez/query69.q.out
index 591f3fc..08ea5a1 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query69.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query69.q.out
@@ -93,16 +93,15 @@ POSTHOOK: type: QUERY
Plan optimized by CBO.
Vertex dependency in root stage
-Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 13 (SIMPLE_EDGE)
-Reducer 12 <- Reducer 11 (SIMPLE_EDGE)
-Reducer 14 <- Map 13 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE)
-Reducer 15 <- Reducer 14 (SIMPLE_EDGE)
-Reducer 16 <- Map 13 (SIMPLE_EDGE), Map 19 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
+Reducer 11 <- Map 10 (SIMPLE_EDGE), Map 12 (SIMPLE_EDGE)
+Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 17 (SIMPLE_EDGE)
+Reducer 14 <- Reducer 13 (SIMPLE_EDGE)
+Reducer 15 <- Map 12 (SIMPLE_EDGE), Map 18 (SIMPLE_EDGE)
+Reducer 16 <- Reducer 15 (SIMPLE_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE)
Reducer 3 <- Map 9 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Reducer 12 (ONE_TO_ONE_EDGE), Reducer 15 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 17 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 4 <- Reducer 11 (SIMPLE_EDGE), Reducer 14 (ONE_TO_ONE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Reducer 16 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE)
Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
@@ -111,158 +110,153 @@ Stage-0
limit:100
Stage-1
Reducer 7
- File Output Operator [FS_76]
- Limit [LIM_75] (rows=100 width=88)
+ File Output Operator [FS_73]
+ Limit [LIM_72] (rows=100 width=88)
Number of rows:100
- Select Operator [SEL_74] (rows=95831279 width=88)
+ Select Operator [SEL_71] (rows=191662559 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"]
<-Reducer 6 [SIMPLE_EDGE]
- SHUFFLE [RS_73]
- Select Operator [SEL_72] (rows=95831279 width=88)
+ SHUFFLE [RS_70]
+ Select Operator [SEL_69] (rows=191662559 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col6"]
- Group By Operator [GBY_71] (rows=95831279 width=88)
+ Group By Operator [GBY_68] (rows=191662559 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count(VALUE._col0)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4
<-Reducer 5 [SIMPLE_EDGE]
- SHUFFLE [RS_70]
+ SHUFFLE [RS_67]
PartitionCols:_col0, _col1, _col2, _col3, _col4
- Group By Operator [GBY_69] (rows=191662559 width=88)
+ Group By Operator [GBY_66] (rows=383325119 width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["count()"],keys:_col6, _col7, _col8, _col9, _col10
- Select Operator [SEL_68] (rows=191662559 width=88)
+ Select Operator [SEL_65] (rows=383325119 width=88)
Output:["_col6","_col7","_col8","_col9","_col10"]
- Filter Operator [FIL_67] (rows=191662559 width=88)
- predicate:_col15 is null
- Merge Join Operator [MERGEJOIN_114] (rows=383325119 width=88)
- Conds:RS_64._col0=RS_65._col0(Left Outer),Output:["_col6","_col7","_col8","_col9","_col10","_col15"]
- <-Reducer 17 [ONE_TO_ONE_EDGE]
- FORWARD [RS_65]
+ Filter Operator [FIL_64] (rows=383325119 width=88)
+ predicate:_col14 is null
+ Merge Join Operator [MERGEJOIN_111] (rows=766650239 width=88)
+ Conds:RS_61._col0=RS_62._col0(Left Outer),Output:["_col6","_col7","_col8","_col9","_col10","_col14"]
+ <-Reducer 16 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_62]
PartitionCols:_col0
- Select Operator [SEL_63] (rows=158394413 width=135)
+ Select Operator [SEL_60] (rows=158394413 width=135)
Output:["_col0","_col1"]
- Group By Operator [GBY_62] (rows=158394413 width=135)
+ Group By Operator [GBY_59] (rows=158394413 width=135)
Output:["_col0"],keys:KEY._col0
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_61]
+ <-Reducer 15 [SIMPLE_EDGE]
+ SHUFFLE [RS_58]
PartitionCols:_col0
- Group By Operator [GBY_60] (rows=316788826 width=135)
+ Group By Operator [GBY_57] (rows=316788826 width=135)
Output:["_col0"],keys:_col1
- Merge Join Operator [MERGEJOIN_112] (rows=316788826 width=135)
- Conds:RS_56._col0=RS_57._col0(Inner),Output:["_col1"]
- <-Map 13 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
+ Merge Join Operator [MERGEJOIN_109] (rows=316788826 width=135)
+ Conds:RS_53._col0=RS_54._col0(Inner),Output:["_col1"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_54]
PartitionCols:_col0
Select Operator [SEL_14] (rows=4058 width=1119)
Output:["_col0"]
- Filter Operator [FIL_103] (rows=4058 width=1119)
+ Filter Operator [FIL_100] (rows=4058 width=1119)
predicate:((d_year = 1999) and d_date_sk is not null and d_moy BETWEEN 1 AND 3)
TableScan [TS_12] (rows=73049 width=1119)
default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"]
- <-Map 19 [SIMPLE_EDGE]
- SHUFFLE [RS_56]
+ <-Map 18 [SIMPLE_EDGE]
+ SHUFFLE [RS_53]
PartitionCols:_col0
- Select Operator [SEL_52] (rows=287989836 width=135)
+ Select Operator [SEL_49] (rows=287989836 width=135)
Output:["_col0","_col1"]
- Filter Operator [FIL_106] (rows=287989836 width=135)
+ Filter Operator [FIL_103] (rows=287989836 width=135)
predicate:(cs_ship_customer_sk is not null and cs_sold_date_sk is not null)
- TableScan [TS_50] (rows=287989836 width=135)
+ TableScan [TS_47] (rows=287989836 width=135)
default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_ship_customer_sk"]
<-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_64]
+ SHUFFLE [RS_61]
PartitionCols:_col0
- Select Operator [SEL_49] (rows=348477374 width=88)
+ Select Operator [SEL_46] (rows=696954748 width=88)
Output:["_col0","_col6","_col7","_col8","_col9","_col10"]
- Filter Operator [FIL_48] (rows=348477374 width=88)
- predicate:_col13 is null
- Select Operator [SEL_47] (rows=696954748 width=88)
- Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col13"]
- Merge Join Operator [MERGEJOIN_113] (rows=696954748 width=88)
- Conds:RS_43._col0=RS_44._col0(Left Outer),RS_43._col0=RS_45._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col12"]
- <-Reducer 12 [ONE_TO_ONE_EDGE]
- FORWARD [RS_44]
- PartitionCols:_col0
- Select Operator [SEL_22] (rows=79201469 width=135)
- Output:["_col0","_col1"]
- Group By Operator [GBY_21] (rows=79201469 width=135)
- Output:["_col0"],keys:KEY._col0
- <-Reducer 11 [SIMPLE_EDGE]
- SHUFFLE [RS_20]
+ Filter Operator [FIL_45] (rows=696954748 width=88)
+ predicate:_col12 is null
+ Merge Join Operator [MERGEJOIN_110] (rows=1393909496 width=88)
+ Conds:RS_41._col0=RS_42._col0(Left Semi),RS_41._col0=RS_43._col0(Left Outer),Output:["_col0","_col6","_col7","_col8","_col9","_col10","_col12"]
+ <-Reducer 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_42]
+ PartitionCols:_col0
+ Group By Operator [GBY_40] (rows=633595212 width=88)
+ Output:["_col0"],keys:_col0
+ Select Operator [SEL_18] (rows=633595212 width=88)
+ Output:["_col0"]
+ Merge Join Operator [MERGEJOIN_107] (rows=633595212 width=88)
+ Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
PartitionCols:_col0
- Group By Operator [GBY_19] (rows=158402938 width=135)
- Output:["_col0"],keys:_col1
- Merge Join Operator [MERGEJOIN_110] (rows=158402938 width=135)
- Conds:RS_15._col0=RS_16._col0(Inner),Output:["_col1"]
- <-Map 13 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
- PartitionCols:_col0
- Please refer to the previous Select Operator [SEL_14]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_15]
- PartitionCols:_col0
- Select Operator [SEL_11] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- Filter Operator [FIL_102] (rows=144002668 width=135)
- predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
- TableScan [TS_9] (rows=144002668 width=135)
- default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"]
- <-Reducer 15 [ONE_TO_ONE_EDGE]
- FORWARD [RS_45]
- PartitionCols:_col0
- Group By Operator [GBY_35] (rows=316797606 width=88)
+ Please refer to the previous Select Operator [SEL_14]
+ <-Map 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_15]
+ PartitionCols:_col0
+ Select Operator [SEL_11] (rows=575995635 width=88)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_99] (rows=575995635 width=88)
+ predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
+ TableScan [TS_9] (rows=575995635 width=88)
+ default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"]
+ <-Reducer 14 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_43]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=79201469 width=135)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_31] (rows=79201469 width=135)
Output:["_col0"],keys:KEY._col0
- <-Reducer 14 [SIMPLE_EDGE]
- SHUFFLE [RS_34]
+ <-Reducer 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_30]
PartitionCols:_col0
- Group By Operator [GBY_33] (rows=633595212 width=88)
+ Group By Operator [GBY_29] (rows=158402938 width=135)
Output:["_col0"],keys:_col1
- Merge Join Operator [MERGEJOIN_111] (rows=633595212 width=88)
- Conds:RS_29._col0=RS_30._col0(Inner),Output:["_col1"]
- <-Map 13 [SIMPLE_EDGE]
- SHUFFLE [RS_30]
+ Merge Join Operator [MERGEJOIN_108] (rows=158402938 width=135)
+ Conds:RS_25._col0=RS_26._col0(Inner),Output:["_col1"]
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_26]
PartitionCols:_col0
Please refer to the previous Select Operator [SEL_14]
- <-Map 18 [SIMPLE_EDGE]
- SHUFFLE [RS_29]
+ <-Map 17 [SIMPLE_EDGE]
+ SHUFFLE [RS_25]
PartitionCols:_col0
- Select Operator [SEL_25] (rows=575995635 width=88)
+ Select Operator [SEL_21] (rows=144002668 width=135)
Output:["_col0","_col1"]
- Filter Operator [FIL_104] (rows=575995635 width=88)
- predicate:(ss_customer_sk is not null and ss_sold_date_sk is not null)
- TableScan [TS_23] (rows=575995635 width=88)
- default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_customer_sk"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_43]
- PartitionCols:_col0
- Merge Join Operator [MERGEJOIN_109] (rows=96800003 width=860)
- Conds:RS_40._col1=RS_41._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10"]
- <-Map 9 [SIMPLE_EDGE]
- SHUFFLE [RS_41]
- PartitionCols:_col0
- Select Operator [SEL_8] (rows=1861800 width=385)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- Filter Operator [FIL_101] (rows=1861800 width=385)
- predicate:cd_demo_sk is not null
- TableScan [TS_6] (rows=1861800 width=385)
- default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status","cd_purchase_estimate","cd_credit_rating"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_40]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_108] (rows=88000001 width=860)
- Conds:RS_37._col2=RS_38._col0(Inner),Output:["_col0","_col1"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_37]
- PartitionCols:_col2
- Select Operator [SEL_2] (rows=80000000 width=860)
- Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_99] (rows=80000000 width=860)
- predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null)
- TableScan [TS_0] (rows=80000000 width=860)
- default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"]
- <-Map 8 [SIMPLE_EDGE]
- SHUFFLE [RS_38]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=20000000 width=1014)
- Output:["_col0"]
- Filter Operator [FIL_100] (rows=20000000 width=1014)
- predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null)
- TableScan [TS_3] (rows=40000000 width=1014)
- default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
+ Filter Operator [FIL_101] (rows=144002668 width=135)
+ predicate:(ws_bill_customer_sk is not null and ws_sold_date_sk is not null)
+ TableScan [TS_19] (rows=144002668 width=135)
+ default@web_sales,web_sales,Tbl:COMPLETE,Col:NONE,Output:["ws_sold_date_sk","ws_bill_customer_sk"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_41]
+ PartitionCols:_col0
+ Merge Join Operator [MERGEJOIN_106] (rows=96800003 width=860)
+ Conds:RS_36._col1=RS_37._col0(Inner),Output:["_col0","_col6","_col7","_col8","_col9","_col10"]
+ <-Map 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_37]
+ PartitionCols:_col0
+ Select Operator [SEL_8] (rows=1861800 width=385)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Filter Operator [FIL_98] (rows=1861800 width=385)
+ predicate:cd_demo_sk is not null
+ TableScan [TS_6] (rows=1861800 width=385)
+ default@customer_demographics,customer_demographics,Tbl:COMPLETE,Col:NONE,Output:["cd_demo_sk","cd_gender","cd_marital_status","cd_education_status","cd_purchase_estimate","cd_credit_rating"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_36]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_105] (rows=88000001 width=860)
+ Conds:RS_33._col2=RS_34._col0(Inner),Output:["_col0","_col1"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_33]
+ PartitionCols:_col2
+ Select Operator [SEL_2] (rows=80000000 width=860)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_96] (rows=80000000 width=860)
+ predicate:(c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_customer_sk is not null)
+ TableScan [TS_0] (rows=80000000 width=860)
+ default@customer,c,Tbl:COMPLETE,Col:NONE,Output:["c_customer_sk","c_current_cdemo_sk","c_current_addr_sk"]
+ <-Map 8 [SIMPLE_EDGE]
+ SHUFFLE [RS_34]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=20000000 width=1014)
+ Output:["_col0"]
+ Filter Operator [FIL_97] (rows=20000000 width=1014)
+ predicate:((ca_state) IN ('CO', 'IL', 'MN') and ca_address_sk is not null)
+ TableScan [TS_3] (rows=40000000 width=1014)
+ default@customer_address,ca,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
http://git-wip-us.apache.org/repos/asf/hive/blob/aee0eaa0/ql/src/test/results/clientpositive/perf/tez/query94.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/tez/query94.q.out b/ql/src/test/results/clientpositive/perf/tez/query94.q.out
index 7674aa7..1c9d16b 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query94.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query94.q.out
@@ -1,4 +1,3 @@
-Warning: Shuffle Join MERGEJOIN[113][tables = [$hdt$_2, $hdt$_3, $hdt$_1, $hdt$_4]] in Stage 'Reducer 18' is a cross product
PREHOOK: query: explain
select
count(distinct ws_order_number) as `order count`
@@ -59,179 +58,125 @@ Plan optimized by CBO.
Vertex dependency in root stage
Reducer 14 <- Map 13 (SIMPLE_EDGE)
-Reducer 16 <- Map 15 (SIMPLE_EDGE), Reducer 19 (SIMPLE_EDGE)
-Reducer 17 <- Reducer 16 (SIMPLE_EDGE)
-Reducer 18 <- Map 15 (CUSTOM_SIMPLE_EDGE), Map 20 (CUSTOM_SIMPLE_EDGE), Map 21 (CUSTOM_SIMPLE_EDGE), Map 22 (CUSTOM_SIMPLE_EDGE)
-Reducer 19 <- Reducer 18 (SIMPLE_EDGE)
-Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE)
-Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
-Reducer 4 <- Map 12 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
-Reducer 5 <- Reducer 14 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE)
-Reducer 6 <- Reducer 17 (ONE_TO_ONE_EDGE), Reducer 5 (SIMPLE_EDGE)
-Reducer 7 <- Reducer 6 (SIMPLE_EDGE)
-Reducer 8 <- Reducer 7 (CUSTOM_SIMPLE_EDGE)
-Reducer 9 <- Reducer 8 (SIMPLE_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)
+Reducer 3 <- Map 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE)
+Reducer 4 <- Map 11 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE)
+Reducer 5 <- Map 12 (SIMPLE_EDGE), Reducer 14 (ONE_TO_ONE_EDGE), Reducer 4 (SIMPLE_EDGE)
+Reducer 6 <- Reducer 5 (SIMPLE_EDGE)
+Reducer 7 <- Reducer 6 (CUSTOM_SIMPLE_EDGE)
+Reducer 8 <- Reducer 7 (SIMPLE_EDGE)
Stage-0
Fetch Operator
limit:-1
Stage-1
- Reducer 9
- File Output Operator [FS_74]
- Limit [LIM_72] (rows=1 width=344)
+ Reducer 8
+ File Output Operator [FS_49]
+ Limit [LIM_47] (rows=1 width=344)
Number of rows:100
- Select Operator [SEL_71] (rows=1 width=344)
+ Select Operator [SEL_46] (rows=1 width=344)
Output:["_col0","_col1","_col2"]
- <-Reducer 8 [SIMPLE_EDGE]
- SHUFFLE [RS_70]
- Select Operator [SEL_69] (rows=1 width=344)
+ <-Reducer 7 [SIMPLE_EDGE]
+ SHUFFLE [RS_45]
+ Select Operator [SEL_44] (rows=1 width=344)
Output:["_col1","_col2","_col3"]
- Group By Operator [GBY_112] (rows=1 width=344)
+ Group By Operator [GBY_77] (rows=1 width=344)
Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"]
- <-Reducer 7 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_111]
- Group By Operator [GBY_110] (rows=1 width=344)
+ <-Reducer 6 [CUSTOM_SIMPLE_EDGE]
+ PARTITION_ONLY_SHUFFLE [RS_76]
+ Group By Operator [GBY_75] (rows=1 width=344)
Output:["_col0","_col1","_col2"],aggregations:["count(_col0)","sum(_col1)","sum(_col2)"]
- Group By Operator [GBY_109] (rows=1395035081047425024 width=1)
+ Group By Operator [GBY_74] (rows=210834322 width=135)
Output:["_col0","_col1","_col2"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0
- <-Reducer 6 [SIMPLE_EDGE]
- SHUFFLE [RS_108]
+ <-Reducer 5 [SIMPLE_EDGE]
+ SHUFFLE [RS_73]
PartitionCols:_col0
- Group By Operator [GBY_107] (rows=1395035081047425024 width=1)
+ Group By Operator [GBY_72] (rows=210834322 width=135)
Output:["_col0","_col2","_col3"],aggregations:["sum(_col5)","sum(_col6)"],keys:_col4
- Select Operator [SEL_65] (rows=1395035081047425024 width=1)
+ Select Operator [SEL_40] (rows=210834322 width=135)
Output:["_col4","_col5","_col6"]
- Filter Operator [FIL_64] (rows=1395035081047425024 width=1)
+ Filter Operator [FIL_39] (rows=210834322 width=135)
predicate:_col16 is null
- Select Operator [SEL_63] (rows=2790070162094850048 width=1)
+ Select Operator [SEL_38] (rows=421668645 width=135)
Output:["_col4","_col5","_col6","_col16"]
- Merge Join Operator [MERGEJOIN_119] (rows=2790070162094850048 width=1)
- Conds:RS_60._col3, _col4=RS_61._col0, _col1(Inner),Output:["_col4","_col5","_col6","_col14"]
- <-Reducer 17 [ONE_TO_ONE_EDGE]
- FORWARD [RS_61]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_46] (rows=2536427365110644736 width=1)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Reducer 16 [SIMPLE_EDGE]
- SHUFFLE [RS_45]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_44] (rows=5072854730221289472 width=1)
- Output:["_col0","_col1"],keys:_col2, _col3
- Select Operator [SEL_43] (rows=5072854730221289472 width=1)
- Output:["_col2","_col3"]
- Filter Operator [FIL_42] (rows=5072854730221289472 width=1)
- predicate:(_col2 <> _col0)
- Merge Join Operator [MERGEJOIN_117] (rows=5072854730221289472 width=1)
- Conds:RS_39._col1=RS_40._col1(Inner),Output:["_col0","_col2","_col3"]
- <-Map 15 [SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_39]
- PartitionCols:_col1
- Select Operator [SEL_20] (rows=144002668 width=135)
- Output:["_col0","_col1"]
- TableScan [TS_19] (rows=144002668 width=135)
- default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
- <-Reducer 19 [SIMPLE_EDGE]
- SHUFFLE [RS_40]
- PartitionCols:_col1
- Select Operator [SEL_38] (rows=4611686018427387903 width=1)
- Output:["_col0","_col1"]
- Group By Operator [GBY_37] (rows=4611686018427387903 width=1)
- Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
- <-Reducer 18 [SIMPLE_EDGE]
- SHUFFLE [RS_36]
- PartitionCols:_col0, _col1
- Group By Operator [GBY_35] (rows=9223372036854775807 width=1)
- Output:["_col0","_col1"],keys:_col4, _col3
- Merge Join Operator [MERGEJOIN_113] (rows=9223372036854775807 width=1)
- Conds:(Inner),(Inner),(Inner),Output:["_col3","_col4"]
- <-Map 15 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_32]
- Please refer to the previous Select Operator [SEL_20]
- <-Map 20 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_29]
- Select Operator [SEL_22] (rows=73049 width=4)
- TableScan [TS_21] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:COMPLETE
- <-Map 21 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_30]
- Select Operator [SEL_24] (rows=84 width=4)
- TableScan [TS_23] (rows=84 width=1850)
- default@web_site,web_site,Tbl:COMPLETE,Col:COMPLETE
- <-Map 22 [CUSTOM_SIMPLE_EDGE]
- PARTITION_ONLY_SHUFFLE [RS_31]
- Select Operator [SEL_26] (rows=40000000 width=4)
- TableScan [TS_25] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:COMPLETE
- <-Reducer 5 [SIMPLE_EDGE]
- SHUFFLE [RS_60]
- PartitionCols:_col3, _col4
- Merge Join Operator [MERGEJOIN_118] (rows=210834322 width=135)
- Conds:RS_57._col4=RS_58._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14"]
- <-Reducer 14 [ONE_TO_ONE_EDGE]
- FORWARD [RS_58]
+ Merge Join Operator [MERGEJOIN_81] (rows=421668645 width=135)
+ Conds:RS_34._col4=RS_35._col0(Left Semi),RS_34._col4=RS_36._col0(Left Outer),Output:["_col3","_col4","_col5","_col6","_col14","_col16"],residual filter predicates:{(_col3 <> _col14)}
+ <-Map 12 [SIMPLE_EDGE]
+ SHUFFLE [RS_35]
+ PartitionCols:_col0
+ Group By Operator [GBY_33] (rows=144002668 width=135)
+ Output:["_col0","_col1"],keys:_col0, _col1
+ Select Operator [SEL_24] (rows=144002668 width=135)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_70] (rows=144002668 width=135)
+ predicate:(ws_order_number is not null and ws_warehouse_sk is not null)
+ TableScan [TS_22] (rows=144002668 width=135)
+ default@web_sales,ws2,Tbl:COMPLETE,Col:NONE,Output:["ws_warehouse_sk","ws_order_number"]
+ <-Reducer 14 [ONE_TO_ONE_EDGE]
+ FORWARD [RS_36]
+ PartitionCols:_col0
+ Select Operator [SEL_31] (rows=7199233 width=92)
+ Output:["_col0","_col1"]
+ Group By Operator [GBY_30] (rows=7199233 width=92)
+ Output:["_col0"],keys:KEY._col0
+ <-Map 13 [SIMPLE_EDGE]
+ SHUFFLE [RS_29]
+ PartitionCols:_col0
+ Group By Operator [GBY_28] (rows=14398467 width=92)
+ Output:["_col0"],keys:wr_order_number
+ Filter Operator [FIL_71] (rows=14398467 width=92)
+ predicate:wr_order_number is not null
+ TableScan [TS_25] (rows=14398467 width=92)
+ default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
+ <-Reducer 4 [SIMPLE_EDGE]
+ SHUFFLE [RS_34]
+ PartitionCols:_col4
+ Merge Join Operator [MERGEJOIN_80] (rows=191667562 width=135)
+ Conds:RS_18._col2=RS_19._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
+ <-Map 11 [SIMPLE_EDGE]
+ SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_18] (rows=7199233 width=92)
- Output:["_col0","_col1"]
- Group By Operator [GBY_17] (rows=7199233 width=92)
- Output:["_col0"],keys:KEY._col0
- <-Map 13 [SIMPLE_EDGE]
- SHUFFLE [RS_16]
- PartitionCols:_col0
- Group By Operator [GBY_15] (rows=14398467 width=92)
- Output:["_col0"],keys:wr_order_number
- Filter Operator [FIL_104] (rows=14398467 width=92)
- predicate:wr_order_number is not null
- TableScan [TS_12] (rows=14398467 width=92)
- default@web_returns,wr1,Tbl:COMPLETE,Col:NONE,Output:["wr_order_number"]
- <-Reducer 4 [SIMPLE_EDGE]
- SHUFFLE [RS_57]
- PartitionCols:_col4
- Merge Join Operator [MERGEJOIN_116] (rows=191667562 width=135)
- Conds:RS_54._col2=RS_55._col0(Inner),Output:["_col3","_col4","_col5","_col6"]
- <-Map 12 [SIMPLE_EDGE]
- SHUFFLE [RS_55]
+ Select Operator [SEL_11] (rows=42 width=1850)
+ Output:["_col0"]
+ Filter Operator [FIL_69] (rows=42 width=1850)
+ predicate:((web_company_name = 'pri') and web_site_sk is not null)
+ TableScan [TS_9] (rows=84 width=1850)
+ default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
+ <-Reducer 3 [SIMPLE_EDGE]
+ SHUFFLE [RS_18]
+ PartitionCols:_col2
+ Merge Join Operator [MERGEJOIN_79] (rows=174243235 width=135)
+ Conds:RS_15._col1=RS_16._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
+ <-Map 10 [SIMPLE_EDGE]
+ SHUFFLE [RS_16]
PartitionCols:_col0
- Select Operator [SEL_11] (rows=42 width=1850)
+ Select Operator [SEL_8] (rows=20000000 width=1014)
Output:["_col0"]
- Filter Operator [FIL_103] (rows=42 width=1850)
- predicate:((web_company_name = 'pri') and web_site_sk is not null)
- TableScan [TS_9] (rows=84 width=1850)
- default@web_site,web_site,Tbl:COMPLETE,Col:NONE,Output:["web_site_sk","web_company_name"]
- <-Reducer 3 [SIMPLE_EDGE]
- SHUFFLE [RS_54]
- PartitionCols:_col2
- Merge Join Operator [MERGEJOIN_115] (rows=174243235 width=135)
- Conds:RS_51._col1=RS_52._col0(Inner),Output:["_col2","_col3","_col4","_col5","_col6"]
- <-Map 11 [SIMPLE_EDGE]
- SHUFFLE [RS_52]
+ Filter Operator [FIL_68] (rows=20000000 width=1014)
+ predicate:((ca_state = 'TX') and ca_address_sk is not null)
+ TableScan [TS_6] (rows=40000000 width=1014)
+ default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
+ <-Reducer 2 [SIMPLE_EDGE]
+ SHUFFLE [RS_15]
+ PartitionCols:_col1
+ Merge Join Operator [MERGEJOIN_78] (rows=158402938 width=135)
+ Conds:RS_12._col0=RS_13._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
+ <-Map 1 [SIMPLE_EDGE]
+ SHUFFLE [RS_12]
PartitionCols:_col0
- Select Operator [SEL_8] (rows=20000000 width=1014)
+ Select Operator [SEL_2] (rows=144002668 width=135)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
+ Filter Operator [FIL_66] (rows=144002668 width=135)
+ predicate:(ws_order_number is not null and ws_ship_addr_sk is not null and ws_ship_date_sk is not null and ws_web_site_sk is not null)
+ TableScan [TS_0] (rows=144002668 width=135)
+ default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
+ <-Map 9 [SIMPLE_EDGE]
+ SHUFFLE [RS_13]
+ PartitionCols:_col0
+ Select Operator [SEL_5] (rows=8116 width=1119)
Output:["_col0"]
- Filter Operator [FIL_102] (rows=20000000 width=1014)
- predicate:((ca_state = 'TX') and ca_address_sk is not null)
- TableScan [TS_6] (rows=40000000 width=1014)
- default@customer_address,customer_address,Tbl:COMPLETE,Col:NONE,Output:["ca_address_sk","ca_state"]
- <-Reducer 2 [SIMPLE_EDGE]
- SHUFFLE [RS_51]
- PartitionCols:_col1
- Merge Join Operator [MERGEJOIN_114] (rows=158402938 width=135)
- Conds:RS_48._col0=RS_49._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"]
- <-Map 1 [SIMPLE_EDGE]
- SHUFFLE [RS_48]
- PartitionCols:_col0
- Select Operator [SEL_2] (rows=144002668 width=135)
- Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- Filter Operator [FIL_100] (rows=144002668 width=135)
- predicate:(ws_ship_addr_sk is not null and ws_ship_date_sk is not null and ws_web_site_sk is not null)
- TableScan [TS_0] (rows=144002668 width=135)
- default@web_sales,ws1,Tbl:COMPLETE,Col:NONE,Output:["ws_ship_date_sk","ws_ship_addr_sk","ws_web_site_sk","ws_warehouse_sk","ws_order_number","ws_ext_ship_cost","ws_net_profit"]
- <-Map 10 [SIMPLE_EDGE]
- SHUFFLE [RS_49]
- PartitionCols:_col0
- Select Operator [SEL_5] (rows=8116 width=1119)
- Output:["_col0"]
- Filter Operator [FIL_101] (rows=8116 width=1119)
- predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
- TableScan [TS_3] (rows=73049 width=1119)
- default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
+ Filter Operator [FIL_67] (rows=8116 width=1119)
+ predicate:(CAST( d_date AS TIMESTAMP) BETWEEN 1999-05-01 00:00:00.0 AND 1999-06-30 00:00:00.0 and d_date_sk is not null)
+ TableScan [TS_3] (rows=73049 width=1119)
+ default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_date"]
http://git-wip-us.apache.org/repos/asf/hive/blob/aee0eaa0/ql/src/test/results/clientpositive/semijoin5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/semijoin5.q.out b/ql/src/test/results/clientpositive/semijoin5.q.out
index 533c077..e42c27f 100644
--- a/ql/src/test/results/clientpositive/semijoin5.q.out
+++ b/ql/src/test/results/clientpositive/semijoin5.q.out
@@ -48,11 +48,10 @@ WHERE (t2.smallint_col_19) IN (SELECT
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1, Stage-7
+ Stage-2 depends on stages: Stage-1, Stage-6
Stage-3 depends on stages: Stage-2
Stage-4 depends on stages: Stage-3
Stage-6 is a root stage
- Stage-7 depends on stages: Stage-6
Stage-0 depends on stages: Stage-4
STAGE PLANS:
@@ -63,7 +62,7 @@ STAGE PLANS:
alias: t1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (bigint_col_7 is not null and decimal2016_col_26 is not null and tinyint_col_3 is not null) (type: boolean)
+ predicate: (bigint_col_7 is not null and decimal2016_col_26 is not null and timestamp_col_9 is not null and tinyint_col_3 is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: tinyint_col_3 (type: tinyint), bigint_col_7 (type: bigint), timestamp_col_9 (type: timestamp), double_col_16 (type: double), decimal2016_col_26 (type: decimal(20,16)), smallint_col_50 (type: smallint)
@@ -79,7 +78,7 @@ STAGE PLANS:
alias: t2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (decimal2709_col_9 is not null and tinyint_col_15 is not null and tinyint_col_20 is not null) (type: boolean)
+ predicate: (decimal2709_col_9 is not null and smallint_col_19 is not null and tinyint_col_15 is not null and tinyint_col_20 is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: decimal2709_col_9 (type: decimal(27,9)), int_col_10 (type: int), tinyint_col_15 (type: tinyint), smallint_col_19 (type: smallint), tinyint_col_20 (type: tinyint)
@@ -119,17 +118,17 @@ STAGE PLANS:
value expressions: _col3 (type: double), _col5 (type: smallint), _col7 (type: int)
TableScan
Reduce Output Operator
- key expressions: _col1 (type: timestamp), -92 (type: int)
+ key expressions: _col0 (type: timestamp), _col1 (type: int)
sort order: ++
- Map-reduce partition columns: _col1 (type: timestamp), -92 (type: int)
+ Map-reduce partition columns: _col0 (type: timestamp), _col1 (type: int)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
0 _col2 (type: timestamp), UDFToInteger(_col9) (type: int)
- 1 _col1 (type: timestamp), -92 (type: int)
+ 1 _col0 (type: timestamp), _col1 (type: int)
outputColumnNames: _col3, _col5, _col7
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
@@ -276,43 +275,21 @@ STAGE PLANS:
1 _col0 (type: decimal(26,12))
outputColumnNames: _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Group By Operator
- keys: _col2 (type: timestamp)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-7
- Map Reduce
- Map Operator Tree:
- TableScan
- Reduce Output Operator
- key expressions: _col0 (type: timestamp)
- sort order: +
- Map-reduce partition columns: _col0 (type: timestamp)
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: timestamp)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col0 (type: timestamp)
- outputColumnNames: _col1
+ expressions: _col2 (type: timestamp), -92 (type: int)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Group By Operator
+ keys: _col0 (type: timestamp), _col1 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/aee0eaa0/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
index b89f9f5..10a880f 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
@@ -104,7 +104,7 @@ STAGE PLANS:
alias: li
Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (l_linenumber = 1) (type: boolean)
+ predicate: ((l_linenumber = 1) and l_orderkey is not null) (type: boolean)
Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int)
@@ -122,10 +122,10 @@ STAGE PLANS:
alias: lineitem
Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((l_linenumber = l_linenumber) and (l_shipmode = 'AIR')) (type: boolean)
+ predicate: ((l_linenumber = 1) and (l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean)
Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: l_orderkey (type: int), l_linenumber (type: int)
+ expressions: l_orderkey (type: int), 1 (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/aee0eaa0/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
index 1c6e380..d116f6a 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_semijoin.q.out
@@ -172,14 +172,32 @@ STAGE PLANS:
outputColumnNames: _col0, _col2
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
+ value expressions: _col2 (type: int)
Map 4
Map Operator Tree:
TableScan
+ alias: table2
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 5
+ Map Operator Tree:
+ TableScan
alias: table3
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -199,46 +217,29 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
- Map 5
- Map Operator Tree:
- TableScan
- alias: table2
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: id is not null (type: boolean)
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: id (type: int), val2 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
- Left Semi Join 0 to 1
+ Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0
+ outputColumnNames: _col0, _col2, _col4
Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col4 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
- Inner Join 0 to 1
+ Left Semi Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col4
Statistics: Num rows: 5 Data size: 121 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/aee0eaa0/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
index 76c74d9..091728a 100644
--- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
@@ -1693,11 +1693,11 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_19]
- Select Operator [SEL_18] (rows=434 width=178)
+ Select Operator [SEL_18] (rows=366 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_17] (rows=434 width=178)
+ Filter Operator [FIL_17] (rows=366 width=179)
predicate:_col3 is null
- Join Operator [JOIN_16] (rows=500 width=178)
+ Join Operator [JOIN_16] (rows=500 width=179)
Output:["_col0","_col1","_col3"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"}
<-Map 1 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_14]
@@ -1709,21 +1709,21 @@ Stage-0
<-Reducer 4 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_15]
PartitionCols:_col0
- Select Operator [SEL_13] (rows=41 width=95)
+ Select Operator [SEL_13] (rows=83 width=95)
Output:["_col0","_col1"]
- Group By Operator [GBY_12] (rows=41 width=91)
+ Group By Operator [GBY_12] (rows=83 width=91)
Output:["_col0"],keys:_col1
- Select Operator [SEL_8] (rows=41 width=178)
+ Select Operator [SEL_8] (rows=83 width=178)
Output:["_col1"]
- Group By Operator [GBY_7] (rows=41 width=178)
+ Group By Operator [GBY_7] (rows=83 width=178)
Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
<-Map 3 [GROUP PARTITION-LEVEL SORT]
GROUP PARTITION-LEVEL SORT [RS_6]
PartitionCols:_col0
- Group By Operator [GBY_5] (rows=41 width=178)
+ Group By Operator [GBY_5] (rows=83 width=178)
Output:["_col0","_col1"],keys:value, key
- Filter Operator [FIL_20] (rows=83 width=178)
- predicate:((value = value) and (value > 'val_2'))
+ Filter Operator [FIL_20] (rows=166 width=178)
+ predicate:(value > 'val_2')
TableScan [TS_2] (rows=500 width=178)
default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -1758,11 +1758,11 @@ Stage-0
Stage-1
Reducer 3
File Output Operator [FS_18]
- Select Operator [SEL_17] (rows=234 width=178)
+ Select Operator [SEL_17] (rows=183 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_16] (rows=234 width=178)
+ Filter Operator [FIL_16] (rows=183 width=179)
predicate:_col4 is null
- Join Operator [JOIN_15] (rows=250 width=178)
+ Join Operator [JOIN_15] (rows=250 width=179)
Output:["_col0","_col1","_col4"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"}
<-Reducer 2 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_13]
@@ -1781,17 +1781,17 @@ Stage-0
<-Reducer 5 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_14]
PartitionCols:_col0, _col1
- Select Operator [SEL_12] (rows=20 width=182)
+ Select Operator [SEL_12] (rows=83 width=182)
Output:["_col0","_col1","_col2"]
- Group By Operator [GBY_11] (rows=20 width=178)
+ Group By Operator [GBY_11] (rows=83 width=178)
Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
<-Map 4 [GROUP]
GROUP [RS_10]
PartitionCols:_col0, _col1
- Group By Operator [GBY_9] (rows=20 width=178)
+ Group By Operator [GBY_9] (rows=83 width=178)
Output:["_col0","_col1"],keys:key, value
- Filter Operator [FIL_19] (rows=41 width=178)
- predicate:((key = key) and (value = value) and (value > 'val_12'))
+ Filter Operator [FIL_19] (rows=166 width=178)
+ predicate:((value > 'val_12') and key is not null)
TableScan [TS_6] (rows=500 width=178)
default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
@@ -1833,26 +1833,28 @@ Stage-0
limit:-1
Stage-1
Reducer 2
- File Output Operator [FS_11]
- Join Operator [JOIN_9] (rows=32 width=178)
+ File Output Operator [FS_12]
+ Join Operator [JOIN_10] (rows=133 width=178)
Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"}
<-Map 1 [PARTITION-LEVEL SORT]
- PARTITION-LEVEL SORT [RS_7]
+ PARTITION-LEVEL SORT [RS_8]
PartitionCols:_col0, _col1
- Select Operator [SEL_1] (rows=500 width=178)
+ Select Operator [SEL_2] (rows=166 width=178)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=500 width=178)
- default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"}
+ Filter Operator [FIL_13] (rows=166 width=178)
+ predicate:((value > 'val_9') and key is not null)
+ TableScan [TS_0] (rows=500 width=178)
+ default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"],properties:{"insideView":"TRUE"}
<-Map 3 [PARTITION-LEVEL SORT]
- PARTITION-LEVEL SORT [RS_8]
+ PARTITION-LEVEL SORT [RS_9]
PartitionCols:_col0, _col1
- Group By Operator [GBY_6] (rows=20 width=178)
+ Group By Operator [GBY_7] (rows=83 width=178)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_4] (rows=41 width=178)
+ Select Operator [SEL_5] (rows=166 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_12] (rows=41 width=178)
- predicate:((key = key) and (value = value) and (value > 'val_9'))
- TableScan [TS_2] (rows=500 width=178)
+ Filter Operator [FIL_14] (rows=166 width=178)
+ predicate:((value > 'val_9') and key is not null)
+ TableScan [TS_3] (rows=500 width=178)
default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
PREHOOK: query: explain select *
@@ -1883,26 +1885,28 @@ Stage-0
limit:-1
Stage-1
Reducer 2
- File Output Operator [FS_11]
- Join Operator [JOIN_9] (rows=32 width=178)
+ File Output Operator [FS_12]
+ Join Operator [JOIN_10] (rows=133 width=178)
Output:["_col0","_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"}
<-Map 1 [PARTITION-LEVEL SORT]
- PARTITION-LEVEL SORT [RS_7]
+ PARTITION-LEVEL SORT [RS_8]
PartitionCols:_col0, _col1
- Select Operator [SEL_1] (rows=500 width=178)
+ Select Operator [SEL_2] (rows=166 width=178)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=500 width=178)
- default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ Filter Operator [FIL_13] (rows=166 width=178)
+ predicate:((value > 'val_9') and key is not null)
+ TableScan [TS_0] (rows=500 width=178)
+ default@src_cbo,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
<-Map 3 [PARTITION-LEVEL SORT]
- PARTITION-LEVEL SORT [RS_8]
+ PARTITION-LEVEL SORT [RS_9]
PartitionCols:_col0, _col1
- Group By Operator [GBY_6] (rows=20 width=178)
+ Group By Operator [GBY_7] (rows=83 width=178)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_4] (rows=41 width=178)
+ Select Operator [SEL_5] (rows=166 width=178)
Output:["_col0","_col1"]
- Filter Operator [FIL_12] (rows=41 width=178)
- predicate:((key = key) and (value = value) and (value > 'val_9'))
- TableScan [TS_2] (rows=500 width=178)
+ Filter Operator [FIL_14] (rows=166 width=178)
+ predicate:((value > 'val_9') and key is not null)
+ TableScan [TS_3] (rows=500 width=178)
default@src_cbo,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
PREHOOK: query: explain select *
@@ -1970,19 +1974,19 @@ Stage-0
Stage-1
Reducer 4
File Output Operator [FS_22]
- Select Operator [SEL_21] (rows=7 width=8)
+ Select Operator [SEL_21] (rows=3 width=8)
Output:["_col0","_col1"]
- Join Operator [JOIN_20] (rows=7 width=8)
+ Join Operator [JOIN_20] (rows=3 width=8)
Output:["_col0","_col3"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1, _col4","1":"_col0, _col1"}
<-Map 6 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_19]
PartitionCols:_col0, _col1
- Group By Operator [GBY_17] (rows=2 width=8)
+ Group By Operator [GBY_17] (rows=1 width=8)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_12] (rows=7 width=8)
+ Select Operator [SEL_12] (rows=2 width=8)
Output:["_col0","_col1"]
- Filter Operator [FIL_25] (rows=7 width=96)
- predicate:((l_linenumber = l_linenumber) and (l_shipmode = 'AIR'))
+ Filter Operator [FIL_25] (rows=2 width=96)
+ predicate:((l_linenumber = 1) and (l_shipmode = 'AIR') and l_orderkey is not null)
TableScan [TS_10] (rows=100 width=96)
default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"]
<-Reducer 3 [PARTITION-LEVEL SORT]
@@ -1996,7 +2000,7 @@ Stage-0
Select Operator [SEL_9] (rows=14 width=16)
Output:["_col0","_col1","_col2","_col3"]
Filter Operator [FIL_24] (rows=14 width=16)
- predicate:((l_linenumber = 1) and l_partkey is not null)
+ predicate:((l_linenumber = 1) and l_orderkey is not null and l_partkey is not null)
TableScan [TS_7] (rows=100 width=16)
default@lineitem,li,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_partkey","l_suppkey","l_linenumber"]
<-Reducer 2 [PARTITION-LEVEL SORT]
@@ -2272,14 +2276,14 @@ Stage-0
File Output Operator [FS_26]
Select Operator [SEL_25] (rows=13 width=223)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_24] (rows=13 width=227)
+ Filter Operator [FIL_24] (rows=13 width=231)
predicate:(not CASE WHEN ((_col4 = 0)) THEN (false) WHEN (_col4 is null) THEN (false) WHEN (_col8 is not null) THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col5 < _col4)) THEN (true) ELSE (false) END)
- Join Operator [JOIN_23] (rows=26 width=227)
+ Join Operator [JOIN_23] (rows=26 width=230)
Output:["_col0","_col1","_col2","_col4","_col5","_col8"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0, _col1","1":"_col0, _col1"}
<-Reducer 2 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_21]
PartitionCols:_col0, _col1
- Join Operator [JOIN_20] (rows=26 width=226)
+ Join Operator [JOIN_20] (rows=26 width=229)
Output:["_col0","_col1","_col2","_col4","_col5"],condition map:[{"":"{\"type\":\"Left Outer\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1","1":"_col0"}
<-Map 1 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_18]
@@ -2291,37 +2295,37 @@ Stage-0
<-Reducer 5 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_19]
PartitionCols:_col0
- Group By Operator [GBY_7] (rows=1 width=114)
+ Group By Operator [GBY_7] (rows=2 width=114)
Output:["_col0","_col1","_col2"],aggregations:["count(VALUE._col0)","count(VALUE._col1)"],keys:KEY._col0
<-Map 4 [GROUP]
GROUP [RS_6]
PartitionCols:_col0
- Group By Operator [GBY_5] (rows=1 width=114)
+ Group By Operator [GBY_5] (rows=2 width=114)
Output:["_col0","_col1","_col2"],aggregations:["count()","count(p_name)"],keys:p_mfgr
- Select Operator [SEL_4] (rows=4 width=223)
+ Select Operator [SEL_4] (rows=8 width=223)
Output:["p_name","p_mfgr"]
- Filter Operator [FIL_27] (rows=4 width=223)
- predicate:((p_mfgr = p_mfgr) and (p_size < 10))
+ Filter Operator [FIL_27] (rows=8 width=223)
+ predicate:((p_size < 10) and p_mfgr is not null)
TableScan [TS_2] (rows=26 width=223)
default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"]
<-Reducer 7 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_22]
PartitionCols:_col0, _col1
- Select Operator [SEL_17] (rows=2 width=223)
+ Select Operator [SEL_17] (rows=4 width=223)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_16] (rows=2 width=219)
+ Filter Operator [FIL_16] (rows=4 width=219)
predicate:_col0 is not null
- Group By Operator [GBY_14] (rows=2 width=219)
+ Group By Operator [GBY_14] (rows=4 width=219)
Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
<-Map 6 [GROUP]
GROUP [RS_13]
PartitionCols:_col0, _col1
- Group By Operator [GBY_12] (rows=2 width=219)
+ Group By Operator [GBY_12] (rows=4 width=219)
Output:["_col0","_col1"],keys:p_name, p_mfgr
- Select Operator [SEL_11] (rows=4 width=223)
+ Select Operator [SEL_11] (rows=8 width=223)
Output:["p_name","p_mfgr"]
- Filter Operator [FIL_28] (rows=4 width=223)
- predicate:((p_mfgr = p_mfgr) and (p_size < 10))
+ Filter Operator [FIL_28] (rows=8 width=223)
+ predicate:((p_size < 10) and p_mfgr is not null)
TableScan [TS_9] (rows=26 width=223)
default@part,part,Tbl:COMPLETE,Col:COMPLETE,Output:["p_name","p_mfgr","p_size"]
@@ -2457,7 +2461,7 @@ Stage-0
Select Operator [SEL_24] (rows=1 width=110)
Output:["_col0","_col1"]
Filter Operator [FIL_23] (rows=1 width=114)
- predicate:(((_col2 - _col1) > 600.0) and (_col1 = _col1))
+ predicate:(((_col2 - _col1) > 600.0) and _col1 is not null)
Group By Operator [GBY_22] (rows=5 width=114)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0
<-Map 6 [GROUP]
@@ -2501,7 +2505,7 @@ Stage-0
Select Operator [SEL_12] (rows=1 width=114)
Output:["_col0","_col1"]
Filter Operator [FIL_11] (rows=1 width=114)
- predicate:(((_col2 - _col1) > 600.0) and (_col1 = _col1))
+ predicate:(((_col2 - _col1) > 600.0) and _col1 is not null)
Group By Operator [GBY_10] (rows=5 width=114)
Output:["_col0","_col1","_col2"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"],keys:KEY._col0
<- Please refer to the previous Map 6 [GROUP]