You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vg...@apache.org on 2018/04/12 17:59:04 UTC
[01/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 91d0a0a2f -> 50f144d1f
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query92.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query92.q.out b/ql/src/test/results/clientpositive/perf/spark/query92.q.out
index 70c4c5a..e7b8632 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query92.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query92.q.out
@@ -67,7 +67,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 7
+ Map 5
Map Operator Tree:
TableScan
alias: date_dim
@@ -90,7 +90,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 10
+ Map 9
Map Operator Tree:
TableScan
alias: date_dim
@@ -112,11 +112,11 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 156), Map 6 (PARTITION-LEVEL SORT, 156)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 270), Reducer 9 (PARTITION-LEVEL SORT, 270)
- Reducer 4 <- Reducer 3 (GROUP, 1)
- Reducer 5 <- Reducer 4 (SORT, 1)
- Reducer 9 <- Map 8 (GROUP, 169)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 261), Reducer 8 (PARTITION-LEVEL SORT, 261)
+ Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 4 <- Reducer 3 (SORT, 1)
+ Reducer 7 <- Map 6 (GROUP, 169)
+ Reducer 8 <- Map 10 (PARTITION-LEVEL SORT, 87), Reducer 7 (PARTITION-LEVEL SORT, 87)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -131,13 +131,25 @@ STAGE PLANS:
expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_discount_amt (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: decimal(7,2))
- Map 6
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
+ input vertices:
+ 1 Map 5
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(7,2))
+ Local Work:
+ Map Reduce Local Work
+ Map 10
Map Operator Tree:
TableScan
alias: item
@@ -154,7 +166,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 6
Map Operator Tree:
TableScan
alias: web_sales
@@ -174,7 +186,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2
input vertices:
- 1 Map 10
+ 1 Map 9
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col2), count(_col2)
@@ -191,50 +203,22 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
Reducer 2
- Local Work:
- Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col3
- input vertices:
- 1 Map 7
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(7,2))
- Reducer 3
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col3 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col2, _col7
- Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
+ 1 _col2 (type: int)
+ outputColumnNames: _col2, _col5
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col2 > _col7) (type: boolean)
- Statistics: Num rows: 63889187 Data size: 8687081792 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col2 > _col5) (type: boolean)
+ Statistics: Num rows: 58081078 Data size: 7897346909 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: decimal(7,2))
outputColumnNames: _col2
- Statistics: Num rows: 63889187 Data size: 8687081792 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 58081078 Data size: 7897346909 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col2)
mode: hash
@@ -244,7 +228,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(17,2))
- Reducer 4
+ Reducer 3
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -260,7 +244,7 @@ STAGE PLANS:
sort order: +
Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
- Reducer 5
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: decimal(17,2))
@@ -276,7 +260,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 9
+ Reducer 7
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1)
@@ -294,6 +278,22 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(38,21))
+ Reducer 8
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 87121617 Data size: 11846020431 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 87121617 Data size: 11846020431 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: decimal(38,21))
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query94.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query94.q.out b/ql/src/test/results/clientpositive/perf/spark/query94.q.out
index 8b853b9..2af8280 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query94.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query94.q.out
@@ -56,7 +56,8 @@ limit 100
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-1 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -82,7 +83,12 @@ STAGE PLANS:
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 9
+
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 8
Map Operator Tree:
TableScan
alias: date_dim
@@ -105,7 +111,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 13 <- Map 12 (GROUP, 11)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 312), Map 8 (PARTITION-LEVEL SORT, 312)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 327), Map 9 (PARTITION-LEVEL SORT, 327)
Reducer 3 <- Map 11 (PARTITION-LEVEL SORT, 357), Reducer 2 (PARTITION-LEVEL SORT, 357)
Reducer 4 <- Reducer 13 (PARTITION-LEVEL SORT, 230), Reducer 3 (PARTITION-LEVEL SORT, 230)
Reducer 5 <- Reducer 4 (GROUP, 124)
@@ -125,12 +131,24 @@ STAGE PLANS:
expressions: ws_ship_date_sk (type: int), ws_ship_addr_sk (type: int), ws_web_site_sk (type: int), ws_warehouse_sk (type: int), ws_order_number (type: int), ws_ext_ship_cost (type: decimal(7,2)), ws_net_profit (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6
+ input vertices:
+ 1 Map 8
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ Local Work:
+ Map Reduce Local Work
Map 11
Map Operator Tree:
TableScan
@@ -172,7 +190,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 9
Map Operator Tree:
TableScan
alias: customer_address
@@ -216,34 +234,24 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col3, _col4, _col5, _col6
input vertices:
- 1 Map 9
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col3, _col4, _col5, _col6
- input vertices:
- 1 Map 10
+ 1 Map 10
+ Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col4 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col4 (type: int)
Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ value expressions: _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
Reducer 3
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query95.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query95.q.out b/ql/src/test/results/clientpositive/perf/spark/query95.q.out
index 2c35b83..72bb8af 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query95.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query95.q.out
@@ -62,7 +62,8 @@ limit 100
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-1 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -70,39 +71,44 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 8
+ Map 9
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ alias: web_site
+ Statistics: Num rows: 84 Data size: 155408 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1999-05-01 00:00:00.0' AND TIMESTAMP'1999-06-30 00:00:00.0' and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((web_company_name = 'pri') and web_site_sk is not null) (type: boolean)
+ Statistics: Num rows: 42 Data size: 77704 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int)
+ expressions: web_site_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 42 Data size: 77704 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 9
+
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 7
Map Operator Tree:
TableScan
- alias: web_site
- Statistics: Num rows: 84 Data size: 155408 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((web_company_name = 'pri') and web_site_sk is not null) (type: boolean)
- Statistics: Num rows: 42 Data size: 77704 Basic stats: COMPLETE Column stats: NONE
+ predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1999-05-01 00:00:00.0' AND TIMESTAMP'1999-06-30 00:00:00.0' and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: web_site_sk (type: int)
+ expressions: d_date_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 42 Data size: 77704 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -115,7 +121,7 @@ STAGE PLANS:
Reducer 15 <- Map 14 (PARTITION-LEVEL SORT, 306), Map 18 (PARTITION-LEVEL SORT, 306)
Reducer 16 <- Map 19 (PARTITION-LEVEL SORT, 179), Reducer 15 (PARTITION-LEVEL SORT, 179)
Reducer 17 <- Reducer 16 (GROUP, 186)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 312), Map 7 (PARTITION-LEVEL SORT, 312)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 327), Map 8 (PARTITION-LEVEL SORT, 327)
Reducer 3 <- Reducer 12 (PARTITION-LEVEL SORT, 381), Reducer 17 (PARTITION-LEVEL SORT, 381), Reducer 2 (PARTITION-LEVEL SORT, 381)
Reducer 4 <- Reducer 3 (GROUP, 448)
Reducer 5 <- Reducer 4 (GROUP, 1)
@@ -134,12 +140,24 @@ STAGE PLANS:
expressions: ws_ship_date_sk (type: int), ws_ship_addr_sk (type: int), ws_web_site_sk (type: int), ws_order_number (type: int), ws_ext_ship_cost (type: decimal(7,2)), ws_net_profit (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5
+ input vertices:
+ 1 Map 7
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
+ Local Work:
+ Map Reduce Local Work
Map 10
Map Operator Tree:
TableScan
@@ -229,7 +247,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- Map 7
+ Map 8
Map Operator Tree:
TableScan
alias: customer_address
@@ -349,34 +367,24 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col3, _col4, _col5
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col2, _col3, _col4, _col5
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col5
+ outputColumnNames: _col3, _col4, _col5
input vertices:
- 1 Map 8
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col3, _col4, _col5
- input vertices:
- 1 Map 9
+ 1 Map 9
+ Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: int)
Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
+ value expressions: _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
Reducer 3
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query97.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query97.q.out b/ql/src/test/results/clientpositive/perf/spark/query97.q.out
index 278fcf1..14a2aed 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query97.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query97.q.out
@@ -102,24 +102,24 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (GROUP, 336)
+ Reducer 2 <- Map 1 (GROUP, 437)
Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 386), Reducer 7 (PARTITION-LEVEL SORT, 386)
Reducer 4 <- Reducer 3 (GROUP, 1)
- Reducer 7 <- Map 6 (GROUP, 437)
+ Reducer 7 <- Map 6 (GROUP, 336)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: catalog_sales
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ alias: store_sales
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: cs_sold_date_sk is not null (type: boolean)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ predicate: ss_sold_date_sk is not null (type: boolean)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_item_sk (type: int)
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -129,31 +129,31 @@ STAGE PLANS:
outputColumnNames: _col1, _col2
input vertices:
1 Map 5
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: _col1 (type: int), _col2 (type: int)
+ keys: _col2 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Local Work:
Map Reduce Local Work
Map 6
Map Operator Tree:
TableScan
- alias: store_sales
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ alias: catalog_sales
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ss_sold_date_sk is not null (type: boolean)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ predicate: cs_sold_date_sk is not null (type: boolean)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int)
+ expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_item_sk (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -163,17 +163,17 @@ STAGE PLANS:
outputColumnNames: _col1, _col2
input vertices:
1 Map 8
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: _col2 (type: int), _col1 (type: int)
+ keys: _col1 (type: int), _col2 (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Local Work:
Map Reduce Local Work
Reducer 2
@@ -182,12 +182,12 @@ STAGE PLANS:
keys: KEY._col0 (type: int), KEY._col1 (type: int)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
Reducer 3
Reduce Operator Tree:
Join Operator
@@ -199,7 +199,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col2
Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: CASE WHEN ((_col2 is not null and _col0 is null)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((_col2 is null and _col0 is not null)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((_col2 is not null and _col0 is not null)) THEN (1) ELSE (0) END (type: int)
+ expressions: CASE WHEN ((_col0 is not null and _col2 is null)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((_col0 is null and _col2 is not null)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((_col0 is not null and _col2 is not null)) THEN (1) ELSE (0) END (type: int)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -235,12 +235,12 @@ STAGE PLANS:
keys: KEY._col0 (type: int), KEY._col1 (type: int)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query99.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query99.q.out b/ql/src/test/results/clientpositive/perf/spark/query99.q.out
index 436eb3c..f59db0a 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query99.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query99.q.out
@@ -97,18 +97,18 @@ STAGE PLANS:
Map 5
Map Operator Tree:
TableScan
- alias: ship_mode
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ alias: call_center
+ Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: sm_ship_mode_sk is not null (type: boolean)
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ predicate: cc_call_center_sk is not null (type: boolean)
+ Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: sm_ship_mode_sk (type: int), sm_type (type: string)
+ expressions: cc_call_center_sk (type: int), cc_name (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col3 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -133,18 +133,18 @@ STAGE PLANS:
Map 7
Map Operator Tree:
TableScan
- alias: call_center
- Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column stats: NONE
+ alias: ship_mode
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: cc_call_center_sk is not null (type: boolean)
- Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column stats: NONE
+ predicate: sm_ship_mode_sk is not null (type: boolean)
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: cc_call_center_sk (type: int), cc_name (type: string)
+ expressions: sm_ship_mode_sk (type: int), sm_type (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -182,34 +182,34 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col3 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col4, _col8
+ outputColumnNames: _col0, _col1, _col3, _col4, _col8
input vertices:
1 Map 5
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col4 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col8, _col10
+ outputColumnNames: _col0, _col1, _col3, _col8, _col10
input vertices:
1 Map 6
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: PARTIAL Column stats: NONE
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col1, _col8, _col10, _col12
input vertices:
1 Map 7
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: substr(_col10, 1, 20) (type: string), _col8 (type: string), _col12 (type: string), CASE WHEN (((_col1 - _col0) <= 30)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 30) and ((_col1 - _col0) <= 60))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 60) and ((_col1 - _col0) <= 90))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 90) and ((_col1 - _col0) <= 120))) THEN (1) ELSE (0) END (type: int), CASE WHEN (((_col1 - _col0) > 120)) THEN (1) ELSE (0) END (type: int)
+ expressions: substr(_col10, 1, 20) (type: string), _col12 (type: string), _col8 (type: string), CASE WHEN (((_col1 - _col0) <= 30)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 30) and ((_col1 - _col0) <= 60))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 60) and ((_col1 - _col0) <= 90))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 90) and ((_col1 - _col0) <= 120))) THEN (1) ELSE (0) END (type: int), CASE WHEN (((_col1 - _col0) > 120)) THEN (1) ELSE (0) END (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: PARTIAL Column stats: NONE
Group By Operator
[02/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query85.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query85.q.out b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
index c82dcf7..d60751c 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query85.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
@@ -175,7 +175,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 15
+ Map 13
Map Operator Tree:
TableScan
alias: reason
@@ -212,21 +212,21 @@ STAGE PLANS:
Statistics: Num rows: 4602 Data size: 2696178 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
- 1 _col2 (type: int)
+ 0 _col10 (type: int)
+ 1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Stage: Stage-1
Spark
Edges:
- Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 20), Map 14 (PARTITION-LEVEL SORT, 20)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 90), Map 8 (PARTITION-LEVEL SORT, 90)
- Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 17), Reducer 2 (PARTITION-LEVEL SORT, 17)
- Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 19), Reducer 3 (PARTITION-LEVEL SORT, 19)
- Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 35), Reducer 4 (PARTITION-LEVEL SORT, 35)
- Reducer 6 <- Reducer 5 (GROUP, 2)
- Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 28), Map 9 (PARTITION-LEVEL SORT, 28)
+ Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 98), Reducer 2 (PARTITION-LEVEL SORT, 98)
+ Reducer 4 <- Map 12 (PARTITION-LEVEL SORT, 5), Reducer 3 (PARTITION-LEVEL SORT, 5)
+ Reducer 5 <- Map 14 (PARTITION-LEVEL SORT, 11), Reducer 4 (PARTITION-LEVEL SORT, 11)
+ Reducer 6 <- Map 15 (PARTITION-LEVEL SORT, 7), Reducer 5 (PARTITION-LEVEL SORT, 7)
+ Reducer 7 <- Reducer 6 (GROUP, 7)
+ Reducer 8 <- Reducer 7 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -242,61 +242,32 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ key expressions: _col0 (type: int), _col5 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col5 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2))
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2))
Map 10
Map Operator Tree:
TableScan
- alias: cd2
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_education_status is not null and cd_marital_status is not null) (type: boolean)
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((ca_country = 'United States') and (ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is not null) (type: boolean)
+ Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cd_demo_sk (type: int), cd_marital_status (type: string), cd_education_status (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ expressions: ca_address_sk (type: int), ca_state (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: string), _col0 (type: int), _col1 (type: string)
- sort order: +++
- Map-reduce partition columns: _col2 (type: string), _col0 (type: int), _col1 (type: string)
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 12
Map Operator Tree:
TableScan
- alias: web_sales
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((ws_net_profit BETWEEN 100 AND 200 or ws_net_profit BETWEEN 150 AND 300 or ws_net_profit BETWEEN 50 AND 250) and (ws_sales_price BETWEEN 100 AND 150 or ws_sales_price BETWEEN 50 AND 100 or ws_sales_price BETWEEN 150 AND 200) and ws_item_sk is not null and ws_order_number is not null and ws_sold_date_sk is not null and ws_web_page_sk is not null) (type: boolean)
- Statistics: Num rows: 16000296 Data size: 2175577518 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_web_page_sk (type: int), ws_order_number (type: int), ws_quantity (type: int), ws_sales_price (type: decimal(7,2)), ws_net_profit (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 16000296 Data size: 2175577518 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col2 (type: int)
- outputColumnNames: _col1, _col2, _col4, _col5, _col6, _col7
- input vertices:
- 0 Map 11
- Statistics: Num rows: 17600325 Data size: 2393135321 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 17600325 Data size: 2393135321 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2))
- Local Work:
- Map Reduce Local Work
- Map 14
- Map Operator Tree:
- TableScan
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -311,28 +282,28 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 14
Map Operator Tree:
TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ alias: cd1
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ca_country = 'United States') and (ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is not null) (type: boolean)
- Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_education_status is not null and cd_marital_status is not null) (type: boolean)
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_state (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
+ expressions: cd_demo_sk (type: int), cd_marital_status (type: string), cd_education_status (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Map 9
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string)
+ Map 15
Map Operator Tree:
TableScan
- alias: cd1
+ alias: cd2
Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_education_status is not null and cd_marital_status is not null) (type: boolean)
@@ -342,145 +313,173 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: string)
Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string)
- Reducer 13
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col4, _col5, _col6, _col7
- Statistics: Num rows: 19360357 Data size: 2632448910 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col2 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2))
- outputColumnNames: _col4, _col6, _col7, _col8, _col9
- Statistics: Num rows: 19360357 Data size: 2632448910 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int), _col6 (type: int)
- sort order: ++
- Map-reduce partition columns: _col4 (type: int), _col6 (type: int)
- Statistics: Num rows: 19360357 Data size: 2632448910 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col7 (type: int), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2))
+ Map 9
+ Map Operator Tree:
+ TableScan
+ alias: web_sales
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((ws_net_profit BETWEEN 100 AND 200 or ws_net_profit BETWEEN 150 AND 300 or ws_net_profit BETWEEN 50 AND 250) and (ws_sales_price BETWEEN 100 AND 150 or ws_sales_price BETWEEN 50 AND 100 or ws_sales_price BETWEEN 150 AND 200) and ws_item_sk is not null and ws_order_number is not null and ws_sold_date_sk is not null and ws_web_page_sk is not null) (type: boolean)
+ Statistics: Num rows: 16000296 Data size: 2175577518 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_web_page_sk (type: int), ws_order_number (type: int), ws_quantity (type: int), ws_sales_price (type: decimal(7,2)), ws_net_profit (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 16000296 Data size: 2175577518 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col3 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: int), _col3 (type: int)
+ Statistics: Num rows: 16000296 Data size: 2175577518 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col6, _col7, _col9
- Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: int), _col5 (type: int)
+ 1 _col1 (type: int), _col3 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10, _col12, _col13, _col14
+ Statistics: Num rows: 17600325 Data size: 2393135321 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col9 (type: string)
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 17600325 Data size: 2393135321 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col3 (type: int), _col4 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: int), _col10 (type: int), _col12 (type: int), _col13 (type: decimal(7,2)), _col14 (type: decimal(7,2))
Reducer 3
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col3, _col4, _col5, _col6, _col7, _col9, _col12, _col13
- Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col13 (type: string), _col3 (type: int), _col12 (type: string)
- sort order: +++
- Map-reduce partition columns: _col13 (type: string), _col3 (type: int), _col12 (type: string)
- Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col9 (type: string)
+ outputColumnNames: _col1, _col3, _col4, _col6, _col7, _col8, _col10, _col12, _col13, _col14, _col16
+ Statistics: Num rows: 19360357 Data size: 2632448910 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((_col16) IN ('KY', 'GA', 'NM') and _col14 BETWEEN 100 AND 200) or ((_col16) IN ('MT', 'OR', 'IN') and _col14 BETWEEN 150 AND 300) or ((_col16) IN ('WI', 'MO', 'WV') and _col14 BETWEEN 50 AND 250)) (type: boolean)
+ Statistics: Num rows: 3226725 Data size: 438741326 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col10 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3, _col4, _col6, _col7, _col8, _col12, _col13
+ input vertices:
+ 1 Map 11
+ Statistics: Num rows: 3549397 Data size: 482615469 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col8 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col8 (type: int)
+ Statistics: Num rows: 3549397 Data size: 482615469 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col3 (type: int), _col4 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col12 (type: int), _col13 (type: decimal(7,2))
Reducer 4
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col13 (type: string), _col3 (type: int), _col12 (type: string)
- 1 _col2 (type: string), _col0 (type: int), _col1 (type: string)
- outputColumnNames: _col0, _col4, _col5, _col6, _col7, _col9, _col12, _col13
- Statistics: Num rows: 19164359 Data size: 1763833572 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col5 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col5 (type: int)
- Statistics: Num rows: 19164359 Data size: 1763833572 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col4 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col9 (type: string), _col12 (type: string), _col13 (type: string)
+ 0 _col8 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3, _col4, _col6, _col7, _col12, _col13
+ Statistics: Num rows: 3904336 Data size: 530877027 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col4 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3, _col6, _col7, _col12, _col13, _col22
+ input vertices:
+ 1 Map 13
+ Statistics: Num rows: 4294769 Data size: 583964742 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 4294769 Data size: 583964742 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col12 (type: int), _col13 (type: decimal(7,2)), _col22 (type: string)
Reducer 5
- Local Work:
- Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int), _col5 (type: int)
- 1 _col4 (type: int), _col6 (type: int)
- outputColumnNames: _col4, _col6, _col7, _col9, _col12, _col13, _col24, _col25, _col26
- Statistics: Num rows: 21296393 Data size: 2895693863 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col3, _col6, _col7, _col12, _col13, _col22, _col24, _col25
+ Statistics: Num rows: 4724246 Data size: 642361230 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((_col12 = 'M') and (_col13 = '4 yr Degree') and _col25 BETWEEN 100 AND 150) or ((_col12 = 'D') and (_col13 = 'Primary') and _col25 BETWEEN 50 AND 100) or ((_col12 = 'U') and (_col13 = 'Advanced Degree') and _col25 BETWEEN 150 AND 200)) and (((_col9) IN ('KY', 'GA', 'NM') and _col26 BETWEEN 100 AND 200) or ((_col9) IN ('MT', 'OR', 'IN') and _col26 BETWEEN 150 AND 300) or ((_col9) IN ('WI', 'MO', 'WV') and _col26 BETWEEN 50 AND 250))) (type: boolean)
- Statistics: Num rows: 295782 Data size: 40217802 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col6, _col7, _col24, _col28
- input vertices:
- 1 Map 15
- Statistics: Num rows: 325360 Data size: 44239583 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col24), count(_col24), sum(_col7), count(_col7), sum(_col6), count(_col6)
- keys: _col28 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 325360 Data size: 44239583 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 325360 Data size: 44239583 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: bigint), _col5 (type: decimal(17,2)), _col6 (type: bigint)
+ predicate: (((_col24 = 'D') and (_col25 = 'Primary') and _col13 BETWEEN 50 AND 100) or ((_col24 = 'M') and (_col25 = '4 yr Degree') and _col13 BETWEEN 100 AND 150) or ((_col24 = 'U') and (_col25 = 'Advanced Degree') and _col13 BETWEEN 150 AND 200)) (type: boolean)
+ Statistics: Num rows: 393687 Data size: 53530079 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int), _col24 (type: string), _col25 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col3 (type: int), _col24 (type: string), _col25 (type: string)
+ Statistics: Num rows: 393687 Data size: 53530079 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col12 (type: int), _col22 (type: string)
Reducer 6
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: int), _col24 (type: string), _col25 (type: string)
+ 1 _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ outputColumnNames: _col6, _col7, _col12, _col22
+ Statistics: Num rows: 2047980 Data size: 788904791 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col12), count(_col12), sum(_col7), count(_col7), sum(_col6), count(_col6)
+ keys: _col22 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 2047980 Data size: 788904791 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2047980 Data size: 788904791 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint), _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: bigint), _col5 (type: decimal(17,2)), _col6 (type: bigint)
+ Reducer 7
+ Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), count(VALUE._col3), sum(VALUE._col4), count(VALUE._col5)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 162680 Data size: 22119791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1023990 Data size: 394452395 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (_col1 / _col2) (type: double), (_col3 / _col4) (type: decimal(37,22)), (_col5 / _col6) (type: decimal(37,22)), substr(_col0, 1, 20) (type: string)
outputColumnNames: _col4, _col5, _col6, _col7
- Statistics: Num rows: 162680 Data size: 22119791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1023990 Data size: 394452395 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col7 (type: string), _col4 (type: double), _col5 (type: decimal(37,22)), _col6 (type: decimal(37,22))
sort order: ++++
- Statistics: Num rows: 162680 Data size: 22119791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1023990 Data size: 394452395 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
- Reducer 7
+ Reducer 8
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: double), KEY.reducesinkkey2 (type: decimal(37,22)), KEY.reducesinkkey3 (type: decimal(37,22))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 162680 Data size: 22119791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1023990 Data size: 394452395 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
- Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 100 Data size: 38500 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 100 Data size: 38500 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query88.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query88.q.out b/ql/src/test/results/clientpositive/perf/spark/query88.q.out
index f8f2b70..be44719 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query88.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query88.q.out
@@ -204,18 +204,18 @@ STAGE PLANS:
Map 10
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -240,18 +240,18 @@ STAGE PLANS:
Map 9
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -294,9 +294,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 14
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -304,7 +304,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 15
@@ -346,9 +346,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 19
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -356,7 +356,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 20
@@ -398,9 +398,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 24
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -408,7 +408,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 25
@@ -450,9 +450,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 29
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -460,7 +460,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 30
@@ -502,9 +502,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 34
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -512,7 +512,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 35
@@ -554,9 +554,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 39
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -564,7 +564,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 40
@@ -606,9 +606,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 9
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -616,7 +616,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 10
@@ -791,36 +791,36 @@ STAGE PLANS:
Map 4
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 5
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -857,9 +857,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col2
input vertices:
1 Map 4
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -867,7 +867,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
input vertices:
1 Map 5
@@ -957,36 +957,36 @@ STAGE PLANS:
Map 14
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 15
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -1016,36 +1016,36 @@ STAGE PLANS:
Map 19
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 20
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -1075,36 +1075,36 @@ STAGE PLANS:
Map 24
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 25
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -1134,36 +1134,36 @@ STAGE PLANS:
Map 29
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 30
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -1193,36 +1193,36 @@ STAGE PLANS:
Map 34
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 35
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -1252,36 +1252,36 @@ STAGE PLANS:
Map 39
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 40
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((((hd_dep_count = 3) and (hd_vehicle_count <= 5)) or ((hd_dep_count = 0) and (hd_vehicle_count <= 2)) or ((hd_dep_count = 1) and (hd_vehicle_count <= 3))) and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_store_name = 'ese') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 3600 Data size: 385200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query89.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query89.q.out b/ql/src/test/results/clientpositive/perf/spark/query89.q.out
index 815c55d..c933fe1 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query89.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query89.q.out
@@ -185,47 +185,47 @@ STAGE PLANS:
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col12 (type: string), _col13 (type: string), _col10 (type: int), _col5 (type: string), _col6 (type: string), _col7 (type: string)
+ keys: _col5 (type: string), _col6 (type: string), _col7 (type: string), _col10 (type: int), _col12 (type: string), _col13 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: string)
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string)
sort order: ++++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: string)
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col6 (type: decimal(17,2))
Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._col4 (type: string), KEY._col5 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col5 (type: string), _col3 (type: string), _col0 (type: string), _col1 (type: string)
+ key expressions: _col2 (type: string), _col0 (type: string), _col4 (type: string), _col5 (type: string)
sort order: ++++
- Map-reduce partition columns: _col5 (type: string), _col3 (type: string), _col0 (type: string), _col1 (type: string)
+ Map-reduce partition columns: _col2 (type: string), _col0 (type: string), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col4 (type: string), _col6 (type: decimal(17,2))
+ value expressions: _col1 (type: string), _col3 (type: int), _col6 (type: decimal(17,2))
Reducer 5
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col2 (type: decimal(17,2))
+ expressions: KEY.reducesinkkey1 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col2 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: int, _col3: string, _col4: string, _col5: string, _col6: decimal(17,2)
+ output shape: _col0: string, _col1: string, _col2: string, _col3: int, _col4: string, _col5: string, _col6: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col5 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST
- partition by: _col5, _col3, _col0, _col1
+ order by: _col2 ASC NULLS FIRST, _col0 ASC NULLS FIRST, _col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST
+ partition by: _col2, _col0, _col4, _col5
raw input shape:
window functions:
window function definition
@@ -236,14 +236,14 @@ STAGE PLANS:
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: decimal(17,2))
+ expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: string), _col5 (type: string), _col6 (type: decimal(17,2))
outputColumnNames: avg_window_0, _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: CASE WHEN ((avg_window_0 <> 0)) THEN (((abs((_col6 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END (type: boolean)
Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col5 (type: string), _col4 (type: string), _col3 (type: string), _col0 (type: string), _col1 (type: string), _col2 (type: int), _col6 (type: decimal(17,2)), avg_window_0 (type: decimal(21,6)), (_col6 - avg_window_0) (type: decimal(22,6))
+ expressions: _col2 (type: string), _col1 (type: string), _col0 (type: string), _col4 (type: string), _col5 (type: string), _col3 (type: int), _col6 (type: decimal(17,2)), avg_window_0 (type: decimal(21,6)), (_col6 - avg_window_0) (type: decimal(22,6))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query90.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query90.q.out b/ql/src/test/results/clientpositive/perf/spark/query90.q.out
index 9978b10..115a2cc 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query90.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query90.q.out
@@ -56,18 +56,18 @@ STAGE PLANS:
Map 10
Map Operator Tree:
TableScan
- alias: web_page
- Statistics: Num rows: 4602 Data size: 2696178 Basic stats: COMPLETE Column stats: NONE
+ alias: time_dim
+ Statistics: Num rows: 86400 Data size: 40694400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) (type: boolean)
- Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
+ predicate: (t_hour BETWEEN 14 AND 15 and t_time_sk is not null) (type: boolean)
+ Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: wp_web_page_sk (type: int)
+ expressions: t_time_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -92,18 +92,18 @@ STAGE PLANS:
Map 9
Map Operator Tree:
TableScan
- alias: time_dim
- Statistics: Num rows: 86400 Data size: 40694400 Basic stats: COMPLETE Column stats: NONE
+ alias: web_page
+ Statistics: Num rows: 4602 Data size: 2696178 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (t_hour BETWEEN 14 AND 15 and t_time_sk is not null) (type: boolean)
- Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
+ predicate: (wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) (type: boolean)
+ Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: t_time_sk (type: int)
+ expressions: wp_web_page_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -130,9 +130,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col1
input vertices:
1 Map 9
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
@@ -140,7 +140,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1
input vertices:
@@ -187,36 +187,36 @@ STAGE PLANS:
Map 4
Map Operator Tree:
TableScan
- alias: time_dim
- Statistics: Num rows: 86400 Data size: 40694400 Basic stats: COMPLETE Column stats: NONE
+ alias: web_page
+ Statistics: Num rows: 4602 Data size: 2696178 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (t_hour BETWEEN 6 AND 7 and t_time_sk is not null) (type: boolean)
- Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
+ predicate: (wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) (type: boolean)
+ Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: t_time_sk (type: int)
+ expressions: wp_web_page_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 5
Map Operator Tree:
TableScan
- alias: web_page
- Statistics: Num rows: 4602 Data size: 2696178 Basic stats: COMPLETE Column stats: NONE
+ alias: time_dim
+ Statistics: Num rows: 86400 Data size: 40694400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (wp_char_count BETWEEN 5000 AND 5200 and wp_web_page_sk is not null) (type: boolean)
- Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
+ predicate: (t_hour BETWEEN 6 AND 7 and t_time_sk is not null) (type: boolean)
+ Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: wp_web_page_sk (type: int)
+ expressions: t_time_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 511 Data size: 299380 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9600 Data size: 4521600 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -262,9 +262,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col1
input vertices:
1 Map 4
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
@@ -272,7 +272,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1
input vertices:
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query91.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query91.q.out b/ql/src/test/results/clientpositive/perf/spark/query91.q.out
index 0ad330f..b4f0650 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query91.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query91.q.out
@@ -234,14 +234,14 @@ STAGE PLANS:
Statistics: Num rows: 106480005 Data size: 91574956652 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col4, _col9, _col10
+ outputColumnNames: _col2, _col7, _col8
Statistics: Num rows: 106480005 Data size: 91574956652 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col4 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col4 (type: int)
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 106480005 Data size: 91574956652 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col9 (type: string), _col10 (type: string)
+ value expressions: _col7 (type: string), _col8 (type: string)
Reducer 2
Local Work:
Map Reduce Local Work
@@ -277,12 +277,12 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col1 (type: int)
- 1 _col4 (type: int)
- outputColumnNames: _col3, _col8, _col9, _col10, _col20, _col21
+ 1 _col2 (type: int)
+ outputColumnNames: _col3, _col8, _col9, _col10, _col18, _col19
Statistics: Num rows: 117128008 Data size: 100732454500 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col8 (type: string), _col9 (type: string), _col10 (type: string), _col20 (type: string), _col21 (type: string)
+ keys: _col8 (type: string), _col9 (type: string), _col10 (type: string), _col18 (type: string), _col19 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 117128008 Data size: 100732454500 Basic stats: COMPLETE Column stats: NONE
[05/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query75.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query75.q.out b/ql/src/test/results/clientpositive/perf/spark/query75.q.out
index f4fca34..4b76570 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query75.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query75.q.out
@@ -144,25 +144,25 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 400), Map 16 (PARTITION-LEVEL SORT, 400)
- Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 438), Reducer 13 (PARTITION-LEVEL SORT, 438)
+ Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 398), Map 16 (PARTITION-LEVEL SORT, 398)
+ Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 440), Reducer 13 (PARTITION-LEVEL SORT, 440)
Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 516), Reducer 14 (PARTITION-LEVEL SORT, 516)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 308), Map 16 (PARTITION-LEVEL SORT, 308)
- Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 156), Map 19 (PARTITION-LEVEL SORT, 156)
- Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 169), Reducer 20 (PARTITION-LEVEL SORT, 169)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 16 (PARTITION-LEVEL SORT, 306)
+ Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 154), Map 19 (PARTITION-LEVEL SORT, 154)
+ Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 171), Reducer 20 (PARTITION-LEVEL SORT, 171)
Reducer 22 <- Map 25 (PARTITION-LEVEL SORT, 196), Reducer 21 (PARTITION-LEVEL SORT, 196)
- Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 308), Map 32 (PARTITION-LEVEL SORT, 308)
- Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 336), Reducer 27 (PARTITION-LEVEL SORT, 336)
+ Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 306), Map 32 (PARTITION-LEVEL SORT, 306)
+ Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 338), Reducer 27 (PARTITION-LEVEL SORT, 338)
Reducer 29 <- Map 34 (PARTITION-LEVEL SORT, 393), Reducer 28 (PARTITION-LEVEL SORT, 393)
- Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 336), Reducer 2 (PARTITION-LEVEL SORT, 336)
+ Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 338), Reducer 2 (PARTITION-LEVEL SORT, 338)
Reducer 30 <- Reducer 29 (GROUP, 934), Reducer 38 (GROUP, 934)
Reducer 31 <- Reducer 30 (GROUP PARTITION-LEVEL SORT, 671), Reducer 45 (GROUP PARTITION-LEVEL SORT, 671)
- Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 400), Map 35 (PARTITION-LEVEL SORT, 400)
- Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 438), Reducer 36 (PARTITION-LEVEL SORT, 438)
+ Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 398), Map 35 (PARTITION-LEVEL SORT, 398)
+ Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 440), Reducer 36 (PARTITION-LEVEL SORT, 440)
Reducer 38 <- Map 41 (PARTITION-LEVEL SORT, 516), Reducer 37 (PARTITION-LEVEL SORT, 516)
Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 393), Reducer 3 (PARTITION-LEVEL SORT, 393)
- Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 156), Map 42 (PARTITION-LEVEL SORT, 156)
- Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 169), Reducer 43 (PARTITION-LEVEL SORT, 169)
+ Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 154), Map 42 (PARTITION-LEVEL SORT, 154)
+ Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 171), Reducer 43 (PARTITION-LEVEL SORT, 171)
Reducer 45 <- Map 48 (PARTITION-LEVEL SORT, 196), Reducer 44 (PARTITION-LEVEL SORT, 196)
Reducer 5 <- Reducer 15 (GROUP, 934), Reducer 4 (GROUP, 934)
Reducer 6 <- Reducer 22 (GROUP PARTITION-LEVEL SORT, 671), Reducer 5 (GROUP PARTITION-LEVEL SORT, 671)
@@ -183,28 +183,29 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 10
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((i_category = 'Sports') and i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null and i_manufact_id is not null) (type: boolean)
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ expressions: i_item_sk (type: int), i_brand_id (type: int), i_class_id (type: int), i_category_id (type: int), i_manufact_id (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col5
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: int)
Map 11
Map Operator Tree:
TableScan
@@ -236,29 +237,28 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 16
Map Operator Tree:
TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((i_category = 'Sports') and i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null and i_manufact_id is not null) (type: boolean)
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: i_item_sk (type: int), i_brand_id (type: int), i_class_id (type: int), i_category_id (type: int), i_manufact_id (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col5
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: int)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Map 18
Map Operator Tree:
TableScan
@@ -290,11 +290,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 25
Map Operator Tree:
TableScan
@@ -326,46 +326,46 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 32
Map Operator Tree:
TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((i_category = 'Sports') and i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null and i_manufact_id is not null) (type: boolean)
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: i_item_sk (type: int), i_brand_id (type: int), i_class_id (type: int), i_category_id (type: int), i_manufact_id (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col5
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: int)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Map 33
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((i_category = 'Sports') and i_brand_id is not null and i_category_id is not null and i_class_id is not null and i_item_sk is not null and i_manufact_id is not null) (type: boolean)
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ expressions: i_item_sk (type: int), i_brand_id (type: int), i_class_id (type: int), i_category_id (type: int), i_manufact_id (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col5
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: int)
Map 34
Map Operator Tree:
TableScan
@@ -397,11 +397,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 41
Map Operator Tree:
TableScan
@@ -433,11 +433,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 48
Map Operator Tree:
TableScan
@@ -462,41 +462,41 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Reducer 14
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col12
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int), _col2 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int)
Reducer 15
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join 0 to 1
- outputColumnNames: _col3, _col4, _col6, _col7, _col8, _col10, _col15, _col16
+ outputColumnNames: _col3, _col4, _col8, _col9, _col10, _col12, _col15, _col16
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
+ expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -515,57 +515,57 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Reducer 20
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Reducer 21
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col12
Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int), _col2 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int)
Reducer 22
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join 0 to 1
- outputColumnNames: _col3, _col4, _col6, _col7, _col8, _col10, _col15, _col16
+ outputColumnNames: _col3, _col4, _col8, _col9, _col10, _col12, _col15, _col16
Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
+ expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -584,41 +584,41 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Reducer 28
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col12
Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int), _col2 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int)
Reducer 29
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join 0 to 1
- outputColumnNames: _col3, _col4, _col6, _col7, _col8, _col10, _col15, _col16
+ outputColumnNames: _col3, _col4, _col8, _col9, _col10, _col12, _col15, _col16
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
+ expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -637,16 +637,16 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col12
Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int), _col2 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int)
Reducer 30
Reduce Operator Tree:
Group By Operator
@@ -689,41 +689,41 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Reducer 37
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col12
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int), _col2 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int)
Reducer 38
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join 0 to 1
- outputColumnNames: _col3, _col4, _col6, _col7, _col8, _col10, _col15, _col16
+ outputColumnNames: _col3, _col4, _col8, _col9, _col10, _col12, _col15, _col16
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
+ expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -741,10 +741,10 @@ STAGE PLANS:
Join Operator
condition map:
Left Outer Join 0 to 1
- outputColumnNames: _col3, _col4, _col6, _col7, _col8, _col10, _col15, _col16
+ outputColumnNames: _col3, _col4, _col8, _col9, _col10, _col12, _col15, _col16
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
+ expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -763,41 +763,41 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Reducer 44
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col10
+ outputColumnNames: _col1, _col2, _col3, _col4, _col8, _col9, _col10, _col12
Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int), _col2 (type: int)
sort order: ++
Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int)
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int)
Reducer 45
Reduce Operator Tree:
Join Operator
condition map:
Left Outer Join 0 to 1
- outputColumnNames: _col3, _col4, _col6, _col7, _col8, _col10, _col15, _col16
+ outputColumnNames: _col3, _col4, _col8, _col9, _col10, _col12, _col15, _col16
Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col6 (type: int), _col7 (type: int), _col8 (type: int), _col10 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
+ expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int), _col12 (type: int), (_col3 - COALESCE(_col15,0)) (type: int), (_col4 - COALESCE(_col16,0)) (type: decimal(8,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query76.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query76.q.out b/ql/src/test/results/clientpositive/perf/spark/query76.q.out
index 5e6b208..0e18f35 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query76.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query76.q.out
@@ -56,10 +56,10 @@ STAGE PLANS:
Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 158), Map 16 (PARTITION-LEVEL SORT, 158)
Reducer 15 <- Map 17 (PARTITION-LEVEL SORT, 169), Reducer 14 (PARTITION-LEVEL SORT, 169)
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 204), Map 6 (PARTITION-LEVEL SORT, 204)
- Reducer 3 <- Map 12 (PARTITION-LEVEL SORT, 219), Reducer 2 (PARTITION-LEVEL SORT, 219)
+ Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 219), Reducer 2 (PARTITION-LEVEL SORT, 219)
Reducer 4 <- Reducer 10 (GROUP, 518), Reducer 15 (GROUP, 518), Reducer 3 (GROUP, 518)
Reducer 5 <- Reducer 4 (SORT, 1)
- Reducer 9 <- Map 1 (PARTITION-LEVEL SORT, 82), Map 11 (PARTITION-LEVEL SORT, 82)
+ Reducer 9 <- Map 11 (PARTITION-LEVEL SORT, 82), Map 8 (PARTITION-LEVEL SORT, 82)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -83,21 +83,21 @@ STAGE PLANS:
Map 11
Map Operator Tree:
TableScan
- alias: web_sales
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (ws_item_sk is not null and ws_sold_date_sk is not null and ws_web_page_sk is null) (type: boolean)
- Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE
+ predicate: i_item_sk is not null (type: boolean)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_sales_price (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE
+ expressions: i_item_sk (type: int), i_category (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col3 (type: decimal(7,2))
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 12
Map Operator Tree:
TableScan
@@ -119,24 +119,6 @@ STAGE PLANS:
Map 13
Map Operator Tree:
TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: i_item_sk is not null (type: boolean)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: i_item_sk (type: int), i_category (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Map 16
- Map Operator Tree:
- TableScan
alias: catalog_sales
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -152,6 +134,24 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 143994918 Data size: 19499804476 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col3 (type: decimal(7,2))
+ Map 16
+ Map Operator Tree:
+ TableScan
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: i_item_sk is not null (type: boolean)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: i_item_sk (type: int), i_category (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Map 17
Map Operator Tree:
TableScan
@@ -188,15 +188,51 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 287997817 Data size: 25407250999 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col3 (type: decimal(7,2))
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: d_date_sk is not null (type: boolean)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int), d_year (type: int), d_qoy (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int)
+ Map 8
+ Map Operator Tree:
+ TableScan
+ alias: web_sales
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (ws_item_sk is not null and ws_sold_date_sk is not null and ws_web_page_sk is null) (type: boolean)
+ Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_sales_price (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col3
+ Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 72001334 Data size: 9790099106 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col3 (type: decimal(7,2))
Reducer 10
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
- outputColumnNames: _col1, _col5, _col7, _col8
+ outputColumnNames: _col3, _col5, _col7, _col8
Statistics: Num rows: 87121617 Data size: 11846020431 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: 'web' (type: string), 'ws_web_page_sk' (type: string), _col7 (type: int), _col8 (type: int), _col1 (type: string), _col5 (type: decimal(7,2))
+ expressions: 'web' (type: string), 'ws_web_page_sk' (type: string), _col7 (type: int), _col8 (type: int), _col5 (type: string), _col3 (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 87121617 Data size: 11846020431 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -218,25 +254,25 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col2 (type: int)
- outputColumnNames: _col1, _col2, _col5
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col3, _col5
Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col5 (type: decimal(7,2))
+ value expressions: _col3 (type: decimal(7,2)), _col5 (type: string)
Reducer 15
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
- outputColumnNames: _col1, _col5, _col7, _col8
+ outputColumnNames: _col3, _col5, _col7, _col8
Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: 'catalog' (type: string), 'cs_warehouse_sk' (type: string), _col7 (type: int), _col8 (type: int), _col1 (type: string), _col5 (type: decimal(7,2))
+ expressions: 'catalog' (type: string), 'cs_warehouse_sk' (type: string), _col7 (type: int), _col8 (type: int), _col5 (type: string), _col3 (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -328,16 +364,16 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col1, _col2, _col5
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col3, _col5
Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col5 (type: decimal(7,2))
+ value expressions: _col3 (type: decimal(7,2)), _col5 (type: string)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query77.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query77.q.out b/ql/src/test/results/clientpositive/perf/spark/query77.q.out
index f30af9f..638bbd9 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query77.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query77.q.out
@@ -421,11 +421,11 @@ STAGE PLANS:
Reducer 11 <- Reducer 10 (GROUP, 43)
Reducer 15 <- Map 14 (GROUP, 336)
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398)
- Reducer 21 <- Map 12 (PARTITION-LEVEL SORT, 11), Map 20 (PARTITION-LEVEL SORT, 11)
- Reducer 22 <- Reducer 21 (GROUP, 13)
+ Reducer 21 <- Map 12 (PARTITION-LEVEL SORT, 154), Map 20 (PARTITION-LEVEL SORT, 154)
+ Reducer 22 <- Reducer 21 (GROUP, 186)
Reducer 23 <- Reducer 22 (PARTITION-LEVEL SORT, 99), Reducer 28 (PARTITION-LEVEL SORT, 99)
- Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 154), Map 29 (PARTITION-LEVEL SORT, 154)
- Reducer 28 <- Reducer 27 (GROUP, 186)
+ Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 11), Map 29 (PARTITION-LEVEL SORT, 11)
+ Reducer 28 <- Reducer 27 (GROUP, 13)
Reducer 3 <- Reducer 2 (GROUP, 481)
Reducer 4 <- Reducer 11 (PARTITION-LEVEL SORT, 262), Reducer 3 (PARTITION-LEVEL SORT, 262)
Reducer 5 <- Reducer 15 (GROUP, 1009), Reducer 23 (GROUP, 1009), Reducer 4 (GROUP, 1009)
@@ -506,38 +506,38 @@ STAGE PLANS:
Map 20
Map Operator Tree:
TableScan
- alias: web_returns
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ alias: web_sales
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (wr_returned_date_sk is not null and wr_web_page_sk is not null) (type: boolean)
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ predicate: (ws_sold_date_sk is not null and ws_web_page_sk is not null) (type: boolean)
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: wr_returned_date_sk (type: int), wr_web_page_sk (type: int), wr_return_amt (type: decimal(7,2)), wr_net_loss (type: decimal(7,2))
+ expressions: ws_sold_date_sk (type: int), ws_web_page_sk (type: int), ws_ext_sales_price (type: decimal(7,2)), ws_net_profit (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2))
Map 26
Map Operator Tree:
TableScan
- alias: web_sales
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ alias: web_returns
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (ws_sold_date_sk is not null and ws_web_page_sk is not null) (type: boolean)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ predicate: (wr_returned_date_sk is not null and wr_web_page_sk is not null) (type: boolean)
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ws_sold_date_sk (type: int), ws_web_page_sk (type: int), ws_ext_sales_price (type: decimal(7,2)), ws_net_profit (type: decimal(7,2))
+ expressions: wr_returned_date_sk (type: int), wr_web_page_sk (type: int), wr_return_amt (type: decimal(7,2)), wr_net_loss (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2))
Map 29
Map Operator Tree:
@@ -721,7 +721,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3
- Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -731,18 +731,18 @@ STAGE PLANS:
outputColumnNames: _col2, _col3, _col6
input vertices:
1 Map 25
- Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col2), sum(_col3)
keys: _col6 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2))
Reducer 22
Reduce Operator Tree:
@@ -751,22 +751,22 @@ STAGE PLANS:
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 8711072 Data size: 801742469 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 8711072 Data size: 801742469 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2))
Reducer 23
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join 0 to 1
- outputColumnNames: _col1, _col2, _col3, _col4, _col5
+ Left Outer Join 0 to 1
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5
Statistics: Num rows: 95833780 Data size: 13030622681 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: 'web channel' (type: string), _col3 (type: int), _col4 (type: decimal(17,2)), COALESCE(_col1,0) (type: decimal(17,2)), (_col5 - COALESCE(_col2,0)) (type: decimal(18,2))
+ expressions: 'web channel' (type: string), _col0 (type: int), _col1 (type: decimal(17,2)), COALESCE(_col4,0) (type: decimal(17,2)), (_col2 - COALESCE(_col5,0)) (type: decimal(18,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 95833780 Data size: 13030622681 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -793,7 +793,7 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -803,18 +803,18 @@ STAGE PLANS:
outputColumnNames: _col2, _col3, _col6
input vertices:
1 Map 30
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col2), sum(_col3)
keys: _col6 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2))
Reducer 28
Reduce Operator Tree:
@@ -823,12 +823,12 @@ STAGE PLANS:
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8711072 Data size: 801742469 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8711072 Data size: 801742469 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(17,2))
Reducer 3
Reduce Operator Tree:
[03/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query81.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query81.q.out b/ql/src/test/results/clientpositive/perf/spark/query81.q.out
index 23998d6..5b2d5b3 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query81.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query81.q.out
@@ -66,37 +66,72 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 25), Map 14 (PARTITION-LEVEL SORT, 25)
- Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 344), Reducer 11 (PARTITION-LEVEL SORT, 344)
- Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349)
- Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 (PARTITION-LEVEL SORT, 697)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 25), Map 8 (PARTITION-LEVEL SORT, 25)
- Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 344), Reducer 2 (PARTITION-LEVEL SORT, 344)
- Reducer 4 <- Reducer 3 (GROUP, 349)
- Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 (PARTITION-LEVEL SORT, 262)
- Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 (PARTITION-LEVEL SORT, 656)
- Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 (PARTITION-LEVEL SORT, 262)
+ Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 25), Map 17 (PARTITION-LEVEL SORT, 25)
+ Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 344), Reducer 14 (PARTITION-LEVEL SORT, 344)
+ Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL SORT, 697)
+ Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 (PARTITION-LEVEL SORT, 656)
+ Reducer 4 <- Reducer 3 (SORT, 1)
+ Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 25), Map 6 (PARTITION-LEVEL SORT, 25)
+ Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 344), Reducer 7 (PARTITION-LEVEL SORT, 344)
+ Reducer 9 <- Reducer 8 (GROUP, 349)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: catalog_returns
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ alias: customer
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cr_returned_date_sk is not null and cr_returning_addr_sk is not null and cr_returning_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cr_returned_date_sk (type: int), cr_returning_customer_sk (type: int), cr_returning_addr_sk (type: int), cr_return_amt_inc_tax (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
+ Map 11
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((d_year = 1998) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
- Map 10
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Map 12
+ Map Operator Tree:
+ TableScan
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ca_address_sk is not null (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ca_address_sk (type: int), ca_state (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 13
Map Operator Tree:
TableScan
alias: catalog_returns
@@ -114,7 +149,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
- Map 14
+ Map 17
Map Operator Tree:
TableScan
alias: date_dim
@@ -131,7 +166,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Map 15
+ Map 18
Map Operator Tree:
TableScan
alias: customer_address
@@ -149,25 +184,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 16
- Map Operator Tree:
- TableScan
- alias: customer
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
- Map 18
+ Map 5
Map Operator Tree:
TableScan
alias: customer_address
@@ -185,42 +202,48 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: decimal(5,2)), _col11 (type: string)
- Map 8
+ Map 6
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_year = 1998) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Map 9
- Map Operator Tree:
- TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ alias: catalog_returns
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ca_address_sk is not null (type: boolean)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ predicate: (cr_returned_date_sk is not null and cr_returning_addr_sk is not null and cr_returning_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_state (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ expressions: cr_returned_date_sk (type: int), cr_returning_customer_sk (type: int), cr_returning_addr_sk (type: int), cr_return_amt_inc_tax (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Reducer 11
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
+ Reducer 10
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col2 (type: string)
+ outputColumnNames: _col0, _col2, _col3, _col4
+ Statistics: Num rows: 24200000 Data size: 24560094211 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END) (type: boolean)
+ Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col2 (type: decimal(17,2))
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(17,2))
+ Reducer 14
Reduce Operator Tree:
Join Operator
condition map:
@@ -236,7 +259,7 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 31678769 Data size: 3362958220 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: decimal(7,2))
- Reducer 12
+ Reducer 15
Reduce Operator Tree:
Join Operator
condition map:
@@ -258,7 +281,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 44000000 Data size: 44654715780 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(17,2))
- Reducer 13
+ Reducer 16
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -286,7 +309,7 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: string)
Statistics: Num rows: 11000000 Data size: 11163678945 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(38,11)), _col1 (type: boolean)
- Reducer 17
+ Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
@@ -302,7 +325,46 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: decimal(5,2)), _col17 (type: string)
- Reducer 2
+ Reducer 3
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3, _col4, _col5, _col7, _col8, _col9, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col20
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: decimal(5,2)), _col17 (type: string), _col20 (type: decimal(17,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: decimal(5,2)), _col13 (type: string), _col14 (type: decimal(17,2))
+ sort order: +++++++++++++++
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: string), KEY.reducesinkkey7 (type: string), KEY.reducesinkkey8 (type: string), KEY.reducesinkkey9 (type: string), KEY.reducesinkkey10 (type: string), KEY.reducesinkkey11 (type: string), KEY.reducesinkkey12 (type: decimal(5,2)), KEY.reducesinkkey13 (type: string), KEY.reducesinkkey14 (type: decimal(17,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 100
+ Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), 'IL' (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: decimal(5,2)), _col13 (type: string), _col14 (type: decimal(17,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
+ Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 7
Reduce Operator Tree:
Join Operator
condition map:
@@ -318,7 +380,7 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 31678769 Data size: 3362958220 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: decimal(7,2))
- Reducer 3
+ Reducer 8
Reduce Operator Tree:
Join Operator
condition map:
@@ -340,7 +402,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
Statistics: Num rows: 44000000 Data size: 44654715780 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(17,2))
- Reducer 4
+ Reducer 9
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -358,64 +420,6 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 22000000 Data size: 22327357890 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: decimal(17,2))
- Reducer 5
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: string)
- 1 _col2 (type: string)
- outputColumnNames: _col0, _col2, _col3, _col4
- Statistics: Num rows: 24200000 Data size: 24560094211 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END) (type: boolean)
- Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Reducer 6
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col7, _col9, _col10, _col11, _col13, _col14, _col15, _col16, _col17, _col18, _col20, _col21, _col22, _col23
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col7 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: decimal(5,2)), _col23 (type: string), _col2 (type: decimal(17,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: decimal(5,2)), _col13 (type: string), _col14 (type: decimal(17,2))
- sort order: +++++++++++++++
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- Reducer 7
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: string), KEY.reducesinkkey7 (type: string), KEY.reducesinkkey8 (type: string), KEY.reducesinkkey9 (type: string), KEY.reducesinkkey10 (type: string), KEY.reducesinkkey11 (type: string), KEY.reducesinkkey12 (type: decimal(5,2)), KEY.reducesinkkey13 (type: string), KEY.reducesinkkey14 (type: decimal(17,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 100
- Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string), 'IL' (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: decimal(5,2)), _col13 (type: string), _col14 (type: decimal(17,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query82.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query82.q.out b/ql/src/test/results/clientpositive/perf/spark/query82.q.out
index f734e87..fb30019 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query82.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query82.q.out
@@ -40,7 +40,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 6
+ Map 7
Map Operator Tree:
TableScan
alias: date_dim
@@ -62,7 +62,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 5 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 5 (PARTITION-LEVEL SORT, 398), Map 6 (PARTITION-LEVEL SORT, 398)
Reducer 3 <- Reducer 2 (GROUP, 874)
Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
@@ -102,7 +102,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 25666 Data size: 36863512 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: decimal(7,2))
- Map 7
+ Map 6
Map Operator Tree:
TableScan
alias: inventory
@@ -120,14 +120,14 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col3
+ outputColumnNames: _col1
input vertices:
- 0 Map 6
+ 1 Map 7
Statistics: Num rows: 4593600 Data size: 72578135 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col3 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 4593600 Data size: 72578135 Basic stats: COMPLETE Column stats: NONE
Local Work:
Map Reduce Local Work
@@ -140,7 +140,7 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- 2 _col3 (type: int)
+ 2 _col1 (type: int)
outputColumnNames: _col2, _col3, _col4
Statistics: Num rows: 1267190424 Data size: 111791907016 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query83.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query83.q.out b/ql/src/test/results/clientpositive/perf/spark/query83.q.out
index d2d91e4..614ae49 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query83.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query83.q.out
@@ -138,47 +138,65 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Reducer 9 (GROUP, 2)
- Reducer 15 <- Map 14 (PARTITION-LEVEL SORT, 2), Reducer 21 (PARTITION-LEVEL SORT, 2)
- Reducer 16 <- Map 23 (PARTITION-LEVEL SORT, 36), Reducer 15 (PARTITION-LEVEL SORT, 36)
- Reducer 17 <- Map 24 (PARTITION-LEVEL SORT, 44), Reducer 16 (PARTITION-LEVEL SORT, 44)
- Reducer 18 <- Reducer 17 (GROUP, 43)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 10 (PARTITION-LEVEL SORT, 2)
- Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 2), Map 22 (PARTITION-LEVEL SORT, 2)
- Reducer 21 <- Reducer 20 (GROUP, 2)
- Reducer 26 <- Map 25 (PARTITION-LEVEL SORT, 2), Reducer 32 (PARTITION-LEVEL SORT, 2)
- Reducer 27 <- Map 34 (PARTITION-LEVEL SORT, 12), Reducer 26 (PARTITION-LEVEL SORT, 12)
- Reducer 28 <- Map 35 (PARTITION-LEVEL SORT, 17), Reducer 27 (PARTITION-LEVEL SORT, 17)
- Reducer 29 <- Reducer 28 (GROUP, 13)
- Reducer 3 <- Map 12 (PARTITION-LEVEL SORT, 25), Reducer 2 (PARTITION-LEVEL SORT, 25)
- Reducer 31 <- Map 30 (PARTITION-LEVEL SORT, 2), Map 33 (PARTITION-LEVEL SORT, 2)
- Reducer 32 <- Reducer 31 (GROUP, 2)
- Reducer 4 <- Map 13 (PARTITION-LEVEL SORT, 32), Reducer 3 (PARTITION-LEVEL SORT, 32)
- Reducer 5 <- Reducer 4 (GROUP, 29)
- Reducer 6 <- Reducer 18 (PARTITION-LEVEL SORT, 42), Reducer 29 (PARTITION-LEVEL SORT, 42), Reducer 5 (PARTITION-LEVEL SORT, 42)
- Reducer 7 <- Reducer 6 (SORT, 1)
- Reducer 9 <- Map 11 (PARTITION-LEVEL SORT, 2), Map 8 (PARTITION-LEVEL SORT, 2)
+ Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 2), Map 13 (PARTITION-LEVEL SORT, 2)
+ Reducer 12 <- Reducer 11 (GROUP, 2)
+ Reducer 15 <- Map 14 (PARTITION-LEVEL SORT, 41), Map 18 (PARTITION-LEVEL SORT, 41)
+ Reducer 16 <- Reducer 15 (PARTITION-LEVEL SORT, 40), Reducer 20 (PARTITION-LEVEL SORT, 40)
+ Reducer 17 <- Reducer 16 (GROUP, 43)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 30), Map 7 (PARTITION-LEVEL SORT, 30)
+ Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 2), Reducer 23 (PARTITION-LEVEL SORT, 2)
+ Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 2), Map 24 (PARTITION-LEVEL SORT, 2)
+ Reducer 23 <- Reducer 22 (GROUP, 2)
+ Reducer 26 <- Map 25 (PARTITION-LEVEL SORT, 16), Map 29 (PARTITION-LEVEL SORT, 16)
+ Reducer 27 <- Reducer 26 (PARTITION-LEVEL SORT, 13), Reducer 31 (PARTITION-LEVEL SORT, 13)
+ Reducer 28 <- Reducer 27 (GROUP, 13)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 27), Reducer 9 (PARTITION-LEVEL SORT, 27)
+ Reducer 31 <- Map 30 (PARTITION-LEVEL SORT, 2), Reducer 34 (PARTITION-LEVEL SORT, 2)
+ Reducer 33 <- Map 32 (PARTITION-LEVEL SORT, 2), Map 35 (PARTITION-LEVEL SORT, 2)
+ Reducer 34 <- Reducer 33 (GROUP, 2)
+ Reducer 4 <- Reducer 3 (GROUP, 29)
+ Reducer 5 <- Reducer 17 (PARTITION-LEVEL SORT, 42), Reducer 28 (PARTITION-LEVEL SORT, 42), Reducer 4 (PARTITION-LEVEL SORT, 42)
+ Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 9 <- Map 8 (PARTITION-LEVEL SORT, 2), Reducer 12 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
+ alias: catalog_returns
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cr_item_sk is not null and cr_returned_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cr_returned_date_sk (type: int), cr_item_sk (type: int), cr_return_quantity (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: int)
+ Map 10
+ Map Operator Tree:
+ TableScan
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (d_date is not null and d_date_sk is not null) (type: boolean)
+ predicate: (d_date is not null and d_week_seq is not null) (type: boolean)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int), d_date (type: string)
+ expressions: d_date (type: string), d_week_seq (type: int)
outputColumnNames: _col0, _col1
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: string)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: string)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
- Map 11
+ value expressions: _col0 (type: string)
+ Map 13
Map Operator Tree:
TableScan
alias: date_dim
@@ -200,25 +218,25 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
- Map 12
+ Map 14
Map Operator Tree:
TableScan
- alias: catalog_returns
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ alias: store_returns
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cr_item_sk is not null and cr_returned_date_sk is not null) (type: boolean)
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ predicate: (sr_item_sk is not null and sr_returned_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cr_returned_date_sk (type: int), cr_item_sk (type: int), cr_return_quantity (type: int)
+ expressions: sr_returned_date_sk (type: int), sr_item_sk (type: int), sr_return_quantity (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int)
- Map 13
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: int)
+ Map 18
Map Operator Tree:
TableScan
alias: item
@@ -236,7 +254,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 14
+ Map 19
Map Operator Tree:
TableScan
alias: date_dim
@@ -254,7 +272,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
- Map 19
+ Map 21
Map Operator Tree:
TableScan
alias: date_dim
@@ -272,7 +290,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Map 22
+ Map 24
Map Operator Tree:
TableScan
alias: date_dim
@@ -294,25 +312,25 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
- Map 23
+ Map 25
Map Operator Tree:
TableScan
- alias: store_returns
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ alias: web_returns
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (sr_item_sk is not null and sr_returned_date_sk is not null) (type: boolean)
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ predicate: (wr_item_sk is not null and wr_returned_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: sr_returned_date_sk (type: int), sr_item_sk (type: int), sr_return_quantity (type: int)
+ expressions: wr_returned_date_sk (type: int), wr_item_sk (type: int), wr_return_quantity (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int)
- Map 24
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: int)
+ Map 29
Map Operator Tree:
TableScan
alias: item
@@ -330,7 +348,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 25
+ Map 30
Map Operator Tree:
TableScan
alias: date_dim
@@ -348,7 +366,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
- Map 30
+ Map 32
Map Operator Tree:
TableScan
alias: date_dim
@@ -366,7 +384,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Map 33
+ Map 35
Map Operator Tree:
TableScan
alias: date_dim
@@ -388,25 +406,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
- Map 34
- Map Operator Tree:
- TableScan
- alias: web_returns
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (wr_item_sk is not null and wr_returned_date_sk is not null) (type: boolean)
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: wr_returned_date_sk (type: int), wr_item_sk (type: int), wr_return_quantity (type: int)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int)
- Map 35
+ Map 7
Map Operator Tree:
TableScan
alias: item
@@ -430,19 +430,39 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (d_date is not null and d_week_seq is not null) (type: boolean)
+ predicate: (d_date is not null and d_date_sk is not null) (type: boolean)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date (type: string), d_week_seq (type: int)
+ expressions: d_date_sk (type: int), d_date (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col1 (type: string)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string)
- Reducer 10
+ value expressions: _col0 (type: int)
+ Reducer 11
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Reducer 12
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
@@ -460,15 +480,16 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col4
+ Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col4 (type: string)
Reducer 16
Reduce Operator Tree:
Join Operator
@@ -477,27 +498,11 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col4, _col5
- Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: int)
- Reducer 17
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col5, _col7
+ outputColumnNames: _col2, _col4
Statistics: Num rows: 69685294 Data size: 5399255980 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col5)
- keys: _col7 (type: string)
+ aggregations: sum(_col2)
+ keys: _col4 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 69685294 Data size: 5399255980 Basic stats: COMPLETE Column stats: NONE
@@ -507,7 +512,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 69685294 Data size: 5399255980 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 18
+ Reducer 17
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -527,6 +532,22 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col4
+ Statistics: Num rows: 31678769 Data size: 3362958220 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 31678769 Data size: 3362958220 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col4 (type: string)
+ Reducer 20
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
0 _col1 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0
@@ -536,7 +557,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reducer 20
+ Reducer 22
Reduce Operator Tree:
Join Operator
condition map:
@@ -556,7 +577,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reducer 21
+ Reducer 23
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
@@ -574,15 +595,16 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col4
+ Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col4 (type: string)
Reducer 27
Reduce Operator Tree:
Join Operator
@@ -591,27 +613,11 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col4, _col5
- Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: int)
- Reducer 28
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col5, _col7
+ outputColumnNames: _col2, _col4
Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col5)
- keys: _col7 (type: string)
+ aggregations: sum(_col2)
+ keys: _col4 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
@@ -621,7 +627,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 17422145 Data size: 1603485031 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 29
+ Reducer 28
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -643,15 +649,36 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col4, _col5
- Statistics: Num rows: 31678769 Data size: 3362958220 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col2, _col4
+ Statistics: Num rows: 34846646 Data size: 3699254122 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col2)
+ keys: _col4 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 34846646 Data size: 3699254122 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 34846646 Data size: 3699254122 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 31
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col4 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 31678769 Data size: 3362958220 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: int)
- Reducer 31
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Reducer 33
Reduce Operator Tree:
Join Operator
condition map:
@@ -671,7 +698,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reducer 32
+ Reducer 34
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
@@ -685,28 +712,6 @@ STAGE PLANS:
Statistics: Num rows: 40176 Data size: 44957448 Basic stats: COMPLETE Column stats: NONE
Reducer 4
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col5, _col7
- Statistics: Num rows: 34846646 Data size: 3699254122 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col5)
- keys: _col7 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 34846646 Data size: 3699254122 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 34846646 Data size: 3699254122 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
- Reducer 5
- Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string)
@@ -719,7 +724,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 17423323 Data size: 1849627061 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 6
+ Reducer 5
Reduce Operator Tree:
Join Operator
condition map:
@@ -741,7 +746,7 @@ STAGE PLANS:
Statistics: Num rows: 76653825 Data size: 5939181706 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: double), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: decimal(25,6))
- Reducer 7
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: double), VALUE._col1 (type: bigint), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: decimal(25,6))
@@ -761,22 +766,17 @@ STAGE PLANS:
Reduce Operator Tree:
Join Operator
condition map:
- Left Semi Join 0 to 1
+ Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
[11/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query29.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query29.q.out b/ql/src/test/results/clientpositive/perf/spark/query29.q.out
index 456fce7..530a625 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query29.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query29.q.out
@@ -124,7 +124,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 14
+ Map 13
Map Operator Tree:
TableScan
alias: d2
@@ -146,51 +146,51 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Map 12 (PARTITION-LEVEL SORT, 306), Map 9 (PARTITION-LEVEL SORT, 306)
- Reducer 11 <- Map 13 (PARTITION-LEVEL SORT, 374), Reducer 10 (PARTITION-LEVEL SORT, 374)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398)
- Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 442), Reducer 2 (PARTITION-LEVEL SORT, 442)
- Reducer 4 <- Reducer 11 (PARTITION-LEVEL SORT, 850), Reducer 3 (PARTITION-LEVEL SORT, 850)
- Reducer 5 <- Reducer 4 (GROUP, 582)
- Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 10 <- Map 14 (PARTITION-LEVEL SORT, 486), Reducer 9 (PARTITION-LEVEL SORT, 486)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 6 (PARTITION-LEVEL SORT, 306)
+ Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 917), Reducer 2 (PARTITION-LEVEL SORT, 917)
+ Reducer 4 <- Reducer 3 (GROUP, 640)
+ Reducer 5 <- Reducer 4 (SORT, 1)
+ Reducer 8 <- Map 11 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398)
+ Reducer 9 <- Map 12 (PARTITION-LEVEL SORT, 476), Reducer 8 (PARTITION-LEVEL SORT, 476)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: store_sales
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ alias: catalog_sales
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null) (type: boolean)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ predicate: (cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_item_sk (type: int), cs_quantity (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int)
- Map 12
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ Map 11
Map Operator Tree:
TableScan
- alias: d3
+ alias: d1
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year) IN (1999, 2000, 2001) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_moy = 4) and (d_year = 1999) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: d_date_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
- Map 13
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Map 12
Map Operator Tree:
TableScan
alias: store_returns
@@ -210,34 +210,17 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col4
input vertices:
- 1 Map 14
+ 1 Map 13
Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int), _col2 (type: int)
- sort order: ++
- Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
+ key expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col3 (type: int)
Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: int)
+ value expressions: _col4 (type: int)
Local Work:
Map Reduce Local Work
- Map 7
- Map Operator Tree:
- TableScan
- alias: d1
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_moy = 4) and (d_year = 1999) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 14
Map Operator Tree:
TableScan
alias: item
@@ -255,56 +238,73 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: string)
- Map 9
+ Map 6
Map Operator Tree:
TableScan
- alias: catalog_sales
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ alias: d3
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_year) IN (1999, 2000, 2001) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_item_sk (type: int), cs_quantity (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: store_sales
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null) (type: boolean)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int)
Reducer 10
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col2 (type: int), _col1 (type: int)
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int)
- Reducer 11
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int), _col1 (type: int)
- 1 _col1 (type: int), _col2 (type: int)
- outputColumnNames: _col3, _col7, _col8, _col9, _col10
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col7 (type: int), _col8 (type: int), _col9 (type: int)
- sort order: +++
- Map-reduce partition columns: _col7 (type: int), _col8 (type: int), _col9 (type: int)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col10 (type: int)
+ outputColumnNames: _col3, _col5, _col10, _col11, _col13, _col18, _col19
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col5, _col10, _col11, _col13, _col18, _col19, _col21, _col22
+ input vertices:
+ 1 Map 15
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col18 (type: string), _col19 (type: string), _col5 (type: int), _col10 (type: int), _col11 (type: int), _col13 (type: int), _col21 (type: string), _col22 (type: string)
+ outputColumnNames: _col1, _col2, _col8, _col13, _col14, _col16, _col21, _col22
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col14 (type: int), _col13 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col14 (type: int), _col13 (type: int)
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string), _col8 (type: int), _col16 (type: int), _col21 (type: string), _col22 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -313,85 +313,57 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col1, _col2, _col3
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int)
+ key expressions: _col1 (type: int), _col2 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: int)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col10, _col11
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int)
- sort order: +++
- Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col4 (type: int)
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: int), _col10 (type: string), _col11 (type: string)
+ 0 _col1 (type: int), _col2 (type: int)
+ 1 _col14 (type: int), _col13 (type: int)
+ outputColumnNames: _col3, _col7, _col8, _col14, _col22, _col27, _col28
+ Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col14), sum(_col22), sum(_col3)
+ keys: _col7 (type: string), _col8 (type: string), _col27 (type: string), _col28 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ sort order: ++++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint)
Reducer 4
- Local Work:
- Map Reduce Local Work
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: int), _col2 (type: int), _col4 (type: int)
- 1 _col7 (type: int), _col8 (type: int), _col9 (type: int)
- outputColumnNames: _col3, _col5, _col10, _col11, _col15, _col22
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col3 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col5, _col10, _col11, _col15, _col22, _col27, _col28
- input vertices:
- 1 Map 15
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col5), sum(_col22), sum(_col15)
- keys: _col10 (type: string), _col11 (type: string), _col27 (type: string), _col28 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
- sort order: ++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint)
- Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
sort order: ++++
- Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint)
- Reducer 6
+ Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: bigint), VALUE._col2 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE
@@ -402,6 +374,38 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 8
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: int), _col5 (type: int)
+ Reducer 9
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ 1 _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ outputColumnNames: _col1, _col3, _col5, _col10, _col11, _col13
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: int), _col5 (type: int), _col10 (type: int), _col11 (type: int), _col13 (type: int)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query30.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query30.q.out b/ql/src/test/results/clientpositive/perf/spark/query30.q.out
index 399251d..6385984 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query30.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query30.q.out
@@ -66,37 +66,72 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 11), Map 14 (PARTITION-LEVEL SORT, 11)
- Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 329), Reducer 11 (PARTITION-LEVEL SORT, 329)
- Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349)
- Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 (PARTITION-LEVEL SORT, 697)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 11), Map 8 (PARTITION-LEVEL SORT, 11)
- Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 329), Reducer 2 (PARTITION-LEVEL SORT, 329)
- Reducer 4 <- Reducer 3 (GROUP, 349)
- Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 (PARTITION-LEVEL SORT, 262)
- Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 (PARTITION-LEVEL SORT, 656)
- Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 (PARTITION-LEVEL SORT, 262)
+ Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 11), Map 17 (PARTITION-LEVEL SORT, 11)
+ Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 329), Reducer 14 (PARTITION-LEVEL SORT, 329)
+ Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL SORT, 697)
+ Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 (PARTITION-LEVEL SORT, 656)
+ Reducer 4 <- Reducer 3 (SORT, 1)
+ Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 11), Map 6 (PARTITION-LEVEL SORT, 11)
+ Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 329), Reducer 7 (PARTITION-LEVEL SORT, 329)
+ Reducer 9 <- Reducer 8 (GROUP, 349)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: web_returns
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ alias: customer
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (wr_returned_date_sk is not null and wr_returning_addr_sk is not null and wr_returning_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: wr_returned_date_sk (type: int), wr_returning_customer_sk (type: int), wr_returning_addr_sk (type: int), wr_return_amt (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string), c_preferred_cust_flag (type: string), c_birth_day (type: int), c_birth_month (type: int), c_birth_year (type: int), c_birth_country (type: string), c_login (type: string), c_email_address (type: string), c_last_review_date (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string)
+ Map 11
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
- Map 10
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Map 12
+ Map Operator Tree:
+ TableScan
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ca_address_sk is not null (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ca_address_sk (type: int), ca_state (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 13
Map Operator Tree:
TableScan
alias: web_returns
@@ -114,7 +149,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
- Map 14
+ Map 17
Map Operator Tree:
TableScan
alias: date_dim
@@ -131,7 +166,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Map 15
+ Map 18
Map Operator Tree:
TableScan
alias: customer_address
@@ -149,25 +184,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 16
- Map Operator Tree:
- TableScan
- alias: customer
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string), c_preferred_cust_flag (type: string), c_birth_day (type: int), c_birth_month (type: int), c_birth_year (type: int), c_birth_country (type: string), c_login (type: string), c_email_address (type: string), c_last_review_date (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string)
- Map 18
+ Map 5
Map Operator Tree:
TableScan
alias: customer_address
@@ -184,42 +201,48 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 6
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Map 9
- Map Operator Tree:
- TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ alias: web_returns
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ca_address_sk is not null (type: boolean)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ predicate: (wr_returned_date_sk is not null and wr_returning_addr_sk is not null and wr_returning_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_state (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ expressions: wr_returned_date_sk (type: int), wr_returning_customer_sk (type: int), wr_returning_addr_sk (type: int), wr_return_amt (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Reducer 11
+ Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
+ Reducer 10
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col2 (type: string)
+ outputColumnNames: _col0, _col2, _col3, _col4
+ Statistics: Num rows: 24200000 Data size: 24560094211 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END) (type: boolean)
+ Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col2 (type: decimal(17,2))
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(17,2))
+ Reducer 14
Reduce Operator Tree:
Join Operator
condition map:
@@ -235,7 +258,7 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: decimal(7,2))
- Reducer 12
+ Reducer 15
Reduce Operator Tree:
Join Operator
condition map:
@@ -257,7 +280,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 44000000 Data size: 44654715780 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(17,2))
- Reducer 13
+ Reducer 16
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -285,7 +308,7 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: string)
Statistics: Num rows: 11000000 Data size: 11163678945 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(38,11)), _col1 (type: boolean)
- Reducer 17
+ Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
@@ -301,7 +324,42 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string)
- Reducer 2
+ Reducer 3
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col18
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col18 (type: decimal(17,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: decimal(17,2))
+ sort order: +++++++++++++
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey6 (type: int), KEY.reducesinkkey7 (type: int), KEY.reducesinkkey8 (type: string), KEY.reducesinkkey9 (type: string), KEY.reducesinkkey10 (type: string), KEY.reducesinkkey11 (type: string), KEY.reducesinkkey12 (type: decimal(17,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 100
+ Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 7
Reduce Operator Tree:
Join Operator
condition map:
@@ -317,7 +375,7 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 15838314 Data size: 1457713633 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col3 (type: decimal(7,2))
- Reducer 3
+ Reducer 8
Reduce Operator Tree:
Join Operator
condition map:
@@ -339,7 +397,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
Statistics: Num rows: 44000000 Data size: 44654715780 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(17,2))
- Reducer 4
+ Reducer 9
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -357,60 +415,6 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 22000000 Data size: 22327357890 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: decimal(17,2))
- Reducer 5
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: string)
- 1 _col2 (type: string)
- outputColumnNames: _col0, _col2, _col3, _col4
- Statistics: Num rows: 24200000 Data size: 24560094211 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col2 > CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END) (type: boolean)
- Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 8066666 Data size: 8186697393 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Reducer 6
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col7, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col7 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: string), _col2 (type: decimal(17,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: decimal(17,2))
- sort order: +++++++++++++
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- Reducer 7
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey6 (type: int), KEY.reducesinkkey7 (type: int), KEY.reducesinkkey8 (type: string), KEY.reducesinkkey9 (type: string), KEY.reducesinkkey10 (type: string), KEY.reducesinkkey11 (type: string), KEY.reducesinkkey12 (type: decimal(17,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 100
- Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 100 Data size: 86000 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query32.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query32.q.out b/ql/src/test/results/clientpositive/perf/spark/query32.q.out
index 0ceea16..eed3a4d 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query32.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query32.q.out
@@ -63,7 +63,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 6
+ Map 4
Map Operator Tree:
TableScan
alias: date_dim
@@ -86,7 +86,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 9
+ Map 8
Map Operator Tree:
TableScan
alias: date_dim
@@ -108,10 +108,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 308), Map 5 (PARTITION-LEVEL SORT, 308)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 537), Reducer 8 (PARTITION-LEVEL SORT, 537)
- Reducer 4 <- Reducer 3 (GROUP, 1)
- Reducer 8 <- Map 7 (GROUP, 336)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 520), Reducer 7 (PARTITION-LEVEL SORT, 520)
+ Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 6 <- Map 5 (GROUP, 336)
+ Reducer 7 <- Map 9 (PARTITION-LEVEL SORT, 171), Reducer 6 (PARTITION-LEVEL SORT, 171)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -126,32 +126,27 @@ STAGE PLANS:
expressions: cs_sold_date_sk (type: int), cs_item_sk (type: int), cs_ext_discount_amt (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: decimal(7,2))
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(7,2))
+ Local Work:
+ Map Reduce Local Work
Map 5
Map Operator Tree:
TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((i_manufact_id = 269) and i_item_sk is not null) (type: boolean)
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: i_item_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- Map 7
- Map Operator Tree:
- TableScan
alias: catalog_sales
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -169,7 +164,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2
input vertices:
- 1 Map 9
+ 1 Map 8
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col2), count(_col2)
@@ -185,51 +180,40 @@ STAGE PLANS:
value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint)
Local Work:
Map Reduce Local Work
+ Map 9
+ Map Operator Tree:
+ TableScan
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((i_manufact_id = 269) and i_item_sk is not null) (type: boolean)
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: i_item_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
Reducer 2
- Local Work:
- Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col3
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col3
- input vertices:
- 1 Map 6
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(7,2))
- Reducer 3
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col3 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col2, _col7
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ 1 _col2 (type: int)
+ outputColumnNames: _col2, _col5
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col2 > _col7) (type: boolean)
- Statistics: Num rows: 127771498 Data size: 17302827584 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col2 > _col5) (type: boolean)
+ Statistics: Num rows: 116155905 Data size: 15729842913 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: decimal(7,2))
outputColumnNames: _col2
- Statistics: Num rows: 127771498 Data size: 17302827584 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 116155905 Data size: 15729842913 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col2)
mode: hash
@@ -240,7 +224,7 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: decimal(17,2))
- Reducer 4
+ Reducer 3
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -257,7 +241,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 8
+ Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1)
@@ -275,6 +259,22 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(38,21))
+ Reducer 7
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: decimal(38,21))
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query34.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query34.q.out b/ql/src/test/results/clientpositive/perf/spark/query34.q.out
index d563a67..80f7ec1 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query34.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query34.q.out
@@ -71,36 +71,36 @@ STAGE PLANS:
Map 8
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_county) IN ('Mobile County', 'Maverick County', 'Huron County', 'Kittitas County', 'Fairfield County', 'Jackson County', 'Barrow County', 'Pennington County') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((hd_buy_potential = '>10000') or (hd_buy_potential = 'unknown')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.2D)) ELSE (null) END and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col3 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 9
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((hd_buy_potential = '>10000') or (hd_buy_potential = 'unknown')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.2D)) ELSE (null) END and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_county) IN ('Mobile County', 'Maverick County', 'Huron County', 'Kittitas County', 'Fairfield County', 'Jackson County', 'Barrow County', 'Pennington County') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -215,9 +215,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col3 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col4
+ outputColumnNames: _col1, _col3, _col4
input vertices:
1 Map 8
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -225,7 +225,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col4
input vertices:
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query35.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query35.q.out b/ql/src/test/results/clientpositive/perf/spark/query35.q.out
index 952e6d1..de7c29d 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query35.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query35.q.out
@@ -123,8 +123,8 @@ STAGE PLANS:
Reducer 14 <- Reducer 13 (GROUP, 169)
Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 305), Map 19 (PARTITION-LEVEL SORT, 305)
Reducer 18 <- Reducer 17 (GROUP, 336)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 544), Map 7 (PARTITION-LEVEL SORT, 544)
- Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 909), Reducer 2 (PARTITION-LEVEL SORT, 909)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 7 (PARTITION-LEVEL SORT, 855)
+ Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 597), Reducer 2 (PARTITION-LEVEL SORT, 597)
Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT, 1009), Reducer 14 (PARTITION-LEVEL SORT, 1009), Reducer 18 (PARTITION-LEVEL SORT, 1009), Reducer 3 (PARTITION-LEVEL SORT, 1009)
Reducer 5 <- Reducer 4 (GROUP, 1009)
Reducer 6 <- Reducer 5 (SORT, 1)
@@ -143,11 +143,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int)
+ value expressions: _col0 (type: int), _col1 (type: int)
Map 11
Map Operator Tree:
TableScan
@@ -238,24 +238,6 @@ STAGE PLANS:
Map 7
Map Operator Tree:
TableScan
- alias: customer_demographics
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: cd_demo_sk is not null (type: boolean)
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: cd_demo_sk (type: int), cd_gender (type: string), cd_marital_status (type: string), cd_dep_count (type: int), cd_dep_employed_count (type: int), cd_dep_college_count (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: int)
- Map 8
- Map Operator Tree:
- TableScan
alias: ca
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -271,6 +253,24 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
+ Map 8
+ Map Operator Tree:
+ TableScan
+ alias: customer_demographics
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cd_demo_sk is not null (type: boolean)
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cd_demo_sk (type: int), cd_gender (type: string), cd_marital_status (type: string), cd_dep_count (type: int), cd_dep_employed_count (type: int), cd_dep_college_count (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: int)
Map 9
Map Operator Tree:
TableScan
@@ -393,36 +393,32 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col4, _col5, _col6, _col7, _col8
+ outputColumnNames: _col0, _col1, _col4
Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: int)
+ value expressions: _col0 (type: int), _col4 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col4, _col5, _col6, _col7, _col8, _col10
+ outputColumnNames: _col0, _col4, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col10 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: int), _col7 (type: int), _col8 (type: int)
- outputColumnNames: _col0, _col4, _col6, _col7, _col8, _col9, _col10
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 96800003 Data size: 83249958789 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col4 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int)
+ value expressions: _col4 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: int), _col9 (type: int), _col10 (type: int)
Reducer 4
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query37.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query37.q.out b/ql/src/test/results/clientpositive/perf/spark/query37.q.out
index b37823c..325861e 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query37.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query37.q.out
@@ -40,7 +40,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 6
+ Map 7
Map Operator Tree:
TableScan
alias: date_dim
@@ -62,7 +62,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 5 (PARTITION-LEVEL SORT, 306), Map 7 (PARTITION-LEVEL SORT, 306)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 5 (PARTITION-LEVEL SORT, 306), Map 6 (PARTITION-LEVEL SORT, 306)
Reducer 3 <- Reducer 2 (GROUP, 671)
Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
@@ -102,7 +102,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 25666 Data size: 36863512 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: decimal(7,2))
- Map 7
+ Map 6
Map Operator Tree:
TableScan
alias: inventory
@@ -120,14 +120,14 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col3
+ outputColumnNames: _col1
input vertices:
- 0 Map 6
+ 1 Map 7
Statistics: Num rows: 4593600 Data size: 72578135 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col3 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 4593600 Data size: 72578135 Basic stats: COMPLETE Column stats: NONE
Local Work:
Map Reduce Local Work
@@ -140,7 +140,7 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- 2 _col3 (type: int)
+ 2 _col1 (type: int)
outputColumnNames: _col2, _col3, _col4
Statistics: Num rows: 633577652 Data size: 85799141554 Basic stats: COMPLETE Column stats: NONE
Group By Operator
[08/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query50.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query50.q.out b/ql/src/test/results/clientpositive/perf/spark/query50.q.out
index b978201..fde0c4e 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query50.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query50.q.out
@@ -124,7 +124,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 6
+ Map 10
Map Operator Tree:
TableScan
alias: store
@@ -138,7 +138,7 @@ STAGE PLANS:
Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col3 (type: int)
+ 0 _col10 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -146,63 +146,16 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 476), Reducer 8 (PARTITION-LEVEL SORT, 476)
- Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 481), Reducer 2 (PARTITION-LEVEL SORT, 481)
- Reducer 4 <- Reducer 3 (GROUP, 529)
- Reducer 5 <- Reducer 4 (SORT, 1)
- Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 36), Map 9 (PARTITION-LEVEL SORT, 36)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 36), Map 7 (PARTITION-LEVEL SORT, 36)
+ Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 436), Reducer 2 (PARTITION-LEVEL SORT, 436)
+ Reducer 4 <- Map 9 (PARTITION-LEVEL SORT, 438), Reducer 3 (PARTITION-LEVEL SORT, 438)
+ Reducer 5 <- Reducer 4 (GROUP, 529)
+ Reducer 6 <- Reducer 5 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: store_sales
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null) (type: boolean)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ticket_number (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col3 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col4, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- input vertices:
- 1 Map 6
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int)
- sort order: +++
- Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col4 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string)
- Local Work:
- Map Reduce Local Work
- Map 10
- Map Operator Tree:
- TableScan
- alias: d1
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: d_date_sk is not null (type: boolean)
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Map 7
- Map Operator Tree:
- TableScan
alias: store_returns
Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -218,7 +171,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int)
- Map 9
+ Map 7
Map Operator Tree:
TableScan
alias: d2
@@ -235,50 +188,113 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Map 8
+ Map Operator Tree:
+ TableScan
+ alias: store_sales
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null) (type: boolean)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ticket_number (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col3 (type: int)
+ Map 9
+ Map Operator Tree:
+ TableScan
+ alias: d1
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: d_date_sk is not null (type: boolean)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int), _col2 (type: int), _col4 (type: int)
- 1 _col1 (type: int), _col2 (type: int), _col3 (type: int)
- outputColumnNames: _col0, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col6 (type: string), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), _col16 (type: int)
+ key expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ 1 _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ outputColumnNames: _col0, _col7, _col10
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col7 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col7 (type: int)
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col10 (type: int)
+ Reducer 4
+ Local Work:
+ Map Reduce Local Work
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col7 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col6 (type: string), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), CASE WHEN (((_col16 - _col0) <= 30)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col16 - _col0) > 30) and ((_col16 - _col0) <= 60))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col16 - _col0) > 60) and ((_col16 - _col0) <= 90))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col16 - _col0) > 90) and ((_col16 - _col0) <= 120))) THEN (1) ELSE (0) END (type: int), CASE WHEN (((_col16 - _col0) > 120)) THEN (1) ELSE (0) END (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
+ outputColumnNames: _col0, _col7, _col10
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col10 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col7, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23
+ input vertices:
+ 1 Map 10
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col10), sum(_col11), sum(_col12), sum(_col13), sum(_col14)
- keys: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
- mode: hash
+ Select Operator
+ expressions: _col14 (type: string), _col15 (type: int), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: string), _col20 (type: string), _col21 (type: string), _col22 (type: string), _col23 (type: string), CASE WHEN (((_col0 - _col7) <= 30)) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col0 - _col7) > 30) and ((_col0 - _col7) <= 60))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col0 - _col7) > 60) and ((_col0 - _col7) <= 90))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col0 - _col7) > 90) and ((_col0 - _col7) <= 120))) THEN (1) ELSE (0) END (type: int), CASE WHEN (((_col0 - _col7) > 120)) THEN (1) ELSE (0) END (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
- sort order: ++++++++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
+ Group By Operator
+ aggregations: sum(_col10), sum(_col11), sum(_col12), sum(_col13), sum(_col14)
+ keys: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: bigint), _col14 (type: bigint)
- Reducer 4
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
+ sort order: ++++++++++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: bigint), _col14 (type: bigint)
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3), sum(VALUE._col4)
@@ -292,7 +308,7 @@ STAGE PLANS:
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col10 (type: bigint), _col11 (type: bigint), _col12 (type: bigint), _col13 (type: bigint), _col14 (type: bigint)
- Reducer 5
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string), KEY.reducesinkkey6 (type: string), KEY.reducesinkkey7 (type: string), KEY.reducesinkkey8 (type: string), KEY.reducesinkkey9 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type: bigint), VALUE._col2 (type: bigint), VALUE._col3 (type: bigint), VALUE._col4 (type: bigint)
@@ -308,22 +324,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 8
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int)
- sort order: +++
- Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col3 (type: int)
- Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query53.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query53.q.out b/ql/src/test/results/clientpositive/perf/spark/query53.q.out
index 3a1e948..3bb21ef 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query53.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query53.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 9
+ Map 8
Map Operator Tree:
TableScan
alias: store
@@ -84,11 +84,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 399), Map 7 (PARTITION-LEVEL SORT, 399)
- Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 438), Reducer 2 (PARTITION-LEVEL SORT, 438)
- Reducer 4 <- Reducer 3 (GROUP, 529)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 265)
- Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 399), Map 6 (PARTITION-LEVEL SORT, 399)
+ Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 438), Reducer 2 (PARTITION-LEVEL SORT, 438)
+ Reducer 4 <- Reducer 3 (GROUP PARTITION-LEVEL SORT, 529)
+ Reducer 5 <- Reducer 4 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -109,7 +108,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
- Map 7
+ Map 6
Map Operator Tree:
TableScan
alias: item
@@ -127,7 +126,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: int)
- Map 8
+ Map 7
Map Operator Tree:
TableScan
alias: date_dim
@@ -181,18 +180,18 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col3, _col8, _col11
input vertices:
- 1 Map 9
+ 1 Map 8
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col11 (type: int), _col8 (type: int)
+ keys: _col8 (type: int), _col11 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(17,2))
Reducer 4
@@ -204,58 +203,46 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: decimal(17,2))
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Reducer 5
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: int), VALUE._col1 (type: decimal(17,2))
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col1: int, _col2: decimal(17,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col1 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: avg_window_0
- arguments: _col2
- name: avg
- window function: GenericUDAFAverageEvaluatorDecimal
- window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ expressions: _col0 (type: int), _col2 (type: decimal(17,2))
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: avg_window_0 (type: decimal(21,6)), _col1 (type: int), _col2 (type: decimal(17,2))
- outputColumnNames: avg_window_0, _col1, _col2
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: int, _col2: decimal(17,2)
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: avg_window_0
+ arguments: _col2
+ name: avg
+ window function: GenericUDAFAverageEvaluatorDecimal
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END (type: boolean)
- Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: int), _col2 (type: decimal(17,2)), avg_window_0 (type: decimal(21,6))
- outputColumnNames: _col0, _col1, _col2
+ Select Operator
+ expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: int), _col2 (type: decimal(17,2))
+ outputColumnNames: avg_window_0, _col0, _col2
+ Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END (type: boolean)
Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: decimal(21,6)), _col1 (type: decimal(17,2)), _col0 (type: int)
- sort order: +++
+ Select Operator
+ expressions: _col0 (type: int), _col2 (type: decimal(17,2)), avg_window_0 (type: decimal(21,6))
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- Reducer 6
+ Reduce Output Operator
+ key expressions: _col2 (type: decimal(21,6)), _col1 (type: decimal(17,2)), _col0 (type: int)
+ sort order: +++
+ Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey2 (type: int), KEY.reducesinkkey1 (type: decimal(17,2)), KEY.reducesinkkey0 (type: decimal(21,6))
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query54.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query54.q.out b/ql/src/test/results/clientpositive/perf/spark/query54.q.out
index 251d7ad..43132bc 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query54.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query54.q.out
@@ -1,7 +1,7 @@
-Warning: Shuffle Join JOIN[111][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 4' is a cross product
-Warning: Shuffle Join JOIN[104][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3]] in Work 'Reducer 14' is a cross product
-Warning: Shuffle Join JOIN[114][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 5' is a cross product
-Warning: Map Join MAPJOIN[143][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[111][tables = [$hdt$_1, $hdt$_2]] in Work 'Reducer 3' is a cross product
+Warning: Shuffle Join JOIN[107][tables = [$hdt$_0, $hdt$_1, $hdt$_2, $hdt$_3, $hdt$_4]] in Work 'Reducer 14' is a cross product
+Warning: Shuffle Join JOIN[114][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product
+Warning: Map Join MAPJOIN[144][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
PREHOOK: query: explain
with my_customers as (
select distinct c_customer_sk
@@ -122,11 +122,11 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 31 <- Map 30 (GROUP, 2)
- Reducer 32 <- Reducer 31 (GROUP, 1)
+ Reducer 29 <- Map 28 (GROUP, 2)
+ Reducer 30 <- Reducer 29 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 30
+ Map 28
Map Operator Tree:
TableScan
alias: date_dim
@@ -148,7 +148,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Reducer 31
+ Reducer 29
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int)
@@ -166,7 +166,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Reducer 32
+ Reducer 30
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -211,23 +211,23 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Map 1 (GROUP, 2)
+ Reducer 10 <- Reducer 9 (GROUP, 1)
Reducer 12 <- Map 11 (PARTITION-LEVEL SORT, 398), Map 15 (PARTITION-LEVEL SORT, 398)
Reducer 13 <- Reducer 12 (PARTITION-LEVEL SORT, 772), Reducer 17 (PARTITION-LEVEL SORT, 772)
- Reducer 14 <- Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 29 (PARTITION-LEVEL SORT, 1)
+ Reducer 14 <- Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 32 (PARTITION-LEVEL SORT, 1)
Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 654), Reducer 23 (PARTITION-LEVEL SORT, 654)
Reducer 2 <- Map 1 (GROUP, 2)
- Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 459), Map 24 (PARTITION-LEVEL SORT, 459), Map 25 (PARTITION-LEVEL SORT, 459)
- Reducer 21 <- Map 26 (PARTITION-LEVEL SORT, 504), Reducer 20 (PARTITION-LEVEL SORT, 504)
+ Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 458), Map 24 (PARTITION-LEVEL SORT, 458), Map 25 (PARTITION-LEVEL SORT, 458)
+ Reducer 21 <- Map 26 (PARTITION-LEVEL SORT, 505), Reducer 20 (PARTITION-LEVEL SORT, 505)
Reducer 22 <- Map 27 (PARTITION-LEVEL SORT, 1009), Reducer 21 (PARTITION-LEVEL SORT, 1009)
Reducer 23 <- Reducer 22 (GROUP, 610)
- Reducer 29 <- Map 28 (GROUP, 2)
- Reducer 3 <- Reducer 2 (GROUP, 1)
- Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 14 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
+ Reducer 32 <- Map 31 (GROUP, 2)
+ Reducer 4 <- Reducer 14 (PARTITION-LEVEL SORT, 1), Reducer 3 (PARTITION-LEVEL SORT, 1)
+ Reducer 5 <- Reducer 4 (GROUP, 1009)
Reducer 6 <- Reducer 5 (GROUP, 1009)
- Reducer 7 <- Reducer 6 (GROUP, 1009)
- Reducer 8 <- Reducer 7 (SORT, 1)
+ Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 9 <- Map 1 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -330,11 +330,11 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 431992504 Data size: 58579807164 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int)
+ value expressions: _col1 (type: int), _col2 (type: int)
Map 24
Map Operator Tree:
TableScan
@@ -348,45 +348,45 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 431992504 Data size: 58579807164 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int)
+ value expressions: _col1 (type: int), _col2 (type: int)
Map 25
Map Operator Tree:
TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((i_category = 'Jewelry') and (i_class = 'consignment') and i_item_sk is not null) (type: boolean)
- Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_moy = 3) and (d_year = 1999) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: i_item_sk (type: int)
+ expressions: d_date_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
Map 26
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_moy = 3) and (d_year = 1999) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((i_category = 'Jewelry') and (i_class = 'consignment') and i_item_sk is not null) (type: boolean)
+ Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int)
+ expressions: i_item_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
Map 27
Map Operator Tree:
TableScan
@@ -405,7 +405,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int)
- Map 28
+ Map 31
Map Operator Tree:
TableScan
alias: date_dim
@@ -430,14 +430,18 @@ STAGE PLANS:
Reducer 10
Reduce Operator Tree:
Group By Operator
- keys: KEY._col0 (type: int)
+ aggregations: count(VALUE._col0)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (sq_count_check(_col0) <= 1) (type: boolean)
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reducer 12
Reduce Operator Tree:
Join Operator
@@ -455,6 +459,8 @@ STAGE PLANS:
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2)), _col4 (type: int)
Reducer 13
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -464,13 +470,21 @@ STAGE PLANS:
1 _col5 (type: int)
outputColumnNames: _col2, _col4, _col10
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(7,2)), _col4 (type: int), _col10 (type: int)
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col2, _col4, _col10
+ input vertices:
+ 1 Reducer 30
+ Statistics: Num rows: 696954748 Data size: 67758142923 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 696954748 Data size: 67758142923 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(7,2)), _col4 (type: int), _col10 (type: int)
Reducer 14
- Local Work:
- Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -478,26 +492,16 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col2, _col4, _col10, _col12
- Statistics: Num rows: 6363893803988 Data size: 7688985619260581 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0
- 1
- outputColumnNames: _col2, _col4, _col10, _col12
- input vertices:
- 1 Reducer 32
+ outputColumnNames: _col2, _col4, _col10, _col13
+ Statistics: Num rows: 6363893803988 Data size: 7746260663496473 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col10 (type: int), _col2 (type: decimal(7,2)), _col4 (type: int), _col13 (type: int)
+ outputColumnNames: _col0, _col4, _col11, _col13
Statistics: Num rows: 6363893803988 Data size: 7746260663496473 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col10 (type: int), _col2 (type: decimal(7,2)), _col4 (type: int), _col12 (type: int)
- outputColumnNames: _col0, _col4, _col11, _col13
+ Reduce Output Operator
+ sort order:
Statistics: Num rows: 6363893803988 Data size: 7746260663496473 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 6363893803988 Data size: 7746260663496473 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col4 (type: decimal(7,2)), _col11 (type: int), _col13 (type: int)
+ value expressions: _col0 (type: int), _col4 (type: decimal(7,2)), _col11 (type: int), _col13 (type: int)
Reducer 17
Reduce Operator Tree:
Join Operator
@@ -520,31 +524,24 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
- Select Operator
+ Reduce Output Operator
+ sort order:
Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ value expressions: _col0 (type: int)
Reducer 20
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 475191764 Data size: 64437789277 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 475191764 Data size: 64437789277 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int)
Reducer 21
@@ -553,7 +550,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1
Statistics: Num rows: 522710951 Data size: 70881569741 Basic stats: COMPLETE Column stats: NONE
@@ -599,47 +596,35 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 287491029 Data size: 38984864202 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
- Reducer 29
- Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
Reducer 3
Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (sq_count_check(_col0) <= 1) (type: boolean)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reducer 4
- Reduce Operator Tree:
Join Operator
condition map:
- Left Outer Join 0 to 1
+ Right Outer Join 0 to 1
+ filter predicates:
+ 0
+ 1 {true}
keys:
0
1
- outputColumnNames: _col1
+ outputColumnNames: _col0
Statistics: Num rows: 9131 Data size: 10299768 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
Statistics: Num rows: 9131 Data size: 10299768 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
- Reducer 5
+ value expressions: _col0 (type: int)
+ Reducer 32
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Reducer 4
Reduce Operator Tree:
Join Operator
condition map:
@@ -647,28 +632,32 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col1, _col2, _col6, _col13, _col15
+ outputColumnNames: _col0, _col2, _col6, _col13, _col15
Statistics: Num rows: 58108714324214428 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col13 BETWEEN _col15 AND _col1 (type: boolean)
- Statistics: Num rows: 6456523813801603 Data size: 1024819115206086144 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col2 (type: int), _col6 (type: decimal(7,2))
- outputColumnNames: _col2, _col6
+ Select Operator
+ expressions: _col2 (type: int), _col6 (type: decimal(7,2)), _col13 (type: int), _col15 (type: int), _col0 (type: int)
+ outputColumnNames: _col0, _col4, _col11, _col13, _col15
+ Statistics: Num rows: 58108714324214428 Data size: 9223372036854775807 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col11 BETWEEN _col13 AND _col15 (type: boolean)
Statistics: Num rows: 6456523813801603 Data size: 1024819115206086144 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col6)
- keys: _col2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
+ Select Operator
+ expressions: _col0 (type: int), _col4 (type: decimal(7,2))
+ outputColumnNames: _col0, _col4
Statistics: Num rows: 6456523813801603 Data size: 1024819115206086144 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Group By Operator
+ aggregations: sum(_col4)
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 6456523813801603 Data size: 1024819115206086144 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2))
- Reducer 6
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6456523813801603 Data size: 1024819115206086144 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(17,2))
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -692,7 +681,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 3228261906900801 Data size: 512409557603043008 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 7
+ Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -710,7 +699,7 @@ STAGE PLANS:
Statistics: Num rows: 1614130953450400 Data size: 256204778801521408 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: int)
- Reducer 8
+ Reducer 7
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: int)
@@ -726,6 +715,24 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 9
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query57.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query57.q.out b/ql/src/test/results/clientpositive/perf/spark/query57.q.out
index 6b30e66..1b66f54 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query57.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query57.q.out
@@ -391,47 +391,47 @@ STAGE PLANS:
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col10 (type: string), _col11 (type: string), _col8 (type: string), _col5 (type: int), _col6 (type: int)
+ keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col10 (type: string), _col11 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string)
sort order: +++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string)
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: decimal(17,2))
Reducer 14
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._col4 (type: int)
+ keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string), _col3 (type: int)
+ key expressions: _col4 (type: string), _col3 (type: string), _col2 (type: string), _col0 (type: int)
sort order: ++++
- Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string), _col3 (type: int)
+ Map-reduce partition columns: _col4 (type: string), _col3 (type: string), _col2 (type: string), _col0 (type: int)
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col4 (type: int), _col5 (type: decimal(17,2))
+ value expressions: _col1 (type: int), _col5 (type: decimal(17,2))
Reducer 15
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: decimal(17,2))
+ expressions: KEY.reducesinkkey3 (type: int), VALUE._col0 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: string, _col3: int, _col4: int, _col5: decimal(17,2)
+ output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string, _col5: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col1 ASC NULLS FIRST, _col0 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col3 ASC NULLS FIRST
- partition by: _col1, _col0, _col2, _col3
+ order by: _col4 ASC NULLS FIRST, _col3 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col0 ASC NULLS FIRST
+ partition by: _col4, _col3, _col2, _col0
raw input shape:
window functions:
window function definition
@@ -442,54 +442,54 @@ STAGE PLANS:
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(17,2))
+ expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: decimal(17,2))
outputColumnNames: avg_window_0, _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ key expressions: _col4 (type: string), _col3 (type: string), _col2 (type: string), _col0 (type: int), _col1 (type: int)
sort order: +++++
- Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string)
+ Map-reduce partition columns: _col4 (type: string), _col3 (type: string), _col2 (type: string)
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
value expressions: avg_window_0 (type: decimal(21,6)), _col5 (type: decimal(17,2))
Reducer 16
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: decimal(21,6)), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col1 (type: decimal(17,2))
+ expressions: VALUE._col0 (type: decimal(21,6)), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: decimal(21,6), _col1: string, _col2: string, _col3: string, _col4: int, _col5: int, _col6: decimal(17,2)
+ output shape: _col0: decimal(21,6), _col1: int, _col2: int, _col3: string, _col4: string, _col5: string, _col6: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST
- partition by: _col2, _col1, _col3
+ order by: _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST
+ partition by: _col5, _col4, _col3
raw input shape:
window functions:
window function definition
alias: rank_window_1
- arguments: _col4, _col5
+ arguments: _col1, _col2
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > 0) and (_col4 = 2000) and rank_window_1 is not null) (type: boolean)
+ predicate: ((_col0 > 0) and (_col1 = 2000) and rank_window_1 is not null) (type: boolean)
Statistics: Num rows: 31942874 Data size: 4325706828 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: rank_window_1 (type: int), _col0 (type: decimal(21,6)), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(17,2))
+ expressions: rank_window_1 (type: int), _col0 (type: decimal(21,6)), _col1 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: decimal(17,2))
outputColumnNames: rank_window_1, _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 31942874 Data size: 4325706828 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: CASE WHEN ((_col0 > 0)) THEN (((abs((_col6 - _col0)) / _col0) > 0.1)) ELSE (null) END (type: boolean)
Statistics: Num rows: 15971437 Data size: 2162853414 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col2 (type: string), _col1 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(17,2)), _col0 (type: decimal(21,6)), rank_window_1 (type: int)
+ expressions: _col5 (type: string), _col4 (type: string), _col3 (type: string), _col1 (type: int), _col2 (type: int), _col6 (type: decimal(17,2)), _col0 (type: decimal(21,6)), rank_window_1 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 15971437 Data size: 2162853414 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -566,52 +566,52 @@ STAGE PLANS:
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col10 (type: string), _col11 (type: string), _col8 (type: string), _col5 (type: int), _col6 (type: int)
+ keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col10 (type: string), _col11 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string)
sort order: +++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string)
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: decimal(17,2))
Reducer 23
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._col4 (type: int)
+ keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ key expressions: _col4 (type: string), _col3 (type: string), _col2 (type: string), _col0 (type: int), _col1 (type: int)
sort order: +++++
- Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string)
+ Map-reduce partition columns: _col4 (type: string), _col3 (type: string), _col2 (type: string)
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: decimal(17,2))
Reducer 24
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col0 (type: decimal(17,2))
+ expressions: KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col0 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: string, _col3: int, _col4: int, _col5: decimal(17,2)
+ output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string, _col5: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST
- partition by: _col1, _col0, _col2
+ order by: _col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST
+ partition by: _col4, _col3, _col2
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: _col3, _col4
+ arguments: _col0, _col1
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
@@ -621,7 +621,7 @@ STAGE PLANS:
predicate: rank_window_0 is not null (type: boolean)
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string), _col5 (type: decimal(17,2)), rank_window_0 (type: int)
+ expressions: _col4 (type: string), _col3 (type: string), _col2 (type: string), _col5 (type: decimal(17,2)), rank_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -642,52 +642,52 @@ STAGE PLANS:
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col10 (type: string), _col11 (type: string), _col8 (type: string), _col5 (type: int), _col6 (type: int)
+ keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col10 (type: string), _col11 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string)
sort order: +++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string)
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: decimal(17,2))
Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: int), KEY._col4 (type: int)
+ keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int)
+ key expressions: _col4 (type: string), _col3 (type: string), _col2 (type: string), _col0 (type: int), _col1 (type: int)
sort order: +++++
- Map-reduce partition columns: _col1 (type: string), _col0 (type: string), _col2 (type: string)
+ Map-reduce partition columns: _col4 (type: string), _col3 (type: string), _col2 (type: string)
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: decimal(17,2))
Reducer 5
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col0 (type: decimal(17,2))
+ expressions: KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col0 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: string, _col3: int, _col4: int, _col5: decimal(17,2)
+ output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string, _col5: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col3 ASC NULLS FIRST, _col4 ASC NULLS FIRST
- partition by: _col1, _col0, _col2
+ order by: _col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST
+ partition by: _col4, _col3, _col2
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: _col3, _col4
+ arguments: _col0, _col1
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
@@ -697,7 +697,7 @@ STAGE PLANS:
predicate: rank_window_0 is not null (type: boolean)
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col1 (type: string), _col0 (type: string), _col2 (type: string), _col5 (type: decimal(17,2)), rank_window_0 (type: int)
+ expressions: _col4 (type: string), _col3 (type: string), _col2 (type: string), _col5 (type: decimal(17,2)), rank_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
[13/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/12a1784d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/12a1784d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/12a1784d
Branch: refs/heads/branch-3
Commit: 12a1784de44f185a963c34462a8f863db53920ce
Parents: 91d0a0a
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Wed Apr 11 20:05:01 2018 -0700
Committer: Vineet Garg <vg...@apache.org>
Committed: Thu Apr 12 10:57:23 2018 -0700
----------------------------------------------------------------------
data/conf/perf-reg/spark/hive-site.xml | 6 +
data/conf/perf-reg/tez/hive-site.xml | 5 +
.../hive/cli/control/CorePerfCliDriver.java | 6 +-
.../hadoop/hive/ql/MetaStoreDumpUtility.java | 231 +++++++++
.../org/apache/hadoop/hive/ql/QTestUtil.java | 221 +--------
.../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
.../clientpositive/perf/spark/query11.q.out | 48 +-
.../clientpositive/perf/spark/query15.q.out | 164 +++----
.../clientpositive/perf/spark/query16.q.out | 70 +--
.../clientpositive/perf/spark/query18.q.out | 216 ++++----
.../clientpositive/perf/spark/query19.q.out | 218 ++++-----
.../clientpositive/perf/spark/query21.q.out | 114 +++--
.../clientpositive/perf/spark/query24.q.out | 282 +++++------
.../clientpositive/perf/spark/query25.q.out | 118 ++---
.../clientpositive/perf/spark/query29.q.out | 280 +++++------
.../clientpositive/perf/spark/query30.q.out | 262 +++++-----
.../clientpositive/perf/spark/query32.q.out | 132 ++---
.../clientpositive/perf/spark/query34.q.out | 34 +-
.../clientpositive/perf/spark/query35.q.out | 74 ++-
.../clientpositive/perf/spark/query37.q.out | 16 +-
.../clientpositive/perf/spark/query4.q.out | 214 ++++----
.../clientpositive/perf/spark/query40.q.out | 116 +++--
.../clientpositive/perf/spark/query44.q.out | 246 +++++-----
.../clientpositive/perf/spark/query45.q.out | 204 ++++----
.../clientpositive/perf/spark/query46.q.out | 104 ++--
.../clientpositive/perf/spark/query47.q.out | 92 ++--
.../clientpositive/perf/spark/query48.q.out | 94 ++--
.../clientpositive/perf/spark/query5.q.out | 38 +-
.../clientpositive/perf/spark/query50.q.out | 196 ++++----
.../clientpositive/perf/spark/query53.q.out | 105 ++--
.../clientpositive/perf/spark/query54.q.out | 277 ++++++-----
.../clientpositive/perf/spark/query57.q.out | 92 ++--
.../clientpositive/perf/spark/query58.q.out | 490 +++++++++----------
.../clientpositive/perf/spark/query6.q.out | 350 +++++++------
.../clientpositive/perf/spark/query61.q.out | 40 +-
.../clientpositive/perf/spark/query63.q.out | 105 ++--
.../clientpositive/perf/spark/query65.q.out | 100 ++--
.../clientpositive/perf/spark/query66.q.out | 20 +-
.../clientpositive/perf/spark/query67.q.out | 137 +++---
.../clientpositive/perf/spark/query68.q.out | 104 ++--
.../clientpositive/perf/spark/query72.q.out | 461 +++++++++--------
.../clientpositive/perf/spark/query73.q.out | 34 +-
.../clientpositive/perf/spark/query75.q.out | 248 +++++-----
.../clientpositive/perf/spark/query76.q.out | 130 +++--
.../clientpositive/perf/spark/query77.q.out | 66 +--
.../clientpositive/perf/spark/query78.q.out | 273 +++++------
.../clientpositive/perf/spark/query79.q.out | 10 +-
.../clientpositive/perf/spark/query8.q.out | 174 +++----
.../clientpositive/perf/spark/query80.q.out | 417 ++++++++--------
.../clientpositive/perf/spark/query81.q.out | 270 +++++-----
.../clientpositive/perf/spark/query82.q.out | 16 +-
.../clientpositive/perf/spark/query83.q.out | 354 +++++++-------
.../clientpositive/perf/spark/query85.q.out | 309 ++++++------
.../clientpositive/perf/spark/query88.q.out | 272 +++++-----
.../clientpositive/perf/spark/query89.q.out | 26 +-
.../clientpositive/perf/spark/query90.q.out | 68 +--
.../clientpositive/perf/spark/query91.q.out | 14 +-
.../clientpositive/perf/spark/query92.q.out | 106 ++--
.../clientpositive/perf/spark/query94.q.out | 70 +--
.../clientpositive/perf/spark/query95.q.out | 100 ++--
.../clientpositive/perf/spark/query97.q.out | 54 +-
.../clientpositive/perf/spark/query99.q.out | 42 +-
63 files changed, 4603 insertions(+), 4536 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/data/conf/perf-reg/spark/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/perf-reg/spark/hive-site.xml b/data/conf/perf-reg/spark/hive-site.xml
index 5ca660d..8a32afe 100644
--- a/data/conf/perf-reg/spark/hive-site.xml
+++ b/data/conf/perf-reg/spark/hive-site.xml
@@ -270,4 +270,10 @@
<value>false</value>
</property>
+<property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>99</value>
+</property>
+
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/data/conf/perf-reg/tez/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/perf-reg/tez/hive-site.xml b/data/conf/perf-reg/tez/hive-site.xml
index 62ecb74..e11f8f8 100644
--- a/data/conf/perf-reg/tez/hive-site.xml
+++ b/data/conf/perf-reg/tez/hive-site.xml
@@ -287,4 +287,9 @@
<value>false</value>
</property>
+<property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>99</value>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 8ef9e0a..3ae691f 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.hadoop.hive.cli.control;
@@ -29,6 +29,7 @@ import com.google.common.base.Strings;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
+import org.apache.hadoop.hive.ql.MetaStoreDumpUtility;
import org.junit.After;
import org.junit.AfterClass;
/**
@@ -70,7 +71,8 @@ public class CorePerfCliDriver extends CliAdapter{
qt.createSources();
// Manually modify the underlying metastore db to reflect statistics corresponding to
// the 30TB TPCDS scale set. This way the optimizer will generate plans for a 30 TB set.
- QTestUtil.setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(qt.getConf());
+ MetaStoreDumpUtility.setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(qt.getConf(),
+ System.getProperty(QTestUtil.TEST_TMP_DIR_PROPERTY));
} catch (Exception e) {
System.err.println("Exception: " + e.getMessage());
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/itests/util/src/main/java/org/apache/hadoop/hive/ql/MetaStoreDumpUtility.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/MetaStoreDumpUtility.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/MetaStoreDumpUtility.java
new file mode 100644
index 0000000..2389c3b
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/MetaStoreDumpUtility.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Stream;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hive.testutils.HiveTestEnvSetup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class which can load an existing metastore dump.
+ *
+ * This can be used to check planning on a large scale database.
+ */
+public class MetaStoreDumpUtility {
+
+ static final Logger LOG = LoggerFactory.getLogger(MetaStoreDumpUtility.class);
+
+ public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf conf, String tmpBaseDir) {
+ Connection conn = null;
+
+ try {
+ Properties props = new Properties(); // connection properties
+ props.put("user", conf.get("javax.jdo.option.ConnectionUserName"));
+ props.put("password", conf.get("javax.jdo.option.ConnectionPassword"));
+ String url = conf.get("javax.jdo.option.ConnectionURL");
+ conn = DriverManager.getConnection(url, props);
+ ResultSet rs = null;
+ Statement s = conn.createStatement();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connected to metastore database ");
+ }
+
+ String mdbPath = HiveTestEnvSetup.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
+
+ // Setup the table column stats
+ BufferedReader br = new BufferedReader(
+ new FileReader(
+ new File(HiveTestEnvSetup.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
+ String command;
+
+ s.execute("DROP TABLE APP.TABLE_PARAMS");
+ s.execute("DROP TABLE APP.TAB_COL_STATS");
+ // Create the column stats table
+ while ((command = br.readLine()) != null) {
+ if (!command.endsWith(";")) {
+ continue;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Going to run command : " + command);
+ }
+ PreparedStatement psCommand = conn.prepareStatement(command.substring(0, command.length() - 1));
+ psCommand.execute();
+ psCommand.close();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully completed " + command);
+ }
+ }
+ br.close();
+
+ java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2");
+ java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2");
+
+ // Set up the foreign key constraints properly in the TAB_COL_STATS data
+ java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt");
+ java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt");
+
+ class MyComp implements Comparator<String> {
+ @Override
+ public int compare(String str1, String str2) {
+ if (str2.length() != str1.length()) {
+ return str2.length() - str1.length();
+ }
+ return str1.compareTo(str2);
+ }
+ }
+
+ final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
+
+ rs = s.executeQuery("SELECT * FROM APP.TBLS");
+ while(rs.next()) {
+ String tblName = rs.getString("TBL_NAME");
+ Integer tblId = rs.getInt("TBL_ID");
+ tableNameToID.put(tblName, tblId);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resultset : " + tblName + " | " + tblId);
+ }
+ }
+
+ final Map<String, Map<String, String>> data = new HashMap<>();
+ rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from "
+ + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a"
+ + " join APP.TBLS on TBLS.SD_ID = a.SD_ID");
+ while (rs.next()) {
+ String tblName = rs.getString(1);
+ String colName = rs.getString(2);
+ String typeName = rs.getString(3);
+ Map<String, String> cols = data.get(tblName);
+ if (null == cols) {
+ cols = new HashMap<>();
+ }
+ cols.put(colName, typeName);
+ data.put(tblName, cols);
+ }
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(
+ new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ))));
+
+ Stream<String> replaced = reader.lines().parallel().map(str-> {
+ String[] splits = str.split(",");
+ String tblName = splits[0];
+ String colName = splits[1];
+ Integer tblID = tableNameToID.get(tblName);
+ StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@");
+ for (int i = 2; i < splits.length; i++) {
+ sb.append(splits[i]+"@");
+ }
+ // Add tbl_id and empty bitvector
+ return sb.append(tblID).append("@").toString();
+ });
+
+ Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator);
+ replaced.close();
+ reader.close();
+
+ BufferedReader reader2 = new BufferedReader(new InputStreamReader(
+ new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ))));
+ final Map<String,String> colStats = new ConcurrentHashMap<>();
+ Stream<String> replacedStream = reader2.lines().parallel().map(str-> {
+ String[] splits = str.split("_@");
+ String tblName = splits[0];
+ Integer tblId = tableNameToID.get(tblName);
+ Map<String,String> cols = data.get(tblName);
+ StringBuilder sb = new StringBuilder();
+ sb.append("{\"COLUMN_STATS\":{");
+ for (String colName : cols.keySet()) {
+ sb.append("\""+colName+"\":\"true\",");
+ }
+ sb.append("},\"BASIC_STATS\":\"true\"}");
+ colStats.put(tblId.toString(), sb.toString());
+
+ return tblId.toString() + "@" + splits[1];
+ });
+
+ Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator);
+ Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream()
+ .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND);
+
+ replacedStream.close();
+ reader2.close();
+ // Load the column stats and table params with 30 TB scale
+ String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" +
+ "', '" + tmpFileLoc1.toAbsolutePath().toString() +
+ "', '@', null, 'UTF-8', 1)";
+ String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" +
+ "', '" + tmpFileLoc2.toAbsolutePath().toString() +
+ "', '@', null, 'UTF-8', 1)";
+
+ PreparedStatement psImport1 = conn.prepareStatement(importStatement1);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Going to execute : " + importStatement1);
+ }
+ psImport1.execute();
+ psImport1.close();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully completed " + importStatement1);
+ }
+ PreparedStatement psImport2 = conn.prepareStatement(importStatement2);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Going to execute : " + importStatement2);
+ }
+ psImport2.execute();
+ psImport2.close();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully completed " + importStatement2);
+ }
+
+ s.execute("ALTER TABLE APP.TAB_COL_STATS ADD COLUMN CAT_NAME VARCHAR(256)");
+ s.execute("update APP.TAB_COL_STATS set CAT_NAME = '" + Warehouse.DEFAULT_CATALOG_NAME + "'");
+
+ s.close();
+
+ conn.close();
+
+ } catch (Exception e) {
+ throw new RuntimeException("error while loading tpcds metastore dump", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 3cdad28..c33851f 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -38,38 +38,23 @@ import java.io.Serializable;
import java.io.StringWriter;
import java.net.URL;
import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.StandardOpenOption;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Comparator;
import java.util.Deque;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import java.util.Set;
-import java.util.SortedMap;
import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Stream;
import junit.framework.TestSuite;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang.StringUtils;
@@ -131,8 +116,6 @@ import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
-
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
@@ -154,12 +137,12 @@ public class QTestUtil {
private static final String CRLF = System.getProperty("line.separator");
public static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
- private static final Logger LOG = LoggerFactory.getLogger("QTestUtil");
+ static final Logger LOG = LoggerFactory.getLogger("QTestUtil");
private final static String defaultInitScript = "q_test_init.sql";
private final static String defaultCleanupScript = "q_test_cleanup.sql";
private final String[] testOnlyCommands = new String[]{"crypto"};
- private static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp
+ public static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp
private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target
public static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables";
@@ -2244,206 +2227,6 @@ public class QTestUtil {
}
}
- public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf conf) {
- Connection conn = null;
- ArrayList<Statement> statements = new ArrayList<Statement>(); // list of Statements, PreparedStatements
-
- try {
- Properties props = new Properties(); // connection properties
- props.put("user", conf.get("javax.jdo.option.ConnectionUserName"));
- props.put("password", conf.get("javax.jdo.option.ConnectionPassword"));
- conn = DriverManager.getConnection(conf.get("javax.jdo.option.ConnectionURL"), props);
- ResultSet rs = null;
- Statement s = conn.createStatement();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Connected to metastore database ");
- }
-
- String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
-
- // Setup the table column stats
- BufferedReader br = new BufferedReader(
- new FileReader(
- new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
- String command;
-
- s.execute("DROP TABLE APP.TABLE_PARAMS");
- s.execute("DROP TABLE APP.TAB_COL_STATS");
- // Create the column stats table
- while ((command = br.readLine()) != null) {
- if (!command.endsWith(";")) {
- continue;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Going to run command : " + command);
- }
- try {
- PreparedStatement psCommand = conn.prepareStatement(command.substring(0, command.length()-1));
- statements.add(psCommand);
- psCommand.execute();
- if (LOG.isDebugEnabled()) {
- LOG.debug("successfully completed " + command);
- }
- } catch (SQLException e) {
- LOG.info("Got SQL Exception " + e.getMessage());
- }
- }
- br.close();
-
- java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2");
- java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2");
-
- // Set up the foreign key constraints properly in the TAB_COL_STATS data
- String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
- java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt");
- java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt");
-
- class MyComp implements Comparator<String> {
- @Override
- public int compare(String str1, String str2) {
- if (str2.length() != str1.length()) {
- return str2.length() - str1.length();
- }
- return str1.compareTo(str2);
- }
- }
-
- final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
-
- rs = s.executeQuery("SELECT * FROM APP.TBLS");
- while(rs.next()) {
- String tblName = rs.getString("TBL_NAME");
- Integer tblId = rs.getInt("TBL_ID");
- tableNameToID.put(tblName, tblId);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Resultset : " + tblName + " | " + tblId);
- }
- }
-
- final Map<String, Map<String, String>> data = new HashMap<>();
- rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from "
- + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a"
- + " join APP.TBLS on TBLS.SD_ID = a.SD_ID");
- while (rs.next()) {
- String tblName = rs.getString(1);
- String colName = rs.getString(2);
- String typeName = rs.getString(3);
- Map<String, String> cols = data.get(tblName);
- if (null == cols) {
- cols = new HashMap<>();
- }
- cols.put(colName, typeName);
- data.put(tblName, cols);
- }
-
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ))));
-
- Stream<String> replaced = reader.lines().parallel().map(str-> {
- String[] splits = str.split(",");
- String tblName = splits[0];
- String colName = splits[1];
- Integer tblID = tableNameToID.get(tblName);
- StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@");
- for (int i = 2; i < splits.length; i++) {
- sb.append(splits[i]+"@");
- }
- // Add tbl_id and empty bitvector
- return sb.append(tblID).append("@").toString();
- });
-
- Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator);
- replaced.close();
- reader.close();
-
- BufferedReader reader2 = new BufferedReader(new InputStreamReader(
- new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ))));
- final Map<String,String> colStats = new ConcurrentHashMap<>();
- Stream<String> replacedStream = reader2.lines().parallel().map(str-> {
- String[] splits = str.split("_@");
- String tblName = splits[0];
- Integer tblId = tableNameToID.get(tblName);
- Map<String,String> cols = data.get(tblName);
- StringBuilder sb = new StringBuilder();
- sb.append("{\"COLUMN_STATS\":{");
- for (String colName : cols.keySet()) {
- sb.append("\""+colName+"\":\"true\",");
- }
- sb.append("},\"BASIC_STATS\":\"true\"}");
- colStats.put(tblId.toString(), sb.toString());
-
- return tblId.toString() + "@" + splits[1];
- });
-
- Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator);
- Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream()
- .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND);
-
- replacedStream.close();
- reader2.close();
- // Load the column stats and table params with 30 TB scale
- String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" +
- "', '" + tmpFileLoc1.toAbsolutePath().toString() +
- "', '@', null, 'UTF-8', 1)";
- String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" +
- "', '" + tmpFileLoc2.toAbsolutePath().toString() +
- "', '@', null, 'UTF-8', 1)";
- try {
- PreparedStatement psImport1 = conn.prepareStatement(importStatement1);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Going to execute : " + importStatement1);
- }
- statements.add(psImport1);
- psImport1.execute();
- if (LOG.isDebugEnabled()) {
- LOG.debug("successfully completed " + importStatement1);
- }
- PreparedStatement psImport2 = conn.prepareStatement(importStatement2);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Going to execute : " + importStatement2);
- }
- statements.add(psImport2);
- psImport2.execute();
- if (LOG.isDebugEnabled()) {
- LOG.debug("successfully completed " + importStatement2);
- }
- } catch (SQLException e) {
- LOG.info("Got SQL Exception " + e.getMessage());
- }
- } catch (FileNotFoundException e1) {
- LOG.info("Got File not found Exception " + e1.getMessage());
- } catch (IOException e1) {
- LOG.info("Got IOException " + e1.getMessage());
- } catch (SQLException e1) {
- LOG.info("Got SQLException " + e1.getMessage());
- } finally {
- // Statements and PreparedStatements
- int i = 0;
- while (!statements.isEmpty()) {
- // PreparedStatement extend Statement
- Statement st = statements.remove(i);
- try {
- if (st != null) {
- st.close();
- st = null;
- }
- } catch (SQLException sqle) {
- }
- }
-
- //Connection
- try {
- if (conn != null) {
- conn.close();
- conn = null;
- }
- } catch (SQLException sqle) {
- }
- }
- }
-
public QOutProcessor getQOutProcessor() {
return qOutProcessor;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index d940cdd..59130ca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -121,7 +121,7 @@ public abstract class BaseSemanticAnalyzer {
protected final Hive db;
protected final HiveConf conf;
protected final QueryState queryState;
- protected List<Task<? extends Serializable>> rootTasks;
+ protected List<Task<?>> rootTasks;
protected FetchTask fetchTask;
protected final Logger LOG;
protected final LogHelper console;
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 10982dd..27efece 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -12211,7 +12211,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
fetchTask = pCtx.getFetchTask();
}
//find all Acid FileSinkOperatorS
- QueryPlanPostProcessor qp = new QueryPlanPostProcessor((List<Task<?>>)rootTasks, acidFileSinks, ctx.getExecutionId());
+ QueryPlanPostProcessor qp = new QueryPlanPostProcessor(rootTasks, acidFileSinks, ctx.getExecutionId());
LOG.info("Completed plan generation");
// 10. put accessed columns to readEntity
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query11.q.out b/ql/src/test/results/clientpositive/perf/spark/query11.q.out
index 227068c..e46aa21 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query11.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query11.q.out
@@ -300,7 +300,7 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
+ predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: d_date_sk (type: int)
@@ -335,7 +335,7 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
+ predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: d_date_sk (type: int)
@@ -577,14 +577,21 @@ STAGE PLANS:
Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col7 (type: decimal(18,2))
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col0, _col7
Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(18,2))
+ Filter Operator
+ predicate: (_col7 > 0) (type: boolean)
+ Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col7 (type: decimal(18,2))
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(18,2))
Reducer 3
Reduce Operator Tree:
Join Operator
@@ -621,21 +628,14 @@ STAGE PLANS:
Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col7 (type: decimal(18,2))
- outputColumnNames: _col0, _col7
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col7 > 0) (type: boolean)
- Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col7 (type: decimal(18,2))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(18,2))
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(18,2))
Reducer 5
Reduce Operator Tree:
Join Operator
@@ -651,7 +651,7 @@ STAGE PLANS:
outputColumnNames: _col1, _col3, _col5, _col6, _col8
Statistics: Num rows: 1149975359 Data size: 101451160012 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col8 / _col1) > (_col6 / _col3)) (type: boolean)
+ predicate: ((_col1 / _col8) > (_col6 / _col3)) (type: boolean)
Statistics: Num rows: 383325119 Data size: 33817053278 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col5 (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query15.q.out b/ql/src/test/results/clientpositive/perf/spark/query15.q.out
index e8ffcb1..c54b95a 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query15.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query15.q.out
@@ -44,16 +44,52 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 305), Map 7 (PARTITION-LEVEL SORT, 305)
- Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 873), Reducer 2 (PARTITION-LEVEL SORT, 873)
- Reducer 4 <- Map 9 (PARTITION-LEVEL SORT, 686), Reducer 3 (PARTITION-LEVEL SORT, 686)
- Reducer 5 <- Reducer 4 (GROUP, 406)
- Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 6 (PARTITION-LEVEL SORT, 855)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 927), Reducer 8 (PARTITION-LEVEL SORT, 927)
+ Reducer 4 <- Reducer 3 (GROUP, 369)
+ Reducer 5 <- Reducer 4 (SORT, 1)
+ Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 305), Map 9 (PARTITION-LEVEL SORT, 305)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
+ alias: customer
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: c_customer_sk (type: int), c_current_addr_sk (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Map 6
+ Map Operator Tree:
+ TableScan
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ca_address_sk is not null (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ca_address_sk (type: int), ca_state (type: string), ca_zip (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string)
+ Map 7
+ Map Operator Tree:
+ TableScan
alias: catalog_sales
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -69,7 +105,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: decimal(7,2))
- Map 7
+ Map 9
Map Operator Tree:
TableScan
alias: date_dim
@@ -86,124 +122,72 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Map 8
- Map Operator Tree:
- TableScan
- alias: customer
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c_customer_sk (type: int), c_current_addr_sk (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
- Map 9
- Map Operator Tree:
- TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ca_address_sk is not null (type: boolean)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ca_address_sk (type: int), ca_state (type: string), ca_zip (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col3, _col4
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(7,2))
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: string), _col4 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col7
+ 0 _col0 (type: int)
+ 1 _col1 (type: int)
+ outputColumnNames: _col3, _col4, _col7
Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col7 (type: int)
- sort order: +
- Map-reduce partition columns: _col7 (type: int)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(7,2))
- Reducer 4
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col7 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col9, _col10
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col2 > 500) or (_col9) IN ('CA', 'WA', 'GA') or (substr(_col10, 1, 5)) IN ('85669', '86197', '88274', '83405', '86475', '85392', '85460', '80348', '81792')) (type: boolean)
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col3) IN ('CA', 'WA', 'GA') or (_col7 > 500) or (substr(_col4, 1, 5)) IN ('85669', '86197', '88274', '83405', '86475', '85392', '85460', '80348', '81792')) (type: boolean)
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col2 (type: decimal(7,2)), _col10 (type: string)
- outputColumnNames: _col2, _col10
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ expressions: _col4 (type: string), _col7 (type: decimal(7,2))
+ outputColumnNames: _col4, _col7
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col2)
- keys: _col10 (type: string)
+ aggregations: sum(_col7)
+ keys: _col4 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: decimal(17,2))
- Reducer 5
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: decimal(17,2))
- Reducer 6
+ Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: decimal(17,2))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 191657247 Data size: 25954241376 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column stats: NONE
@@ -214,6 +198,22 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 8
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(7,2))
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query16.q.out b/ql/src/test/results/clientpositive/perf/spark/query16.q.out
index 625faff..743f27d 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query16.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query16.q.out
@@ -60,7 +60,8 @@ limit 100
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-1 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -86,7 +87,12 @@ STAGE PLANS:
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 9
+
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 8
Map Operator Tree:
TableScan
alias: date_dim
@@ -109,7 +115,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 13 <- Map 12 (GROUP, 24)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 464), Map 8 (PARTITION-LEVEL SORT, 464)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 494), Map 9 (PARTITION-LEVEL SORT, 494)
Reducer 3 <- Map 11 (PARTITION-LEVEL SORT, 711), Reducer 2 (PARTITION-LEVEL SORT, 711)
Reducer 4 <- Reducer 13 (PARTITION-LEVEL SORT, 459), Reducer 3 (PARTITION-LEVEL SORT, 459)
Reducer 5 <- Reducer 4 (GROUP, 246)
@@ -129,12 +135,24 @@ STAGE PLANS:
expressions: cs_ship_date_sk (type: int), cs_ship_addr_sk (type: int), cs_call_center_sk (type: int), cs_warehouse_sk (type: int), cs_order_number (type: int), cs_ext_ship_cost (type: decimal(7,2)), cs_net_profit (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6
+ input vertices:
+ 1 Map 8
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ Local Work:
+ Map Reduce Local Work
Map 11
Map Operator Tree:
TableScan
@@ -176,7 +194,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 9
Map Operator Tree:
TableScan
alias: customer_address
@@ -220,34 +238,24 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col3, _col4, _col5, _col6
input vertices:
- 1 Map 9
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col3, _col4, _col5, _col6
- input vertices:
- 1 Map 10
+ 1 Map 10
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col4 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col4 (type: int)
Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ value expressions: _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
Reducer 3
Reduce Operator Tree:
Join Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query18.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query18.q.out b/ql/src/test/results/clientpositive/perf/spark/query18.q.out
index c7dee3d..cb3c114 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query18.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query18.q.out
@@ -72,53 +72,52 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 307), Map 13 (PARTITION-LEVEL SORT, 307)
- Reducer 12 <- Map 14 (PARTITION-LEVEL SORT, 336), Reducer 11 (PARTITION-LEVEL SORT, 336)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 428), Map 8 (PARTITION-LEVEL SORT, 428)
- Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 302), Reducer 2 (PARTITION-LEVEL SORT, 302)
- Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 694), Reducer 3 (PARTITION-LEVEL SORT, 694)
- Reducer 5 <- Map 15 (PARTITION-LEVEL SORT, 411), Reducer 4 (PARTITION-LEVEL SORT, 411)
- Reducer 6 <- Reducer 5 (GROUP, 1009)
- Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 10 <- Map 13 (PARTITION-LEVEL SORT, 306), Map 9 (PARTITION-LEVEL SORT, 306)
+ Reducer 11 <- Map 14 (PARTITION-LEVEL SORT, 337), Reducer 10 (PARTITION-LEVEL SORT, 337)
+ Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 374), Reducer 11 (PARTITION-LEVEL SORT, 374)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 428), Map 7 (PARTITION-LEVEL SORT, 428)
+ Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 302), Reducer 2 (PARTITION-LEVEL SORT, 302)
+ Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 731), Reducer 3 (PARTITION-LEVEL SORT, 731)
+ Reducer 5 <- Reducer 4 (GROUP, 1009)
+ Reducer 6 <- Reducer 5 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ alias: customer
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ca_state) IN ('ND', 'WI', 'AL', 'NC', 'OK', 'MS', 'TN') and ca_address_sk is not null) (type: boolean)
- Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((c_birth_month) IN (9, 5, 12, 4, 1, 10) and c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_county (type: string), ca_state (type: string), ca_country (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
+ expressions: c_customer_sk (type: int), c_current_cdemo_sk (type: int), c_current_addr_sk (type: int), c_birth_year (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col4
+ Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
- Map 10
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: int), _col4 (type: int)
+ Map 13
Map Operator Tree:
TableScan
- alias: catalog_sales
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cs_bill_cdemo_sk is not null and cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_bill_cdemo_sk (type: int), cs_item_sk (type: int), cs_quantity (type: int), cs_list_price (type: decimal(7,2)), cs_sales_price (type: decimal(7,2)), cs_coupon_amt (type: decimal(7,2)), cs_net_profit (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2))
- Map 13
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Map 14
Map Operator Tree:
TableScan
alias: cd1
@@ -136,23 +135,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 465450 Data size: 179296539 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: int)
- Map 14
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Map 15
Map Operator Tree:
TableScan
@@ -171,25 +153,25 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 8
+ Map 7
Map Operator Tree:
TableScan
- alias: customer
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((c_birth_month) IN (9, 5, 12, 4, 1, 10) and c_current_addr_sk is not null and c_current_cdemo_sk is not null and c_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((ca_state) IN ('ND', 'WI', 'AL', 'NC', 'OK', 'MS', 'TN') and ca_address_sk is not null) (type: boolean)
+ Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: c_customer_sk (type: int), c_current_cdemo_sk (type: int), c_current_addr_sk (type: int), c_birth_year (type: int)
- outputColumnNames: _col0, _col1, _col2, _col4
- Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
+ expressions: ca_address_sk (type: int), ca_county (type: string), ca_state (type: string), ca_country (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col4 (type: int)
- Map 9
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ Map 8
Map Operator Tree:
TableScan
alias: cd2
@@ -206,102 +188,120 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
- Reducer 11
+ Map 9
+ Map Operator Tree:
+ TableScan
+ alias: catalog_sales
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cs_bill_cdemo_sk is not null and cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_bill_cdemo_sk (type: int), cs_item_sk (type: int), cs_quantity (type: int), cs_list_price (type: decimal(7,2)), cs_sales_price (type: decimal(7,2)), cs_coupon_amt (type: decimal(7,2)), cs_net_profit (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2))
+ Reducer 10
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col6, _col7, _col8, _col12
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col12 (type: int)
- Reducer 12
+ value expressions: _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2))
+ Reducer 11
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col3, _col4, _col5, _col6, _col7, _col8, _col12
+ outputColumnNames: _col1, _col3, _col4, _col5, _col6, _col7, _col8, _col14
Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col12 (type: int)
- outputColumnNames: _col3, _col5, _col6, _col7, _col8, _col9, _col10, _col14
+ Reduce Output Operator
+ key expressions: _col3 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: int)
Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col14 (type: int)
+ Reducer 12
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col4, _col5, _col6, _col7, _col8, _col14, _col16
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col16 (type: string), _col1 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col14 (type: int)
+ outputColumnNames: _col1, _col3, _col6, _col7, _col8, _col9, _col10, _col16
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: int)
sort order: +
Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: int), _col6 (type: int), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col14 (type: int)
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col6 (type: int), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col16 (type: int)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col2 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col8
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col4, _col6, _col7, _col8
Statistics: Num rows: 44000000 Data size: 37840889538 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col5 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col5 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 44000000 Data size: 37840889538 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col8 (type: int)
+ value expressions: _col0 (type: int), _col4 (type: int), _col6 (type: string), _col7 (type: string), _col8 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col5 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col8
+ outputColumnNames: _col0, _col4, _col6, _col7, _col8
Statistics: Num rows: 48400001 Data size: 41624979393 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col4 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col4 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 48400001 Data size: 41624979393 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col8 (type: int)
+ value expressions: _col4 (type: int), _col6 (type: string), _col7 (type: string), _col8 (type: string)
Reducer 4
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col4 (type: int)
+ 0 _col0 (type: int)
1 _col3 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col8, _col15, _col16, _col17, _col18, _col19, _col20, _col24
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col15 (type: int)
- sort order: +
- Map-reduce partition columns: _col15 (type: int)
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col8 (type: int), _col16 (type: int), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col24 (type: int)
- Reducer 5
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col15 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col8, _col16, _col17, _col18, _col19, _col20, _col24, _col26
+ outputColumnNames: _col4, _col6, _col7, _col8, _col11, _col16, _col17, _col18, _col19, _col20, _col26
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col26 (type: string), _col3 (type: string), _col2 (type: string), _col1 (type: string), CAST( _col16 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col17 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col19 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col18 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col20 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col8 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col24 AS decimal(12,2)) (type: decimal(12,2))
+ expressions: _col11 (type: string), _col8 (type: string), _col7 (type: string), _col6 (type: string), CAST( _col16 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col17 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col19 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col18 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col20 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col4 AS decimal(12,2)) (type: decimal(12,2)), CAST( _col26 AS decimal(12,2)) (type: decimal(12,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
Group By Operator
@@ -316,7 +316,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: bigint)
Statistics: Num rows: 2108229765 Data size: 285496662075 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: decimal(22,2)), _col6 (type: bigint), _col7 (type: decimal(22,2)), _col8 (type: bigint), _col9 (type: decimal(22,2)), _col10 (type: bigint), _col11 (type: decimal(22,2)), _col12 (type: bigint), _col13 (type: decimal(22,2)), _col14 (type: bigint), _col15 (type: decimal(22,2)), _col16 (type: bigint), _col17 (type: decimal(22,2)), _col18 (type: bigint)
- Reducer 6
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1), sum(VALUE._col2), count(VALUE._col3), sum(VALUE._col4), count(VALUE._col5), sum(VALUE._col6), count(VALUE._col7), sum(VALUE._col8), count(VALUE._col9), sum(VALUE._col10), count(VALUE._col11), sum(VALUE._col12), count(VALUE._col13)
@@ -335,7 +335,7 @@ STAGE PLANS:
Statistics: Num rows: 1054114882 Data size: 142748330969 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col4 (type: decimal(38,18)), _col5 (type: decimal(38,18)), _col6 (type: decimal(38,18)), _col7 (type: decimal(38,18)), _col8 (type: decimal(38,18)), _col9 (type: decimal(38,18)), _col10 (type: decimal(38,18))
- Reducer 7
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey3 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: decimal(38,18)), VALUE._col1 (type: decimal(38,18)), VALUE._col2 (type: decimal(38,18)), VALUE._col3 (type: decimal(38,18)), VALUE._col4 (type: decimal(38,18)), VALUE._col5 (type: decimal(38,18)), VALUE._col6 (type: decimal(38,18))
[04/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query78.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query78.q.out b/ql/src/test/results/clientpositive/perf/spark/query78.q.out
index c2e8577..1467c5f 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query78.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query78.q.out
@@ -120,41 +120,57 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 164), Map 14 (PARTITION-LEVEL SORT, 164)
- Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 85), Reducer 11 (PARTITION-LEVEL SORT, 85)
- Reducer 13 <- Reducer 12 (GROUP, 93)
- Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85)
+ Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 14 (PARTITION-LEVEL SORT, 85)
+ Reducer 12 <- Reducer 11 (GROUP, 93)
+ Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 164), Map 15 (PARTITION-LEVEL SORT, 164)
+ Reducer 17 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85)
Reducer 18 <- Reducer 17 (GROUP, 93)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 8 (PARTITION-LEVEL SORT, 432)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 219), Reducer 8 (PARTITION-LEVEL SORT, 219)
Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 177), Map 21 (PARTITION-LEVEL SORT, 177)
- Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 219), Reducer 2 (PARTITION-LEVEL SORT, 219)
- Reducer 4 <- Reducer 3 (GROUP, 241)
- Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 167), Reducer 4 (PARTITION-LEVEL SORT, 167)
- Reducer 6 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 5 (PARTITION-LEVEL SORT, 91)
- Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 241)
+ Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 167), Reducer 3 (PARTITION-LEVEL SORT, 167)
+ Reducer 5 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 4 (PARTITION-LEVEL SORT, 91)
+ Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 432), Map 9 (PARTITION-LEVEL SORT, 432)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: store_sales
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ss_sold_date_sk is not null (type: boolean)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_sales_price (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int), _col3 (type: int)
- sort order: ++
- Map-reduce partition columns: _col1 (type: int), _col3 (type: int)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Map 10
Map Operator Tree:
TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Map 13
+ Map Operator Tree:
+ TableScan
alias: web_sales
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -170,7 +186,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int), _col3 (type: int)
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
- Map 14
+ Map 15
Map Operator Tree:
TableScan
alias: web_returns
@@ -184,40 +200,6 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
- Map 15
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Map 16
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Map 19
Map Operator Tree:
TableScan
@@ -250,7 +232,25 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: store_sales
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ss_sold_date_sk is not null (type: boolean)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_sales_price (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col3 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col1 (type: int), _col3 (type: int)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ Map 9
Map Operator Tree:
TableScan
alias: store_returns
@@ -264,59 +264,19 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
- Map 9
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reducer 11
Reduce Operator Tree:
Join Operator
condition map:
- Left Outer Join 0 to 1
- keys:
- 0 _col1 (type: int), _col3 (type: int)
- 1 _col0 (type: int), _col1 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col8
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col8 is null (type: boolean)
- Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6
- Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
- Reducer 12
- Reduce Operator Tree:
- Join Operator
- condition map:
Inner Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col4, _col5, _col6
+ outputColumnNames: _col3, _col4, _col6, _col7, _col8
Statistics: Num rows: 87121617 Data size: 11846020431 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col4), sum(_col5), sum(_col6)
- keys: _col2 (type: int), _col1 (type: int)
+ aggregations: sum(_col6), sum(_col7), sum(_col8)
+ keys: _col4 (type: int), _col3 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 87121617 Data size: 11846020431 Basic stats: COMPLETE Column stats: NONE
@@ -326,7 +286,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 87121617 Data size: 11846020431 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2))
- Reducer 13
+ Reducer 12
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
@@ -344,6 +304,29 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int), _col0 (type: int)
Statistics: Num rows: 43560808 Data size: 5923010147 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2))
+ Reducer 14
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join 0 to 1
+ keys:
+ 0 _col1 (type: int), _col3 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col8
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col8 is null (type: boolean)
+ Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6
+ Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 79201469 Data size: 10769109250 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
Reducer 17
Reduce Operator Tree:
Join Operator
@@ -388,25 +371,24 @@ STAGE PLANS:
Reduce Operator Tree:
Join Operator
condition map:
- Left Outer Join 0 to 1
+ Inner Join 0 to 1
keys:
- 0 _col1 (type: int), _col3 (type: int)
- 1 _col0 (type: int), _col1 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col8
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col8 is null (type: boolean)
- Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6
- Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col3, _col4, _col6, _col7, _col8
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col6), sum(_col7), sum(_col8)
+ keys: _col4 (type: int), _col3 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2))
Reducer 20
Reduce Operator Tree:
Join Operator
@@ -432,28 +414,6 @@ STAGE PLANS:
value expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
Reducer 3
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col4, _col5, _col6
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col4), sum(_col5), sum(_col6)
- keys: _col2 (type: int), _col1 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2))
- Reducer 4
- Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
keys: KEY._col0 (type: int), KEY._col1 (type: int)
@@ -470,7 +430,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int), _col0 (type: int)
Statistics: Num rows: 174238687 Data size: 15371387547 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2))
- Reducer 5
+ Reducer 4
Reduce Operator Tree:
Join Operator
condition map:
@@ -489,7 +449,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 63887519 Data size: 5636175497 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: bigint), _col3 (type: decimal(17,2)), _col4 (type: decimal(17,2)), _col7 (type: bigint), _col8 (type: decimal(17,2)), _col9 (type: decimal(17,2))
- Reducer 6
+ Reducer 5
Reduce Operator Tree:
Join Operator
condition map:
@@ -511,7 +471,7 @@ STAGE PLANS:
sort order: ++---++++
Statistics: Num rows: 23425424 Data size: 2066597727 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
- Reducer 7
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: 2000 (type: int), KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey8 (type: double), KEY.reducesinkkey2 (type: bigint), KEY.reducesinkkey3 (type: decimal(17,2)), KEY.reducesinkkey4 (type: decimal(17,2)), KEY.reducesinkkey5 (type: bigint), KEY.reducesinkkey6 (type: decimal(18,2)), KEY.reducesinkkey7 (type: decimal(18,2))
@@ -527,6 +487,29 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 8
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join 0 to 1
+ keys:
+ 0 _col1 (type: int), _col3 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6, _col8
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: _col8 is null (type: boolean)
+ Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5, _col6
+ Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query79.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query79.q.out b/ql/src/test/results/clientpositive/perf/spark/query79.q.out
index 8dcb7f1..f010369 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query79.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query79.q.out
@@ -221,26 +221,26 @@ STAGE PLANS:
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col6), sum(_col7)
- keys: _col1 (type: int), _col13 (type: string), _col3 (type: int), _col5 (type: int)
+ keys: _col1 (type: int), _col3 (type: int), _col5 (type: int), _col13 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string)
sort order: ++++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int)
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: string)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2))
Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1)
- keys: KEY._col0 (type: int), KEY._col1 (type: string), KEY._col2 (type: int), KEY._col3 (type: int)
+ keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int), KEY._col3 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col3 (type: int), _col0 (type: int), _col1 (type: string), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2))
+ expressions: _col2 (type: int), _col0 (type: int), _col3 (type: string), _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query8.q.out b/ql/src/test/results/clientpositive/perf/spark/query8.q.out
index 677746b..583abbc 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query8.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query8.q.out
@@ -222,14 +222,31 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP, 159)
- Reducer 4 <- Reducer 3 (GROUP, 109), Reducer 8 (GROUP, 109)
- Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 586), Map 9 (PARTITION-LEVEL SORT, 586)
- Reducer 7 <- Reducer 6 (GROUP, 349)
- Reducer 8 <- Reducer 7 (GROUP, 59)
+ Reducer 10 <- Map 13 (PARTITION-LEVEL SORT, 586), Map 9 (PARTITION-LEVEL SORT, 586)
+ Reducer 11 <- Reducer 10 (GROUP, 349)
+ Reducer 12 <- Reducer 11 (GROUP, 59)
+ Reducer 7 <- Map 6 (GROUP, 159)
+ Reducer 8 <- Reducer 12 (GROUP, 109), Reducer 7 (GROUP, 109)
#### A masked pattern was here ####
Vertices:
- Map 2
+ Map 13
+ Map Operator Tree:
+ TableScan
+ alias: customer
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((c_preferred_cust_flag = 'Y') and c_current_addr_sk is not null) (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: c_current_addr_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
+ Map 6
Map Operator Tree:
TableScan
alias: customer_address
@@ -253,7 +270,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 20000000 Data size: 20297597642 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 5
+ Map 9
Map Operator Tree:
TableScan
alias: customer_address
@@ -271,65 +288,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 9
- Map Operator Tree:
- TableScan
- alias: customer
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((c_preferred_cust_flag = 'Y') and c_current_addr_sk is not null) (type: boolean)
- Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: c_current_addr_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 40000000 Data size: 34400807926 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(_col1)
- keys: _col0 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 13666666 Data size: 13870024459 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 13666666 Data size: 13870024459 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
- Reducer 4
- Local Work:
- Map Reduce Local Work
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 6833333 Data size: 6935012229 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col1 = 2L) (type: boolean)
- Statistics: Num rows: 1 Data size: 1014 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 1014 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 substr(_col2, 1, 2) (type: string)
- 1 substr(_col0, 1, 2) (type: string)
- Reducer 6
+ Reducer 10
Reduce Operator Tree:
Join Operator
condition map:
@@ -351,7 +310,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 44000000 Data size: 44654715780 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 7
+ Reducer 11
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -378,7 +337,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 7333333 Data size: 7442452291 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 8
+ Reducer 12
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -398,12 +357,53 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 13666666 Data size: 13870024459 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
+ Reducer 7
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(_col1)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 13666666 Data size: 13870024459 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 13666666 Data size: 13870024459 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
+ Reducer 8
+ Local Work:
+ Map Reduce Local Work
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6833333 Data size: 6935012229 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col1 = 2L) (type: boolean)
+ Statistics: Num rows: 1 Data size: 1014 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 1014 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 substr(_col0, 1, 2) (type: string)
+ 1 substr(_col2, 1, 2) (type: string)
Stage: Stage-2
Spark
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 14
Map Operator Tree:
TableScan
alias: store
@@ -419,15 +419,15 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 substr(_col2, 1, 2) (type: string)
- 1 substr(_col0, 1, 2) (type: string)
- outputColumnNames: _col0, _col1
+ 0 substr(_col0, 1, 2) (type: string)
+ 1 substr(_col2, 1, 2) (type: string)
+ outputColumnNames: _col1, _col2
input vertices:
- 1 Reducer 4
+ 0 Reducer 8
Statistics: Num rows: 1874 Data size: 3581903 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col1 (type: int)
Local Work:
Map Reduce Local Work
@@ -435,12 +435,12 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 398), Map 14 (PARTITION-LEVEL SORT, 398)
- Reducer 12 <- Reducer 11 (GROUP, 481)
- Reducer 13 <- Reducer 12 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 5 (PARTITION-LEVEL SORT, 398)
+ Reducer 3 <- Reducer 2 (GROUP, 481)
+ Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
Vertices:
- Map 10
+ Map 1
Map Operator Tree:
TableScan
alias: store_sales
@@ -458,7 +458,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: decimal(7,2))
- Map 14
+ Map 5
Map Operator Tree:
TableScan
alias: date_dim
@@ -475,7 +475,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Reducer 11
+ Reducer 2
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -491,15 +491,15 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col1 (type: int)
- outputColumnNames: _col1, _col6
+ outputColumnNames: _col2, _col8
input vertices:
- 0 Map 1
+ 1 Map 14
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col6)
- keys: _col1 (type: string)
+ aggregations: sum(_col2)
+ keys: _col8 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -510,7 +510,7 @@ STAGE PLANS:
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: decimal(17,2))
- Reducer 12
+ Reducer 3
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -524,7 +524,7 @@ STAGE PLANS:
Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: decimal(17,2))
- Reducer 13
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: decimal(17,2))
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query80.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query80.q.out b/ql/src/test/results/clientpositive/perf/spark/query80.q.out
index 051bfd1..9b008eb 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query80.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query80.q.out
@@ -194,7 +194,8 @@ STAGE DEPENDENCIES:
Stage-4 depends on stages: Stage-3
Stage-5 depends on stages: Stage-4
Stage-6 depends on stages: Stage-5
- Stage-1 depends on stages: Stage-6
+ Stage-7 depends on stages: Stage-6
+ Stage-1 depends on stages: Stage-7
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -202,7 +203,25 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 12
+ Map 10
+ Map Operator Tree:
+ TableScan
+ alias: promotion
+ Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((p_channel_tv = 'N') and p_promo_sk is not null) (type: boolean)
+ Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: p_promo_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 11
Map Operator Tree:
TableScan
alias: store
@@ -225,7 +244,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 10
+ Map 8
Map Operator Tree:
TableScan
alias: date_dim
@@ -248,7 +267,30 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 21
+ Map 20
+ Map Operator Tree:
+ TableScan
+ alias: promotion
+ Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((p_channel_tv = 'N') and p_promo_sk is not null) (type: boolean)
+ Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: p_promo_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-5
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 18
Map Operator Tree:
TableScan
alias: date_dim
@@ -267,11 +309,29 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
- Stage: Stage-5
+ Stage: Stage-6
Spark
#### A masked pattern was here ####
Vertices:
- Map 33
+ Map 29
+ Map Operator Tree:
+ TableScan
+ alias: promotion
+ Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((p_channel_tv = 'N') and p_promo_sk is not null) (type: boolean)
+ Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: p_promo_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 30
Map Operator Tree:
TableScan
alias: web_site
@@ -290,11 +350,11 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
- Stage: Stage-6
+ Stage: Stage-7
Spark
#### A masked pattern was here ####
Vertices:
- Map 31
+ Map 27
Map Operator Tree:
TableScan
alias: date_dim
@@ -316,21 +376,18 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 329), Map 19 (PARTITION-LEVEL SORT, 329)
- Reducer 15 <- Map 20 (PARTITION-LEVEL SORT, 336), Reducer 14 (PARTITION-LEVEL SORT, 336)
- Reducer 16 <- Map 11 (PARTITION-LEVEL SORT, 408), Reducer 15 (PARTITION-LEVEL SORT, 408)
- Reducer 17 <- Map 23 (PARTITION-LEVEL SORT, 447), Reducer 16 (PARTITION-LEVEL SORT, 447)
- Reducer 18 <- Reducer 17 (GROUP, 491)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 8 (PARTITION-LEVEL SORT, 432)
- Reducer 25 <- Map 24 (PARTITION-LEVEL SORT, 164), Map 29 (PARTITION-LEVEL SORT, 164)
- Reducer 26 <- Map 30 (PARTITION-LEVEL SORT, 169), Reducer 25 (PARTITION-LEVEL SORT, 169)
- Reducer 27 <- Map 32 (PARTITION-LEVEL SORT, 206), Reducer 26 (PARTITION-LEVEL SORT, 206)
- Reducer 28 <- Reducer 27 (GROUP, 247)
- Reducer 3 <- Map 20 (PARTITION-LEVEL SORT, 437), Reducer 2 (PARTITION-LEVEL SORT, 437)
- Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 531), Reducer 3 (PARTITION-LEVEL SORT, 531)
- Reducer 5 <- Reducer 4 (GROUP, 640)
- Reducer 6 <- Reducer 18 (GROUP, 1009), Reducer 28 (GROUP, 1009), Reducer 5 (GROUP, 1009)
- Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 329), Map 17 (PARTITION-LEVEL SORT, 329)
+ Reducer 14 <- Map 19 (PARTITION-LEVEL SORT, 371), Reducer 13 (PARTITION-LEVEL SORT, 371)
+ Reducer 15 <- Map 21 (PARTITION-LEVEL SORT, 447), Reducer 14 (PARTITION-LEVEL SORT, 447)
+ Reducer 16 <- Reducer 15 (GROUP, 491)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 7 (PARTITION-LEVEL SORT, 432)
+ Reducer 23 <- Map 22 (PARTITION-LEVEL SORT, 164), Map 26 (PARTITION-LEVEL SORT, 164)
+ Reducer 24 <- Map 28 (PARTITION-LEVEL SORT, 187), Reducer 23 (PARTITION-LEVEL SORT, 187)
+ Reducer 25 <- Reducer 24 (GROUP, 247)
+ Reducer 3 <- Map 19 (PARTITION-LEVEL SORT, 483), Reducer 2 (PARTITION-LEVEL SORT, 483)
+ Reducer 4 <- Reducer 3 (GROUP, 640)
+ Reducer 5 <- Reducer 16 (GROUP, 1009), Reducer 25 (GROUP, 1009), Reducer 4 (GROUP, 1009)
+ Reducer 6 <- Reducer 5 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -351,24 +408,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int), _col4 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
- Map 11
- Map Operator Tree:
- TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((i_current_price > 50) and i_item_sk is not null) (type: boolean)
- Statistics: Num rows: 154000 Data size: 221186819 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: i_item_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 154000 Data size: 221186819 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 154000 Data size: 221186819 Basic stats: COMPLETE Column stats: NONE
- Map 13
+ Map 12
Map Operator Tree:
TableScan
alias: catalog_sales
@@ -386,7 +426,7 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: int), _col4 (type: int)
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
- Map 19
+ Map 17
Map Operator Tree:
TableScan
alias: catalog_returns
@@ -404,24 +444,24 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2))
- Map 20
+ Map 19
Map Operator Tree:
TableScan
- alias: promotion
- Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((p_channel_tv = 'N') and p_promo_sk is not null) (type: boolean)
- Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((i_current_price > 50) and i_item_sk is not null) (type: boolean)
+ Statistics: Num rows: 154000 Data size: 221186819 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: p_promo_sk (type: int)
+ expressions: i_item_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 154000 Data size: 221186819 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
- Map 23
+ Statistics: Num rows: 154000 Data size: 221186819 Basic stats: COMPLETE Column stats: NONE
+ Map 21
Map Operator Tree:
TableScan
alias: catalog_page
@@ -439,7 +479,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 46000 Data size: 21198808 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 24
+ Map 22
Map Operator Tree:
TableScan
alias: web_sales
@@ -457,7 +497,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int), _col4 (type: int)
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
- Map 29
+ Map 26
Map Operator Tree:
TableScan
alias: web_returns
@@ -475,24 +515,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2))
- Map 30
- Map Operator Tree:
- TableScan
- alias: promotion
- Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((p_channel_tv = 'N') and p_promo_sk is not null) (type: boolean)
- Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: p_promo_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1150 Data size: 1356710 Basic stats: COMPLETE Column stats: NONE
- Map 32
+ Map 28
Map Operator Tree:
TableScan
alias: item
@@ -509,7 +532,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 154000 Data size: 221186819 Basic stats: COMPLETE Column stats: NONE
- Map 8
+ Map 7
Map Operator Tree:
TableScan
alias: store_returns
@@ -527,7 +550,9 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2))
- Reducer 14
+ Reducer 13
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -537,41 +562,25 @@ STAGE PLANS:
1 _col0 (type: int), _col1 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col5, _col6, _col9, _col10
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
- Reducer 15
- Local Work:
- Map Reduce Local Work
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col3 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col9, _col10
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col5, _col6, _col9, _col10
+ outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col9, _col10
input vertices:
- 1 Map 21
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ 1 Map 18
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int)
sort order: +
Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
- Reducer 16
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
+ Reducer 14
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -579,15 +588,25 @@ STAGE PLANS:
keys:
0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col5, _col6, _col9, _col10
- Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ outputColumnNames: _col1, _col3, _col5, _col6, _col9, _col10
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col5, _col6, _col9, _col10
+ input vertices:
+ 1 Map 20
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
- Reducer 17
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
+ Reducer 15
Reduce Operator Tree:
Join Operator
condition map:
@@ -613,7 +632,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 463810558 Data size: 62809267017 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(22,2)), _col3 (type: decimal(23,2))
- Reducer 18
+ Reducer 16
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
@@ -639,6 +658,8 @@ STAGE PLANS:
TopN Hash Memory Usage: 0.1
value expressions: _col3 (type: decimal(27,2)), _col4 (type: decimal(32,2)), _col5 (type: decimal(33,2))
Reducer 2
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -648,13 +669,25 @@ STAGE PLANS:
1 _col0 (type: int), _col1 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col5, _col6, _col9, _col10
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
- Reducer 25
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col9, _col10
+ input vertices:
+ 1 Map 8
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
+ Reducer 23
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -664,41 +697,23 @@ STAGE PLANS:
1 _col0 (type: int), _col1 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col5, _col6, _col9, _col10
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
- Reducer 26
- Local Work:
- Map Reduce Local Work
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col3 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col9, _col10
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col5, _col6, _col9, _col10
+ outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col9, _col10
input vertices:
- 1 Map 31
- Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
+ 1 Map 27
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
- Reducer 27
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
+ Reducer 24
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -708,35 +723,45 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col5, _col6, _col9, _col10
- Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col2, _col3, _col5, _col6, _col9, _col10
+ Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col5, _col6, _col9, _col10, _col18
+ outputColumnNames: _col2, _col5, _col6, _col9, _col10
input vertices:
- 1 Map 33
- Statistics: Num rows: 231917759 Data size: 31534108438 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col18 (type: string), _col5 (type: decimal(7,2)), COALESCE(_col9,0) (type: decimal(12,2)), (_col6 - COALESCE(_col10,0)) (type: decimal(13,2))
- outputColumnNames: _col0, _col1, _col2, _col3
+ 1 Map 29
+ Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col5, _col6, _col9, _col10, _col18
+ input vertices:
+ 1 Map 30
Statistics: Num rows: 231917759 Data size: 31534108438 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col1), sum(_col2), sum(_col3)
- keys: _col0 (type: string)
- mode: hash
+ Select Operator
+ expressions: _col18 (type: string), _col5 (type: decimal(7,2)), COALESCE(_col9,0) (type: decimal(12,2)), (_col6 - COALESCE(_col10,0)) (type: decimal(13,2))
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 231917759 Data size: 31534108438 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: sum(_col1), sum(_col2), sum(_col3)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 231917759 Data size: 31534108438 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(22,2)), _col3 (type: decimal(23,2))
- Reducer 28
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 231917759 Data size: 31534108438 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(22,2)), _col3 (type: decimal(23,2))
+ Reducer 25
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
@@ -769,65 +794,47 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col3 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col5, _col6, _col9, _col10
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col5, _col6, _col9, _col10
- input vertices:
- 1 Map 10
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2))
- Reducer 4
- Local Work:
- Map Reduce Local Work
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col5, _col6, _col9, _col10
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col2, _col3, _col5, _col6, _col9, _col10
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col5, _col6, _col9, _col10, _col18
+ outputColumnNames: _col2, _col5, _col6, _col9, _col10
input vertices:
- 1 Map 12
- Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col18 (type: string), _col5 (type: decimal(7,2)), COALESCE(_col9,0) (type: decimal(12,2)), (_col6 - COALESCE(_col10,0)) (type: decimal(13,2))
- outputColumnNames: _col0, _col1, _col2, _col3
+ 1 Map 10
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col5, _col6, _col9, _col10, _col18
+ input vertices:
+ 1 Map 11
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col1), sum(_col2), sum(_col3)
- keys: _col0 (type: string)
- mode: hash
+ Select Operator
+ expressions: _col18 (type: string), _col5 (type: decimal(7,2)), COALESCE(_col9,0) (type: decimal(12,2)), (_col6 - COALESCE(_col10,0)) (type: decimal(13,2))
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Group By Operator
+ aggregations: sum(_col1), sum(_col2), sum(_col3)
+ keys: _col0 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(22,2)), _col3 (type: decimal(23,2))
- Reducer 5
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(17,2)), _col2 (type: decimal(22,2)), _col3 (type: decimal(23,2))
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
@@ -852,7 +859,7 @@ STAGE PLANS:
Statistics: Num rows: 2435062716 Data size: 264270971781 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col3 (type: decimal(27,2)), _col4 (type: decimal(32,2)), _col5 (type: decimal(33,2))
- Reducer 6
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
@@ -871,7 +878,7 @@ STAGE PLANS:
Statistics: Num rows: 1217531358 Data size: 132135485890 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: decimal(27,2)), _col3 (type: decimal(32,2)), _col4 (type: decimal(33,2))
- Reducer 7
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: decimal(27,2)), VALUE._col1 (type: decimal(32,2)), VALUE._col2 (type: decimal(33,2))
[06/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query65.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query65.q.out b/ql/src/test/results/clientpositive/perf/spark/query65.q.out
index 575fc5c..3b3baef 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query65.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query65.q.out
@@ -57,8 +57,7 @@ POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
Stage-3 depends on stages: Stage-2
- Stage-4 depends on stages: Stage-3
- Stage-1 depends on stages: Stage-4
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -66,29 +65,6 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 11
- Map Operator Tree:
- TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: s_store_sk is not null (type: boolean)
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: s_store_sk (type: int), s_store_name (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- Local Work:
- Map Reduce Local Work
-
- Stage: Stage-3
- Spark
-#### A masked pattern was here ####
- Vertices:
Map 6
Map Operator Tree:
TableScan
@@ -108,7 +84,7 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
- Stage: Stage-4
+ Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
@@ -135,8 +111,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP, 437)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328)
- Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 86), Reducer 3 (PARTITION-LEVEL SORT, 86)
+ Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 328), Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328)
+ Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 166), Reducer 3 (PARTITION-LEVEL SORT, 166)
Reducer 5 <- Reducer 4 (SORT, 1)
Reducer 8 <- Map 7 (GROUP PARTITION-LEVEL SORT, 437)
#### A masked pattern was here ####
@@ -180,6 +156,24 @@ STAGE PLANS:
Map 10
Map Operator Tree:
TableScan
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: s_store_sk is not null (type: boolean)
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: s_store_sk (type: int), s_store_name (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 11
+ Map Operator Tree:
+ TableScan
alias: item
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -250,23 +244,23 @@ STAGE PLANS:
Join Operator
condition map:
Inner Join 0 to 1
+ Inner Join 0 to 2
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col4
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
+ 2 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col4, _col6
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col2 <= (0.1 * _col4)) (type: boolean)
- Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 232318249 Data size: 20495183367 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 116159124 Data size: 10247591639 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: decimal(17,2))
+ Statistics: Num rows: 232318249 Data size: 20495183367 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(17,2)), _col6 (type: string)
Reducer 4
- Local Work:
- Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -274,34 +268,24 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col6, _col7, _col8, _col9
- Statistics: Num rows: 127775039 Data size: 11272351047 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col6, _col7, _col8, _col9, _col11
- input vertices:
- 1 Map 11
- Statistics: Num rows: 140552545 Data size: 12399586420 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col11 (type: string), _col6 (type: string), _col2 (type: decimal(17,2)), _col7 (type: decimal(7,2)), _col8 (type: decimal(7,2)), _col9 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 140552545 Data size: 12399586420 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Statistics: Num rows: 140552545 Data size: 12399586420 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col2 (type: decimal(17,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: string)
+ outputColumnNames: _col2, _col6, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 255550079 Data size: 22544702192 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col6 (type: string), _col8 (type: string), _col2 (type: decimal(17,2)), _col9 (type: decimal(7,2)), _col10 (type: decimal(7,2)), _col11 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 255550079 Data size: 22544702192 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 255550079 Data size: 22544702192 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col2 (type: decimal(17,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: string)
Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: decimal(17,2)), VALUE._col1 (type: decimal(7,2)), VALUE._col2 (type: decimal(7,2)), VALUE._col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 140552545 Data size: 12399586420 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 255550079 Data size: 22544702192 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query66.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query66.q.out b/ql/src/test/results/clientpositive/perf/spark/query66.q.out
index 15654f9..17e6ac7 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query66.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query66.q.out
@@ -582,7 +582,7 @@ STAGE PLANS:
Edges:
Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 336), Map 14 (PARTITION-LEVEL SORT, 336)
Reducer 12 <- Reducer 11 (GROUP, 447)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 169), Map 14 (PARTITION-LEVEL SORT, 169)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 169), Map 7 (PARTITION-LEVEL SORT, 169)
Reducer 3 <- Reducer 2 (GROUP, 224)
Reducer 4 <- Reducer 12 (GROUP, 336), Reducer 3 (GROUP, 336)
Reducer 5 <- Reducer 4 (SORT, 1)
@@ -666,6 +666,24 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: int)
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int), d_moy (type: int)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int)
Reducer 11
Local Work:
Map Reduce Local Work
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query67.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query67.q.out b/ql/src/test/results/clientpositive/perf/spark/query67.q.out
index a3252b6..26f6775 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query67.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query67.q.out
@@ -86,8 +86,7 @@ limit 100
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-3 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -114,36 +113,14 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
- Stage: Stage-3
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 6
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223) (type: boolean)
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int), d_year (type: int), d_moy (type: int), d_qoy (type: int)
- outputColumnNames: _col0, _col2, _col3, _col4
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- Local Work:
- Map Reduce Local Work
-
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 442), Map 7 (PARTITION-LEVEL SORT, 442)
- Reducer 3 <- Reducer 2 (GROUP, 1009)
- Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1009)
- Reducer 5 <- Reducer 4 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398)
+ Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 486), Reducer 2 (PARTITION-LEVEL SORT, 486)
+ Reducer 4 <- Reducer 3 (GROUP, 1009)
+ Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1009)
+ Reducer 6 <- Reducer 5 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -158,27 +135,33 @@ STAGE PLANS:
expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_store_sk (type: int), ss_quantity (type: int), ss_sales_price (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col7, _col8, _col9
- input vertices:
- 1 Map 6
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col7 (type: int), _col8 (type: int), _col9 (type: int)
- Local Work:
- Map Reduce Local Work
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 7
Map Operator Tree:
TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223) (type: boolean)
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int), d_year (type: int), d_moy (type: int), d_qoy (type: int)
+ outputColumnNames: _col0, _col2, _col3, _col4
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int)
+ Map 9
+ Map Operator Tree:
+ TableScan
alias: item
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -202,37 +185,53 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col7, _col8, _col9, _col11, _col12, _col13, _col14
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col1, _col2, _col3, _col4, _col7, _col8, _col9
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col3, _col4, _col7, _col8, _col9, _col11, _col12, _col13, _col14, _col16
+ outputColumnNames: _col1, _col3, _col4, _col7, _col8, _col9, _col11
input vertices:
1 Map 8
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col11 (type: string)
+ Reducer 3
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col3, _col4, _col7, _col8, _col9, _col11, _col13, _col14, _col15, _col16
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col15 (type: string), _col14 (type: string), _col13 (type: string), _col16 (type: string), _col7 (type: int), _col9 (type: int), _col8 (type: int), _col11 (type: string), COALESCE((_col4 * CAST( _col3 AS decimal(10,0))),0) (type: decimal(18,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col13 (type: string), _col12 (type: string), _col11 (type: string), _col14 (type: string), _col7 (type: int), _col9 (type: int), _col8 (type: int), _col16 (type: string), COALESCE((_col4 * CAST( _col3 AS decimal(10,0))),0) (type: decimal(18,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col8)
- keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), 0L (type: bigint)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+ Group By Operator
+ aggregations: sum(_col8)
+ keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), 0L (type: bigint)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
+ Statistics: Num rows: 6899852151 Data size: 608706960084 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: bigint)
+ sort order: +++++++++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: bigint)
Statistics: Num rows: 6899852151 Data size: 608706960084 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: bigint)
- sort order: +++++++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: bigint)
- Statistics: Num rows: 6899852151 Data size: 608706960084 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col9 (type: decimal(28,2))
- Reducer 3
+ value expressions: _col9 (type: decimal(28,2))
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -252,7 +251,7 @@ STAGE PLANS:
Statistics: Num rows: 3449926075 Data size: 304353479997 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string)
- Reducer 4
+ Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int), VALUE._col6 (type: string), KEY.reducesinkkey1 (type: decimal(28,2))
@@ -291,7 +290,7 @@ STAGE PLANS:
sort order: ++++++++++
Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
- Reducer 5
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey6 (type: int), KEY.reducesinkkey7 (type: string), KEY.reducesinkkey8 (type: decimal(28,2)), KEY.reducesinkkey9 (type: int)
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query68.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query68.q.out b/ql/src/test/results/clientpositive/perf/spark/query68.q.out
index 5585a54..585aa40 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query68.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query68.q.out
@@ -93,6 +93,24 @@ STAGE PLANS:
Map 10
Map Operator Tree:
TableScan
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((s_city) IN ('Cedar Grove', 'Wildwood') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: s_store_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col4 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 11
+ Map Operator Tree:
+ TableScan
alias: household_demographics
Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -108,7 +126,7 @@ STAGE PLANS:
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 8
+ Map 9
Map Operator Tree:
TableScan
alias: date_dim
@@ -126,33 +144,15 @@ STAGE PLANS:
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 9
- Map Operator Tree:
- TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((s_city) IN ('Cedar Grove', 'Wildwood') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: s_store_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- Local Work:
- Map Reduce Local Work
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 829), Reducer 7 (PARTITION-LEVEL SORT, 829)
- Reducer 3 <- Map 12 (PARTITION-LEVEL SORT, 637), Reducer 2 (PARTITION-LEVEL SORT, 637)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 5 (PARTITION-LEVEL SORT, 855)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 882), Reducer 8 (PARTITION-LEVEL SORT, 882)
Reducer 4 <- Reducer 3 (SORT, 1)
- Reducer 6 <- Map 11 (PARTITION-LEVEL SORT, 846), Map 5 (PARTITION-LEVEL SORT, 846)
- Reducer 7 <- Reducer 6 (GROUP, 582)
+ Reducer 7 <- Map 12 (PARTITION-LEVEL SORT, 846), Map 6 (PARTITION-LEVEL SORT, 846)
+ Reducer 8 <- Reducer 7 (GROUP, 582)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -168,12 +168,12 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: string)
- Map 11
+ value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string)
+ Map 12
Map Operator Tree:
TableScan
alias: customer_address
@@ -191,7 +191,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 12
+ Map 5
Map Operator Tree:
TableScan
alias: current_addr
@@ -209,7 +209,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 5
+ Map 6
Map Operator Tree:
TableScan
alias: store_sales
@@ -229,7 +229,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
input vertices:
- 1 Map 8
+ 1 Map 9
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -239,7 +239,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col7, _col8
input vertices:
- 1 Map 9
+ 1 Map 10
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -249,7 +249,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col3, _col5, _col6, _col7, _col8
input vertices:
- 1 Map 10
+ 1 Map 11
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: int)
@@ -265,37 +265,37 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8, _col9
- Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col3, _col5
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: int), _col6 (type: string), _col7 (type: decimal(17,2)), _col8 (type: decimal(17,2)), _col9 (type: decimal(17,2))
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string), _col3 (type: string), _col5 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col6, _col7, _col8, _col9, _col11
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: int)
+ 1 _col1 (type: int)
+ outputColumnNames: _col2, _col3, _col5, _col6, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col11 <> _col6) (type: boolean)
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col5 <> _col8) (type: boolean)
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col3 (type: string), _col2 (type: string), _col11 (type: string), _col6 (type: string), _col4 (type: int), _col7 (type: decimal(17,2)), _col9 (type: decimal(17,2)), _col8 (type: decimal(17,2))
+ expressions: _col3 (type: string), _col2 (type: string), _col5 (type: string), _col8 (type: string), _col6 (type: int), _col9 (type: decimal(17,2)), _col11 (type: decimal(17,2)), _col10 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col4 (type: int)
sort order: ++
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string), _col5 (type: decimal(17,2)), _col6 (type: decimal(17,2)), _col7 (type: decimal(17,2))
Reducer 4
@@ -303,7 +303,7 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), KEY.reducesinkkey1 (type: int), VALUE._col3 (type: decimal(17,2)), VALUE._col4 (type: decimal(17,2)), VALUE._col5 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE
@@ -314,7 +314,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 6
+ Reducer 7
Reduce Operator Tree:
Join Operator
condition map:
@@ -336,7 +336,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int)
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2)), _col6 (type: decimal(17,2))
- Reducer 7
+ Reducer 8
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query72.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query72.q.out b/ql/src/test/results/clientpositive/perf/spark/query72.q.out
index bb71c10..d204803 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query72.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query72.q.out
@@ -61,8 +61,7 @@ POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
Stage-3 depends on stages: Stage-2
- Stage-4 depends on stages: Stage-3
- Stage-1 depends on stages: Stage-4
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -70,27 +69,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 20
- Map Operator Tree:
- TableScan
- alias: promotion
- Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: p_promo_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
- Spark HashTable Sink Operator
- keys:
- 0 _col13 (type: int)
- 1 _col0 (type: int)
- Local Work:
- Map Reduce Local Work
-
- Stage: Stage-3
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 11
+ Map 7
Map Operator Tree:
TableScan
alias: warehouse
@@ -109,11 +88,11 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
- Stage: Stage-4
+ Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
- Map 16
+ Map 15
Map Operator Tree:
TableScan
alias: household_demographics
@@ -131,75 +110,85 @@ STAGE PLANS:
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
+ Map 16
+ Map Operator Tree:
+ TableScan
+ alias: promotion
+ Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: p_promo_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2300 Data size: 2713420 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 _col5 (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Map 19 (PARTITION-LEVEL SORT, 165), Reducer 9 (PARTITION-LEVEL SORT, 165)
- Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 308), Map 15 (PARTITION-LEVEL SORT, 308)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 90), Reducer 10 (PARTITION-LEVEL SORT, 90)
- Reducer 3 <- Reducer 2 (GROUP, 73)
- Reducer 4 <- Reducer 3 (SORT, 1)
- Reducer 6 <- Map 12 (PARTITION-LEVEL SORT, 6), Map 5 (PARTITION-LEVEL SORT, 6)
- Reducer 7 <- Reducer 14 (PARTITION-LEVEL SORT, 375), Reducer 6 (PARTITION-LEVEL SORT, 375)
- Reducer 8 <- Map 17 (PARTITION-LEVEL SORT, 136), Reducer 7 (PARTITION-LEVEL SORT, 136)
- Reducer 9 <- Map 18 (PARTITION-LEVEL SORT, 154), Reducer 8 (PARTITION-LEVEL SORT, 154)
+ Reducer 10 <- Map 14 (PARTITION-LEVEL SORT, 338), Reducer 9 (PARTITION-LEVEL SORT, 338)
+ Reducer 11 <- Map 17 (PARTITION-LEVEL SORT, 452), Reducer 10 (PARTITION-LEVEL SORT, 452)
+ Reducer 12 <- Map 18 (PARTITION-LEVEL SORT, 492), Reducer 11 (PARTITION-LEVEL SORT, 492)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 186), Reducer 12 (PARTITION-LEVEL SORT, 186)
+ Reducer 3 <- Map 19 (PARTITION-LEVEL SORT, 67), Reducer 2 (PARTITION-LEVEL SORT, 67)
+ Reducer 4 <- Map 20 (PARTITION-LEVEL SORT, 97), Reducer 3 (PARTITION-LEVEL SORT, 97)
+ Reducer 5 <- Reducer 4 (GROUP, 80)
+ Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 9 <- Map 13 (PARTITION-LEVEL SORT, 306), Map 8 (PARTITION-LEVEL SORT, 306)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: catalog_returns
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ alias: inventory
+ Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: cr_item_sk is not null (type: boolean)
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ predicate: (inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) (type: boolean)
+ Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cr_item_sk (type: int), cr_order_number (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
- Map 12
+ expressions: inv_date_sk (type: int), inv_item_sk (type: int), inv_warehouse_sk (type: int), inv_quantity_on_hand (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col3, _col5
+ input vertices:
+ 1 Map 7
+ Statistics: Num rows: 41342400 Data size: 653203228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 41342400 Data size: 653203228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col3 (type: int), _col5 (type: string)
+ Local Work:
+ Map Reduce Local Work
+ Map 13
Map Operator Tree:
TableScan
- alias: d2
+ alias: d1
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (d_date_sk is not null and d_week_seq is not null) (type: boolean)
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((d_year = 2001) and d_date_sk is not null and d_week_seq is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int), d_week_seq (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ expressions: d_date_sk (type: int), d_date (type: string), d_week_seq (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
- Map 13
- Map Operator Tree:
- TableScan
- alias: catalog_sales
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (cs_bill_cdemo_sk is not null and cs_bill_hdemo_sk is not null and cs_item_sk is not null and cs_ship_date_sk is not null and cs_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: cs_sold_date_sk (type: int), cs_ship_date_sk (type: int), cs_bill_cdemo_sk (type: int), cs_bill_hdemo_sk (type: int), cs_item_sk (type: int), cs_promo_sk (type: int), cs_order_number (type: int), cs_quantity (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int)
- Map 15
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: int)
+ Map 14
Map Operator Tree:
TableScan
alias: customer_demographics
@@ -219,24 +208,6 @@ STAGE PLANS:
Map 17
Map Operator Tree:
TableScan
- alias: d1
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_year = 2001) and d_date_sk is not null and d_week_seq is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int), d_date (type: string), d_week_seq (type: int)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int), _col0 (type: int)
- sort order: ++
- Map-reduce partition columns: _col2 (type: int), _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Map 18
- Map Operator Tree:
- TableScan
alias: item
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -252,7 +223,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 19
+ Map 18
Map Operator Tree:
TableScan
alias: d3
@@ -270,36 +241,58 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 5
+ Map 19
Map Operator Tree:
TableScan
- alias: inventory
- Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
+ alias: d2
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (inv_date_sk is not null and inv_item_sk is not null and inv_warehouse_sk is not null) (type: boolean)
- Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
+ predicate: (d_date_sk is not null and d_week_seq is not null) (type: boolean)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: inv_date_sk (type: int), inv_item_sk (type: int), inv_warehouse_sk (type: int), inv_quantity_on_hand (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col3, _col5
- input vertices:
- 1 Map 11
- Statistics: Num rows: 41342400 Data size: 653203228 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 41342400 Data size: 653203228 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col3 (type: int), _col5 (type: string)
- Local Work:
- Map Reduce Local Work
+ expressions: d_date_sk (type: int), d_week_seq (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Map 20
+ Map Operator Tree:
+ TableScan
+ alias: catalog_returns
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: cr_item_sk is not null (type: boolean)
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cr_item_sk (type: int), cr_order_number (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
+ Map 8
+ Map Operator Tree:
+ TableScan
+ alias: catalog_sales
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (cs_bill_cdemo_sk is not null and cs_bill_hdemo_sk is not null and cs_item_sk is not null and cs_ship_date_sk is not null and cs_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cs_sold_date_sk (type: int), cs_ship_date_sk (type: int), cs_bill_cdemo_sk (type: int), cs_bill_hdemo_sk (type: int), cs_item_sk (type: int), cs_promo_sk (type: int), cs_order_number (type: int), cs_quantity (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int)
Reducer 10
Local Work:
Map Reduce Local Work
@@ -308,107 +301,160 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col9 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col5, _col12, _col13, _col14, _col21, _col22, _col25, _col27
- Statistics: Num rows: 170063873 Data size: 23030065010 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (UDFToDouble(_col27) > (UDFToDouble(_col21) + 5.0D)) (type: boolean)
- Statistics: Num rows: 56687957 Data size: 7676688246 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col1, _col3, _col4, _col5, _col6, _col7, _col9, _col10
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col4, _col5, _col6, _col7, _col9, _col10
+ input vertices:
+ 1 Map 15
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Left Outer Join 0 to 1
keys:
- 0 _col13 (type: int)
+ 0 _col5 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col5, _col12, _col14, _col22, _col25, _col28
+ outputColumnNames: _col1, _col4, _col6, _col7, _col9, _col10, _col16
input vertices:
- 1 Map 20
- Statistics: Num rows: 62356754 Data size: 8444357253 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col12 (type: int), _col14 (type: int), _col5 (type: string), _col25 (type: string), _col22 (type: int), _col28 (type: int)
- outputColumnNames: _col4, _col6, _col13, _col15, _col22, _col28
- Statistics: Num rows: 62356754 Data size: 8444357253 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int), _col6 (type: int)
- sort order: ++
- Map-reduce partition columns: _col4 (type: int), _col6 (type: int)
- Statistics: Num rows: 62356754 Data size: 8444357253 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col13 (type: string), _col15 (type: string), _col22 (type: int), _col28 (type: int)
- Reducer 14
- Local Work:
- Map Reduce Local Work
+ 1 Map 16
+ Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col4 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col4 (type: int)
+ Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col6 (type: int), _col7 (type: int), _col9 (type: string), _col10 (type: int), _col16 (type: int)
+ Reducer 11
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col4 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col3 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col5, _col6, _col7
- input vertices:
- 1 Map 16
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int)
+ outputColumnNames: _col1, _col4, _col6, _col7, _col9, _col10, _col16, _col18
+ Statistics: Num rows: 463810558 Data size: 62809267017 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 463810558 Data size: 62809267017 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col4 (type: int), _col6 (type: int), _col7 (type: int), _col9 (type: string), _col10 (type: int), _col16 (type: int), _col18 (type: string)
+ Reducer 12
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col4, _col6, _col7, _col9, _col10, _col16, _col18, _col20
+ Statistics: Num rows: 510191624 Data size: 69090195216 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (UDFToDouble(_col20) > (UDFToDouble(_col9) + 5.0D)) (type: boolean)
+ Statistics: Num rows: 170063874 Data size: 23030064981 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col18 (type: string), _col4 (type: int), _col6 (type: int), _col7 (type: int), _col10 (type: int), _col16 (type: int)
+ outputColumnNames: _col3, _col8, _col10, _col11, _col14, _col20
+ Statistics: Num rows: 170063874 Data size: 23030064981 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col8 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col8 (type: int)
+ Statistics: Num rows: 170063874 Data size: 23030064981 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: string), _col10 (type: int), _col11 (type: int), _col14 (type: int), _col20 (type: int)
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
- Right Outer Join 0 to 1
+ Inner Join 0 to 1
keys:
- 0 _col0 (type: int), _col1 (type: int)
- 1 _col4 (type: int), _col6 (type: int)
- outputColumnNames: _col15, _col17, _col24, _col30
- Statistics: Num rows: 68592430 Data size: 9288793179 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col8 (type: int)
+ outputColumnNames: _col0, _col3, _col5, _col9, _col14, _col16, _col17, _col20, _col26
+ Statistics: Num rows: 187070265 Data size: 25333072028 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col3 < _col17) (type: boolean)
+ Statistics: Num rows: 62356755 Data size: 8444357342 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col20 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col20 (type: int)
+ Statistics: Num rows: 62356755 Data size: 8444357342 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: string), _col9 (type: string), _col14 (type: int), _col16 (type: int), _col26 (type: int)
+ Reducer 3
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int), _col20 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col5, _col9, _col14, _col16, _col20, _col26
+ Statistics: Num rows: 68592431 Data size: 9288793277 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col17 (type: string), _col15 (type: string), _col24 (type: int), CASE WHEN (_col30 is null) THEN (1) ELSE (0) END (type: int), CASE WHEN (_col30 is not null) THEN (1) ELSE (0) END (type: int)
+ expressions: _col14 (type: int), _col16 (type: int), _col5 (type: string), _col9 (type: string), _col20 (type: int), _col26 (type: int)
+ outputColumnNames: _col4, _col6, _col13, _col15, _col22, _col28
+ Statistics: Num rows: 68592431 Data size: 9288793277 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col4 (type: int), _col6 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col4 (type: int), _col6 (type: int)
+ Statistics: Num rows: 68592431 Data size: 9288793277 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col13 (type: string), _col15 (type: string), _col22 (type: int), _col28 (type: int)
+ Reducer 4
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join 0 to 1
+ keys:
+ 0 _col4 (type: int), _col6 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col13, _col15, _col22, _col28
+ Statistics: Num rows: 75451675 Data size: 10217672826 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col15 (type: string), _col13 (type: string), _col22 (type: int), CASE WHEN (_col28 is null) THEN (1) ELSE (0) END (type: int), CASE WHEN (_col28 is not null) THEN (1) ELSE (0) END (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 68592430 Data size: 9288793179 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 75451675 Data size: 10217672826 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col3), count(_col4), count()
keys: _col0 (type: string), _col1 (type: string), _col2 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 68592430 Data size: 9288793179 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 75451675 Data size: 10217672826 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int)
sort order: +++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: int)
- Statistics: Num rows: 68592430 Data size: 9288793179 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 75451675 Data size: 10217672826 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint), _col4 (type: bigint), _col5 (type: bigint)
- Reducer 3
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0), count(VALUE._col1), count(VALUE._col2)
keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: int)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 34296215 Data size: 4644396589 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 37725837 Data size: 5108836345 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col5 (type: bigint), _col0 (type: string), _col1 (type: string), _col2 (type: int)
sort order: -+++
- Statistics: Num rows: 34296215 Data size: 4644396589 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 37725837 Data size: 5108836345 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col3 (type: bigint), _col4 (type: bigint)
- Reducer 4
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: int), VALUE._col0 (type: bigint), VALUE._col1 (type: bigint), KEY.reducesinkkey0 (type: bigint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 34296215 Data size: 4644396589 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 37725837 Data size: 5108836345 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column stats: NONE
@@ -419,73 +465,22 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 6
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col3, _col5, _col7
- Statistics: Num rows: 45476640 Data size: 718523566 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 45476640 Data size: 718523566 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: string), _col7 (type: int)
- Reducer 7
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: int)
- 1 _col4 (type: int)
- outputColumnNames: _col3, _col5, _col7, _col8, _col9, _col12, _col13, _col14, _col15
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col3 < _col15) (type: boolean)
- Statistics: Num rows: 127771498 Data size: 17302827584 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col7 (type: int), _col8 (type: int)
- sort order: ++
- Map-reduce partition columns: _col7 (type: int), _col8 (type: int)
- Statistics: Num rows: 127771498 Data size: 17302827584 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: string), _col9 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int)
- Reducer 8
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col7 (type: int), _col8 (type: int)
- 1 _col2 (type: int), _col0 (type: int)
- outputColumnNames: _col5, _col9, _col12, _col13, _col14, _col21, _col22
- Statistics: Num rows: 140548650 Data size: 19033110754 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col12 (type: int)
- sort order: +
- Map-reduce partition columns: _col12 (type: int)
- Statistics: Num rows: 140548650 Data size: 19033110754 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: string), _col9 (type: int), _col13 (type: int), _col14 (type: int), _col21 (type: string), _col22 (type: int)
Reducer 9
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col12 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col5, _col9, _col12, _col13, _col14, _col21, _col22, _col25
- Statistics: Num rows: 154603518 Data size: 20936422283 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col9, _col10
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col9 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col9 (type: int)
- Statistics: Num rows: 154603518 Data size: 20936422283 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: string), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col21 (type: string), _col22 (type: int), _col25 (type: string)
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: int), _col9 (type: string), _col10 (type: int)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query73.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query73.q.out b/ql/src/test/results/clientpositive/perf/spark/query73.q.out
index 9847cef..7fec0e1 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query73.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query73.q.out
@@ -83,36 +83,36 @@ STAGE PLANS:
Map 7
Map Operator Tree:
TableScan
- alias: store
- Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
+ alias: household_demographics
+ Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((s_county) IN ('Mobile County', 'Maverick County', 'Huron County', 'Kittitas County') and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ predicate: (((hd_buy_potential = '>10000') or (hd_buy_potential = 'unknown')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.0D)) ELSE (null) END and hd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: hd_demo_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col3 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
Map 8
Map Operator Tree:
TableScan
- alias: household_demographics
- Statistics: Num rows: 7200 Data size: 770400 Basic stats: COMPLETE Column stats: NONE
+ alias: store
+ Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((hd_buy_potential = '>10000') or (hd_buy_potential = 'unknown')) and (hd_vehicle_count > 0) and CASE WHEN ((hd_vehicle_count > 0)) THEN (((UDFToDouble(hd_dep_count) / UDFToDouble(hd_vehicle_count)) > 1.0D)) ELSE (null) END and hd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((s_county) IN ('Mobile County', 'Maverick County', 'Huron County', 'Kittitas County') and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hd_demo_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 1200 Data size: 128400 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 852 Data size: 1628138 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -169,9 +169,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col3 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col4
+ outputColumnNames: _col1, _col3, _col4
input vertices:
1 Map 7
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -179,7 +179,7 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col4
input vertices:
[09/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query45.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query45.q.out b/ql/src/test/results/clientpositive/perf/spark/query45.q.out
index 7e1cc88..b674400 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query45.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query45.q.out
@@ -46,10 +46,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 6 <- Map 5 (GROUP, 1)
+ Reducer 16 <- Map 15 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 5
+ Map 15
Map Operator Tree:
TableScan
alias: item
@@ -70,7 +70,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint), _col1 (type: bigint)
- Reducer 6
+ Reducer 16
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -88,45 +88,33 @@ STAGE PLANS:
Spark
Edges:
Reducer 11 <- Map 10 (GROUP, 3)
- Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 154), Map 15 (PARTITION-LEVEL SORT, 154)
- Reducer 14 <- Map 16 (PARTITION-LEVEL SORT, 706), Reducer 13 (PARTITION-LEVEL SORT, 706)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 527), Reducer 9 (PARTITION-LEVEL SORT, 527)
- Reducer 3 <- Reducer 2 (GROUP, 224)
- Reducer 4 <- Reducer 3 (SORT, 1)
+ Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 154), Map 14 (PARTITION-LEVEL SORT, 154)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 6 (PARTITION-LEVEL SORT, 855)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 777), Reducer 9 (PARTITION-LEVEL SORT, 777)
+ Reducer 4 <- Reducer 3 (GROUP, 230)
+ Reducer 5 <- Reducer 4 (SORT, 1)
Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 7), Reducer 11 (PARTITION-LEVEL SORT, 7)
- Reducer 9 <- Reducer 14 (PARTITION-LEVEL SORT, 191), Reducer 8 (PARTITION-LEVEL SORT, 191)
+ Reducer 9 <- Reducer 13 (PARTITION-LEVEL SORT, 174), Reducer 8 (PARTITION-LEVEL SORT, 174)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ alias: customer
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ca_address_sk is not null (type: boolean)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_county (type: string), ca_zip (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0
- 1
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
- input vertices:
- 1 Reducer 6
- Statistics: Num rows: 40000000 Data size: 41275195284 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 40000000 Data size: 41275195284 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint), _col4 (type: bigint)
- Local Work:
- Map Reduce Local Work
+ expressions: c_customer_sk (type: int), c_current_addr_sk (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
Map 10
Map Operator Tree:
TableScan
@@ -167,7 +155,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
- Map 15
+ Map 14
Map Operator Tree:
TableScan
alias: date_dim
@@ -184,24 +172,24 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Map 16
+ Map 6
Map Operator Tree:
TableScan
- alias: customer
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ predicate: ca_address_sk is not null (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: c_customer_sk (type: int), c_current_addr_sk (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ expressions: ca_address_sk (type: int), ca_county (type: string), ca_zip (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string)
Map 7
Map Operator Tree:
TableScan
@@ -248,91 +236,99 @@ STAGE PLANS:
outputColumnNames: _col1, _col2, _col3
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col2 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col3 (type: decimal(7,2))
- Reducer 14
+ value expressions: _col2 (type: int), _col3 (type: decimal(7,2))
+ Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col3, _col8
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col8 (type: int), _col1 (type: int), _col3 (type: decimal(7,2))
- outputColumnNames: _col1, _col3, _col5
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col5 (type: decimal(7,2))
- Reducer 2
+ outputColumnNames: _col0, _col3, _col4
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: string), _col4 (type: string)
+ Reducer 3
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
0 _col0 (type: int)
- 1 _col5 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col8, _col14
- Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col14 (type: decimal(7,2)), _col1 (type: string), _col2 (type: string), _col6 (type: string), _col3 (type: bigint), _col4 (type: bigint), _col8 (type: boolean)
- outputColumnNames: _col3, _col7, _col8, _col13, _col14, _col15, _col17
- Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((substr(_col8, 1, 5)) IN ('85669', '86197', '88274', '83405', '86475', '85392', '85460', '80348', '81792') or CASE WHEN ((_col14 = 0L)) THEN (false) WHEN (_col17 is not null) THEN (true) WHEN (_col13 is null) THEN (null) WHEN ((_col15 < _col14)) THEN (null) ELSE (false) END) (type: boolean)
- Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col3 (type: decimal(7,2)), _col7 (type: string), _col8 (type: string)
- outputColumnNames: _col3, _col7, _col8
- Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col3)
- keys: _col8 (type: string), _col7 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 210834322 Data size: 28667370686 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col2 (type: decimal(17,2))
- Reducer 3
+ 1 _col6 (type: int)
+ outputColumnNames: _col3, _col4, _col6, _col8, _col12
+ Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col3, _col4, _col6, _col8, _col12, _col16, _col17
+ input vertices:
+ 1 Reducer 16
+ Statistics: Num rows: 191667562 Data size: 29319594068 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col12 (type: decimal(7,2)), _col3 (type: string), _col4 (type: string), _col6 (type: string), _col16 (type: bigint), _col17 (type: bigint), _col8 (type: boolean)
+ outputColumnNames: _col3, _col7, _col8, _col13, _col14, _col15, _col17
+ Statistics: Num rows: 191667562 Data size: 29319594068 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((substr(_col8, 1, 5)) IN ('85669', '86197', '88274', '83405', '86475', '85392', '85460', '80348', '81792') or CASE WHEN ((_col14 = 0L)) THEN (false) WHEN (_col17 is not null) THEN (true) WHEN (_col13 is null) THEN (null) WHEN ((_col15 < _col14)) THEN (null) ELSE (false) END) (type: boolean)
+ Statistics: Num rows: 191667562 Data size: 29319594068 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col3 (type: decimal(7,2)), _col7 (type: string), _col8 (type: string)
+ outputColumnNames: _col3, _col7, _col8
+ Statistics: Num rows: 191667562 Data size: 29319594068 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col3)
+ keys: _col8 (type: string), _col7 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 191667562 Data size: 29319594068 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Statistics: Num rows: 191667562 Data size: 29319594068 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col2 (type: decimal(17,2))
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 105417161 Data size: 14333685343 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 95833781 Data size: 14659797034 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 105417161 Data size: 14333685343 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 95833781 Data size: 14659797034 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: decimal(17,2))
- Reducer 4
+ Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 105417161 Data size: 14333685343 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 95833781 Data size: 14659797034 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
- Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 100 Data size: 15200 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 100 Data size: 15200 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -360,15 +356,15 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col0 (type: int)
- 1 _col3 (type: int)
- outputColumnNames: _col1, _col3, _col5, _col9
- Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
+ 1 _col1 (type: int)
+ outputColumnNames: _col1, _col3, _col6, _col7
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col5 (type: int)
+ key expressions: _col6 (type: int)
sort order: +
- Map-reduce partition columns: _col5 (type: int)
- Statistics: Num rows: 191667562 Data size: 26061245514 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col3 (type: boolean), _col9 (type: decimal(7,2))
+ Map-reduce partition columns: _col6 (type: int)
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col3 (type: boolean), _col7 (type: decimal(7,2))
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query46.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query46.q.out b/ql/src/test/results/clientpositive/perf/spark/query46.q.out
index 6705f50..8b0525d 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query46.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query46.q.out
@@ -76,7 +76,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 10
+ Map 11
Map Operator Tree:
TableScan
alias: store
@@ -94,7 +94,7 @@ STAGE PLANS:
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 11
+ Map 12
Map Operator Tree:
TableScan
alias: household_demographics
@@ -116,12 +116,12 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 829), Reducer 8 (PARTITION-LEVEL SORT, 829)
- Reducer 3 <- Map 13 (PARTITION-LEVEL SORT, 637), Reducer 2 (PARTITION-LEVEL SORT, 637)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 5 (PARTITION-LEVEL SORT, 855)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 882), Reducer 9 (PARTITION-LEVEL SORT, 882)
Reducer 4 <- Reducer 3 (SORT, 1)
- Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 398), Map 9 (PARTITION-LEVEL SORT, 398)
- Reducer 7 <- Map 12 (PARTITION-LEVEL SORT, 846), Reducer 6 (PARTITION-LEVEL SORT, 846)
- Reducer 8 <- Reducer 7 (GROUP, 582)
+ Reducer 7 <- Map 10 (PARTITION-LEVEL SORT, 398), Map 6 (PARTITION-LEVEL SORT, 398)
+ Reducer 8 <- Map 13 (PARTITION-LEVEL SORT, 846), Reducer 7 (PARTITION-LEVEL SORT, 846)
+ Reducer 9 <- Reducer 8 (GROUP, 582)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -137,12 +137,29 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string)
+ Map 10
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((d_dow) IN (6, 0) and (d_year) IN (1998, 1999, 2000) and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 18263 Data size: 20436297 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 18263 Data size: 20436297 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: string)
- Map 12
+ Statistics: Num rows: 18263 Data size: 20436297 Basic stats: COMPLETE Column stats: NONE
+ Map 13
Map Operator Tree:
TableScan
alias: customer_address
@@ -160,7 +177,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 13
+ Map 5
Map Operator Tree:
TableScan
alias: current_addr
@@ -178,7 +195,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 5
+ Map 6
Map Operator Tree:
TableScan
alias: store_sales
@@ -196,60 +213,43 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2))
- Map 9
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_dow) IN (6, 0) and (d_year) IN (1998, 1999, 2000) and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 18263 Data size: 20436297 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 18263 Data size: 20436297 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 18263 Data size: 20436297 Basic stats: COMPLETE Column stats: NONE
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col6, _col7, _col8
- Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col3, _col5
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: string), _col3 (type: string), _col4 (type: int), _col6 (type: string), _col7 (type: decimal(17,2)), _col8 (type: decimal(17,2))
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string), _col3 (type: string), _col5 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col6, _col7, _col8, _col10
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: int)
+ 1 _col1 (type: int)
+ outputColumnNames: _col2, _col3, _col5, _col6, _col8, _col9, _col10
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (_col10 <> _col6) (type: boolean)
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ predicate: (_col5 <> _col8) (type: boolean)
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col3 (type: string), _col2 (type: string), _col10 (type: string), _col6 (type: string), _col4 (type: int), _col7 (type: decimal(17,2)), _col8 (type: decimal(17,2))
+ expressions: _col3 (type: string), _col2 (type: string), _col5 (type: string), _col8 (type: string), _col6 (type: int), _col9 (type: decimal(17,2)), _col10 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int)
sort order: +++++
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col5 (type: decimal(17,2)), _col6 (type: decimal(17,2))
Reducer 4
@@ -257,7 +257,7 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: int), VALUE._col0 (type: decimal(17,2)), VALUE._col1 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 510205766 Data size: 45010500864 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 463823414 Data size: 40918636263 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE
@@ -268,7 +268,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 6
+ Reducer 7
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -288,7 +288,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col5, _col6, _col7
input vertices:
- 1 Map 10
+ 1 Map 11
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -298,7 +298,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col3, _col5, _col6, _col7
input vertices:
- 1 Map 11
+ 1 Map 12
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: int)
@@ -306,7 +306,7 @@ STAGE PLANS:
Map-reduce partition columns: _col3 (type: int)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2))
- Reducer 7
+ Reducer 8
Reduce Operator Tree:
Join Operator
condition map:
@@ -328,7 +328,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: string), _col2 (type: int), _col3 (type: int)
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: decimal(17,2)), _col5 (type: decimal(17,2))
- Reducer 8
+ Reducer 9
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), sum(VALUE._col1)
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query47.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query47.q.out b/ql/src/test/results/clientpositive/perf/spark/query47.q.out
index bc97fdf..5175f80 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query47.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query47.q.out
@@ -397,47 +397,47 @@ STAGE PLANS:
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col11 (type: string), _col12 (type: string), _col8 (type: string), _col9 (type: string), _col5 (type: int), _col6 (type: int)
+ keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col9 (type: string), _col11 (type: string), _col12 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
sort order: ++++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int)
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col6 (type: decimal(17,2))
Reducer 14
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: int)
+ keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int)
+ key expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int)
sort order: +++++
- Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int)
+ Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: int), _col6 (type: decimal(17,2))
+ value expressions: _col1 (type: int), _col6 (type: decimal(17,2))
Reducer 15
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey4 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: decimal(17,2))
+ expressions: KEY.reducesinkkey4 (type: int), VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col1 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: string, _col3: string, _col4: int, _col5: int, _col6: decimal(17,2)
+ output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string, _col5: string, _col6: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col3 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST, _col4 ASC NULLS FIRST
- partition by: _col3, _col2, _col0, _col1, _col4
+ order by: _col3 ASC NULLS FIRST, _col2 ASC NULLS FIRST, _col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST, _col0 ASC NULLS FIRST
+ partition by: _col3, _col2, _col4, _col5, _col0
raw input shape:
window functions:
window function definition
@@ -448,54 +448,54 @@ STAGE PLANS:
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(17,2))
+ expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: decimal(17,2))
outputColumnNames: avg_window_0, _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int), _col5 (type: int)
+ key expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int), _col1 (type: int)
sort order: ++++++
- Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string)
+ Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
value expressions: avg_window_0 (type: decimal(21,6)), _col6 (type: decimal(17,2))
Reducer 16
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: decimal(21,6)), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey5 (type: int), VALUE._col1 (type: decimal(17,2))
+ expressions: VALUE._col0 (type: decimal(21,6)), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col1 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: decimal(21,6), _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: int, _col7: decimal(17,2)
+ output shape: _col0: decimal(21,6), _col1: int, _col2: int, _col3: string, _col4: string, _col5: string, _col6: string, _col7: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col5 ASC NULLS FIRST, _col6 ASC NULLS FIRST
- partition by: _col4, _col3, _col1, _col2
+ order by: _col1 ASC NULLS FIRST, _col2 ASC NULLS FIRST
+ partition by: _col4, _col3, _col5, _col6
raw input shape:
window functions:
window function definition
alias: rank_window_1
- arguments: _col5, _col6
+ arguments: _col1, _col2
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
isPivotResult: true
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > 0) and (_col5 = 2000) and rank_window_1 is not null) (type: boolean)
+ predicate: ((_col0 > 0) and (_col1 = 2000) and rank_window_1 is not null) (type: boolean)
Statistics: Num rows: 63887519 Data size: 5636175475 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: rank_window_1 (type: int), _col0 (type: decimal(21,6)), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: decimal(17,2))
+ expressions: rank_window_1 (type: int), _col0 (type: decimal(21,6)), _col1 (type: int), _col2 (type: int), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string), _col7 (type: decimal(17,2))
outputColumnNames: rank_window_1, _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 63887519 Data size: 5636175475 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: CASE WHEN ((_col0 > 0)) THEN (((abs((_col7 - _col0)) / _col0) > 0.1)) ELSE (null) END (type: boolean)
Statistics: Num rows: 31943759 Data size: 2818087693 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col4 (type: string), _col3 (type: string), _col1 (type: string), _col2 (type: string), _col5 (type: int), _col6 (type: int), _col7 (type: decimal(17,2)), _col0 (type: decimal(21,6)), rank_window_1 (type: int)
+ expressions: _col4 (type: string), _col3 (type: string), _col5 (type: string), _col6 (type: string), _col1 (type: int), _col2 (type: int), _col7 (type: decimal(17,2)), _col0 (type: decimal(21,6)), rank_window_1 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
Statistics: Num rows: 31943759 Data size: 2818087693 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -560,52 +560,52 @@ STAGE PLANS:
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col11 (type: string), _col12 (type: string), _col8 (type: string), _col9 (type: string), _col5 (type: int), _col6 (type: int)
+ keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col9 (type: string), _col11 (type: string), _col12 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
sort order: ++++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int)
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col6 (type: decimal(17,2))
Reducer 23
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: int)
+ keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int), _col5 (type: int)
+ key expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int), _col1 (type: int)
sort order: ++++++
- Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string)
+ Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
value expressions: _col6 (type: decimal(17,2))
Reducer 24
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey5 (type: int), VALUE._col0 (type: decimal(17,2))
+ expressions: KEY.reducesinkkey4 (type: int), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col0 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: string, _col3: string, _col4: int, _col5: int, _col6: decimal(17,2)
+ output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string, _col5: string, _col6: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST
- partition by: _col3, _col2, _col0, _col1
+ order by: _col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST
+ partition by: _col3, _col2, _col4, _col5
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: _col4, _col5
+ arguments: _col0, _col1
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
@@ -615,7 +615,7 @@ STAGE PLANS:
predicate: rank_window_0 is not null (type: boolean)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col6 (type: decimal(17,2)), rank_window_0 (type: int)
+ expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: decimal(17,2)), rank_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
@@ -648,52 +648,52 @@ STAGE PLANS:
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col11 (type: string), _col12 (type: string), _col8 (type: string), _col9 (type: string), _col5 (type: int), _col6 (type: int)
+ keys: _col5 (type: int), _col6 (type: int), _col8 (type: string), _col9 (type: string), _col11 (type: string), _col12 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int)
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
sort order: ++++++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int)
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col6 (type: decimal(17,2))
Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: int)
+ keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: int), _col5 (type: int)
+ key expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col0 (type: int), _col1 (type: int)
sort order: ++++++
- Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string)
+ Map-reduce partition columns: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
value expressions: _col6 (type: decimal(17,2))
Reducer 5
Reduce Operator Tree:
Select Operator
- expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey4 (type: int), KEY.reducesinkkey5 (type: int), VALUE._col0 (type: decimal(17,2))
+ expressions: KEY.reducesinkkey4 (type: int), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), VALUE._col0 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
PTF Operator
Function definitions:
Input definition
input alias: ptf_0
- output shape: _col0: string, _col1: string, _col2: string, _col3: string, _col4: int, _col5: int, _col6: decimal(17,2)
+ output shape: _col0: int, _col1: int, _col2: string, _col3: string, _col4: string, _col5: string, _col6: decimal(17,2)
type: WINDOWING
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col4 ASC NULLS FIRST, _col5 ASC NULLS FIRST
- partition by: _col3, _col2, _col0, _col1
+ order by: _col0 ASC NULLS FIRST, _col1 ASC NULLS FIRST
+ partition by: _col3, _col2, _col4, _col5
raw input shape:
window functions:
window function definition
alias: rank_window_0
- arguments: _col4, _col5
+ arguments: _col0, _col1
name: rank
window function: GenericUDAFRankEvaluator
window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
@@ -703,7 +703,7 @@ STAGE PLANS:
predicate: rank_window_0 is not null (type: boolean)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col3 (type: string), _col2 (type: string), _col0 (type: string), _col1 (type: string), _col6 (type: decimal(17,2)), rank_window_0 (type: int)
+ expressions: _col3 (type: string), _col2 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: decimal(17,2)), rank_window_0 (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query48.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query48.q.out b/ql/src/test/results/clientpositive/perf/spark/query48.q.out
index 505b178..a2b3663 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query48.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query48.q.out
@@ -161,8 +161,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 49), Map 7 (PARTITION-LEVEL SORT, 49)
- Reducer 4 <- Map 8 (PARTITION-LEVEL SORT, 133), Reducer 3 (PARTITION-LEVEL SORT, 133)
- Reducer 5 <- Map 9 (PARTITION-LEVEL SORT, 12), Reducer 4 (PARTITION-LEVEL SORT, 12)
+ Reducer 4 <- Map 8 (PARTITION-LEVEL SORT, 55), Reducer 3 (PARTITION-LEVEL SORT, 55)
+ Reducer 5 <- Map 9 (PARTITION-LEVEL SORT, 138), Reducer 4 (PARTITION-LEVEL SORT, 138)
Reducer 6 <- Reducer 5 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
@@ -216,38 +216,38 @@ STAGE PLANS:
Map 8
Map Operator Tree:
TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ alias: customer_demographics
+ Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ca_country = 'United States') and (ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is not null) (type: boolean)
- Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((cd_education_status = '4 yr Degree') and (cd_marital_status = 'M') and cd_demo_sk is not null) (type: boolean)
+ Statistics: Num rows: 465450 Data size: 179296539 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ca_address_sk (type: int), ca_state (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
+ expressions: cd_demo_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 465450 Data size: 179296539 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
+ Statistics: Num rows: 465450 Data size: 179296539 Basic stats: COMPLETE Column stats: NONE
Map 9
Map Operator Tree:
TableScan
- alias: customer_demographics
- Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((cd_education_status = '4 yr Degree') and (cd_marital_status = 'M') and cd_demo_sk is not null) (type: boolean)
- Statistics: Num rows: 465450 Data size: 179296539 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((ca_country = 'United States') and (ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is not null) (type: boolean)
+ Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cd_demo_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 465450 Data size: 179296539 Basic stats: COMPLETE Column stats: NONE
+ expressions: ca_address_sk (type: int), ca_state (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 465450 Data size: 179296539 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10000000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
@@ -259,49 +259,53 @@ STAGE PLANS:
outputColumnNames: _col2, _col3, _col5, _col7
Statistics: Num rows: 77439413 Data size: 6831727584 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col3 (type: int)
+ key expressions: _col2 (type: int)
sort order: +
- Map-reduce partition columns: _col3 (type: int)
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 77439413 Data size: 6831727584 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col5 (type: int), _col7 (type: decimal(7,2))
+ value expressions: _col3 (type: int), _col5 (type: int), _col7 (type: decimal(7,2))
Reducer 4
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col3 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col5, _col7, _col11
+ outputColumnNames: _col3, _col5, _col7
Statistics: Num rows: 85183356 Data size: 7514900505 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (((_col11) IN ('KY', 'GA', 'NM') and _col7 BETWEEN 0 AND 2000) or ((_col11) IN ('MT', 'OR', 'IN') and _col7 BETWEEN 150 AND 3000) or ((_col11) IN ('WI', 'MO', 'WV') and _col7 BETWEEN 50 AND 25000)) (type: boolean)
- Statistics: Num rows: 14197224 Data size: 1252483241 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 14197224 Data size: 1252483241 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: int)
+ Reduce Output Operator
+ key expressions: _col3 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: int)
+ Statistics: Num rows: 85183356 Data size: 7514900505 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: int), _col7 (type: decimal(7,2))
Reducer 5
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col2 (type: int)
+ 0 _col3 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col5
- Statistics: Num rows: 15616946 Data size: 1377731594 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col5)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ outputColumnNames: _col5, _col7, _col14
+ Statistics: Num rows: 93701693 Data size: 8266390734 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((_col14) IN ('KY', 'GA', 'NM') and _col7 BETWEEN 0 AND 2000) or ((_col14) IN ('MT', 'OR', 'IN') and _col7 BETWEEN 150 AND 3000) or ((_col14) IN ('WI', 'MO', 'WV') and _col7 BETWEEN 50 AND 25000)) (type: boolean)
+ Statistics: Num rows: 15616947 Data size: 1377731627 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: int)
+ outputColumnNames: _col5
+ Statistics: Num rows: 15616947 Data size: 1377731627 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col5)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reducer 6
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query5.q.out b/ql/src/test/results/clientpositive/perf/spark/query5.q.out
index a0b70a7..90b45c1 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query5.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query5.q.out
@@ -311,9 +311,9 @@ STAGE PLANS:
Reducer 10 <- Map 13 (PARTITION-LEVEL SORT, 329), Map 14 (PARTITION-LEVEL SORT, 329), Map 9 (PARTITION-LEVEL SORT, 329)
Reducer 11 <- Map 15 (PARTITION-LEVEL SORT, 362), Reducer 10 (PARTITION-LEVEL SORT, 362)
Reducer 12 <- Reducer 11 (GROUP, 398)
- Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 322), Map 22 (PARTITION-LEVEL SORT, 322), Reducer 20 (PARTITION-LEVEL SORT, 322)
+ Reducer 17 <- Map 14 (PARTITION-LEVEL SORT, 322), Map 16 (PARTITION-LEVEL SORT, 322), Reducer 20 (PARTITION-LEVEL SORT, 322)
Reducer 18 <- Reducer 17 (GROUP, 389)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 14 (PARTITION-LEVEL SORT, 432), Map 6 (PARTITION-LEVEL SORT, 432)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 6 (PARTITION-LEVEL SORT, 432), Map 7 (PARTITION-LEVEL SORT, 432)
Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 164), Map 21 (PARTITION-LEVEL SORT, 164)
Reducer 3 <- Reducer 2 (GROUP, 523)
Reducer 4 <- Reducer 12 (GROUP, 1009), Reducer 18 (GROUP, 1009), Reducer 3 (GROUP, 1009)
@@ -445,23 +445,6 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int), _col2 (type: int)
Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2))
- Map 22
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-08-04 00:00:00.0' AND TIMESTAMP'1998-08-18 00:00:00.0' and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_date_sk (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Map 6
Map Operator Tree:
TableScan
@@ -480,6 +463,23 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 633586785 Data size: 55276696920 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-08-04 00:00:00.0' AND TIMESTAMP'1998-08-18 00:00:00.0' and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_date_sk (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Map 9
Map Operator Tree:
TableScan
[10/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query4.q.out b/ql/src/test/results/clientpositive/perf/spark/query4.q.out
index 5074d49..75c4fbb 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query4.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query4.q.out
@@ -225,20 +225,20 @@ STAGE PLANS:
Reducer 10 <- Map 13 (PARTITION-LEVEL SORT, 398), Map 9 (PARTITION-LEVEL SORT, 398)
Reducer 11 <- Map 14 (PARTITION-LEVEL SORT, 975), Reducer 10 (PARTITION-LEVEL SORT, 975)
Reducer 12 <- Reducer 11 (GROUP, 481)
- Reducer 16 <- Map 15 (PARTITION-LEVEL SORT, 306), Map 19 (PARTITION-LEVEL SORT, 306)
- Reducer 17 <- Map 20 (PARTITION-LEVEL SORT, 873), Reducer 16 (PARTITION-LEVEL SORT, 873)
- Reducer 18 <- Reducer 17 (GROUP, 369)
+ Reducer 16 <- Map 15 (PARTITION-LEVEL SORT, 398), Map 19 (PARTITION-LEVEL SORT, 398)
+ Reducer 17 <- Map 20 (PARTITION-LEVEL SORT, 975), Reducer 16 (PARTITION-LEVEL SORT, 975)
+ Reducer 18 <- Reducer 17 (GROUP, 481)
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 7 (PARTITION-LEVEL SORT, 306)
- Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 154), Map 25 (PARTITION-LEVEL SORT, 154)
- Reducer 23 <- Map 26 (PARTITION-LEVEL SORT, 706), Reducer 22 (PARTITION-LEVEL SORT, 706)
- Reducer 24 <- Reducer 23 (GROUP, 186)
+ Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 306), Map 25 (PARTITION-LEVEL SORT, 306)
+ Reducer 23 <- Map 26 (PARTITION-LEVEL SORT, 873), Reducer 22 (PARTITION-LEVEL SORT, 873)
+ Reducer 24 <- Reducer 23 (GROUP, 369)
Reducer 28 <- Map 27 (PARTITION-LEVEL SORT, 154), Map 31 (PARTITION-LEVEL SORT, 154)
Reducer 29 <- Map 32 (PARTITION-LEVEL SORT, 706), Reducer 28 (PARTITION-LEVEL SORT, 706)
Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 873), Reducer 2 (PARTITION-LEVEL SORT, 873)
Reducer 30 <- Reducer 29 (GROUP, 186)
- Reducer 34 <- Map 33 (PARTITION-LEVEL SORT, 398), Map 37 (PARTITION-LEVEL SORT, 398)
- Reducer 35 <- Map 38 (PARTITION-LEVEL SORT, 975), Reducer 34 (PARTITION-LEVEL SORT, 975)
- Reducer 36 <- Reducer 35 (GROUP, 481)
+ Reducer 34 <- Map 33 (PARTITION-LEVEL SORT, 154), Map 37 (PARTITION-LEVEL SORT, 154)
+ Reducer 35 <- Map 38 (PARTITION-LEVEL SORT, 706), Reducer 34 (PARTITION-LEVEL SORT, 706)
+ Reducer 36 <- Reducer 35 (GROUP, 186)
Reducer 4 <- Reducer 3 (GROUP, 369)
Reducer 5 <- Reducer 12 (PARTITION-LEVEL SORT, 690), Reducer 18 (PARTITION-LEVEL SORT, 690), Reducer 24 (PARTITION-LEVEL SORT, 690), Reducer 30 (PARTITION-LEVEL SORT, 690), Reducer 36 (PARTITION-LEVEL SORT, 690), Reducer 4 (PARTITION-LEVEL SORT, 690)
Reducer 6 <- Reducer 5 (SORT, 1)
@@ -300,20 +300,20 @@ STAGE PLANS:
Map 15
Map Operator Tree:
TableScan
- alias: catalog_sales
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ alias: store_sales
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cs_bill_customer_sk is not null and cs_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ predicate: (ss_customer_sk is not null and ss_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_ext_discount_amt (type: decimal(7,2)), cs_ext_sales_price (type: decimal(7,2)), cs_ext_wholesale_cost (type: decimal(7,2)), cs_ext_list_price (type: decimal(7,2))
+ expressions: ss_sold_date_sk (type: int), ss_customer_sk (type: int), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
Map 19
Map Operator Tree:
@@ -353,20 +353,20 @@ STAGE PLANS:
Map 21
Map Operator Tree:
TableScan
- alias: web_sales
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ alias: catalog_sales
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (ws_bill_customer_sk is not null and ws_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ predicate: (cs_bill_customer_sk is not null and cs_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ws_sold_date_sk (type: int), ws_bill_customer_sk (type: int), ws_ext_discount_amt (type: decimal(7,2)), ws_ext_sales_price (type: decimal(7,2)), ws_ext_wholesale_cost (type: decimal(7,2)), ws_ext_list_price (type: decimal(7,2))
+ expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_ext_discount_amt (type: decimal(7,2)), cs_ext_sales_price (type: decimal(7,2)), cs_ext_wholesale_cost (type: decimal(7,2)), cs_ext_list_price (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
Map 25
Map Operator Tree:
@@ -374,7 +374,7 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
+ predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: d_date_sk (type: int)
@@ -427,7 +427,7 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
+ predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: d_date_sk (type: int)
@@ -459,20 +459,20 @@ STAGE PLANS:
Map 33
Map Operator Tree:
TableScan
- alias: store_sales
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ alias: web_sales
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (ss_customer_sk is not null and ss_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ predicate: (ws_bill_customer_sk is not null and ws_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_sold_date_sk (type: int), ss_customer_sk (type: int), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2))
+ expressions: ws_sold_date_sk (type: int), ws_bill_customer_sk (type: int), ws_ext_discount_amt (type: decimal(7,2)), ws_ext_sales_price (type: decimal(7,2)), ws_ext_wholesale_cost (type: decimal(7,2)), ws_ext_list_price (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
Map 37
Map Operator Tree:
@@ -480,7 +480,7 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
+ predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: d_date_sk (type: int)
@@ -515,7 +515,7 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_year = 2001) and d_date_sk is not null) (type: boolean)
+ predicate: ((d_year = 2002) and d_date_sk is not null) (type: boolean)
Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: d_date_sk (type: int)
@@ -638,12 +638,12 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
Reducer 17
Reduce Operator Tree:
@@ -654,22 +654,22 @@ STAGE PLANS:
0 _col1 (type: int)
1 _col0 (type: int)
outputColumnNames: _col2, _col3, _col4, _col5, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), ((((_col5 - _col4) - _col2) + _col3) / 2) (type: decimal(14,6))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col7)
keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
sort order: +++++++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
value expressions: _col7 (type: decimal(24,6))
Reducer 18
Reduce Operator Tree:
@@ -678,17 +678,17 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string), KEY._col6 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: string), _col7 (type: decimal(24,6))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ expressions: _col0 (type: string), _col3 (type: string), _col7 (type: decimal(24,6))
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(24,6))
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: decimal(24,6))
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -714,12 +714,12 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
Reducer 23
Reduce Operator Tree:
@@ -730,22 +730,22 @@ STAGE PLANS:
0 _col1 (type: int)
1 _col0 (type: int)
outputColumnNames: _col2, _col3, _col4, _col5, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), ((((_col5 - _col4) - _col2) + _col3) / 2) (type: decimal(14,6))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col7)
keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
sort order: +++++++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
- Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
value expressions: _col7 (type: decimal(24,6))
Reducer 24
Reduce Operator Tree:
@@ -754,17 +754,24 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string), KEY._col6 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col7 (type: decimal(24,6))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(24,6))
+ outputColumnNames: _col0, _col7
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col7 > 0) (type: boolean)
+ Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col7 (type: decimal(24,6))
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(24,6))
Reducer 28
Reduce Operator Tree:
Join Operator
@@ -843,21 +850,14 @@ STAGE PLANS:
Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col7 (type: decimal(24,6))
- outputColumnNames: _col0, _col7
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col7 > 0) (type: boolean)
- Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col7 (type: decimal(24,6))
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(24,6))
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(24,6))
Reducer 34
Reduce Operator Tree:
Join Operator
@@ -867,12 +867,12 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2)), _col3 (type: decimal(7,2)), _col4 (type: decimal(7,2)), _col5 (type: decimal(7,2))
Reducer 35
Reduce Operator Tree:
@@ -883,22 +883,22 @@ STAGE PLANS:
0 _col1 (type: int)
1 _col0 (type: int)
outputColumnNames: _col2, _col3, _col4, _col5, _col9, _col10, _col11, _col12, _col13, _col14, _col15
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string), ((((_col5 - _col4) - _col2) + _col3) / 2) (type: decimal(14,6))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col7)
keys: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
sort order: +++++++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string), _col6 (type: string)
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
value expressions: _col7 (type: decimal(24,6))
Reducer 36
Reduce Operator Tree:
@@ -907,42 +907,42 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string), KEY._col6 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col3 (type: string), _col7 (type: decimal(24,6))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: decimal(24,6))
- Reducer 4
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string), KEY._col6 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
- Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col7 (type: decimal(24,6))
outputColumnNames: _col0, _col7
- Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 87121617 Data size: 11846020363 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col7 > 0) (type: boolean)
- Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col7 (type: decimal(24,6))
outputColumnNames: _col0, _col1
- Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 58077952 Data size: 7864921389 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 29040539 Data size: 3948673454 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(24,6))
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: string), KEY._col6 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col7 (type: decimal(24,6))
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(24,6))
Reducer 5
Reduce Operator Tree:
Join Operator
@@ -959,13 +959,13 @@ STAGE PLANS:
3 _col0 (type: string)
4 _col0 (type: string)
5 _col0 (type: string)
- outputColumnNames: _col1, _col3, _col5, _col7, _col9, _col11, _col12
+ outputColumnNames: _col1, _col3, _col5, _col6, _col8, _col10, _col12
Statistics: Num rows: 1916625598 Data size: 169085266687 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col5 / _col1) > (_col12 / _col3)) and ((_col5 / _col1) > (_col7 / _col9))) (type: boolean)
+ predicate: (((_col1 / _col8) > (_col10 / _col12)) and ((_col1 / _col8) > (_col6 / _col3))) (type: boolean)
Statistics: Num rows: 212958399 Data size: 18787251785 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col11 (type: string)
+ expressions: _col5 (type: string)
outputColumnNames: _col0
Statistics: Num rows: 212958399 Data size: 18787251785 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query40.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query40.q.out b/ql/src/test/results/clientpositive/perf/spark/query40.q.out
index 5360385..f286294 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query40.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query40.q.out
@@ -54,7 +54,8 @@ limit 100
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-1 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -62,39 +63,44 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 8
+ Map 9
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ alias: warehouse
+ Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ predicate: w_warehouse_sk is not null (type: boolean)
+ Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int), d_date (type: string)
+ expressions: w_warehouse_sk (type: int), w_state (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 9
+
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 7
Map Operator Tree:
TableScan
- alias: warehouse
- Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: w_warehouse_sk is not null (type: boolean)
- Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
+ predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: w_warehouse_sk (type: int), w_state (type: string)
+ expressions: d_date_sk (type: int), d_date (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -103,7 +109,7 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 329), Map 6 (PARTITION-LEVEL SORT, 329)
- Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 336), Reducer 2 (PARTITION-LEVEL SORT, 336)
+ Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 370), Reducer 2 (PARTITION-LEVEL SORT, 370)
Reducer 4 <- Reducer 3 (GROUP, 447)
Reducer 5 <- Reducer 4 (SORT, 1)
#### A masked pattern was here ####
@@ -144,7 +150,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(7,2))
- Map 7
+ Map 8
Map Operator Tree:
TableScan
alias: item
@@ -163,6 +169,8 @@ STAGE PLANS:
Statistics: Num rows: 51333 Data size: 73728460 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Reducer 2
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -172,12 +180,22 @@ STAGE PLANS:
1 _col0 (type: int), _col1 (type: int)
outputColumnNames: _col0, _col1, _col2, _col4, _col7
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col4 (type: decimal(7,2)), _col7 (type: decimal(7,2))
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col4, _col7, _col9
+ input vertices:
+ 1 Map 7
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col4 (type: decimal(7,2)), _col7 (type: decimal(7,2)), _col9 (type: string)
Reducer 3
Local Work:
Map Reduce Local Work
@@ -188,45 +206,35 @@ STAGE PLANS:
keys:
0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col7, _col9
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col1, _col4, _col7, _col9, _col11
+ Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1, _col4, _col7, _col9, _col12
+ outputColumnNames: _col4, _col7, _col9, _col11, _col14
input vertices:
- 1 Map 8
- Statistics: Num rows: 383314495 Data size: 51908482889 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col4, _col7, _col9, _col12, _col14
- input vertices:
- 1 Map 9
+ 1 Map 9
+ Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col14 (type: string), _col11 (type: string), CASE WHEN ((CAST( _col9 AS DATE) < DATE'1998-04-08')) THEN ((_col4 - COALESCE(_col7,0))) ELSE (0) END (type: decimal(13,2)), CASE WHEN ((CAST( _col9 AS DATE) >= DATE'1998-04-08')) THEN ((_col4 - COALESCE(_col7,0))) ELSE (0) END (type: decimal(13,2))
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col14 (type: string), _col9 (type: string), CASE WHEN ((CAST( _col12 AS DATE) < DATE'1998-04-08')) THEN ((_col4 - COALESCE(_col7,0))) ELSE (0) END (type: decimal(13,2)), CASE WHEN ((CAST( _col12 AS DATE) >= DATE'1998-04-08')) THEN ((_col4 - COALESCE(_col7,0))) ELSE (0) END (type: decimal(13,2))
+ Group By Operator
+ aggregations: sum(_col2), sum(_col3)
+ keys: _col0 (type: string), _col1 (type: string)
+ mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col2), sum(_col3)
- keys: _col0 (type: string), _col1 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 421645953 Data size: 57099332415 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col2 (type: decimal(23,2)), _col3 (type: decimal(23,2))
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col2 (type: decimal(23,2)), _col3 (type: decimal(23,2))
Reducer 4
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query44.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query44.q.out b/ql/src/test/results/clientpositive/perf/spark/query44.q.out
index b432c16..4ca41fb 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query44.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query44.q.out
@@ -1,5 +1,5 @@
-Warning: Shuffle Join JOIN[33][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Work 'Reducer 4' is a cross product
-Warning: Shuffle Join JOIN[78][tables = [$hdt$_3, $hdt$_4, $hdt$_2]] in Work 'Reducer 17' is a cross product
+Warning: Shuffle Join JOIN[36][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Work 'Reducer 8' is a cross product
+Warning: Shuffle Join JOIN[81][tables = [$hdt$_4, $hdt$_5, $hdt$_3]] in Work 'Reducer 19' is a cross product
PREHOOK: query: explain
select asceding.rnk, i1.i_product_name best_performing, i2.i_product_name worst_performing
from(select *
@@ -76,43 +76,65 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Map 20 (GROUP, 100)
- Reducer 12 <- Map 11 (GROUP, 199)
- Reducer 15 <- Map 1 (GROUP, 100)
- Reducer 16 <- Reducer 15 (GROUP, 1)
- Reducer 17 <- Reducer 16 (PARTITION-LEVEL SORT, 1), Reducer 21 (PARTITION-LEVEL SORT, 1), Reducer 23 (PARTITION-LEVEL SORT, 1)
- Reducer 18 <- Reducer 17 (PARTITION-LEVEL SORT, 1009)
- Reducer 19 <- Map 24 (PARTITION-LEVEL SORT, 1009), Reducer 18 (PARTITION-LEVEL SORT, 1009)
- Reducer 21 <- Map 20 (GROUP, 100)
- Reducer 23 <- Map 11 (GROUP, 199)
- Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT, 1), Reducer 12 (PARTITION-LEVEL SORT, 1), Reducer 16 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1009)
- Reducer 6 <- Map 13 (PARTITION-LEVEL SORT, 1009), Reducer 5 (PARTITION-LEVEL SORT, 1009)
- Reducer 7 <- Reducer 19 (PARTITION-LEVEL SORT, 1009), Reducer 6 (PARTITION-LEVEL SORT, 1009)
- Reducer 8 <- Reducer 7 (SORT, 1)
+ Reducer 11 <- Map 10 (GROUP, 100)
+ Reducer 13 <- Map 12 (GROUP, 199)
+ Reducer 15 <- Map 14 (PARTITION-LEVEL SORT, 1009), Reducer 20 (PARTITION-LEVEL SORT, 1009)
+ Reducer 17 <- Map 16 (GROUP, 100)
+ Reducer 18 <- Reducer 17 (GROUP, 1)
+ Reducer 19 <- Reducer 18 (PARTITION-LEVEL SORT, 1), Reducer 22 (PARTITION-LEVEL SORT, 1), Reducer 24 (PARTITION-LEVEL SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1009), Reducer 9 (PARTITION-LEVEL SORT, 1009)
+ Reducer 20 <- Reducer 19 (PARTITION-LEVEL SORT, 1009)
+ Reducer 22 <- Map 10 (GROUP, 100)
+ Reducer 24 <- Map 12 (GROUP, 199)
+ Reducer 3 <- Reducer 15 (PARTITION-LEVEL SORT, 1009), Reducer 2 (PARTITION-LEVEL SORT, 1009)
+ Reducer 4 <- Reducer 3 (SORT, 1)
+ Reducer 8 <- Reducer 11 (PARTITION-LEVEL SORT, 1), Reducer 13 (PARTITION-LEVEL SORT, 1), Reducer 18 (PARTITION-LEVEL SORT, 1)
+ Reducer 9 <- Reducer 8 (PARTITION-LEVEL SORT, 1009)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
+ alias: i1
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: i_item_sk is not null (type: boolean)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: i_item_sk (type: int), i_product_name (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 10
+ Map Operator Tree:
+ TableScan
alias: store_sales
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((ss_store_sk = 410) and ss_hdemo_sk is null) (type: boolean)
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
Select Operator
+ expressions: ss_net_profit (type: decimal(7,2))
+ outputColumnNames: _col1
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
Group By Operator
+ aggregations: sum(_col1), count(_col1)
keys: 410 (type: int)
mode: hash
- outputColumnNames: _col0
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
- Map 11
+ value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint)
+ Map 12
Map Operator Tree:
TableScan
alias: ss1
@@ -136,7 +158,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 287997817 Data size: 25407250999 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint)
- Map 13
+ Map 14
Map Operator Tree:
TableScan
alias: i2
@@ -154,7 +176,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 20
+ Map 16
Map Operator Tree:
TableScan
alias: store_sales
@@ -163,40 +185,18 @@ STAGE PLANS:
predicate: ((ss_store_sk = 410) and ss_hdemo_sk is null) (type: boolean)
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_net_profit (type: decimal(7,2))
- outputColumnNames: _col1
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col1), count(_col1)
keys: 410 (type: int)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint)
- Map 24
- Map Operator Tree:
- TableScan
- alias: i1
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: i_item_sk is not null (type: boolean)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: i_item_sk (type: int), i_product_name (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Reducer 10
+ Reducer 11
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1)
@@ -212,7 +212,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(37,22))
- Reducer 12
+ Reducer 13
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1)
@@ -230,6 +230,22 @@ STAGE PLANS:
value expressions: _col0 (type: int), _col1 (type: decimal(37,22))
Reducer 15
Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3
+ Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: int)
+ Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reducer 17
+ Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int)
mode: mergepartial
@@ -246,7 +262,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Reducer 16
+ Reducer 18
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -261,7 +277,7 @@ STAGE PLANS:
Reduce Output Operator
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reducer 17
+ Reducer 19
Reduce Operator Tree:
Join Operator
condition map:
@@ -278,12 +294,28 @@ STAGE PLANS:
Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 0 (type: int), _col3 (type: decimal(37,22))
- sort order: ++
+ sort order: +-
Map-reduce partition columns: 0 (type: int)
Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: int)
- Reducer 18
+ Reducer 2
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col3
+ Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: int)
+ Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reducer 20
Reduce Operator Tree:
Select Operator
expressions: VALUE._col2 (type: int), KEY.reducesinkkey1 (type: decimal(37,22))
@@ -298,7 +330,7 @@ STAGE PLANS:
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col3 ASC NULLS FIRST
+ order by: _col3 DESC NULLS LAST
partition by: 0
raw input shape:
window functions:
@@ -323,23 +355,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int)
- Reducer 19
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col3
- Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: string)
- Reducer 21
+ Reducer 22
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1)
@@ -355,7 +371,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 71999454 Data size: 6351812727 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: decimal(37,22))
- Reducer 23
+ Reducer 24
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1)
@@ -371,8 +387,44 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 143998908 Data size: 12703625455 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: decimal(37,22))
+ Reducer 3
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col3 (type: int)
+ 1 _col3 (type: int)
+ outputColumnNames: _col1, _col3, _col5
+ Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col3 (type: int), _col1 (type: string), _col5 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ value expressions: _col1 (type: string), _col2 (type: string)
Reducer 4
Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 100
+ Statistics: Num rows: 100 Data size: 18500 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 100 Data size: 18500 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 8
+ Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
@@ -388,12 +440,12 @@ STAGE PLANS:
Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: 0 (type: int), _col3 (type: decimal(37,22))
- sort order: +-
+ sort order: ++
Map-reduce partition columns: 0 (type: int)
Statistics: Num rows: 3455947584198744 Data size: 640872925954123264 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: int)
- Reducer 5
+ Reducer 9
Reduce Operator Tree:
Select Operator
expressions: VALUE._col2 (type: int), KEY.reducesinkkey1 (type: decimal(37,22))
@@ -408,7 +460,7 @@ STAGE PLANS:
Windowing table definition
input alias: ptf_1
name: windowingtablefunction
- order by: _col3 DESC NULLS LAST
+ order by: _col3 ASC NULLS FIRST
partition by: 0
raw input shape:
window functions:
@@ -433,58 +485,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 1151982528066248 Data size: 213624308651374400 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int)
- Reducer 6
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col3
- Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 1267180808338276 Data size: 234986744609712256 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: string)
- Reducer 7
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col1 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col3, _col5, _col7
- Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col5 (type: int), _col7 (type: string), _col3 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- value expressions: _col1 (type: string), _col2 (type: string)
- Reducer 8
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string), VALUE._col1 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1393898919384048 Data size: 258485424673204064 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 100
- Statistics: Num rows: 100 Data size: 18500 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 100 Data size: 18500 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
[07/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query58.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query58.q.out b/ql/src/test/results/clientpositive/perf/spark/query58.q.out
index eb5dffe..f06cbef 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query58.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query58.q.out
@@ -1,6 +1,6 @@
-Warning: Map Join MAPJOIN[180][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[181][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[182][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[183][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[184][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
PREHOOK: query: explain
with ss_items as
(select i_item_id item_id
@@ -140,10 +140,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 12 <- Map 11 (GROUP, 1)
+ Reducer 11 <- Map 10 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 11
+ Map 10
Map Operator Tree:
TableScan
alias: date_dim
@@ -162,7 +162,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Reducer 12
+ Reducer 11
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -184,10 +184,10 @@ STAGE PLANS:
Stage: Stage-3
Spark
Edges:
- Reducer 25 <- Map 24 (GROUP, 1)
+ Reducer 24 <- Map 23 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 24
+ Map 23
Map Operator Tree:
TableScan
alias: date_dim
@@ -206,7 +206,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Reducer 25
+ Reducer 24
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -228,10 +228,10 @@ STAGE PLANS:
Stage: Stage-4
Spark
Edges:
- Reducer 38 <- Map 37 (GROUP, 1)
+ Reducer 37 <- Map 36 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 37
+ Map 36
Map Operator Tree:
TableScan
alias: date_dim
@@ -250,7 +250,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Reducer 38
+ Reducer 37
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -272,47 +272,76 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Reducer 9 (GROUP, 2)
- Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 2), Reducer 23 (PARTITION-LEVEL SORT, 2)
- Reducer 18 <- Map 27 (PARTITION-LEVEL SORT, 398), Reducer 17 (PARTITION-LEVEL SORT, 398)
- Reducer 19 <- Map 28 (PARTITION-LEVEL SORT, 442), Reducer 18 (PARTITION-LEVEL SORT, 442)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 10 (PARTITION-LEVEL SORT, 2)
- Reducer 20 <- Reducer 19 (GROUP, 481)
- Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 2), Map 26 (PARTITION-LEVEL SORT, 2)
- Reducer 23 <- Reducer 22 (GROUP, 2)
- Reducer 3 <- Map 14 (PARTITION-LEVEL SORT, 306), Reducer 2 (PARTITION-LEVEL SORT, 306)
- Reducer 30 <- Map 29 (PARTITION-LEVEL SORT, 2), Reducer 36 (PARTITION-LEVEL SORT, 2)
- Reducer 31 <- Map 40 (PARTITION-LEVEL SORT, 154), Reducer 30 (PARTITION-LEVEL SORT, 154)
- Reducer 32 <- Map 41 (PARTITION-LEVEL SORT, 174), Reducer 31 (PARTITION-LEVEL SORT, 174)
- Reducer 33 <- Reducer 32 (GROUP, 186)
- Reducer 35 <- Map 34 (PARTITION-LEVEL SORT, 2), Map 39 (PARTITION-LEVEL SORT, 2)
- Reducer 36 <- Reducer 35 (GROUP, 2)
- Reducer 4 <- Map 15 (PARTITION-LEVEL SORT, 341), Reducer 3 (PARTITION-LEVEL SORT, 341)
- Reducer 5 <- Reducer 4 (GROUP, 369)
- Reducer 6 <- Reducer 20 (PARTITION-LEVEL SORT, 518), Reducer 33 (PARTITION-LEVEL SORT, 518), Reducer 5 (PARTITION-LEVEL SORT, 518)
- Reducer 7 <- Reducer 6 (SORT, 1)
- Reducer 9 <- Map 13 (PARTITION-LEVEL SORT, 2), Map 8 (PARTITION-LEVEL SORT, 2)
+ Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 2), Map 15 (PARTITION-LEVEL SORT, 2)
+ Reducer 14 <- Reducer 13 (GROUP, 2)
+ Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 403), Map 20 (PARTITION-LEVEL SORT, 403)
+ Reducer 18 <- Reducer 17 (PARTITION-LEVEL SORT, 438), Reducer 22 (PARTITION-LEVEL SORT, 438)
+ Reducer 19 <- Reducer 18 (GROUP, 481)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 310), Map 7 (PARTITION-LEVEL SORT, 310)
+ Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 2), Reducer 27 (PARTITION-LEVEL SORT, 2)
+ Reducer 26 <- Map 25 (PARTITION-LEVEL SORT, 2), Map 28 (PARTITION-LEVEL SORT, 2)
+ Reducer 27 <- Reducer 26 (GROUP, 2)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 336), Reducer 9 (PARTITION-LEVEL SORT, 336)
+ Reducer 30 <- Map 29 (PARTITION-LEVEL SORT, 159), Map 33 (PARTITION-LEVEL SORT, 159)
+ Reducer 31 <- Reducer 30 (PARTITION-LEVEL SORT, 169), Reducer 35 (PARTITION-LEVEL SORT, 169)
+ Reducer 32 <- Reducer 31 (GROUP, 186)
+ Reducer 35 <- Map 34 (PARTITION-LEVEL SORT, 2), Reducer 40 (PARTITION-LEVEL SORT, 2)
+ Reducer 39 <- Map 38 (PARTITION-LEVEL SORT, 2), Map 41 (PARTITION-LEVEL SORT, 2)
+ Reducer 4 <- Reducer 3 (GROUP, 369)
+ Reducer 40 <- Reducer 39 (GROUP, 2)
+ Reducer 5 <- Reducer 19 (PARTITION-LEVEL SORT, 518), Reducer 32 (PARTITION-LEVEL SORT, 518), Reducer 4 (PARTITION-LEVEL SORT, 518)
+ Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 9 <- Map 8 (PARTITION-LEVEL SORT, 2), Reducer 14 (PARTITION-LEVEL SORT, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ alias: catalog_sales
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (d_date is not null and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ predicate: (cs_item_sk is not null and cs_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int), d_date (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ expressions: cs_sold_date_sk (type: int), cs_item_sk (type: int), cs_ext_sales_price (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: string)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int)
- Map 13
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: decimal(7,2))
+ Map 12
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((d_date = '1998-02-19') and d_week_seq is not null) (type: boolean)
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_week_seq (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col1
+ input vertices:
+ 0 Reducer 11
+ Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
+ Map 15
Map Operator Tree:
TableScan
alias: date_dim
@@ -330,25 +359,25 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Map 14
+ Map 16
Map Operator Tree:
TableScan
- alias: catalog_sales
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ alias: store_sales
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cs_item_sk is not null and cs_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ predicate: (ss_item_sk is not null and ss_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: cs_sold_date_sk (type: int), cs_item_sk (type: int), cs_ext_sales_price (type: decimal(7,2))
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_ext_sales_price (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: decimal(7,2))
- Map 15
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: decimal(7,2))
+ Map 20
Map Operator Tree:
TableScan
alias: item
@@ -366,7 +395,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 16
+ Map 21
Map Operator Tree:
TableScan
alias: date_dim
@@ -384,7 +413,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
- Map 21
+ Map 25
Map Operator Tree:
TableScan
alias: date_dim
@@ -402,18 +431,18 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col0
+ outputColumnNames: _col1
input vertices:
- 1 Reducer 25
+ 0 Reducer 24
Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
Local Work:
Map Reduce Local Work
- Map 26
+ Map 28
Map Operator Tree:
TableScan
alias: date_dim
@@ -431,25 +460,25 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Map 27
+ Map 29
Map Operator Tree:
TableScan
- alias: store_sales
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ alias: web_sales
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (ss_item_sk is not null and ss_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ predicate: (ws_item_sk is not null and ws_sold_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_ext_sales_price (type: decimal(7,2))
+ expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_sales_price (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: decimal(7,2))
- Map 28
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: decimal(7,2))
+ Map 33
Map Operator Tree:
TableScan
alias: item
@@ -467,7 +496,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Map 29
+ Map 34
Map Operator Tree:
TableScan
alias: date_dim
@@ -485,7 +514,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
- Map 34
+ Map 38
Map Operator Tree:
TableScan
alias: date_dim
@@ -503,18 +532,18 @@ STAGE PLANS:
keys:
0
1
- outputColumnNames: _col0
+ outputColumnNames: _col1
input vertices:
- 1 Reducer 38
+ 0 Reducer 37
Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
Local Work:
Map Reduce Local Work
- Map 39
+ Map 41
Map Operator Tree:
TableScan
alias: date_dim
@@ -532,25 +561,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Map 40
- Map Operator Tree:
- TableScan
- alias: web_sales
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (ws_item_sk is not null and ws_sold_date_sk is not null) (type: boolean)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_sales_price (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: decimal(7,2))
- Map 41
+ Map 7
Map Operator Tree:
TableScan
alias: item
@@ -574,30 +585,39 @@ STAGE PLANS:
alias: date_dim
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((d_date = '1998-02-19') and d_week_seq is not null) (type: boolean)
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
+ predicate: (d_date is not null and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_week_seq (type: int)
- outputColumnNames: _col0
- Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0
- 1
- outputColumnNames: _col0
- input vertices:
- 1 Reducer 12
- Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 36524 Data size: 41199072 Basic stats: COMPLETE Column stats: NONE
- Local Work:
- Map Reduce Local Work
- Reducer 10
+ expressions: d_date_sk (type: int), d_date (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Reducer 13
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col1 (type: int)
+ outputColumnNames: _col2
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col2 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Reducer 14
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
@@ -615,15 +635,16 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col4
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(7,2)), _col4 (type: string)
Reducer 18
Reduce Operator Tree:
Join Operator
@@ -632,27 +653,11 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col4, _col5
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: decimal(7,2))
- Reducer 19
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col5, _col7
+ outputColumnNames: _col2, _col4
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col5)
- keys: _col7 (type: string)
+ aggregations: sum(_col2)
+ keys: _col4 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
@@ -662,12 +667,42 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2))
+ Reducer 19
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ keys: KEY._col0 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(17,2))
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col4
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(7,2)), _col4 (type: string)
+ Reducer 22
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
0 _col1 (type: string)
1 _col0 (type: string)
outputColumnNames: _col0
@@ -677,27 +712,13 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reducer 20
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2))
- Reducer 22
+ Reducer 26
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: int)
1 _col1 (type: int)
outputColumnNames: _col2
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
@@ -711,7 +732,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reducer 23
+ Reducer 27
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
@@ -731,29 +752,36 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col4, _col5
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: decimal(7,2))
+ outputColumnNames: _col2, _col4
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col2)
+ keys: _col4 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(17,2))
Reducer 30
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2, _col4
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: decimal(7,2)), _col4 (type: string)
Reducer 31
Reduce Operator Tree:
Join Operator
@@ -762,27 +790,11 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col4, _col5
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col4 (type: int)
- sort order: +
- Map-reduce partition columns: _col4 (type: int)
- Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col5 (type: decimal(7,2))
- Reducer 32
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col5, _col7
+ outputColumnNames: _col2, _col4
Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col5)
- keys: _col7 (type: string)
+ aggregations: sum(_col2)
+ keys: _col4 (type: string)
mode: hash
outputColumnNames: _col0, _col1
Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
@@ -792,7 +804,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 174243235 Data size: 23692040863 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: decimal(17,2))
- Reducer 33
+ Reducer 32
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
@@ -812,7 +824,22 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
+ Reducer 39
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
1 _col1 (type: int)
outputColumnNames: _col2
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
@@ -826,55 +853,33 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reducer 36
+ Reducer 4
Reduce Operator Tree:
Group By Operator
+ aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 40176 Data size: 44957448 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 40176 Data size: 44957448 Basic stats: COMPLETE Column stats: NONE
- Reducer 4
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col4 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col5, _col7
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col5)
- keys: _col7 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2))
- Reducer 5
+ Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(17,2))
+ Reducer 40
Reduce Operator Tree:
Group By Operator
- aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0
+ Statistics: Num rows: 40176 Data size: 44957448 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 174233858 Data size: 23594764438 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2))
- Reducer 6
+ Statistics: Num rows: 40176 Data size: 44957448 Basic stats: COMPLETE Column stats: NONE
+ Reducer 5
Reduce Operator Tree:
Join Operator
condition map:
@@ -899,7 +904,7 @@ STAGE PLANS:
Statistics: Num rows: 1442 Data size: 127213 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: decimal(38,17)), _col3 (type: decimal(17,2)), _col4 (type: decimal(38,17)), _col5 (type: decimal(17,2)), _col6 (type: decimal(38,17)), _col7 (type: decimal(23,6))
- Reducer 7
+ Reducer 6
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: decimal(17,2)), VALUE._col0 (type: decimal(38,17)), VALUE._col1 (type: decimal(17,2)), VALUE._col2 (type: decimal(38,17)), VALUE._col3 (type: decimal(17,2)), VALUE._col4 (type: decimal(38,17)), VALUE._col5 (type: decimal(23,6))
@@ -921,20 +926,15 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col2
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col2 (type: string)
- mode: hash
- outputColumnNames: _col0
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query6.q.out b/ql/src/test/results/clientpositive/perf/spark/query6.q.out
index 8090c8a..954bacb 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query6.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query6.q.out
@@ -58,11 +58,11 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 6 <- Map 5 (GROUP, 2)
- Reducer 7 <- Reducer 6 (GROUP, 1)
+ Reducer 18 <- Map 17 (GROUP, 2)
+ Reducer 19 <- Reducer 18 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 5
+ Map 17
Map Operator Tree:
TableScan
alias: date_dim
@@ -84,7 +84,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Reducer 6
+ Reducer 18
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: int)
@@ -102,7 +102,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: bigint)
- Reducer 7
+ Reducer 19
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -124,71 +124,21 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 439), Reducer 9 (PARTITION-LEVEL SORT, 439)
- Reducer 11 <- Map 20 (PARTITION-LEVEL SORT, 1009), Reducer 10 (PARTITION-LEVEL SORT, 1009)
- Reducer 13 <- Map 12 (GROUP, 6)
- Reducer 15 <- Map 14 (PARTITION-LEVEL SORT, 2), Reducer 18 (PARTITION-LEVEL SORT, 2)
- Reducer 16 <- Map 19 (PARTITION-LEVEL SORT, 398), Reducer 15 (PARTITION-LEVEL SORT, 398)
- Reducer 18 <- Map 17 (GROUP, 2)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 849), Reducer 11 (PARTITION-LEVEL SORT, 849)
- Reducer 3 <- Reducer 2 (GROUP, 582)
- Reducer 4 <- Reducer 3 (SORT, 1)
- Reducer 9 <- Map 8 (PARTITION-LEVEL SORT, 8), Reducer 13 (PARTITION-LEVEL SORT, 8)
+ Reducer 12 <- Map 11 (PARTITION-LEVEL SORT, 855), Map 13 (PARTITION-LEVEL SORT, 855)
+ Reducer 15 <- Map 14 (GROUP, 6)
+ Reducer 16 <- Map 20 (PARTITION-LEVEL SORT, 8), Reducer 15 (PARTITION-LEVEL SORT, 8)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 9 (PARTITION-LEVEL SORT, 2)
+ Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 398), Reducer 2 (PARTITION-LEVEL SORT, 398)
+ Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 1009), Reducer 3 (PARTITION-LEVEL SORT, 1009)
+ Reducer 5 <- Reducer 16 (PARTITION-LEVEL SORT, 483), Reducer 4 (PARTITION-LEVEL SORT, 483)
+ Reducer 6 <- Reducer 5 (GROUP, 529)
+ Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 9 <- Map 8 (GROUP, 2)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: a
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ca_address_sk is not null (type: boolean)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ca_address_sk (type: int), ca_state (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0
- 1
- outputColumnNames: _col0, _col1
- input vertices:
- 1 Reducer 7
- Statistics: Num rows: 40000000 Data size: 40955195284 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 40000000 Data size: 40955195284 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Local Work:
- Map Reduce Local Work
- Map 12
- Map Operator Tree:
- TableScan
- alias: j
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: i_category is not null (type: boolean)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(i_current_price), count(i_current_price)
- keys: i_category (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint)
- Map 14
- Map Operator Tree:
- TableScan
alias: d
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -204,29 +154,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int)
- Map 17
- Map Operator Tree:
- TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((d_moy = 2) and (d_year = 2000) and d_month_seq is not null) (type: boolean)
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: d_month_seq (type: int)
- outputColumnNames: d_month_seq
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: d_month_seq (type: int)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Map 19
+ Map 10
Map Operator Tree:
TableScan
alias: s
@@ -244,7 +172,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int)
- Map 20
+ Map 11
Map Operator Tree:
TableScan
alias: c
@@ -257,12 +185,50 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ Map 13
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ca_address_sk is not null (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ca_address_sk (type: int), ca_state (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
- Map 8
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 14
+ Map Operator Tree:
+ TableScan
+ alias: j
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: i_category is not null (type: boolean)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(i_current_price), count(i_current_price)
+ keys: i_category (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(17,2)), _col2 (type: bigint)
+ Map 20
Map Operator Tree:
TableScan
alias: i
@@ -280,41 +246,47 @@ STAGE PLANS:
Map-reduce partition columns: _col2 (type: string)
Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col1 (type: decimal(7,2))
- Reducer 10
+ Map 8
+ Map Operator Tree:
+ TableScan
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((d_moy = 2) and (d_year = 2000) and d_month_seq is not null) (type: boolean)
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: d_month_seq (type: int)
+ outputColumnNames: d_month_seq
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: d_month_seq (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
+ Reducer 12
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col8
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col3
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col8 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col8 (type: int)
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- Reducer 11
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col8 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col13
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col13 (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Reducer 13
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: string)
+ Reducer 15
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0), count(VALUE._col1)
@@ -326,13 +298,45 @@ STAGE PLANS:
expressions: (_col1 / _col2) (type: decimal(37,22)), true (type: boolean), _col0 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: string)
- sort order: +
- Map-reduce partition columns: _col2 (type: string)
- Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: decimal(37,22)), _col1 (type: boolean)
- Reducer 15
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0
+ 1
+ outputColumnNames: _col0, _col1, _col2
+ input vertices:
+ 1 Reducer 19
+ Statistics: Num rows: 231000 Data size: 333859228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: string)
+ Statistics: Num rows: 231000 Data size: 333859228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: decimal(37,22)), _col1 (type: boolean)
+ Reducer 16
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: string)
+ 1 _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col4, _col5
+ Statistics: Num rows: 508200 Data size: 729916518 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (_col5 > (1.2 * CASE WHEN (_col1 is null) THEN (null) ELSE (_col0) END)) (type: boolean)
+ Statistics: Num rows: 169400 Data size: 243305506 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col4 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 169400 Data size: 243305506 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 169400 Data size: 243305506 Basic stats: COMPLETE Column stats: NONE
+ Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
@@ -347,7 +351,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 80353 Data size: 89916016 Basic stats: COMPLETE Column stats: NONE
- Reducer 16
+ Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
@@ -357,73 +361,73 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col4, _col5
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col4 (type: int), _col5 (type: int)
- outputColumnNames: _col1, _col2
+ Reduce Output Operator
+ key expressions: _col5 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col5 (type: int)
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int)
- Reducer 18
+ value expressions: _col4 (type: int)
+ Reducer 4
Reduce Operator Tree:
- Group By Operator
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col5 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col4, _col9
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col4 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
- Reducer 2
+ Map-reduce partition columns: _col4 (type: int)
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col9 (type: string)
+ Reducer 5
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col1 (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ 0 _col4 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col9
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
- keys: _col1 (type: string)
+ keys: _col9 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Reducer 3
+ Reducer 6
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col1 >= 10L) (type: boolean)
- Statistics: Num rows: 140552546 Data size: 12399586418 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 127775039 Data size: 11272351038 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: bigint)
sort order: +
- Statistics: Num rows: 140552546 Data size: 12399586418 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 127775039 Data size: 11272351038 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col0 (type: string)
- Reducer 4
+ Reducer 7
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 140552546 Data size: 12399586418 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 127775039 Data size: 11272351038 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE
@@ -436,22 +440,16 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Reducer 9
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: string)
- 1 _col2 (type: string)
- outputColumnNames: _col0, _col1, _col3, _col4
- Statistics: Num rows: 508200 Data size: 729916518 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col1 > (1.2 * CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END)) (type: boolean)
- Statistics: Num rows: 169400 Data size: 243305506 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 169400 Data size: 243305506 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 9131 Data size: 10217589 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query61.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query61.q.out b/ql/src/test/results/clientpositive/perf/spark/query61.q.out
index c01612a..5062307 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query61.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query61.q.out
@@ -235,11 +235,11 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col0 (type: int)
- 1 _col6 (type: int)
- outputColumnNames: _col12
+ 1 _col2 (type: int)
+ outputColumnNames: _col8
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col12)
+ aggregations: sum(_col8)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
@@ -298,16 +298,12 @@ STAGE PLANS:
input vertices:
1 Map 24
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col2 (type: int), _col4 (type: decimal(7,2))
- outputColumnNames: _col6, _col8
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col6 (type: int)
- sort order: +
- Map-reduce partition columns: _col6 (type: int)
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col8 (type: decimal(7,2))
+ value expressions: _col4 (type: decimal(7,2))
Stage: Stage-4
Spark
@@ -470,11 +466,11 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col0 (type: int)
- 1 _col10 (type: int)
- outputColumnNames: _col17
+ 1 _col2 (type: int)
+ outputColumnNames: _col9
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col17)
+ aggregations: sum(_col9)
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: NONE
@@ -575,16 +571,12 @@ STAGE PLANS:
input vertices:
1 Map 13
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col2 (type: int), _col5 (type: decimal(7,2))
- outputColumnNames: _col10, _col13
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col10 (type: int)
- sort order: +
- Map-reduce partition columns: _col10 (type: int)
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col13 (type: decimal(7,2))
+ value expressions: _col5 (type: decimal(7,2))
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query63.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query63.q.out b/ql/src/test/results/clientpositive/perf/spark/query63.q.out
index 1f62e87..3c934de 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query63.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query63.q.out
@@ -64,7 +64,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 9
+ Map 8
Map Operator Tree:
TableScan
alias: store
@@ -86,11 +86,10 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 399), Map 7 (PARTITION-LEVEL SORT, 399)
- Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 438), Reducer 2 (PARTITION-LEVEL SORT, 438)
- Reducer 4 <- Reducer 3 (GROUP, 529)
- Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 265)
- Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 399), Map 6 (PARTITION-LEVEL SORT, 399)
+ Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 438), Reducer 2 (PARTITION-LEVEL SORT, 438)
+ Reducer 4 <- Reducer 3 (GROUP PARTITION-LEVEL SORT, 529)
+ Reducer 5 <- Reducer 4 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -111,7 +110,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: decimal(7,2))
- Map 7
+ Map 6
Map Operator Tree:
TableScan
alias: item
@@ -129,7 +128,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 115500 Data size: 165890114 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: int)
- Map 8
+ Map 7
Map Operator Tree:
TableScan
alias: date_dim
@@ -183,18 +182,18 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col3, _col8, _col11
input vertices:
- 1 Map 9
+ 1 Map 8
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col3)
- keys: _col11 (type: int), _col8 (type: int)
+ keys: _col8 (type: int), _col11 (type: int)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: decimal(17,2))
Reducer 4
@@ -206,58 +205,46 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col1 (type: int), _col2 (type: decimal(17,2))
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: decimal(17,2))
- Reducer 5
- Reduce Operator Tree:
- Select Operator
- expressions: KEY.reducesinkkey0 (type: int), VALUE._col1 (type: decimal(17,2))
- outputColumnNames: _col1, _col2
- Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- PTF Operator
- Function definitions:
- Input definition
- input alias: ptf_0
- output shape: _col1: int, _col2: decimal(17,2)
- type: WINDOWING
- Windowing table definition
- input alias: ptf_1
- name: windowingtablefunction
- order by: _col1 ASC NULLS FIRST
- partition by: _col1
- raw input shape:
- window functions:
- window function definition
- alias: avg_window_0
- arguments: _col2
- name: avg
- window function: GenericUDAFAverageEvaluatorDecimal
- window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+ expressions: _col0 (type: int), _col2 (type: decimal(17,2))
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: avg_window_0 (type: decimal(21,6)), _col1 (type: int), _col2 (type: decimal(17,2))
- outputColumnNames: avg_window_0, _col1, _col2
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: int, _col2: decimal(17,2)
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col0 ASC NULLS FIRST
+ partition by: _col0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: avg_window_0
+ arguments: _col2
+ name: avg
+ window function: GenericUDAFAverageEvaluatorDecimal
+ window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END (type: boolean)
- Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: int), _col2 (type: decimal(17,2)), avg_window_0 (type: decimal(21,6))
- outputColumnNames: _col0, _col1, _col2
+ Select Operator
+ expressions: avg_window_0 (type: decimal(21,6)), _col0 (type: int), _col2 (type: decimal(17,2))
+ outputColumnNames: avg_window_0, _col0, _col2
+ Statistics: Num rows: 383325119 Data size: 33817053293 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: CASE WHEN ((avg_window_0 > 0)) THEN (((abs((_col2 - avg_window_0)) / avg_window_0) > 0.1)) ELSE (null) END (type: boolean)
Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col2 (type: decimal(21,6)), _col1 (type: decimal(17,2))
- sort order: +++
+ Select Operator
+ expressions: _col0 (type: int), _col2 (type: decimal(17,2)), avg_window_0 (type: decimal(21,6))
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.1
- Reducer 6
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col2 (type: decimal(21,6)), _col1 (type: decimal(17,2))
+ sort order: +++
+ Statistics: Num rows: 191662559 Data size: 16908526602 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.1
+ Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey2 (type: decimal(17,2)), KEY.reducesinkkey1 (type: decimal(21,6))
[14/15] hive git commit: HIVE-19156 :
TestMiniLlapLocalCliDriver.vectorized_dynamic_semijoin_reduction.q is broken
(Jason Dere via Deepak Jaiswal)
Posted by vg...@apache.org.
HIVE-19156 : TestMiniLlapLocalCliDriver.vectorized_dynamic_semijoin_reduction.q is broken (Jason Dere via Deepak Jaiswal)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5a95a826
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5a95a826
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5a95a826
Branch: refs/heads/branch-3
Commit: 5a95a826a1ce6db56a5a721ef99e3f76a9360abf
Parents: 12a1784
Author: Jason Dere <jd...@hortonworks.com>
Authored: Thu Apr 12 08:47:25 2018 -0700
Committer: Vineet Garg <vg...@apache.org>
Committed: Thu Apr 12 10:58:15 2018 -0700
----------------------------------------------------------------------
.../ql/exec/tez/DynamicValueRegistryTez.java | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5a95a826/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java
index ec1e84b..2d99f50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DynamicValueRegistryTez.java
@@ -61,6 +61,11 @@ public class DynamicValueRegistryTez implements DynamicValueRegistry {
}
}
+ static class NullValue {
+ }
+
+ static final NullValue NULL_VALUE = new NullValue();
+
protected Map<String, Object> values = new ConcurrentHashMap<>();
public DynamicValueRegistryTez() {
@@ -71,11 +76,21 @@ public class DynamicValueRegistryTez implements DynamicValueRegistry {
if (!values.containsKey(key)) {
throw new NoDynamicValuesException("Value does not exist in registry: " + key);
}
- return values.get(key);
+ Object val = values.get(key);
+
+ if (val == NULL_VALUE) {
+ return null;
+ }
+ return val;
}
protected void setValue(String key, Object value) {
- values.put(key, value);
+ if (value == null) {
+ // ConcurrentHashMap does not allow null - use a substitute value.
+ values.put(key, NULL_VALUE);
+ } else {
+ values.put(key, value);
+ }
}
@Override
[12/15] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T
missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Posted by vg...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query19.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query19.q.out b/ql/src/test/results/clientpositive/perf/spark/query19.q.out
index 453bc56..281445c 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query19.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query19.q.out
@@ -70,7 +70,7 @@ STAGE PLANS:
Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col3 (type: int)
+ 0 _col7 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -78,35 +78,17 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 8 (PARTITION-LEVEL SORT, 398)
- Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 440), Reducer 2 (PARTITION-LEVEL SORT, 440)
- Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 1009), Reducer 3 (PARTITION-LEVEL SORT, 1009)
- Reducer 5 <- Map 11 (PARTITION-LEVEL SORT, 846), Reducer 4 (PARTITION-LEVEL SORT, 846)
- Reducer 6 <- Reducer 5 (GROUP, 640)
- Reducer 7 <- Reducer 6 (SORT, 1)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 6 (PARTITION-LEVEL SORT, 855)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1009), Reducer 9 (PARTITION-LEVEL SORT, 1009)
+ Reducer 4 <- Reducer 3 (GROUP, 582)
+ Reducer 5 <- Reducer 4 (SORT, 1)
+ Reducer 8 <- Map 10 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398)
+ Reducer 9 <- Map 11 (PARTITION-LEVEL SORT, 440), Reducer 8 (PARTITION-LEVEL SORT, 440)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
- alias: store_sales
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ext_sales_price (type: decimal(7,2))
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
- Map 10
- Map Operator Tree:
- TableScan
alias: customer
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -117,30 +99,12 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: _col1 (type: int)
sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int)
- Map 11
- Map Operator Tree:
- TableScan
- alias: customer_address
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ca_address_sk is not null (type: boolean)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: ca_address_sk (type: int), ca_zip (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string)
- Map 8
+ value expressions: _col0 (type: int)
+ Map 10
Map Operator Tree:
TableScan
alias: date_dim
@@ -157,7 +121,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 18262 Data size: 20435178 Basic stats: COMPLETE Column stats: NONE
- Map 9
+ Map 11
Map Operator Tree:
TableScan
alias: item
@@ -175,55 +139,59 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: int), _col4 (type: string)
+ Map 6
+ Map Operator Tree:
+ TableScan
+ alias: customer_address
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ca_address_sk is not null (type: boolean)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ca_address_sk (type: int), ca_zip (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: store_sales
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ext_sales_price (type: decimal(7,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Reducer 2
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
- Reducer 3
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col9, _col10, _col11, _col12
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: int)
- sort order: +
- Map-reduce partition columns: _col2 (type: int)
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col9 (type: int), _col10 (type: string), _col11 (type: int), _col12 (type: string)
- Reducer 4
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col3, _col4, _col9, _col10, _col11, _col12, _col15
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col0, _col3
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col15 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col15 (type: int)
- Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col9 (type: int), _col10 (type: string), _col11 (type: int), _col12 (type: string)
- Reducer 5
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 88000001 Data size: 75681779077 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: string)
+ Reducer 3
Local Work:
Map Reduce Local Work
Reduce Operator Tree:
@@ -231,62 +199,62 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col15 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col3, _col4, _col9, _col10, _col11, _col12, _col17
- Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
+ 0 _col0 (type: int)
+ 1 _col2 (type: int)
+ outputColumnNames: _col3, _col7, _col8, _col13, _col14, _col15, _col16
+ Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col3 (type: int)
+ 0 _col7 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col4, _col9, _col10, _col11, _col12, _col17, _col19
+ outputColumnNames: _col3, _col8, _col13, _col14, _col15, _col16, _col19
input vertices:
1 Map 12
- Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (substr(_col17, 1, 5) <> substr(_col19, 1, 5)) (type: boolean)
- Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ predicate: (substr(_col3, 1, 5) <> substr(_col19, 1, 5)) (type: boolean)
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col4 (type: decimal(7,2)), _col9 (type: int), _col10 (type: string), _col11 (type: int), _col12 (type: string)
- outputColumnNames: _col4, _col9, _col10, _col11, _col12
- Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ expressions: _col8 (type: decimal(7,2)), _col13 (type: int), _col14 (type: string), _col15 (type: int), _col16 (type: string)
+ outputColumnNames: _col8, _col13, _col14, _col15, _col16
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col4)
- keys: _col10 (type: string), _col9 (type: int), _col11 (type: int), _col12 (type: string)
+ aggregations: sum(_col8)
+ keys: _col14 (type: string), _col13 (type: int), _col15 (type: int), _col16 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), _col3 (type: string)
sort order: ++++
Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int), _col3 (type: string)
- Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: decimal(17,2))
- Reducer 6
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: int), KEY._col3 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: int), _col3 (type: string), _col4 (type: decimal(17,2)), _col0 (type: string), _col1 (type: int)
outputColumnNames: _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col4 (type: decimal(17,2)), _col5 (type: string), _col6 (type: int), _col2 (type: int), _col3 (type: string)
sort order: -++++
- Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
- Reducer 7
+ Reducer 5
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey2 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey0 (type: decimal(17,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
- Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 421657640 Data size: 37198759433 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Statistics: Num rows: 100 Data size: 8800 Basic stats: COMPLETE Column stats: NONE
@@ -297,6 +265,38 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 8
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
+ Reducer 9
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col2, _col3, _col4, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col9 (type: int), _col10 (type: string), _col11 (type: int), _col12 (type: string)
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query21.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query21.q.out b/ql/src/test/results/clientpositive/perf/spark/query21.q.out
index fc9055a..5c3d2fb 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query21.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query21.q.out
@@ -58,7 +58,8 @@ select *
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-1 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
@@ -66,39 +67,44 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 6
+ Map 7
Map Operator Tree:
TableScan
- alias: date_dim
- Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
+ alias: warehouse
+ Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean)
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ predicate: w_warehouse_sk is not null (type: boolean)
+ Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: d_date_sk (type: int), d_date (type: string)
+ expressions: w_warehouse_sk (type: int), w_warehouse_name (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
- Map 7
+
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 5
Map Operator Tree:
TableScan
- alias: warehouse
- Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
+ alias: date_dim
+ Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: w_warehouse_sk is not null (type: boolean)
- Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
+ predicate: (CAST( d_date AS TIMESTAMP) BETWEEN TIMESTAMP'1998-03-08 23:00:00.0' AND TIMESTAMP'1998-05-08 00:00:00.0' and d_date_sk is not null) (type: boolean)
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: w_warehouse_sk (type: int), w_warehouse_name (type: string)
+ expressions: d_date_sk (type: int), d_date (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
- 0 _col2 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
Local Work:
Map Reduce Local Work
@@ -106,7 +112,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 6), Map 5 (PARTITION-LEVEL SORT, 6)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 6), Map 6 (PARTITION-LEVEL SORT, 6)
Reducer 3 <- Reducer 2 (GROUP, 7)
Reducer 4 <- Reducer 3 (SORT, 1)
#### A masked pattern was here ####
@@ -123,13 +129,25 @@ STAGE PLANS:
expressions: inv_date_sk (type: int), inv_item_sk (type: int), inv_warehouse_sk (type: int), inv_quantity_on_hand (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 37584000 Data size: 593821104 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int)
- Map 5
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col5
+ input vertices:
+ 1 Map 5
+ Statistics: Num rows: 41342400 Data size: 653203228 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 41342400 Data size: 653203228 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: int), _col3 (type: int), _col5 (type: string)
+ Local Work:
+ Map Reduce Local Work
+ Map 6
Map Operator Tree:
TableScan
alias: item
@@ -157,44 +175,34 @@ STAGE PLANS:
keys:
0 _col1 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col2, _col3, _col5
- Statistics: Num rows: 41342400 Data size: 653203228 Basic stats: COMPLETE Column stats: NONE
+ outputColumnNames: _col2, _col3, _col5, _col7
+ Statistics: Num rows: 45476640 Data size: 718523566 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
+ 0 _col2 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col3, _col5, _col8
+ outputColumnNames: _col3, _col5, _col7, _col10
input vertices:
- 1 Map 6
- Statistics: Num rows: 45476640 Data size: 718523566 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col3, _col5, _col8, _col10
- input vertices:
- 1 Map 7
+ 1 Map 7
+ Statistics: Num rows: 50024305 Data size: 790375939 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col10 (type: string), _col7 (type: string), CASE WHEN ((CAST( _col5 AS DATE) < DATE'1998-04-08')) THEN (_col3) ELSE (0) END (type: int), CASE WHEN ((CAST( _col5 AS DATE) >= DATE'1998-04-08')) THEN (_col3) ELSE (0) END (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 50024305 Data size: 790375939 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col10 (type: string), _col5 (type: string), CASE WHEN ((CAST( _col8 AS DATE) < DATE'1998-04-08')) THEN (_col3) ELSE (0) END (type: int), CASE WHEN ((CAST( _col8 AS DATE) >= DATE'1998-04-08')) THEN (_col3) ELSE (0) END (type: int)
+ Group By Operator
+ aggregations: sum(_col2), sum(_col3)
+ keys: _col0 (type: string), _col1 (type: string)
+ mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 50024305 Data size: 790375939 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col2), sum(_col3)
- keys: _col0 (type: string), _col1 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 50024305 Data size: 790375939 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
- sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 50024305 Data size: 790375939 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: bigint), _col3 (type: bigint)
+ value expressions: _col2 (type: bigint), _col3 (type: bigint)
Reducer 3
Reduce Operator Tree:
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query24.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query24.q.out b/ql/src/test/results/clientpositive/perf/spark/query24.q.out
index 13ac1e8..1f291c0 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query24.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query24.q.out
@@ -131,10 +131,10 @@ STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 472), Map 20 (PARTITION-LEVEL SORT, 472)
- Reducer 14 <- Map 21 (PARTITION-LEVEL SORT, 1009), Reducer 13 (PARTITION-LEVEL SORT, 1009)
- Reducer 15 <- Map 22 (PARTITION-LEVEL SORT, 846), Reducer 14 (PARTITION-LEVEL SORT, 846)
- Reducer 16 <- Map 23 (PARTITION-LEVEL SORT, 587), Reducer 15 (PARTITION-LEVEL SORT, 587)
+ Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 975), Map 20 (PARTITION-LEVEL SORT, 975)
+ Reducer 14 <- Map 21 (PARTITION-LEVEL SORT, 486), Reducer 13 (PARTITION-LEVEL SORT, 486)
+ Reducer 15 <- Map 22 (PARTITION-LEVEL SORT, 564), Reducer 14 (PARTITION-LEVEL SORT, 564)
+ Reducer 16 <- Map 23 (PARTITION-LEVEL SORT, 899), Reducer 15 (PARTITION-LEVEL SORT, 899)
Reducer 17 <- Reducer 16 (GROUP, 640)
Reducer 18 <- Reducer 17 (GROUP, 1)
#### A masked pattern was here ####
@@ -162,33 +162,16 @@ STAGE PLANS:
1 Map 19
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int), _col3 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col3 (type: int)
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string)
+ value expressions: _col0 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string)
Local Work:
Map Reduce Local Work
Map 20
Map Operator Tree:
TableScan
- alias: store_returns
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean)
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: sr_item_sk (type: int), sr_ticket_number (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
- Map 21
- Map Operator Tree:
- TableScan
alias: customer
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -204,9 +187,44 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ Map 21
+ Map Operator Tree:
+ TableScan
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: i_item_sk is not null (type: boolean)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: i_item_sk (type: int), i_current_price (type: decimal(7,2)), i_size (type: string), i_color (type: string), i_units (type: string), i_manager_id (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: decimal(7,2)), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int)
Map 22
Map Operator Tree:
TableScan
+ alias: store_returns
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean)
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: sr_item_sk (type: int), sr_ticket_number (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ Map 23
+ Map Operator Tree:
+ TableScan
alias: customer_address
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -222,99 +240,81 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string), upper(_col2) (type: string)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Map 23
- Map Operator Tree:
- TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: i_item_sk is not null (type: boolean)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: i_item_sk (type: int), i_current_price (type: decimal(7,2)), i_size (type: string), i_color (type: string), i_units (type: string), i_manager_id (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: decimal(7,2)), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int)
Reducer 13
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int), _col3 (type: int)
- 1 _col0 (type: int), _col1 (type: int)
- outputColumnNames: _col0, _col1, _col4, _col6, _col8, _col9
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col3, _col4, _col6, _col8, _col9, _col11, _col12, _col13
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
+ key expressions: _col0 (type: int)
sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string)
+ value expressions: _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string)
Reducer 14
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
+ 0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col4, _col6, _col8, _col9, _col13, _col14, _col15
+ outputColumnNames: _col0, _col3, _col4, _col6, _col8, _col9, _col11, _col12, _col13, _col15, _col16, _col17, _col18, _col19
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col9 (type: string), _col15 (type: string)
+ key expressions: _col0 (type: int), _col3 (type: int)
sort order: ++
- Map-reduce partition columns: _col9 (type: string), _col15 (type: string)
+ Map-reduce partition columns: _col0 (type: int), _col3 (type: int)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col13 (type: string), _col14 (type: string)
+ value expressions: _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col15 (type: decimal(7,2)), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: int)
Reducer 15
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col9 (type: string), _col15 (type: string)
- 1 _col1 (type: string), upper(_col2) (type: string)
- outputColumnNames: _col0, _col4, _col6, _col8, _col13, _col14, _col16
+ 0 _col0 (type: int), _col3 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col4, _col6, _col8, _col9, _col11, _col12, _col13, _col15, _col16, _col17, _col18, _col19
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ key expressions: _col9 (type: string), _col13 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col9 (type: string), _col13 (type: string)
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col13 (type: string), _col14 (type: string), _col16 (type: string)
+ value expressions: _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col11 (type: string), _col12 (type: string), _col15 (type: decimal(7,2)), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: int)
Reducer 16
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col4, _col6, _col8, _col13, _col14, _col16, _col20, _col21, _col22, _col23, _col24
+ 0 _col9 (type: string), _col13 (type: string)
+ 1 _col1 (type: string), upper(_col2) (type: string)
+ outputColumnNames: _col4, _col6, _col8, _col11, _col12, _col15, _col16, _col17, _col18, _col19, _col22
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col4)
- keys: _col20 (type: decimal(7,2)), _col21 (type: string), _col22 (type: string), _col23 (type: string), _col24 (type: int), _col16 (type: string), _col13 (type: string), _col14 (type: string), _col6 (type: string), _col8 (type: string)
+ keys: _col11 (type: string), _col12 (type: string), _col6 (type: string), _col8 (type: string), _col15 (type: decimal(7,2)), _col16 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: int), _col22 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: decimal(7,2)), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: decimal(7,2)), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: int), _col9 (type: string)
sort order: ++++++++++
- Map-reduce partition columns: _col0 (type: decimal(7,2)), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: string), _col9 (type: string)
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: decimal(7,2)), _col5 (type: string), _col6 (type: string), _col7 (type: string), _col8 (type: int), _col9 (type: string)
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
value expressions: _col10 (type: decimal(17,2))
Reducer 17
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: decimal(7,2)), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: string), KEY._col6 (type: string), KEY._col7 (type: string), KEY._col8 (type: string), KEY._col9 (type: string)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: decimal(7,2)), KEY._col5 (type: string), KEY._col6 (type: string), KEY._col7 (type: string), KEY._col8 (type: int), KEY._col9 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
@@ -353,7 +353,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 7
+ Map 8
Map Operator Tree:
TableScan
alias: store
@@ -375,9 +375,9 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 440), Map 8 (PARTITION-LEVEL SORT, 440)
- Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 516), Reducer 2 (PARTITION-LEVEL SORT, 516)
- Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 1009), Reducer 3 (PARTITION-LEVEL SORT, 1009)
+ Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 400), Map 7 (PARTITION-LEVEL SORT, 400)
+ Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 1009), Reducer 2 (PARTITION-LEVEL SORT, 1009)
+ Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 564), Reducer 3 (PARTITION-LEVEL SORT, 564)
Reducer 5 <- Map 11 (PARTITION-LEVEL SORT, 899), Reducer 4 (PARTITION-LEVEL SORT, 899)
Reducer 6 <- Reducer 5 (GROUP PARTITION-LEVEL SORT, 640)
#### A masked pattern was here ####
@@ -394,42 +394,29 @@ STAGE PLANS:
expressions: ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ticket_number (type: int), ss_sales_price (type: decimal(7,2))
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col3, _col4, _col6, _col8, _col9
- input vertices:
- 1 Map 7
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string)
- Local Work:
- Map Reduce Local Work
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2))
Map 10
Map Operator Tree:
TableScan
- alias: customer
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ alias: store_returns
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (c_birth_country is not null and c_customer_sk is not null) (type: boolean)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean)
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: c_customer_sk (type: int), c_first_name (type: string), c_last_name (type: string), c_birth_country (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ expressions: sr_item_sk (type: int), sr_ticket_number (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ key expressions: _col0 (type: int), _col1 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
Map 11
Map Operator Tree:
TableScan
@@ -448,7 +435,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string), upper(_col2) (type: string)
Statistics: Num rows: 40000000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Map 8
+ Map 7
Map Operator Tree:
TableScan
alias: item
@@ -469,21 +456,24 @@ STAGE PLANS:
Map 9
Map Operator Tree:
TableScan
- alias: store_returns
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ alias: customer
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (sr_item_sk is not null and sr_ticket_number is not null) (type: boolean)
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ predicate: (c_birth_country is not null and c_customer_sk is not null) (type: boolean)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: sr_item_sk (type: int), sr_ticket_number (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ expressions: c_customer_sk (type: int), c_first_name (type: string), c_last_name (type: string), c_birth_country (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 57591150 Data size: 4462194832 Basic stats: COMPLETE Column stats: NONE
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 80000000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
Reducer 2
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Join Operator
condition map:
@@ -491,64 +481,74 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col3, _col4, _col6, _col8, _col9, _col11, _col12, _col14, _col15
- Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int), _col3 (type: int)
- sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col3 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col6, _col7, _col9, _col10
+ Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col3, _col4, _col6, _col7, _col9, _col10, _col12, _col14, _col15
+ input vertices:
+ 1 Map 8
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: int), _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string), _col11 (type: decimal(7,2)), _col12 (type: string), _col14 (type: string), _col15 (type: int)
+ Reduce Output Operator
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
+ Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: string), _col9 (type: string), _col10 (type: int), _col12 (type: string), _col14 (type: string), _col15 (type: string)
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col0 (type: int), _col3 (type: int)
- 1 _col0 (type: int), _col1 (type: int)
- outputColumnNames: _col1, _col4, _col6, _col8, _col9, _col11, _col12, _col14, _col15
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col3, _col4, _col6, _col7, _col9, _col10, _col12, _col14, _col15, _col17, _col18, _col19
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ key expressions: _col0 (type: int), _col3 (type: int)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col3 (type: int)
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col9 (type: string), _col11 (type: decimal(7,2)), _col12 (type: string), _col14 (type: string), _col15 (type: int)
+ value expressions: _col4 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: string), _col9 (type: string), _col10 (type: int), _col12 (type: string), _col14 (type: string), _col15 (type: string), _col17 (type: string), _col18 (type: string), _col19 (type: string)
Reducer 4
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col4, _col6, _col8, _col9, _col11, _col12, _col14, _col15, _col19, _col20, _col21
+ 0 _col0 (type: int), _col3 (type: int)
+ 1 _col0 (type: int), _col1 (type: int)
+ outputColumnNames: _col4, _col6, _col7, _col9, _col10, _col12, _col14, _col15, _col17, _col18, _col19
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col9 (type: string), _col21 (type: string)
+ key expressions: _col15 (type: string), _col19 (type: string)
sort order: ++
- Map-reduce partition columns: _col9 (type: string), _col21 (type: string)
+ Map-reduce partition columns: _col15 (type: string), _col19 (type: string)
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col4 (type: decimal(7,2)), _col6 (type: string), _col8 (type: string), _col11 (type: decimal(7,2)), _col12 (type: string), _col14 (type: string), _col15 (type: int), _col19 (type: string), _col20 (type: string)
+ value expressions: _col4 (type: decimal(7,2)), _col6 (type: decimal(7,2)), _col7 (type: string), _col9 (type: string), _col10 (type: int), _col12 (type: string), _col14 (type: string), _col17 (type: string), _col18 (type: string)
Reducer 5
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col9 (type: string), _col21 (type: string)
+ 0 _col15 (type: string), _col19 (type: string)
1 _col1 (type: string), upper(_col2) (type: string)
- outputColumnNames: _col4, _col6, _col8, _col11, _col12, _col14, _col15, _col19, _col20, _col22
+ outputColumnNames: _col4, _col6, _col7, _col9, _col10, _col12, _col14, _col17, _col18, _col22
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col4)
- keys: _col19 (type: string), _col20 (type: string), _col6 (type: string), _col22 (type: string), _col8 (type: string), _col11 (type: decimal(7,2)), _col12 (type: string), _col14 (type: string), _col15 (type: int)
+ keys: _col17 (type: string), _col18 (type: string), _col12 (type: string), _col22 (type: string), _col6 (type: decimal(7,2)), _col7 (type: string), _col9 (type: string), _col10 (type: int), _col14 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: decimal(7,2)), _col6 (type: string), _col7 (type: string), _col8 (type: int)
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: decimal(7,2)), _col5 (type: string), _col6 (type: string), _col7 (type: int), _col8 (type: string)
sort order: +++++++++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
Statistics: Num rows: 927646829 Data size: 81837272625 Basic stats: COMPLETE Column stats: NONE
@@ -559,17 +559,17 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: string), KEY._col5 (type: decimal(7,2)), KEY._col6 (type: string), KEY._col7 (type: string), KEY._col8 (type: int)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: decimal(7,2)), KEY._col5 (type: string), KEY._col6 (type: string), KEY._col7 (type: int), KEY._col8 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col9 (type: decimal(17,2))
- outputColumnNames: _col1, _col2, _col3, _col9
+ outputColumnNames: _col1, _col2, _col7, _col9
Statistics: Num rows: 463823414 Data size: 40918636268 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col9)
- keys: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ keys: _col1 (type: string), _col2 (type: string), _col7 (type: string)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 231911707 Data size: 20459318134 Basic stats: COMPLETE Column stats: NONE
http://git-wip-us.apache.org/repos/asf/hive/blob/12a1784d/ql/src/test/results/clientpositive/perf/spark/query25.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query25.q.out b/ql/src/test/results/clientpositive/perf/spark/query25.q.out
index ddfcb83..3ecb2e5 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query25.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query25.q.out
@@ -127,7 +127,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 11
+ Map 10
Map Operator Tree:
TableScan
alias: d3
@@ -150,7 +150,7 @@ STAGE PLANS:
Spark
#### A masked pattern was here ####
Vertices:
- Map 13
+ Map 12
Map Operator Tree:
TableScan
alias: d2
@@ -172,12 +172,12 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 10 <- Map 12 (PARTITION-LEVEL SORT, 374), Map 9 (PARTITION-LEVEL SORT, 374)
Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398)
- Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 442), Reducer 2 (PARTITION-LEVEL SORT, 442)
- Reducer 4 <- Reducer 10 (PARTITION-LEVEL SORT, 850), Reducer 3 (PARTITION-LEVEL SORT, 850)
+ Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 806), Reducer 9 (PARTITION-LEVEL SORT, 806)
+ Reducer 4 <- Map 13 (PARTITION-LEVEL SORT, 486), Reducer 3 (PARTITION-LEVEL SORT, 486)
Reducer 5 <- Reducer 4 (GROUP, 582)
Reducer 6 <- Reducer 5 (SORT, 1)
+ Reducer 9 <- Map 11 (PARTITION-LEVEL SORT, 374), Map 8 (PARTITION-LEVEL SORT, 374)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -198,7 +198,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2))
- Map 12
+ Map 11
Map Operator Tree:
TableScan
alias: store_returns
@@ -218,7 +218,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3, _col4
input vertices:
- 1 Map 13
+ 1 Map 12
Statistics: Num rows: 63350266 Data size: 4908414421 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: int), _col2 (type: int)
@@ -228,6 +228,24 @@ STAGE PLANS:
value expressions: _col3 (type: int), _col4 (type: decimal(7,2))
Local Work:
Map Reduce Local Work
+ Map 13
+ Map Operator Tree:
+ TableScan
+ alias: item
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: i_item_sk is not null (type: boolean)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: i_item_sk (type: int), i_item_id (type: string), i_item_desc (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col2 (type: string)
Map 7
Map Operator Tree:
TableScan
@@ -248,24 +266,6 @@ STAGE PLANS:
Map 8
Map Operator Tree:
TableScan
- alias: item
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: i_item_sk is not null (type: boolean)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: i_item_sk (type: int), i_item_id (type: string), i_item_desc (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col2 (type: string)
- Map 9
- Map Operator Tree:
- TableScan
alias: catalog_sales
Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE
Filter Operator
@@ -283,7 +283,7 @@ STAGE PLANS:
1 _col0 (type: int)
outputColumnNames: _col1, _col2, _col3
input vertices:
- 1 Map 11
+ 1 Map 10
Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: int), _col1 (type: int)
@@ -293,22 +293,6 @@ STAGE PLANS:
value expressions: _col3 (type: decimal(7,2))
Local Work:
Map Reduce Local Work
- Reducer 10
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col2 (type: int), _col1 (type: int)
- 1 _col1 (type: int), _col2 (type: int)
- outputColumnNames: _col3, _col8, _col9, _col10, _col11
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int)
- sort order: +++
- Map-reduce partition columns: _col8 (type: int), _col9 (type: int), _col10 (type: int)
- Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: decimal(7,2)), _col11 (type: decimal(7,2))
Reducer 2
Reduce Operator Tree:
Join Operator
@@ -320,27 +304,27 @@ STAGE PLANS:
outputColumnNames: _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int)
- sort order: +
- Map-reduce partition columns: _col1 (type: int)
+ key expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col4 (type: int)
Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: decimal(7,2))
+ value expressions: _col3 (type: int), _col5 (type: decimal(7,2))
Reducer 3
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int)
- 1 _col0 (type: int)
- outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col10, _col11
+ 0 _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ 1 _col8 (type: int), _col9 (type: int), _col10 (type: int)
+ outputColumnNames: _col1, _col3, _col5, _col12, _col20
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int)
- sort order: +++
- Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col4 (type: int)
+ key expressions: _col1 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: int)
Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col3 (type: int), _col5 (type: decimal(7,2)), _col10 (type: string), _col11 (type: string)
+ value expressions: _col3 (type: int), _col5 (type: decimal(7,2)), _col12 (type: decimal(7,2)), _col20 (type: decimal(7,2))
Reducer 4
Local Work:
Map Reduce Local Work
@@ -349,9 +333,9 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 _col1 (type: int), _col2 (type: int), _col4 (type: int)
- 1 _col8 (type: int), _col9 (type: int), _col10 (type: int)
- outputColumnNames: _col3, _col5, _col10, _col11, _col15, _col23
+ 0 _col1 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col3, _col5, _col12, _col20, _col25, _col26
Statistics: Num rows: 766650239 Data size: 67634106676 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -359,13 +343,13 @@ STAGE PLANS:
keys:
0 _col3 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col5, _col10, _col11, _col15, _col23, _col28, _col29
+ outputColumnNames: _col5, _col12, _col20, _col25, _col26, _col28, _col29
input vertices:
1 Map 14
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum(_col5), sum(_col23), sum(_col15)
- keys: _col10 (type: string), _col11 (type: string), _col28 (type: string), _col29 (type: string)
+ aggregations: sum(_col5), sum(_col20), sum(_col12)
+ keys: _col25 (type: string), _col26 (type: string), _col28 (type: string), _col29 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
Statistics: Num rows: 843315281 Data size: 74397518956 Basic stats: COMPLETE Column stats: NONE
@@ -406,6 +390,22 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Reducer 9
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col2 (type: int), _col1 (type: int)
+ 1 _col1 (type: int), _col2 (type: int)
+ outputColumnNames: _col3, _col8, _col9, _col10, _col11
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col8 (type: int), _col9 (type: int), _col10 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col8 (type: int), _col9 (type: int), _col10 (type: int)
+ Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col3 (type: decimal(7,2)), _col11 (type: decimal(7,2))
Stage: Stage-0
Fetch Operator
[15/15] hive git commit: HIVE-19175 :
TestMiniLlapLocalCliDriver.testCliDriver update_access_time_non_current_db
failing (Vineet Garg via Peter Vary)
Posted by vg...@apache.org.
HIVE-19175 : TestMiniLlapLocalCliDriver.testCliDriver update_access_time_non_current_db failing (Vineet Garg via Peter Vary)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/50f144d1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/50f144d1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/50f144d1
Branch: refs/heads/branch-3
Commit: 50f144d1f62b6c74d7d2b5f6a20232ab77f32be0
Parents: 5a95a82
Author: Vineet Garg <vg...@apache.org>
Authored: Thu Apr 12 09:34:31 2018 -0700
Committer: Vineet Garg <vg...@apache.org>
Committed: Thu Apr 12 10:58:27 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 2 +-
.../update_access_time_non_current_db.q.out | 47 ++++++++++++++++++++
.../update_access_time_non_current_db.q.out | 47 --------------------
3 files changed, 48 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/50f144d1/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 48d62a8..c5d4e9f 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -285,7 +285,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
union8.q,\
union9.q,\
union_stats.q,\
- update_access_time_non_current_db.q, \
update_after_multiple_inserts.q,\
update_all_non_partitioned.q,\
update_all_partitioned.q,\
@@ -739,6 +738,7 @@ minillaplocal.query.files=\
union_fast_stats.q,\
union_remove_26.q,\
union_top_level.q,\
+ update_access_time_non_current_db.q, \
vector_annotate_stats_select.q,\
vector_auto_smb_mapjoin_14.q,\
vector_char_varchar_1.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/50f144d1/ql/src/test/results/clientpositive/llap/update_access_time_non_current_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/update_access_time_non_current_db.q.out b/ql/src/test/results/clientpositive/llap/update_access_time_non_current_db.q.out
new file mode 100644
index 0000000..7fda3ac
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/update_access_time_non_current_db.q.out
@@ -0,0 +1,47 @@
+PREHOOK: query: create database temp1
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:temp1
+POSTHOOK: query: create database temp1
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:temp1
+PREHOOK: query: use temp1
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:temp1
+POSTHOOK: query: use temp1
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:temp1
+PREHOOK: query: create table test1(id int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:temp1
+PREHOOK: Output: temp1@test1
+POSTHOOK: query: create table test1(id int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:temp1
+POSTHOOK: Output: temp1@test1
+PREHOOK: query: create database temp2
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:temp2
+POSTHOOK: query: create database temp2
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:temp2
+PREHOOK: query: use temp2
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:temp2
+POSTHOOK: query: use temp2
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:temp2
+PREHOOK: query: create table test2(id int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:temp2
+PREHOOK: Output: temp2@test2
+POSTHOOK: query: create table test2(id int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:temp2
+POSTHOOK: Output: temp2@test2
+POSTHOOK: query: use temp1
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:temp1
+POSTHOOK: query: desc temp2.test2
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: temp2@test2
+id int
http://git-wip-us.apache.org/repos/asf/hive/blob/50f144d1/ql/src/test/results/clientpositive/update_access_time_non_current_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/update_access_time_non_current_db.q.out b/ql/src/test/results/clientpositive/update_access_time_non_current_db.q.out
deleted file mode 100644
index 7fda3ac..0000000
--- a/ql/src/test/results/clientpositive/update_access_time_non_current_db.q.out
+++ /dev/null
@@ -1,47 +0,0 @@
-PREHOOK: query: create database temp1
-PREHOOK: type: CREATEDATABASE
-PREHOOK: Output: database:temp1
-POSTHOOK: query: create database temp1
-POSTHOOK: type: CREATEDATABASE
-POSTHOOK: Output: database:temp1
-PREHOOK: query: use temp1
-PREHOOK: type: SWITCHDATABASE
-PREHOOK: Input: database:temp1
-POSTHOOK: query: use temp1
-POSTHOOK: type: SWITCHDATABASE
-POSTHOOK: Input: database:temp1
-PREHOOK: query: create table test1(id int)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:temp1
-PREHOOK: Output: temp1@test1
-POSTHOOK: query: create table test1(id int)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:temp1
-POSTHOOK: Output: temp1@test1
-PREHOOK: query: create database temp2
-PREHOOK: type: CREATEDATABASE
-PREHOOK: Output: database:temp2
-POSTHOOK: query: create database temp2
-POSTHOOK: type: CREATEDATABASE
-POSTHOOK: Output: database:temp2
-PREHOOK: query: use temp2
-PREHOOK: type: SWITCHDATABASE
-PREHOOK: Input: database:temp2
-POSTHOOK: query: use temp2
-POSTHOOK: type: SWITCHDATABASE
-POSTHOOK: Input: database:temp2
-PREHOOK: query: create table test2(id int)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:temp2
-PREHOOK: Output: temp2@test2
-POSTHOOK: query: create table test2(id int)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:temp2
-POSTHOOK: Output: temp2@test2
-POSTHOOK: query: use temp1
-POSTHOOK: type: SWITCHDATABASE
-POSTHOOK: Input: database:temp1
-POSTHOOK: query: desc temp2.test2
-POSTHOOK: type: DESCTABLE
-POSTHOOK: Input: temp2@test2
-id int