You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/11/25 14:03:23 UTC
svn commit: r1641614 [21/43] - in /hive/branches/spark:
itests/src/test/resources/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/
ql/src/java/org/apache/hadoop/hive/ql/parse/spark/ ql/sr...
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/join_star.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/join_star.q.out?rev=1641614&r1=1641613&r2=1641614&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/join_star.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/join_star.q.out Tue Nov 25 13:03:18 2014
@@ -131,14 +131,13 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select m1, m2, f2 from fact join dim1 on fact.d1=dim1.f1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -149,13 +148,21 @@ STAGE PLANS:
Filter Operator
predicate: f1 is not null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f1 (type: int)
- sort order: +
- Map-reduce partition columns: f1 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: f2 (type: int)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {m1} {m2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
Map Operator Tree:
TableScan
alias: fact
@@ -163,33 +170,32 @@ STAGE PLANS:
Filter Operator
predicate: d1 is not null (type: boolean)
Statistics: Num rows: 4 Data size: 49 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: d1 (type: int)
- sort order: +
- Map-reduce partition columns: d1 (type: int)
- Statistics: Num rows: 4 Data size: 49 Basic stats: COMPLETE Column stats: NONE
- value expressions: m1 (type: int), m2 (type: int)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8
- Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {m1} {m2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ outputColumnNames: _col0, _col1, _col8
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -222,15 +228,13 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select m1, m2, f2, f4 from fact join dim1 on fact.d1=dim1.f1 join dim2 on fact.d2=dim2.f3
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -241,13 +245,16 @@ STAGE PLANS:
Filter Operator
predicate: f3 is not null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f3 (type: int)
- sort order: +
- Map-reduce partition columns: f3 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: f4 (type: int)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col3 (type: int)
+ 1 f3 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
Map Operator Tree:
TableScan
alias: dim1
@@ -255,13 +262,21 @@ STAGE PLANS:
Filter Operator
predicate: f1 is not null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f1 (type: int)
- sort order: +
- Map-reduce partition columns: f1 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: f2 (type: int)
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {m1} {m2} {d2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
Map Operator Tree:
TableScan
alias: fact
@@ -269,49 +284,45 @@ STAGE PLANS:
Filter Operator
predicate: (d1 is not null and d2 is not null) (type: boolean)
Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: d1 (type: int)
- sort order: +
- Map-reduce partition columns: d1 (type: int)
- Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE
- value expressions: m1 (type: int), m2 (type: int), d2 (type: int)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col7}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8, _col13
- Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 4
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col2}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col3, _col8
- Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int)
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {m1} {m2} {d2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ outputColumnNames: _col0, _col1, _col3, _col8
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 2 Data size: 35 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col3 (type: int)
+ 1 f3 (type: int)
+ outputColumnNames: _col0, _col1, _col8, _col13
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 38 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -346,15 +357,13 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select m1, m2, f2, f4 from fact join dim1 on fact.d1= dim1.f1 join dim2 on dim1.f2 = dim2.f3
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -365,13 +374,16 @@ STAGE PLANS:
Filter Operator
predicate: f3 is not null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f3 (type: int)
- sort order: +
- Map-reduce partition columns: f3 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: f4 (type: int)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col8 (type: int)
+ 1 f3 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
Map Operator Tree:
TableScan
alias: dim1
@@ -379,13 +391,21 @@ STAGE PLANS:
Filter Operator
predicate: (f1 is not null and f2 is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f1 (type: int)
- sort order: +
- Map-reduce partition columns: f1 (type: int)
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: f2 (type: int)
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {m1} {m2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
Map Operator Tree:
TableScan
alias: fact
@@ -393,49 +413,45 @@ STAGE PLANS:
Filter Operator
predicate: d1 is not null (type: boolean)
Statistics: Num rows: 4 Data size: 49 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: d1 (type: int)
- sort order: +
- Map-reduce partition columns: d1 (type: int)
- Statistics: Num rows: 4 Data size: 49 Basic stats: COMPLETE Column stats: NONE
- value expressions: m1 (type: int), m2 (type: int)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {KEY.reducesinkkey0}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8, _col13
- Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 4
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8
- Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col8 (type: int)
- sort order: +
- Map-reduce partition columns: _col8 (type: int)
- Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int)
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {m1} {m2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ outputColumnNames: _col0, _col1, _col8
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 4 Data size: 53 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col8 (type: int)
+ 1 f3 (type: int)
+ outputColumnNames: _col0, _col1, _col8, _col13
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 58 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -470,15 +486,13 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select m1, m2, f2, f4 from fact Left outer join dim1 on fact.d1= dim1.f1 Left outer join dim2 on dim1.f2 = dim2.f3
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 4 <- Map 3 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -486,71 +500,78 @@ STAGE PLANS:
TableScan
alias: dim2
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f3 (type: int)
- sort order: +
- Map-reduce partition columns: f3 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f4 (type: int)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col8 (type: int)
+ 1 f3 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
Map Operator Tree:
TableScan
alias: dim1
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f1 (type: int)
- sort order: +
- Map-reduce partition columns: f1 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f2 (type: int)
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {m1} {m2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
Map Operator Tree:
TableScan
alias: fact
Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: d1 (type: int)
- sort order: +
- Map-reduce partition columns: d1 (type: int)
- Statistics: Num rows: 8 Data size: 98 Basic stats: COMPLETE Column stats: NONE
- value expressions: m1 (type: int), m2 (type: int)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {KEY.reducesinkkey0}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8, _col13
- Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 4
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8
- Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col8 (type: int)
- sort order: +
- Map-reduce partition columns: _col8 (type: int)
- Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int)
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {m1} {m2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ outputColumnNames: _col0, _col1, _col8
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 8 Data size: 107 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col8 (type: int)
+ 1 f3 (type: int)
+ outputColumnNames: _col0, _col1, _col8, _col13
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 8 Data size: 117 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -601,19 +622,13 @@ POSTHOOK: query: explain Select m1, m2,
Left outer Join dim7 on dim6.f12 = dim7.f13
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 3 <- Map 12 (PARTITION-LEVEL SORT, 1), Map 14 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Map 13 (PARTITION-LEVEL SORT, 1), Reducer 3 (PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Map 11 (PARTITION-LEVEL SORT, 1), Reducer 4 (PARTITION-LEVEL SORT, 1)
- Reducer 9 <- Map 10 (PARTITION-LEVEL SORT, 1), Map 8 (PARTITION-LEVEL SORT, 1)
- Reducer 7 <- Map 6 (PARTITION-LEVEL SORT, 1), Reducer 9 (PARTITION-LEVEL SORT, 1)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 7 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -621,192 +636,208 @@ STAGE PLANS:
TableScan
alias: dim3
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f5 (type: int)
- sort order: +
- Map-reduce partition columns: f5 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f6 (type: int)
- Map 10
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13}
+ 1 {f6}
+ keys:
+ 0 _col3 (type: int)
+ 1 f5 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
Map Operator Tree:
TableScan
- alias: fact
- Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: d1 (type: int)
- sort order: +
- Map-reduce partition columns: d1 (type: int)
- Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE
- value expressions: m1 (type: int), m2 (type: int), d2 (type: int)
- Map 11
+ alias: dim2
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col3} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col8 (type: int)
+ 1 f3 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: dim1
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {m1} {m2} {d2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 5
Map Operator Tree:
TableScan
alias: dim7
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f13 (type: int)
- sort order: +
- Map-reduce partition columns: f13 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f14 (type: int)
- Map 12
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13} {_col18} {_col23} {_col28} {_col33}
+ 1 {f14}
+ keys:
+ 0 _col28 (type: int)
+ 1 f13 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 6
Map Operator Tree:
TableScan
alias: dim6
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f11 (type: int)
- sort order: +
- Map-reduce partition columns: f11 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f12 (type: int)
- Map 13
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13} {_col18}
+ 1 {f8}
+ 2 {f12}
+ keys:
+ 0 _col18 (type: int)
+ 1 f7 (type: int)
+ 2 f11 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 7
Map Operator Tree:
TableScan
alias: dim5
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f9 (type: int)
- sort order: +
- Map-reduce partition columns: f9 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f10 (type: int)
- Map 14
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13} {_col18} {_col23} {_col28}
+ 1 {f10}
+ keys:
+ 0 _col23 (type: int)
+ 1 f9 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 8
Map Operator Tree:
TableScan
alias: dim4
Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f7 (type: int)
- sort order: +
- Map-reduce partition columns: f7 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f8 (type: int)
- Map 6
- Map Operator Tree:
- TableScan
- alias: dim2
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f3 (type: int)
- sort order: +
- Map-reduce partition columns: f3 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f4 (type: int)
- Map 8
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13} {_col18}
+ 1 {f8}
+ 2 {f12}
+ keys:
+ 0 _col18 (type: int)
+ 1 f7 (type: int)
+ 2 f11 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 4
Map Operator Tree:
TableScan
- alias: dim1
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: f1 (type: int)
- sort order: +
- Map-reduce partition columns: f1 (type: int)
- Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: f2 (type: int)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col7} {VALUE._col12}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8, _col13, _col18
- Statistics: Num rows: 6 Data size: 128 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col18 (type: int)
- sort order: +
- Map-reduce partition columns: _col18 (type: int)
- Statistics: Num rows: 6 Data size: 128 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
- Reducer 3
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- Left Outer Join0 to 2
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col8} {VALUE._col13} {KEY.reducesinkkey0}
- 1 {VALUE._col0}
- 2 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8, _col13, _col18, _col23, _col28
- Statistics: Num rows: 13 Data size: 281 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col23 (type: int)
- sort order: +
- Map-reduce partition columns: _col23 (type: int)
- Statistics: Num rows: 13 Data size: 281 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int), _col18 (type: int), _col28 (type: int)
- Reducer 4
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col8} {VALUE._col13} {VALUE._col18} {KEY.reducesinkkey0} {VALUE._col27}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8, _col13, _col18, _col23, _col28, _col33
- Statistics: Num rows: 14 Data size: 309 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col28 (type: int)
- sort order: +
- Map-reduce partition columns: _col28 (type: int)
- Statistics: Num rows: 14 Data size: 309 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int), _col18 (type: int), _col23 (type: int), _col33 (type: int)
- Reducer 5
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col8} {VALUE._col13} {VALUE._col18} {VALUE._col23} {KEY.reducesinkkey0} {VALUE._col32}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col8, _col13, _col18, _col23, _col28, _col33, _col38
- Statistics: Num rows: 15 Data size: 339 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int), _col18 (type: int), _col23 (type: int), _col33 (type: int), _col28 (type: int), _col38 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 15 Data size: 339 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 15 Data size: 339 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 7
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col3} {KEY.reducesinkkey0}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col3, _col8, _col13
- Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col3 (type: int)
- sort order: +
- Map-reduce partition columns: _col3 (type: int)
- Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int)
- Reducer 9
- Reduce Operator Tree:
- Join Operator
- condition map:
- Left Outer Join0 to 1
- condition expressions:
- 0 {VALUE._col0} {VALUE._col1} {VALUE._col2}
- 1 {VALUE._col0}
- outputColumnNames: _col0, _col1, _col3, _col8
- Statistics: Num rows: 6 Data size: 107 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col8 (type: int)
- sort order: +
- Map-reduce partition columns: _col8 (type: int)
- Statistics: Num rows: 6 Data size: 107 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int)
+ alias: fact
+ Statistics: Num rows: 6 Data size: 98 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {m1} {m2} {d2}
+ 1 {f2}
+ keys:
+ 0 d1 (type: int)
+ 1 f1 (type: int)
+ outputColumnNames: _col0, _col1, _col3, _col8
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6 Data size: 107 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1} {_col3} {_col8}
+ 1 {f4}
+ keys:
+ 0 _col8 (type: int)
+ 1 f3 (type: int)
+ outputColumnNames: _col0, _col1, _col3, _col8, _col13
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 6 Data size: 117 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13}
+ 1 {f6}
+ keys:
+ 0 _col3 (type: int)
+ 1 f5 (type: int)
+ outputColumnNames: _col0, _col1, _col8, _col13, _col18
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 6 Data size: 128 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Left Outer Join0 to 2
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13} {_col18}
+ 1 {f8}
+ 2 {f12}
+ keys:
+ 0 _col18 (type: int)
+ 1 f7 (type: int)
+ 2 f11 (type: int)
+ outputColumnNames: _col0, _col1, _col8, _col13, _col18, _col23, _col28
+ input vertices:
+ 1 Map 8
+ 2 Map 6
+ Statistics: Num rows: 13 Data size: 281 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13} {_col18} {_col23} {_col28}
+ 1 {f10}
+ keys:
+ 0 _col23 (type: int)
+ 1 f9 (type: int)
+ outputColumnNames: _col0, _col1, _col8, _col13, _col18, _col23, _col28, _col33
+ input vertices:
+ 1 Map 7
+ Statistics: Num rows: 14 Data size: 309 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ condition expressions:
+ 0 {_col0} {_col1} {_col8} {_col13} {_col18} {_col23} {_col28} {_col33}
+ 1 {f14}
+ keys:
+ 0 _col28 (type: int)
+ 1 f13 (type: int)
+ outputColumnNames: _col0, _col1, _col8, _col13, _col18, _col23, _col28, _col33, _col38
+ input vertices:
+ 1 Map 5
+ Statistics: Num rows: 15 Data size: 339 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col8 (type: int), _col13 (type: int), _col18 (type: int), _col23 (type: int), _col33 (type: int), _col28 (type: int), _col38 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 15 Data size: 339 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 15 Data size: 339 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/join_thrift.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/join_thrift.q.out?rev=1641614&r1=1641613&r2=1641614&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/join_thrift.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/join_thrift.q.out Tue Nov 25 13:03:18 2014
@@ -31,14 +31,13 @@ JOIN src_thrift s2
ON s1.aint = s2.aint
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -49,13 +48,21 @@ STAGE PLANS:
Filter Operator
predicate: aint is not null (type: boolean)
Statistics: Num rows: 6 Data size: 1674 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: aint (type: int)
- sort order: +
- Map-reduce partition columns: aint (type: int)
- Statistics: Num rows: 6 Data size: 1674 Basic stats: COMPLETE Column stats: NONE
- value expressions: lintstring (type: array<struct<myint:int,mystring:string,underscore_int:int>>)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {aint}
+ 1 {lintstring}
+ keys:
+ 0 aint (type: int)
+ 1 aint (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
Map Operator Tree:
TableScan
alias: s1
@@ -63,32 +70,32 @@ STAGE PLANS:
Filter Operator
predicate: aint is not null (type: boolean)
Statistics: Num rows: 6 Data size: 1674 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: aint (type: int)
- sort order: +
- Map-reduce partition columns: aint (type: int)
- Statistics: Num rows: 6 Data size: 1674 Basic stats: COMPLETE Column stats: NONE
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0}
- 1 {VALUE._col3}
- outputColumnNames: _col0, _col17
- Statistics: Num rows: 6 Data size: 1841 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: int), _col17 (type: array<struct<myint:int,mystring:string,underscore_int:int>>)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 1841 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 6 Data size: 1841 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {aint}
+ 1 {lintstring}
+ keys:
+ 0 aint (type: int)
+ 1 aint (type: int)
+ outputColumnNames: _col0, _col17
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 6 Data size: 1841 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col17 (type: array<struct<myint:int,mystring:string,underscore_int:int>>)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6 Data size: 1841 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 1841 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/join_vc.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/join_vc.q.out?rev=1641614&r1=1641613&r2=1641614&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/join_vc.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/join_vc.q.out Tue Nov 25 13:03:18 2014
@@ -11,16 +11,13 @@ POSTHOOK: query: -- see HIVE-4033 earlie
explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value limit 3
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 5 <- Map 4 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1)
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -31,13 +28,16 @@ STAGE PLANS:
Filter Operator
predicate: value is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: value (type: string)
- sort order: +
- Map-reduce partition columns: value (type: string)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: key (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint)
- Map 4
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0
+ 1 {key} {BLOCK__OFFSET__INSIDE__FILE}
+ keys:
+ 0 _col6 (type: string)
+ 1 value (type: string)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
Map Operator Tree:
TableScan
alias: t2
@@ -45,13 +45,23 @@ STAGE PLANS:
Filter Operator
predicate: (key is not null and value is not null) (type: boolean)
Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 6
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0
+ 1 {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 4 <- Map 3 (SORT, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
Map Operator Tree:
TableScan
alias: t1
@@ -59,30 +69,43 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1}
- outputColumnNames: _col10, _col11, _col12
- Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col12 (type: bigint), _col10 (type: string), _col11 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string)
- sort order: +++
- Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col6
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {key} {value} {BLOCK__OFFSET__INSIDE__FILE}
+ keys:
+ 0 _col6 (type: string)
+ 1 value (type: string)
+ outputColumnNames: _col10, _col11, _col12
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col12 (type: bigint), _col10 (type: string), _col11 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: bigint), _col1 (type: string), _col2 (type: string)
+ sort order: +++
+ Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: bigint), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string)
@@ -98,21 +121,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 5
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1 {VALUE._col0}
- outputColumnNames: _col6
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col6 (type: string)
- sort order: +
- Map-reduce partition columns: _col6 (type: string)
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Stage: Stage-0
Fetch Operator
@@ -140,14 +148,13 @@ select t2.BLOCK__OFFSET__INSIDE__FILE
from src t1 join src t2 on t1.key = t2.key where t1.key < 100
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -158,13 +165,21 @@ STAGE PLANS:
Filter Operator
predicate: (key < 100) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: BLOCK__OFFSET__INSIDE__FILE (type: bigint)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0
+ 1 {BLOCK__OFFSET__INSIDE__FILE}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
Map Operator Tree:
TableScan
alias: t1
@@ -172,32 +187,32 @@ STAGE PLANS:
Filter Operator
predicate: (key < 100) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1 {VALUE._col1}
- outputColumnNames: _col7
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col7 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {BLOCK__OFFSET__INSIDE__FILE}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col7
+ input vertices:
+ 1 Map 1
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col7 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out?rev=1641614&r1=1641613&r2=1641614&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/join_view.q.out Tue Nov 25 13:03:18 2014
@@ -50,32 +50,11 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
+ Map 2 <- Map 1 (NONE, 0)
#### A masked pattern was here ####
Vertices:
Map 1
- Map 3
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col1}
- 1 {VALUE._col0}
- outputColumnNames: _col1, _col6
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Select Operator
- expressions: _col1 (type: string), _col6 (type: int), '2011-09-01' (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map 2
Stage: Stage-0
Fetch Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out?rev=1641614&r1=1641613&r2=1641614&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out Tue Nov 25 13:03:18 2014
@@ -806,17 +806,15 @@ join
on subq.key=subq2.key limit 4
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-3
Spark
Edges:
Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 5 <- Map 4 (GROUP, 1)
- Reducer 6 <- Reducer 5 (GROUP, 1)
- Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 6 (PARTITION-LEVEL SORT, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -841,29 +839,9 @@ STAGE PLANS:
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
value expressions: _col1 (type: bigint)
- Map 4
- Map Operator Tree:
- TableScan
- alias: src
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: key
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count(1)
- keys: key (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- TopN Hash Memory Usage: 0.3
- value expressions: _col1 (type: bigint)
Reducer 2
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -881,37 +859,44 @@ STAGE PLANS:
Filter Operator
predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {_col1}
+ 1 {_col0} {_col1}
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 4 <- Map 3 (GROUP, 1)
+ Reducer 5 <- Reducer 4 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: key
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(1)
+ keys: key (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ TopN Hash Memory Usage: 0.3
value expressions: _col1 (type: bigint)
- Reducer 3
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 4
- Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 5
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
@@ -931,7 +916,9 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
value expressions: _col0 (type: string), _col1 (type: bigint)
- Reducer 6
+ Reducer 5
+ Local Work:
+ Map Reduce Local Work
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint)
@@ -943,12 +930,33 @@ STAGE PLANS:
Filter Operator
predicate: _col0 is not null (type: boolean)
Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 20 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {_col0} {_col1}
+ 1 {_col0} {_col1}
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ input vertices:
+ 0 Reducer 2
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 4
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out?rev=1641614&r1=1641613&r2=1641614&view=diff
==============================================================================
Files hive/branches/spark/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/spark/louter_join_ppr.q.out Tue Nov 25 13:03:18 2014 differ