You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/12/13 18:44:42 UTC
svn commit: r1645338 [8/9] - in /hive/branches/spark: data/conf/spark/
itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/test/results/clientpositive/spark/
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin1.q.out?rev=1645338&r1=1645337&r2=1645338&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin1.q.out Sat Dec 13 17:44:41 2014
@@ -30,14 +30,13 @@ EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -45,50 +44,60 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {true}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
Map Operator Tree:
TableScan
alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- filter predicates:
- 0
- 1 {true}
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {true}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -104,16 +113,16 @@ POSTHOOK: query: SELECT /*+ MAPJOIN(a) *
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-105 val_105 105 val_105
+238 val_238 238 val_238
+238 val_238 238 val_238
+86 val_86 86 val_86
+311 val_311 311 val_311
+311 val_311 311 val_311
+311 val_311 311 val_311
+27 val_27 27 val_27
+165 val_165 165 val_165
+165 val_165 165 val_165
+409 val_409 409 val_409
PREHOOK: query: -- func filter on outer join
EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
@@ -123,14 +132,13 @@ EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -138,50 +146,60 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {((key * 10) < '1000')}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
Map Operator Tree:
TableScan
alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- filter predicates:
- 0
- 1 {((KEY.reducesinkkey0 * 10) < '1000')}
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ filter predicates:
+ 0
+ 1 {((key * 10) < '1000')}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -197,16 +215,16 @@ POSTHOOK: query: SELECT /*+ MAPJOIN(a) *
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-NULL NULL 105 val_105
+NULL NULL 238 val_238
+86 val_86 86 val_86
+NULL NULL 311 val_311
+27 val_27 27 val_27
+NULL NULL 165 val_165
+NULL NULL 409 val_409
+NULL NULL 255 val_255
+NULL NULL 278 val_278
+98 val_98 98 val_98
+98 val_98 98 val_98
PREHOOK: query: -- field filter on outer join
EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
@@ -218,14 +236,35 @@ SELECT /*+ MAPJOIN(a) */ * FROM src a RI
(select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {_col0} {_col1}
+ filter predicates:
+ 0
+ 1 {(_col1.key > 200)}
+ keys:
+ 0 key (type: string)
+ 1 _col0 (type: string)
+ Local Work:
+ Map Reduce Local Work
+
Stage: Stage-1
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -237,50 +276,38 @@ STAGE PLANS:
expressions: key (type: string), named_struct('key',key,'value',value) (type: struct<key:string,value:string>)
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: struct<key:string,value:string>)
- Map 3
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- filter predicates:
- 0
- 1 {(VALUE._col0.key > 200)}
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: struct<key:string,value:string>)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {_col0} {_col1}
+ filter predicates:
+ 0
+ 1 {(_col1.key > 200)}
+ keys:
+ 0 key (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 0 Map 2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: struct<key:string,value:string>)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -298,16 +325,16 @@ POSTHOOK: query: SELECT /*+ MAPJOIN(a) *
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-NULL NULL 0 {"key":"0","value":"val_0"}
-NULL NULL 0 {"key":"0","value":"val_0"}
-NULL NULL 0 {"key":"0","value":"val_0"}
-NULL NULL 105 {"key":"105","value":"val_105"}
-NULL NULL 111 {"key":"111","value":"val_111"}
-NULL NULL 114 {"key":"114","value":"val_114"}
-NULL NULL 12 {"key":"12","value":"val_12"}
-NULL NULL 12 {"key":"12","value":"val_12"}
-NULL NULL 120 {"key":"120","value":"val_120"}
-NULL NULL 120 {"key":"120","value":"val_120"}
+238 val_238 238 {"key":"238","value":"val_238"}
+238 val_238 238 {"key":"238","value":"val_238"}
+NULL NULL 86 {"key":"86","value":"val_86"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+NULL NULL 27 {"key":"27","value":"val_27"}
+NULL NULL 165 {"key":"165","value":"val_165"}
+409 val_409 409 {"key":"409","value":"val_409"}
+409 val_409 409 {"key":"409","value":"val_409"}
PREHOOK: query: EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
PREHOOK: type: QUERY
@@ -315,14 +342,13 @@ POSTHOOK: query: EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND true limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -330,47 +356,54 @@ STAGE PLANS:
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {key} {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
Map Operator Tree:
TableScan
alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -386,16 +419,16 @@ POSTHOOK: query: SELECT /*+ MAPJOIN(a) *
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-105 val_105 105 val_105
+238 val_238 238 val_238
+238 val_238 238 val_238
+86 val_86 86 val_86
+311 val_311 311 val_311
+311 val_311 311 val_311
+311 val_311 311 val_311
+27 val_27 27 val_27
+165 val_165 165 val_165
+165 val_165 165 val_165
+409 val_409 409 val_409
PREHOOK: query: EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
PREHOOK: type: QUERY
@@ -403,14 +436,13 @@ POSTHOOK: query: EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN src b on a.key=b.key AND b.key * 10 < '1000' limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -421,13 +453,21 @@ STAGE PLANS:
Filter Operator
predicate: ((key * 10) < '1000') (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 3
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {key} {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
Map Operator Tree:
TableScan
alias: b
@@ -435,36 +475,35 @@ STAGE PLANS:
Filter Operator
predicate: ((key * 10) < '1000') (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 0 Map 1
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -480,16 +519,16 @@ POSTHOOK: query: SELECT /*+ MAPJOIN(a) *
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-0 val_0 0 val_0
-12 val_12 12 val_12
+86 val_86 86 val_86
+27 val_27 27 val_27
+98 val_98 98 val_98
+98 val_98 98 val_98
+66 val_66 66 val_66
+37 val_37 37 val_37
+37 val_37 37 val_37
+15 val_15 15 val_15
+15 val_15 15 val_15
+82 val_82 82 val_82
PREHOOK: query: EXPLAIN
SELECT /*+ MAPJOIN(a) */ * FROM src a RIGHT OUTER JOIN
(select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
@@ -499,14 +538,32 @@ SELECT /*+ MAPJOIN(a) */ * FROM src a RI
(select key, named_struct('key', key, 'value', value) as kv from src) b on a.key=b.key AND b.kv.key > 200 limit 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {_col0} {_col1}
+ keys:
+ 0 key (type: string)
+ 1 _col0 (type: string)
+ Local Work:
+ Map Reduce Local Work
+
Stage: Stage-1
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -521,47 +578,35 @@ STAGE PLANS:
Filter Operator
predicate: (_col1.key > 200) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: struct<key:string,value:string>)
- Map 3
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col1, _col5, _col6
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: struct<key:string,value:string>)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 10
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1 {_col0} {_col1}
+ keys:
+ 0 key (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 0 Map 2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: struct<key:string,value:string>)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 10
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 10 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
Stage: Stage-0
Fetch Operator
@@ -579,13 +624,13 @@ POSTHOOK: query: SELECT /*+ MAPJOIN(a) *
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-201 val_201 201 {"key":"201","value":"val_201"}
-207 val_207 207 {"key":"207","value":"val_207"}
-207 val_207 207 {"key":"207","value":"val_207"}
-207 val_207 207 {"key":"207","value":"val_207"}
-207 val_207 207 {"key":"207","value":"val_207"}
-213 val_213 213 {"key":"213","value":"val_213"}
-213 val_213 213 {"key":"213","value":"val_213"}
-213 val_213 213 {"key":"213","value":"val_213"}
-213 val_213 213 {"key":"213","value":"val_213"}
-216 val_216 216 {"key":"216","value":"val_216"}
+238 val_238 238 {"key":"238","value":"val_238"}
+238 val_238 238 {"key":"238","value":"val_238"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+311 val_311 311 {"key":"311","value":"val_311"}
+409 val_409 409 {"key":"409","value":"val_409"}
+409 val_409 409 {"key":"409","value":"val_409"}
+409 val_409 409 {"key":"409","value":"val_409"}
+255 val_255 255 {"key":"255","value":"val_255"}
+255 val_255 255 {"key":"255","value":"val_255"}
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out?rev=1645338&r1=1645337&r2=1645338&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out Sat Dec 13 17:44:41 2014
@@ -11,67 +11,75 @@ ON ( c.key=d.key AND c.ds='2008-04-08' A
SELECT /*+ MAPJOIN(d) */ DISTINCT c.value
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1)
- Reducer 4 <- Reducer 3 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 4
Map Operator Tree:
TableScan
- alias: c
+ alias: d
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
Map Operator Tree:
TableScan
- alias: d
+ alias: c
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: rand() (type: double)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0}
- 1
- outputColumnNames: _col1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col1 (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: rand() (type: double)
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: partials
@@ -82,7 +90,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reducer 4
+ Reducer 3
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
@@ -148,66 +156,74 @@ ON ( c.key=d.key AND c.ds='2008-04-08' A
SELECT /*+ MAPJOIN(d) */ DISTINCT c.value
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
- alias: c
+ alias: d
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 4
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 3)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
Map Operator Tree:
TableScan
- alias: d
+ alias: c
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0}
- 1
- outputColumnNames: _col1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- keys: _col1 (type: string)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: mergepartial
@@ -272,62 +288,70 @@ ON ( c.key=d.key AND c.ds='2008-04-08' A
SELECT /*+ MAPJOIN(d) */ DISTINCT c.value
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3)
- Reducer 4 <- Reducer 3 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 4
Map Operator Tree:
TableScan
- alias: c
+ alias: d
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3)
+ Reducer 3 <- Reducer 2 (GROUP, 3)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
Map Operator Tree:
TableScan
- alias: d
+ alias: c
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: rand() (type: double)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0}
- 1
- outputColumnNames: _col1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: string)
- sort order: +
- Map-reduce partition columns: rand() (type: double)
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: partial1
@@ -338,7 +362,7 @@ STAGE PLANS:
sort order: +
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reducer 4
+ Reducer 3
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
@@ -404,61 +428,69 @@ ON ( c.key=d.key AND c.ds='2008-04-08' A
SELECT /*+ MAPJOIN(d) */ DISTINCT c.value
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 3)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
- alias: c
+ alias: d
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 4
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 3)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
Map Operator Tree:
TableScan
- alias: d
+ alias: c
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {value}
+ 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ outputColumnNames: _col1
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0 {VALUE._col0}
- 1
- outputColumnNames: _col1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col1 (type: string)
- sort order: +
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: string)
mode: complete
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out?rev=1645338&r1=1645337&r2=1645338&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out Sat Dec 13 17:44:41 2014
@@ -53,15 +53,13 @@ SELECT /*+ mapjoin(src1, src2) */ * FROM
SORT BY src1.key, src2.key, src3.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -72,13 +70,22 @@ STAGE PLANS:
Filter Operator
predicate: ((key < 300) and (key < 10)) (type: boolean)
Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 15 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 4
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1 {(key > 10)}
+ 2
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
Map Operator Tree:
TableScan
alias: src2
@@ -86,13 +93,29 @@ STAGE PLANS:
Filter Operator
predicate: (key < 300) (type: boolean)
Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1 {(key > 10)}
+ 2
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 4 <- Map 3 (SORT, 3)
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
Map Operator Tree:
TableScan
alias: src3
@@ -100,38 +123,39 @@ STAGE PLANS:
Filter Operator
predicate: (key < 300) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- Inner Join 1 to 2
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- 2 {KEY.reducesinkkey0} {VALUE._col0}
- filter predicates:
- 0
- 1 {(KEY.reducesinkkey0 > 10)}
- 2
- outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
- Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col2 (type: string), _col4 (type: string)
- sort order: +++
- Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: string), _col3 (type: string), _col5 (type: string)
- Reducer 3
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ Inner Join 1 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ filter predicates:
+ 0
+ 1 {(key > 10)}
+ 2
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 0 Map 1
+ 1 Map 2
+ Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col2 (type: string), _col4 (type: string)
+ sort order: +++
+ Statistics: Num rows: 365 Data size: 3878 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string), _col3 (type: string), _col5 (type: string)
+ Local Work:
+ Map Reduce Local Work
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: string), VALUE._col2 (type: string)
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out?rev=1645338&r1=1645337&r2=1645338&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out Sat Dec 13 17:44:41 2014
@@ -244,15 +244,13 @@ SELECT /*+ mapjoin(src1, src2) */ * FROM
SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -260,55 +258,73 @@ STAGE PLANS:
TableScan
alias: src1
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 4
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+ Map 2
Map Operator Tree:
TableScan
alias: src2
Statistics: Num rows: 9 Data size: 40 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 9 Data size: 40 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {key} {value}
+ 1 {value}
+ 2 {key} {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 4 <- Map 3 (SORT, 3)
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
Map Operator Tree:
TableScan
alias: src3
Statistics: Num rows: 9 Data size: 40 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 9 Data size: 40 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
- Reducer 2
- Reduce Operator Tree:
- Join Operator
- condition map:
- Right Outer Join0 to 1
- Inner Join 1 to 2
- condition expressions:
- 0 {KEY.reducesinkkey0} {VALUE._col0}
- 1 {KEY.reducesinkkey0} {VALUE._col0}
- 2 {KEY.reducesinkkey0} {VALUE._col0}
- outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
- Statistics: Num rows: 55 Data size: 420 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 55 Data size: 420 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
- sort order: ++++++
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ Inner Join 1 to 2
+ condition expressions:
+ 0 {key} {value}
+ 1 {key} {value}
+ 2 {key} {value}
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ 2 key (type: string)
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 0 Map 1
+ 1 Map 2
Statistics: Num rows: 55 Data size: 420 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col5 (type: string), _col6 (type: string), _col10 (type: string), _col11 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 55 Data size: 420 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
+ sort order: ++++++
+ Statistics: Num rows: 55 Data size: 420 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: string), KEY.reducesinkkey5 (type: string)
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/semijoin.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/semijoin.q.out?rev=1645338&r1=1645337&r2=1645338&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/semijoin.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/semijoin.q.out Sat Dec 13 17:44:41 2014
@@ -931,31 +931,16 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select /*+ mapjoin(b) */ a.key from t3 a left semi join t1 b on a.key = b.key sort by a.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
- Map 1
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE
- Map 4
+ Map 3
Map Operator Tree:
TableScan
alias: b
@@ -972,27 +957,51 @@ STAGE PLANS:
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ keys:
+ 0 key (type: int)
+ 1 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (SORT, 3)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ condition expressions:
+ 0 {key}
+ 1
+ keys:
+ 0 key (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Left Semi Join 0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey0}
- 1
- outputColumnNames: _col0
- Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Statistics: Num rows: 12 Data size: 89 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0
@@ -1387,31 +1396,16 @@ PREHOOK: type: QUERY
POSTHOOK: query: explain select /*+ mapjoin(b, c) */ a.key from t3 a left semi join t1 b on a.key = b.key left semi join t2 c on a.key = c.key sort by a.key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (SORT, 3)
#### A masked pattern was here ####
Vertices:
- Map 1
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE
- Map 4
+ Map 3
Map Operator Tree:
TableScan
alias: b
@@ -1428,12 +1422,18 @@ STAGE PLANS:
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 6 Data size: 43 Basic stats: COMPLETE Column stats: NONE
- Map 5
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2
+ keys:
+ 0 key (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+ Map 4
Map Operator Tree:
TableScan
alias: c
@@ -1450,29 +1450,57 @@ STAGE PLANS:
mode: hash
outputColumnNames: _col0
Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ condition expressions:
+ 0 {key}
+ 1
+ 2
+ keys:
+ 0 key (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (SORT, 3)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 22 Data size: 163 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 11 Data size: 81 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ Left Semi Join 0 to 2
+ condition expressions:
+ 0 {key}
+ 1
+ 2
+ keys:
+ 0 key (type: int)
+ 1 _col0 (type: int)
+ 2 _col0 (type: int)
+ outputColumnNames: _col0
+ input vertices:
+ 1 Map 3
+ 2 Map 4
+ Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Left Semi Join 0 to 1
- Left Semi Join 0 to 2
- condition expressions:
- 0 {KEY.reducesinkkey0}
- 1
- 2
- outputColumnNames: _col0
- Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Statistics: Num rows: 24 Data size: 178 Basic stats: COMPLETE Column stats: NONE
- Reducer 3
- Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: int)
outputColumnNames: _col0