You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/12/29 03:14:00 UTC
svn commit: r1648280 [23/25] - in /hive/branches/spark: data/conf/spark/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/
ql/src/test/results/clientpositive/spark/
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out Mon Dec 29 02:13:57 2014
@@ -77,10 +77,6 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
Inner Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -141,10 +137,6 @@ STAGE PLANS:
condition map:
Left Outer Join0 to 1
Inner Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -205,10 +197,6 @@ STAGE PLANS:
condition map:
Left Outer Join0 to 1
Left Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -274,10 +262,6 @@ STAGE PLANS:
condition map:
Left Outer Join0 to 1
Right Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -344,10 +328,6 @@ STAGE PLANS:
condition map:
Left Outer Join0 to 1
Outer Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -419,10 +399,6 @@ STAGE PLANS:
condition map:
Right Outer Join0 to 1
Inner Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -485,10 +461,6 @@ STAGE PLANS:
condition map:
Right Outer Join0 to 1
Left Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -553,10 +525,6 @@ STAGE PLANS:
condition map:
Right Outer Join0 to 1
Right Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -623,10 +591,6 @@ STAGE PLANS:
condition map:
Right Outer Join0 to 1
Outer Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -695,10 +659,6 @@ STAGE PLANS:
condition map:
Outer Join 0 to 1
Inner Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -761,10 +721,6 @@ STAGE PLANS:
condition map:
Outer Join 0 to 1
Left Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -834,10 +790,6 @@ STAGE PLANS:
condition map:
Outer Join 0 to 1
Right Outer Join1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -904,10 +856,6 @@ STAGE PLANS:
condition map:
Outer Join 0 to 1
Outer Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_6.q.out Mon Dec 29 02:13:57 2014
@@ -83,9 +83,6 @@ STAGE PLANS:
Sorted Merge Bucket Map Join Operator
condition map:
Inner Join 0 to 1
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -1230,9 +1227,6 @@ STAGE PLANS:
Sorted Merge Bucket Map Join Operator
condition map:
Inner Join 0 to 1
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -2393,9 +2387,6 @@ STAGE PLANS:
Sorted Merge Bucket Map Join Operator
condition map:
Inner Join 0 to 1
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -2472,9 +2463,6 @@ STAGE PLANS:
Sorted Merge Bucket Map Join Operator
condition map:
Inner Join 0 to 1
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
@@ -2549,10 +2537,6 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
Inner Join 1 to 2
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
- 2 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out Mon Dec 29 02:13:57 2014
@@ -632,9 +632,6 @@ STAGE PLANS:
Sorted Merge Bucket Map Join Operator
condition map:
Outer Join 0 to 1
- condition expressions:
- 0 {key} {value}
- 1 {key} {value}
keys:
0 key (type: int)
1 key (type: int)
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_1.q.out Mon Dec 29 02:13:57 2014
@@ -62,8 +62,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -74,44 +73,24 @@ STAGE PLANS:
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Map 4
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (key < 10) (type: boolean)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Sorted Merge Bucket Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Reducer 3
- Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
mode: mergepartial
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_2.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_2.q.out Mon Dec 29 02:13:57 2014
@@ -70,8 +70,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -82,44 +81,24 @@ STAGE PLANS:
Filter Operator
predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean)
Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Map 4
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Sorted Merge Bucket Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: string), value (type: string)
+ 1 key (type: string), value (type: string)
+ Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Reducer 3
- Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
mode: mergepartial
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_3.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_3.q.out Mon Dec 29 02:13:57 2014
@@ -70,8 +70,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -82,44 +81,24 @@ STAGE PLANS:
Filter Operator
predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean)
Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Map 4
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Sorted Merge Bucket Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: string), value (type: string)
+ 1 key (type: string), value (type: string)
+ Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Reducer 3
- Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
mode: mergepartial
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_4.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_4.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_4.q.out Mon Dec 29 02:13:57 2014
@@ -61,63 +61,66 @@ select /*+ mapjoin(b) */ count(*) from t
on a.key=b.key and a.value=b.value where a.key < 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean)
Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Map 4
+ Spark HashTable Sink Operator
+ keys:
+ 0 key (type: string), value (type: string)
+ 1 key (type: string), value (type: string)
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((key is not null and value is not null) and (key < 10)) (type: boolean)
Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value (type: string)
- Statistics: Num rows: 41 Data size: 435 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: string), value (type: string)
+ 1 key (type: string), value (type: string)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 45 Data size: 478 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Reducer 3
- Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
mode: mergepartial
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_5.q.out Mon Dec 29 02:13:57 2014
@@ -119,8 +119,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -133,13 +132,25 @@ STAGE PLANS:
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Sorted Merge Bucket Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ Position of Big Table: 0
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -155,7 +166,7 @@ STAGE PLANS:
bucket_count 1
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
@@ -178,7 +189,7 @@ STAGE PLANS:
bucket_count 1
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
name default.srcbucket_mapjoin_part_1
@@ -193,98 +204,7 @@ STAGE PLANS:
name: default.srcbucket_mapjoin_part_1
Truncated Path -> Alias:
/srcbucket_mapjoin_part_1/part=1 [a]
- Map 4
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: part=1
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- part 1
- properties:
- COLUMN_STATS_ACCURATE true
- bucket_count 1
- bucket_field_name key
- columns key,value
- columns.comments
- columns.types int:string
-#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
- numFiles 1
- numRows 500
- partition_columns part
- partition_columns.types string
- rawDataSize 5312
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 1
- bucket_field_name key
- columns key,value
- columns.comments
- columns.types int:string
-#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
- partition_columns part
- partition_columns.types string
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcbucket_mapjoin_part_2
- name: default.srcbucket_mapjoin_part_2
- Truncated Path -> Alias:
- /srcbucket_mapjoin_part_2/part=1 [b]
Reducer 2
- Needs Tagging: true
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Reducer 3
Needs Tagging: false
Reduce Operator Tree:
Group By Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_6.q.out Mon Dec 29 02:13:57 2014
@@ -112,34 +112,37 @@ TOK_QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Spark HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ Position of Big Table: 0
+ Local Work:
+ Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -155,16 +158,16 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_1
+ name default.srcbucket_mapjoin_part_2
numFiles 2
numRows 500
partition_columns part
partition_columns.types string
rawDataSize 5312
- serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -174,42 +177,70 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
- SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_1
+ name default.srcbucket_mapjoin_part_2
partition_columns part
partition_columns.types string
- serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcbucket_mapjoin_part_1
- name: default.srcbucket_mapjoin_part_1
+ name: default.srcbucket_mapjoin_part_2
+ name: default.srcbucket_mapjoin_part_2
Truncated Path -> Alias:
- /srcbucket_mapjoin_part_1/part=1 [a]
- Map 4
+ /srcbucket_mapjoin_part_2/part=1 [b]
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- auto parallelism: false
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ input vertices:
+ 1 Map 3
+ Position of Big Table: 0
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ BucketMapJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Local Work:
+ Map Reduce Local Work
+ Bucket Mapjoin Context:
+ Alias Bucket File Name Mapping:
+#### A masked pattern was here ####
+ Alias Bucket Output File Name Mapping:
+#### A masked pattern was here ####
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -225,16 +256,16 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
+ name default.srcbucket_mapjoin_part_1
numFiles 2
numRows 500
partition_columns part
partition_columns.types string
rawDataSize 5312
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -244,46 +275,26 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
+ SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
+ name default.srcbucket_mapjoin_part_1
partition_columns part
partition_columns.types string
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcbucket_mapjoin_part_2
- name: default.srcbucket_mapjoin_part_2
+ name: default.srcbucket_mapjoin_part_1
+ name: default.srcbucket_mapjoin_part_1
Truncated Path -> Alias:
- /srcbucket_mapjoin_part_2/part=1 [b]
+ /srcbucket_mapjoin_part_1/part=1 [a]
Reducer 2
- Needs Tagging: true
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Reducer 3
Needs Tagging: false
Reduce Operator Tree:
Group By Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_7.q.out Mon Dec 29 02:13:57 2014
@@ -148,34 +148,32 @@ TOK_QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
- Stage: Stage-1
+ Stage: Stage-2
Spark
- Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
- Map 1
+ Map 3
Map Operator Tree:
TableScan
- alias: a
+ alias: b
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- tag: 0
- auto parallelism: false
+ Spark HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ Position of Big Table: 0
+ Local Work:
+ Map Reduce Local Work
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -191,16 +189,16 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_1
+ name default.srcbucket_mapjoin_part_2
numFiles 2
numRows 500
partition_columns part
partition_columns.types string
rawDataSize 5312
- serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -213,19 +211,19 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_1
+ name default.srcbucket_mapjoin_part_2
partition_columns part
partition_columns.types string
- serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcbucket_mapjoin_part_1
- name: default.srcbucket_mapjoin_part_1
+ name: default.srcbucket_mapjoin_part_2
+ name: default.srcbucket_mapjoin_part_2
#### A masked pattern was here ####
Partition
base file name: part=2
@@ -238,16 +236,16 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_1
+ name default.srcbucket_mapjoin_part_2
numFiles 2
numRows 500
partition_columns part
partition_columns.types string
rawDataSize 5312
- serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -260,39 +258,62 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_1
+ name default.srcbucket_mapjoin_part_2
partition_columns part
partition_columns.types string
- serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcbucket_mapjoin_part_1
- name: default.srcbucket_mapjoin_part_1
+ name: default.srcbucket_mapjoin_part_2
+ name: default.srcbucket_mapjoin_part_2
Truncated Path -> Alias:
- /srcbucket_mapjoin_part_1/part=1 [a]
- /srcbucket_mapjoin_part_1/part=2 [a]
- Map 4
+ /srcbucket_mapjoin_part_2/part=1 [b]
+ /srcbucket_mapjoin_part_2/part=2 [b]
+
+ Stage: Stage-1
+ Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- tag: 1
- auto parallelism: false
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ input vertices:
+ 1 Map 3
+ Position of Big Table: 0
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ tag: -1
+ value expressions: _col0 (type: bigint)
+ auto parallelism: false
+ Local Work:
+ Map Reduce Local Work
Path -> Alias:
#### A masked pattern was here ####
Path -> Partition:
@@ -308,16 +329,16 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
+ name default.srcbucket_mapjoin_part_1
numFiles 2
numRows 500
partition_columns part
partition_columns.types string
rawDataSize 5312
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -330,19 +351,19 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
+ name default.srcbucket_mapjoin_part_1
partition_columns part
partition_columns.types string
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcbucket_mapjoin_part_2
- name: default.srcbucket_mapjoin_part_2
+ name: default.srcbucket_mapjoin_part_1
+ name: default.srcbucket_mapjoin_part_1
#### A masked pattern was here ####
Partition
base file name: part=2
@@ -355,16 +376,16 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
+ name default.srcbucket_mapjoin_part_1
numFiles 2
numRows 500
partition_columns part
partition_columns.types string
rawDataSize 5312
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
totalSize 5812
@@ -377,44 +398,23 @@ STAGE PLANS:
bucket_count 2
bucket_field_name key
columns key,value
- columns.comments
+ columns.comments
columns.types int:string
#### A masked pattern was here ####
- name default.srcbucket_mapjoin_part_2
+ name default.srcbucket_mapjoin_part_1
partition_columns part
partition_columns.types string
- serialization.ddl struct srcbucket_mapjoin_part_2 { i32 key, string value}
+ serialization.ddl struct srcbucket_mapjoin_part_1 { i32 key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
#### A masked pattern was here ####
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.srcbucket_mapjoin_part_2
- name: default.srcbucket_mapjoin_part_2
+ name: default.srcbucket_mapjoin_part_1
+ name: default.srcbucket_mapjoin_part_1
Truncated Path -> Alias:
- /srcbucket_mapjoin_part_2/part=1 [b]
- /srcbucket_mapjoin_part_2/part=2 [b]
+ /srcbucket_mapjoin_part_1/part=1 [a]
+ /srcbucket_mapjoin_part_1/part=2 [a]
Reducer 2
- Needs Tagging: true
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Reducer 3
Needs Tagging: false
Reduce Operator Tree:
Group By Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/sort_merge_join_desc_8.q.out Mon Dec 29 02:13:57 2014
@@ -115,8 +115,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -127,44 +126,24 @@ STAGE PLANS:
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Map 4
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (key < 10) (type: boolean)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string)
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+ Sorted Merge Bucket Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: string)
+ 1 key (type: string)
+ Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Reducer 3
- Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
mode: mergepartial
@@ -214,15 +193,34 @@ select /*+ mapjoin(b) */ count(*) from t
on a.key=b.key and a.value2=b.value2 where a.key < 10
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
Stage-0 depends on stages: Stage-1
STAGE PLANS:
+ Stage: Stage-2
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((key is not null and value2 is not null) and (key < 10)) (type: boolean)
+ Statistics: Num rows: 41 Data size: 517 Basic stats: COMPLETE Column stats: NONE
+ Spark HashTable Sink Operator
+ keys:
+ 0 key (type: string), value2 (type: string)
+ 1 key (type: string), value2 (type: string)
+ Local Work:
+ Map Reduce Local Work
+
Stage: Stage-1
Spark
Edges:
- Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
- Reducer 3 <- Reducer 2 (GROUP, 1)
+ Reducer 2 <- Map 1 (GROUP, 1)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -233,44 +231,28 @@ STAGE PLANS:
Filter Operator
predicate: ((key is not null and value2 is not null) and (key < 10)) (type: boolean)
Statistics: Num rows: 41 Data size: 837 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value2 (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value2 (type: string)
- Statistics: Num rows: 41 Data size: 837 Basic stats: COMPLETE Column stats: NONE
- Map 4
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((key is not null and value2 is not null) and (key < 10)) (type: boolean)
- Statistics: Num rows: 41 Data size: 517 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: string), value2 (type: string)
- sort order: ++
- Map-reduce partition columns: key (type: string), value2 (type: string)
- Statistics: Num rows: 41 Data size: 517 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: string), value2 (type: string)
+ 1 key (type: string), value2 (type: string)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 45 Data size: 920 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
Reducer 2
Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- condition expressions:
- 0
- 1
- Statistics: Num rows: 45 Data size: 920 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
- Reducer 3
- Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
mode: mergepartial
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/stats0.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/stats0.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
Binary files hive/branches/spark/ql/src/test/results/clientpositive/spark/stats0.q.out (original) and hive/branches/spark/ql/src/test/results/clientpositive/spark/stats0.q.out Mon Dec 29 02:13:57 2014 differ
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/stats12.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/stats12.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/stats12.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/stats12.q.out Mon Dec 29 02:13:57 2014
@@ -84,7 +84,7 @@ STAGE PLANS:
COLUMN_STATS_ACCURATE false
bucket_count -1
columns key,value
- columns.comments default default
+ columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
name default.analyze_srcpart
@@ -105,7 +105,7 @@ STAGE PLANS:
properties:
bucket_count -1
columns key,value
- columns.comments default default
+ columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
name default.analyze_srcpart
@@ -130,7 +130,7 @@ STAGE PLANS:
COLUMN_STATS_ACCURATE false
bucket_count -1
columns key,value
- columns.comments default default
+ columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
name default.analyze_srcpart
@@ -151,7 +151,7 @@ STAGE PLANS:
properties:
bucket_count -1
columns key,value
- columns.comments default default
+ columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
name default.analyze_srcpart
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/stats13.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/stats13.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/stats13.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/stats13.q.out Mon Dec 29 02:13:57 2014
@@ -85,7 +85,7 @@ STAGE PLANS:
COLUMN_STATS_ACCURATE false
bucket_count -1
columns key,value
- columns.comments default default
+ columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
name default.analyze_srcpart
@@ -106,7 +106,7 @@ STAGE PLANS:
properties:
bucket_count -1
columns key,value
- columns.comments default default
+ columns.comments 'default','default'
columns.types string:string
#### A masked pattern was here ####
name default.analyze_srcpart
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_exists.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_exists.q.out?rev=1648280&r1=1648279&r2=1648280&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_exists.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/subquery_exists.q.out Mon Dec 29 02:13:57 2014
@@ -73,9 +73,9 @@ STAGE PLANS:
Join Operator
condition map:
Left Semi Join 0 to 1
- condition expressions:
- 0 {KEY.reducesinkkey1} {KEY.reducesinkkey0}
- 1
+ keys:
+ 0 value (type: string), key (type: string)
+ 1 _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
Select Operator