You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/09/16 04:02:49 UTC
svn commit: r1625203 [2/2] - in /hive/branches/spark/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/unionproc/
java/org/apache/hadoop/hive/ql/parse/spark/ test/queries/clientpositive/
test/results/clientpositive/spark/
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_19.q.out Tue Sep 16 02:02:48 2014
@@ -74,7 +74,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -98,7 +99,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -126,39 +127,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
@@ -279,7 +282,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -306,7 +310,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 15 Data size: 15 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -337,39 +341,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 7 Data size: 7 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
@@ -452,7 +458,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -476,7 +483,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -504,53 +511,51 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
- expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
+ expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Filter Operator
- predicate: (_col0 >= 7.0) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Select Operator
- expressions: _col0 (type: double), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate: (_col0 >= 7.0) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
- expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
+ expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Filter Operator
- predicate: (_col0 >= 7.0) (type: boolean)
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Select Operator
- expressions: _col0 (type: double), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: (_col0 + _col0) (type: double), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate: (_col0 >= 7.0) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_2.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_2.q.out Tue Sep 16 02:02:48 2014
@@ -75,27 +75,28 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP)
+ Reducer 4 <- Map 3 (GROUP)
+ Union 2 <- Map 1 (NONE), Map 5 (NONE), Reducer 4 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(2) (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Map 2
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Map 3
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -116,43 +117,45 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(1) (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 3
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 2
+ Vertex: Union 2
Stage: Stage-2
Dependency Collection
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_20.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_20.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_20.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_20.q.out Tue Sep 16 02:02:48 2014
@@ -72,7 +72,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -96,7 +97,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -124,39 +125,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
- expressions: _col1 (type: bigint), _col0 (type: string)
+ expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col1 (type: bigint), _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
- expressions: _col1 (type: bigint), _col0 (type: string)
+ expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col1 (type: bigint), _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_21.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_21.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_21.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_21.q.out Tue Sep 16 02:02:48 2014
@@ -72,7 +72,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -96,7 +97,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -124,39 +125,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string)
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_24.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_24.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_24.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_24.q.out Tue Sep 16 02:02:48 2014
@@ -68,7 +68,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -92,7 +93,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -120,39 +121,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: UDFToDouble(UDFToLong(_col0)) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: UDFToDouble(_col0) (type: double), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: double), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_25.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_25.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_25.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_25.q.out Tue Sep 16 02:02:48 2014
@@ -86,7 +86,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -110,7 +111,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -138,39 +139,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
@@ -298,7 +301,9 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 4 <- Union 3 (GROUP SORT)
+ Reducer 6 <- Map 5 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 6 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -317,7 +322,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
- Map 3
+ Map 5
Map Operator Tree:
TableScan
alias: srcpart
@@ -338,43 +343,45 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 500
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl2
+ Reduce Output Operator
+ key expressions: _col2 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: string)
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
Reducer 4
Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl2
+ Reducer 6
+ Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 500
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl2
+ Reduce Output Operator
+ key expressions: _col2 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: string)
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string)
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
@@ -449,7 +456,7 @@ Protect Mode: None
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE false
- numFiles 2
+ numFiles 1
numRows -1
rawDataSize -1
totalSize 6826
@@ -491,7 +498,9 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 4 <- Union 3 (GROUP SORT)
+ Reducer 6 <- Map 5 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 6 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -510,7 +519,7 @@ STAGE PLANS:
sort order:
Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string)
- Map 3
+ Map 5
Map Operator Tree:
TableScan
alias: srcpart
@@ -531,43 +540,45 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 1000
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl3
+ Reduce Output Operator
+ key expressions: _col2 (type: string), _col3 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col2 (type: string), _col3 (type: string)
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string)
Reducer 4
Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 116 Data size: 23248 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl3
+ Reducer 6
+ Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 1000
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint), _col2 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 58 Data size: 11624 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl3
+ Reduce Output Operator
+ key expressions: _col2 (type: string), _col3 (type: string)
+ sort order: ++
+ Map-reduce partition columns: _col2 (type: string), _col3 (type: string)
+ value expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: string), _col3 (type: string)
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
@@ -648,7 +659,7 @@ Protect Mode: None
#### A masked pattern was here ####
Partition Parameters:
COLUMN_STATS_ACCURATE false
- numFiles 2
+ numFiles 1
numRows -1
rawDataSize -1
totalSize 6812
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_4.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_4.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_4.q.out Tue Sep 16 02:02:48 2014
@@ -62,15 +62,21 @@ FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
STAGE PLANS:
Stage: Stage-1
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -94,7 +100,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -122,39 +128,50 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
Stage: Stage-2
Dependency Collection
@@ -169,6 +186,42 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Merge
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+
+ Stage: Stage-5
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Merge
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
PREHOOK: query: insert overwrite table outputTbl1
SELECT *
FROM (
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_5.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_5.q.out Tue Sep 16 02:02:48 2014
@@ -70,34 +70,40 @@ FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP)
+ Reducer 4 <- Map 3 (GROUP)
+ Union 2 <- Map 1 (NONE), Map 5 (NONE), Reducer 4 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(2) (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Map 2
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Map 3
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -118,43 +124,54 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(1) (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
- Reducer 3
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+ Union 2
+ Vertex: Union 2
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
Stage: Stage-2
Dependency Collection
@@ -169,6 +186,42 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.outputtbl1
+ Stage: Stage-3
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Merge
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+
+ Stage: Stage-5
+ Spark
+#### A masked pattern was here ####
+ Vertices:
+ Merge
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.outputtbl1
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
PREHOOK: query: insert overwrite table outputTbl1
SELECT *
FROM (
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_7.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_7.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_7.q.out Tue Sep 16 02:02:48 2014
@@ -74,7 +74,8 @@ STAGE PLANS:
Spark
Edges:
Reducer 2 <- Map 1 (GROUP)
- Reducer 4 <- Map 3 (GROUP)
+ Reducer 5 <- Map 4 (GROUP)
+ Union 3 <- Reducer 2 (NONE), Reducer 5 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -98,7 +99,7 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 3
+ Map 4
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -126,39 +127,41 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
- Reducer 4
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
+ Reducer 5
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
+ Union 3
+ Vertex: Union 3
Stage: Stage-2
Dependency Collection
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_8.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_8.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_8.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_8.q.out Tue Sep 16 02:02:48 2014
@@ -79,27 +79,28 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
- Reducer 3 <- Map 2 (GROUP)
+ Reducer 4 <- Map 3 (GROUP)
+ Union 2 <- Map 1 (NONE), Map 5 (NONE), Reducer 4 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(2) (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
- Map 2
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
+ Map 3
Map Operator Tree:
TableScan
alias: inputtbl1
@@ -120,43 +121,45 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
value expressions: _col1 (type: bigint)
- Map 4
+ Map 5
Map Operator Tree:
TableScan
alias: inputtbl1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), UDFToLong(1) (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
- Reducer 3
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
+ Reducer 4
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
+ Union 2
+ Vertex: Union 2
Stage: Stage-2
Dependency Collection
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_9.q.out?rev=1625203&r1=1625202&r2=1625203&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_9.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_9.q.out Tue Sep 16 02:02:48 2014
@@ -74,15 +74,20 @@ select * FROM (
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-2 depends on stages: Stage-4, Stage-3, Stage-6
Stage-0 depends on stages: Stage-2
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
STAGE PLANS:
Stage: Stage-1
Spark
Edges:
Reducer 4 <- Map 3 (GROUP)
- Union 2 <- Map 1 (NONE), Map 5 (NONE)
+ Union 2 <- Map 1 (NONE), Map 5 (NONE), Reducer 4 (NONE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -95,13 +100,16 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
Map 3
Map Operator Tree:
TableScan
@@ -133,13 +141,16 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: string), UDFToLong(_col1) (type: bigint)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
Reducer 4
Reduce Operator Tree:
Group By Operator
@@ -147,22 +158,31 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- name: default.outputtbl1
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.outputtbl1
Union 2
Vertex: Union 2
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
Stage: Stage-2
Dependency Collection
@@ -176,6 +196,22 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
name: default.outputtbl1
+ Stage: Stage-3
+ Merge Work
+ merge level: block
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+
+ Stage: Stage-5
+ Merge Work
+ merge level: block
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
PREHOOK: query: insert overwrite table outputTbl1
SELECT * FROM
(