You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/10/21 04:45:05 UTC
svn commit: r1633268 [5/7] - in /hive/branches/spark:
itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/
ql/src/java/org/apache/hadoop/hive/ql/parse/spark/
ql/src/java/org/apache/had...
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out?rev=1633268&r1=1633267&r2=1633268&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out Tue Oct 21 02:45:04 2014
@@ -30,38 +30,22 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -77,6 +61,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
+ Filter Operator
+ predicate: ((key > 10) and (key < 20)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
Stage: Stage-3
Dependency Collection
@@ -91,7 +90,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -104,31 +103,8 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
- Stats-Aggr Operator
-
Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
+ Stats-Aggr Operator
PREHOOK: query: from src
insert overwrite table src_multi1 select * where key < 10
@@ -197,38 +173,22 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -244,6 +204,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
+ Filter Operator
+ predicate: ((key > 10) and (key < 20)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
Stage: Stage-3
Dependency Collection
@@ -258,7 +233,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -271,31 +246,8 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
- Stats-Aggr Operator
-
Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
+ Stats-Aggr Operator
PREHOOK: query: from src
insert overwrite table src_multi1 select * where key < 10
@@ -364,38 +316,22 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -411,6 +347,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
+ Filter Operator
+ predicate: ((key > 10) and (key < 20)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
Stage: Stage-3
Dependency Collection
@@ -425,7 +376,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -438,31 +389,8 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
- Stats-Aggr Operator
-
Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
+ Stats-Aggr Operator
PREHOOK: query: from src
insert overwrite table src_multi1 select * where key < 10
@@ -531,38 +459,22 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -578,6 +490,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
+ Filter Operator
+ predicate: ((key > 10) and (key < 20)) (type: boolean)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
Stage: Stage-3
Dependency Collection
@@ -592,7 +519,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -605,31 +532,8 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
- Stats-Aggr Operator
-
Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((key > 10) and (key < 20)) (type: boolean)
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
+ Stats-Aggr Operator
PREHOOK: query: from src
insert overwrite table src_multi1 select * where key < 10
@@ -1354,72 +1258,83 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Union 4 <- Map 3 (NONE, 0), Map 5 (NONE, 0)
+ Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0)
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Map 5
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Map 3
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Union 4
- Vertex: Union 4
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: (_col0 < 10) (type: boolean)
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi1
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Union 2
+ Vertex: Union 2
Stage: Stage-3
Dependency Collection
@@ -1434,7 +1349,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -1447,32 +1362,9 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
+ Stage: Stage-5
Stats-Aggr Operator
- Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
-
PREHOOK: query: from (select * from src union all select * from src) s
insert overwrite table src_multi1 select * where key < 10
insert overwrite table src_multi2 select * where key > 10 and key < 20
@@ -1559,72 +1451,83 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Union 4 <- Map 3 (NONE, 0), Map 5 (NONE, 0)
+ Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0)
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Map 5
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Map 3
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Union 4
- Vertex: Union 4
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: (_col0 < 10) (type: boolean)
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi1
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Union 2
+ Vertex: Union 2
Stage: Stage-3
Dependency Collection
@@ -1639,7 +1542,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -1652,31 +1555,8 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
- Stats-Aggr Operator
-
Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
+ Stats-Aggr Operator
PREHOOK: query: from (select * from src union all select * from src) s
insert overwrite table src_multi1 select * where key < 10
@@ -1764,72 +1644,83 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Union 4 <- Map 3 (NONE, 0), Map 5 (NONE, 0)
+ Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0)
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Map 5
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Map 3
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Union 4
- Vertex: Union 4
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: (_col0 < 10) (type: boolean)
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi1
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Union 2
+ Vertex: Union 2
Stage: Stage-3
Dependency Collection
@@ -1844,7 +1735,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -1857,31 +1748,8 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
- Stats-Aggr Operator
-
Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
+ Stats-Aggr Operator
PREHOOK: query: from (select * from src union all select * from src) s
insert overwrite table src_multi1 select * where key < 10
@@ -1969,72 +1837,83 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
- Stage-4 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-4, Stage-5
+ Stage-3 depends on stages: Stage-2
Stage-0 depends on stages: Stage-3
- Stage-6 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
Stage-1 depends on stages: Stage-3
- Stage-7 depends on stages: Stage-1
- Stage-5 depends on stages: Stage-2
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
Spark
Edges:
- Union 4 <- Map 3 (NONE, 0), Map 5 (NONE, 0)
+ Union 2 <- Map 1 (NONE, 0), Map 3 (NONE, 0)
#### A masked pattern was here ####
Vertices:
- Map 3
+ Map 1
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Map 5
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Map 3
Map Operator Tree:
TableScan
alias: src
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Union 4
- Vertex: Union 4
-
- Stage: Stage-4
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: (_col0 < 10) (type: boolean)
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 333 Data size: 3537 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi1
+ Filter Operator
+ predicate: (_col0 < 10) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+ Filter Operator
+ predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+ Union 2
+ Vertex: Union 2
Stage: Stage-3
Dependency Collection
@@ -2049,7 +1928,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
- Stage: Stage-6
+ Stage: Stage-4
Stats-Aggr Operator
Stage: Stage-1
@@ -2062,31 +1941,8 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
- Stage: Stage-7
- Stats-Aggr Operator
-
Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
- Filter Operator
- predicate: ((_col0 > 10) and (_col0 < 20)) (type: boolean)
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 111 Data size: 1179 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
+ Stats-Aggr Operator
PREHOOK: query: from (select * from src union all select * from src) s
insert overwrite table src_multi1 select * where key < 10
@@ -2173,38 +2029,21 @@ from src
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-3 is a root stage
- Stage-5 depends on stages: Stage-3
- Stage-4 depends on stages: Stage-5, Stage-6, Stage-7
- Stage-0 depends on stages: Stage-5
- Stage-6 depends on stages: Stage-3
- Stage-1 depends on stages: Stage-6
- Stage-7 depends on stages: Stage-3
- Stage-2 depends on stages: Stage-7
+ Stage-4 depends on stages: Stage-3
+ Stage-0 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-3
+ Stage-2 depends on stages: Stage-3
STAGE PLANS:
Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2219,23 +2058,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-4
- Dependency Collection
-
- Stage: Stage-0
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 2) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2250,20 +2072,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-1
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-7
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 4) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2279,6 +2087,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Stage: Stage-4
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
+ Stage: Stage-1
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
Stage: Stage-2
Move Operator
files:
@@ -2307,38 +2130,21 @@ from src
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-3 is a root stage
- Stage-5 depends on stages: Stage-3
- Stage-4 depends on stages: Stage-5, Stage-6, Stage-7
- Stage-0 depends on stages: Stage-5
- Stage-6 depends on stages: Stage-3
- Stage-1 depends on stages: Stage-6
- Stage-7 depends on stages: Stage-3
- Stage-2 depends on stages: Stage-7
+ Stage-4 depends on stages: Stage-3
+ Stage-0 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-3
+ Stage-2 depends on stages: Stage-3
STAGE PLANS:
Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2353,23 +2159,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-4
- Dependency Collection
-
- Stage: Stage-0
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 2) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2384,20 +2173,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-1
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-7
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 4) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2413,6 +2188,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Stage: Stage-4
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
+ Stage: Stage-1
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
Stage: Stage-2
Move Operator
files:
@@ -2441,38 +2231,21 @@ from src
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-3 is a root stage
- Stage-5 depends on stages: Stage-3
- Stage-4 depends on stages: Stage-5, Stage-6, Stage-7
- Stage-0 depends on stages: Stage-5
- Stage-6 depends on stages: Stage-3
- Stage-1 depends on stages: Stage-6
- Stage-7 depends on stages: Stage-3
- Stage-2 depends on stages: Stage-7
+ Stage-4 depends on stages: Stage-3
+ Stage-0 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-3
+ Stage-2 depends on stages: Stage-3
STAGE PLANS:
Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2487,23 +2260,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-4
- Dependency Collection
-
- Stage: Stage-0
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 2) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2518,20 +2274,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-1
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-7
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 4) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2547,6 +2289,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Stage: Stage-4
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
+ Stage: Stage-1
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
Stage: Stage-2
Move Operator
files:
@@ -2575,38 +2332,21 @@ from src
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-3 is a root stage
- Stage-5 depends on stages: Stage-3
- Stage-4 depends on stages: Stage-5, Stage-6, Stage-7
- Stage-0 depends on stages: Stage-5
- Stage-6 depends on stages: Stage-3
- Stage-1 depends on stages: Stage-6
- Stage-7 depends on stages: Stage-3
- Stage-2 depends on stages: Stage-7
+ Stage-4 depends on stages: Stage-3
+ Stage-0 depends on stages: Stage-3
+ Stage-1 depends on stages: Stage-3
+ Stage-2 depends on stages: Stage-3
STAGE PLANS:
Stage: Stage-3
Spark
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-5
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 0) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2621,23 +2361,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-4
- Dependency Collection
-
- Stage: Stage-0
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 2) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2652,20 +2375,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-1
- Move Operator
- files:
- hdfs directory: false
-#### A masked pattern was here ####
-
- Stage: Stage-7
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key = 4) (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
@@ -2681,6 +2390,21 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Stage: Stage-4
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
+ Stage: Stage-1
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
Stage: Stage-2
Move Operator
files:
@@ -3249,41 +2973,28 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-4 is a root stage
- Stage-6 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-6, Stage-7, Stage-8
+ Stage-5 depends on stages: Stage-4
Stage-0 depends on stages: Stage-5
- Stage-9 depends on stages: Stage-0
+ Stage-6 depends on stages: Stage-0
Stage-1 depends on stages: Stage-5
- Stage-10 depends on stages: Stage-1
- Stage-7 depends on stages: Stage-4
- Stage-8 depends on stages: Stage-4
- Stage-2 depends on stages: Stage-8
- Stage-3 depends on stages: Stage-8
+ Stage-7 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-4
+ Stage-3 depends on stages: Stage-4
STAGE PLANS:
Stage: Stage-4
Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP SORT, 1)
+ Reducer 4 <- Reducer 2 (GROUP SORT, 1)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -3299,43 +3010,6 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
-
- Stage: Stage-5
- Dependency Collection
-
- Stage: Stage-0
- Move Operator
- tables:
- replace: true
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi1
-
- Stage: Stage-9
- Stats-Aggr Operator
-
- Stage: Stage-1
- Move Operator
- tables:
- replace: true
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
-
- Stage: Stage-10
- Stats-Aggr Operator
-
- Stage: Stage-7
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
Filter Operator
predicate: ((key > 10) and (key < 20)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
@@ -3351,18 +3025,6 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
-
- Stage: Stage-8
- Spark
- Edges:
- Reducer 5 <- Map 3 (GROUP SORT, 1)
- Reducer 6 <- Reducer 5 (GROUP SORT, 1)
- Reducer 7 <- Reducer 5 (GROUP SORT, 1)
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
Filter Operator
predicate: ((key < 10) or ((key > 10) and (key < 20))) (type: boolean)
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3375,7 +3037,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: key (type: string), value (type: string)
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
- Reducer 5
+ Reducer 2
Reduce Operator Tree:
Forward
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3415,7 +3077,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Reducer 6
+ Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
@@ -3428,7 +3090,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 7
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
@@ -3442,6 +3104,35 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+
+ Stage: Stage-6
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
Stage: Stage-2
Move Operator
files:
@@ -3529,41 +3220,28 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-4 is a root stage
- Stage-6 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-6, Stage-7, Stage-8
+ Stage-5 depends on stages: Stage-4
Stage-0 depends on stages: Stage-5
- Stage-9 depends on stages: Stage-0
+ Stage-6 depends on stages: Stage-0
Stage-1 depends on stages: Stage-5
- Stage-10 depends on stages: Stage-1
- Stage-7 depends on stages: Stage-4
- Stage-8 depends on stages: Stage-4
- Stage-2 depends on stages: Stage-8
- Stage-3 depends on stages: Stage-8
+ Stage-7 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-4
+ Stage-3 depends on stages: Stage-4
STAGE PLANS:
Stage: Stage-4
Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP SORT, 1)
+ Reducer 4 <- Reducer 2 (GROUP SORT, 1)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -3579,43 +3257,6 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
-
- Stage: Stage-5
- Dependency Collection
-
- Stage: Stage-0
- Move Operator
- tables:
- replace: true
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi1
-
- Stage: Stage-9
- Stats-Aggr Operator
-
- Stage: Stage-1
- Move Operator
- tables:
- replace: true
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
-
- Stage: Stage-10
- Stats-Aggr Operator
-
- Stage: Stage-7
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
Filter Operator
predicate: ((key > 10) and (key < 20)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
@@ -3631,18 +3272,6 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
-
- Stage: Stage-8
- Spark
- Edges:
- Reducer 5 <- Map 3 (GROUP SORT, 1)
- Reducer 6 <- Reducer 5 (GROUP SORT, 1)
- Reducer 7 <- Reducer 5 (GROUP SORT, 1)
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
Filter Operator
predicate: ((key < 10) or ((key > 10) and (key < 20))) (type: boolean)
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3655,7 +3284,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: key (type: string), value (type: string)
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
- Reducer 5
+ Reducer 2
Reduce Operator Tree:
Forward
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3695,7 +3324,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Reducer 6
+ Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
@@ -3708,7 +3337,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 7
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
@@ -3722,6 +3351,35 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+
+ Stage: Stage-6
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
Stage: Stage-2
Move Operator
files:
@@ -3809,41 +3467,28 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-4 is a root stage
- Stage-6 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-6, Stage-7, Stage-8
+ Stage-5 depends on stages: Stage-4
Stage-0 depends on stages: Stage-5
- Stage-9 depends on stages: Stage-0
+ Stage-6 depends on stages: Stage-0
Stage-1 depends on stages: Stage-5
- Stage-10 depends on stages: Stage-1
- Stage-7 depends on stages: Stage-4
- Stage-8 depends on stages: Stage-4
- Stage-2 depends on stages: Stage-8
- Stage-3 depends on stages: Stage-8
+ Stage-7 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-4
+ Stage-3 depends on stages: Stage-4
STAGE PLANS:
Stage: Stage-4
Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP SORT, 1)
+ Reducer 4 <- Reducer 2 (GROUP SORT, 1)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
@@ -3859,43 +3504,6 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi1
-
- Stage: Stage-5
- Dependency Collection
-
- Stage: Stage-0
- Move Operator
- tables:
- replace: true
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi1
-
- Stage: Stage-9
- Stats-Aggr Operator
-
- Stage: Stage-1
- Move Operator
- tables:
- replace: true
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.src_multi2
-
- Stage: Stage-10
- Stats-Aggr Operator
-
- Stage: Stage-7
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 2
- Map Operator Tree:
- TableScan
Filter Operator
predicate: ((key > 10) and (key < 20)) (type: boolean)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
@@ -3911,18 +3519,6 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.src_multi2
-
- Stage: Stage-8
- Spark
- Edges:
- Reducer 5 <- Map 3 (GROUP SORT, 1)
- Reducer 6 <- Reducer 5 (GROUP SORT, 1)
- Reducer 7 <- Reducer 5 (GROUP SORT, 1)
-#### A masked pattern was here ####
- Vertices:
- Map 3
- Map Operator Tree:
- TableScan
Filter Operator
predicate: ((key < 10) or ((key > 10) and (key < 20))) (type: boolean)
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3935,7 +3531,7 @@ STAGE PLANS:
sort order: ++
Map-reduce partition columns: key (type: string), value (type: string)
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
- Reducer 5
+ Reducer 2
Reduce Operator Tree:
Forward
Statistics: Num rows: 221 Data size: 2347 Basic stats: COMPLETE Column stats: NONE
@@ -3975,7 +3571,7 @@ STAGE PLANS:
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string)
- Reducer 6
+ Reducer 3
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
@@ -3988,7 +3584,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Reducer 7
+ Reducer 4
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
@@ -4002,6 +3598,35 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Stage: Stage-5
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi1
+
+ Stage: Stage-6
+ Stats-Aggr Operator
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src_multi2
+
+ Stage: Stage-7
+ Stats-Aggr Operator
+
Stage: Stage-2
Move Operator
files:
@@ -4089,41 +3714,28 @@ insert overwrite table src_multi2 select
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-4 is a root stage
- Stage-6 depends on stages: Stage-4
- Stage-5 depends on stages: Stage-6, Stage-7, Stage-8
+ Stage-5 depends on stages: Stage-4
Stage-0 depends on stages: Stage-5
- Stage-9 depends on stages: Stage-0
+ Stage-6 depends on stages: Stage-0
Stage-1 depends on stages: Stage-5
- Stage-10 depends on stages: Stage-1
- Stage-7 depends on stages: Stage-4
- Stage-8 depends on stages: Stage-4
- Stage-2 depends on stages: Stage-8
- Stage-3 depends on stages: Stage-8
+ Stage-7 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-4
+ Stage-3 depends on stages: Stage-4
STAGE PLANS:
Stage: Stage-4
Spark
+ Edges:
+ Reducer 2 <- Map 1 (GROUP SORT, 1)
+ Reducer 3 <- Reducer 2 (GROUP SORT, 1)
+ Reducer 4 <- Reducer 2 (GROUP SORT, 1)
#### A masked pattern was here ####
Vertices:
- Map 4
+ Map 1
Map Operator Tree:
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-6
- Spark
-#### A masked pattern was here ####
- Vertices:
- Map 1
- Map Operator Tree:
- TableScan
Filter Operator
predicate: (key < 10) (type: boolean)
[... 128 lines stripped ...]