You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/02/25 00:00:59 UTC
svn commit: r1571481 - in /hive/branches/tez/ql/src:
java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
test/results/clientpositive/tez/insert1.q.out
test/results/clientpositive/tez/load_dyn_part1.q.out
Author: gunther
Date: Mon Feb 24 23:00:59 2014
New Revision: 1571481
URL: http://svn.apache.org/r1571481
Log:
HIVE-6442: load_dyn_part1 is flaky on Tez because it doesn't have the stage re-arranger (Gunther Hagleitner)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
hive/branches/tez/ql/src/test/results/clientpositive/tez/insert1.q.out
hive/branches/tez/ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1571481&r1=1571480&r2=1571481&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Mon Feb 24 23:00:59 2014
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer;
+import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MoveWork;
@@ -251,6 +252,9 @@ public class TezCompiler extends TaskCom
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
(new Vectorizer()).resolve(physicalCtx);
}
+ if (!"none".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVESTAGEIDREARRANGE))) {
+ (new StageIDsRearranger()).resolve(physicalCtx);
+ }
return;
}
}
Modified: hive/branches/tez/ql/src/test/results/clientpositive/tez/insert1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/tez/insert1.q.out?rev=1571481&r1=1571480&r2=1571481&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/tez/insert1.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/tez/insert1.q.out Mon Feb 24 23:00:59 2014
@@ -264,10 +264,10 @@ POSTHOOK: Lineage: insert1.value SIMPLE
STAGE DEPENDENCIES:
Stage-2 is a root stage
Stage-3 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-3
- Stage-4 depends on stages: Stage-1
Stage-0 depends on stages: Stage-3
- Stage-5 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-3
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -313,28 +313,28 @@ STAGE PLANS:
Stage: Stage-3
Dependency Collection
- Stage: Stage-1
+ Stage: Stage-0
Move Operator
tables:
- replace: true
+ replace: false
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: x.insert1
+ name: default.insert1
Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-0
+ Stage: Stage-1
Move Operator
tables:
- replace: false
+ replace: true
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.insert1
+ name: x.insert1
Stage: Stage-5
Stats-Aggr Operator
Modified: hive/branches/tez/ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out?rev=1571481&r1=1571480&r2=1571481&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/tez/load_dyn_part1.q.out Mon Feb 24 23:00:59 2014
@@ -45,10 +45,10 @@ POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-2 is a root stage
Stage-3 depends on stages: Stage-2
- Stage-1 depends on stages: Stage-3
- Stage-4 depends on stages: Stage-1
Stage-0 depends on stages: Stage-3
- Stage-5 depends on stages: Stage-0
+ Stage-4 depends on stages: Stage-0
+ Stage-1 depends on stages: Stage-3
+ Stage-5 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-2
@@ -94,34 +94,34 @@ STAGE PLANS:
Stage: Stage-3
Dependency Collection
- Stage: Stage-1
+ Stage: Stage-0
Move Operator
tables:
partition:
- ds 2008-12-31
+ ds
hr
replace: true
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.nzhang_part2
+ name: default.nzhang_part1
Stage: Stage-4
Stats-Aggr Operator
- Stage: Stage-0
+ Stage: Stage-1
Move Operator
tables:
partition:
- ds
+ ds 2008-12-31
hr
replace: true
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.nzhang_part1
+ name: default.nzhang_part2
Stage: Stage-5
Stats-Aggr Operator