You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/04/19 19:57:25 UTC
[3/5] hive git commit: HIVE-16473 : Hive-on-Tez may fail to write to
an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
HIVE-16473 : Hive-on-Tez may fail to write to an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/48002e7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/48002e7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/48002e7d
Branch: refs/heads/branch-2.2
Commit: 48002e7d4356657b15e41f5795dc9df9e574932f
Parents: f13de0b
Author: sergey <se...@apache.org>
Authored: Wed Apr 19 12:27:04 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Apr 19 12:32:12 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/48002e7d/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index cd616c6..66015c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -319,6 +323,14 @@ public class TezTask extends Task<TezWork> {
}
}
+ void checkOutputSpec(BaseWork work, JobConf jc) throws IOException {
+ for (Operator<?> op : work.getAllOperators()) {
+ if (op instanceof FileSinkOperator) {
+ ((FileSinkOperator) op).checkOutputSpecs(null, jc);
+ }
+ }
+ }
+
DAG build(JobConf conf, TezWork work, Path scratchDir,
LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
throws Exception {
@@ -352,7 +364,6 @@ public class TezTask extends Task<TezWork> {
setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf);
for (BaseWork w: ws) {
-
boolean isFinal = work.getLeaves().contains(w);
// translate work to vertex
@@ -374,6 +385,8 @@ public class TezTask extends Task<TezWork> {
children.add(v);
}
}
+ JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+ checkOutputSpec(w, parentConf);
// create VertexGroup
Vertex[] vertexArray = new Vertex[unionWorkItems.size()];
@@ -386,7 +399,7 @@ public class TezTask extends Task<TezWork> {
// For a vertex group, all Outputs use the same Key-class, Val-class and partitioner.
// Pick any one source vertex to figure out the Edge configuration.
- JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+
// now hook up the children
for (BaseWork v: children) {
@@ -399,6 +412,7 @@ public class TezTask extends Task<TezWork> {
} else {
// Regular vertices
JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
+ checkOutputSpec(w, wxConf);
Vertex wx =
utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
work, work.getVertexType(w));