You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/04/19 19:57:23 UTC
[1/5] hive git commit: HIVE-16473 : Hive-on-Tez may fail to write to
an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/branch-1 f8db95803 -> bf5a718b3
refs/heads/branch-2 318252417 -> 1542e39b8
refs/heads/branch-2.2 f13de0bf2 -> 48002e7d4
refs/heads/branch-2.3 d9927aa02 -> 1d5374a9a
refs/heads/master 5c018a5d9 -> 788c04fec
HIVE-16473 : Hive-on-Tez may fail to write to an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/788c04fe
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/788c04fe
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/788c04fe
Branch: refs/heads/master
Commit: 788c04fec4813ca60e0270874e22dcd633e85ab7
Parents: 5c018a5
Author: sergey <se...@apache.org>
Authored: Wed Apr 19 12:27:04 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Apr 19 12:27:04 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/788c04fe/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 6c8bf29..1c84c6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -324,6 +328,14 @@ public class TezTask extends Task<TezWork> {
}
}
+ void checkOutputSpec(BaseWork work, JobConf jc) throws IOException {
+ for (Operator<?> op : work.getAllOperators()) {
+ if (op instanceof FileSinkOperator) {
+ ((FileSinkOperator) op).checkOutputSpecs(null, jc);
+ }
+ }
+ }
+
DAG build(JobConf conf, TezWork work, Path scratchDir,
LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
throws Exception {
@@ -357,7 +369,6 @@ public class TezTask extends Task<TezWork> {
setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf);
for (BaseWork w: ws) {
-
boolean isFinal = work.getLeaves().contains(w);
// translate work to vertex
@@ -379,6 +390,8 @@ public class TezTask extends Task<TezWork> {
children.add(v);
}
}
+ JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+ checkOutputSpec(w, parentConf);
// create VertexGroup
Vertex[] vertexArray = new Vertex[unionWorkItems.size()];
@@ -391,7 +404,7 @@ public class TezTask extends Task<TezWork> {
// For a vertex group, all Outputs use the same Key-class, Val-class and partitioner.
// Pick any one source vertex to figure out the Edge configuration.
- JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+
// now hook up the children
for (BaseWork v: children) {
@@ -404,6 +417,7 @@ public class TezTask extends Task<TezWork> {
} else {
// Regular vertices
JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
+ checkOutputSpec(w, wxConf);
Vertex wx =
utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
work, work.getVertexType(w));
[2/5] hive git commit: HIVE-16473 : Hive-on-Tez may fail to write to
an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Posted by se...@apache.org.
HIVE-16473 : Hive-on-Tez may fail to write to an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1d5374a9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1d5374a9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1d5374a9
Branch: refs/heads/branch-2.3
Commit: 1d5374a9a64edef01c3628858078475a39f78883
Parents: d9927aa
Author: sergey <se...@apache.org>
Authored: Wed Apr 19 12:27:04 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Apr 19 12:31:50 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1d5374a9/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 6c8bf29..1c84c6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -324,6 +328,14 @@ public class TezTask extends Task<TezWork> {
}
}
+ void checkOutputSpec(BaseWork work, JobConf jc) throws IOException {
+ for (Operator<?> op : work.getAllOperators()) {
+ if (op instanceof FileSinkOperator) {
+ ((FileSinkOperator) op).checkOutputSpecs(null, jc);
+ }
+ }
+ }
+
DAG build(JobConf conf, TezWork work, Path scratchDir,
LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
throws Exception {
@@ -357,7 +369,6 @@ public class TezTask extends Task<TezWork> {
setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf);
for (BaseWork w: ws) {
-
boolean isFinal = work.getLeaves().contains(w);
// translate work to vertex
@@ -379,6 +390,8 @@ public class TezTask extends Task<TezWork> {
children.add(v);
}
}
+ JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+ checkOutputSpec(w, parentConf);
// create VertexGroup
Vertex[] vertexArray = new Vertex[unionWorkItems.size()];
@@ -391,7 +404,7 @@ public class TezTask extends Task<TezWork> {
// For a vertex group, all Outputs use the same Key-class, Val-class and partitioner.
// Pick any one source vertex to figure out the Edge configuration.
- JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+
// now hook up the children
for (BaseWork v: children) {
@@ -404,6 +417,7 @@ public class TezTask extends Task<TezWork> {
} else {
// Regular vertices
JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
+ checkOutputSpec(w, wxConf);
Vertex wx =
utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
work, work.getVertexType(w));
[5/5] hive git commit: HIVE-16473 : Hive-on-Tez may fail to write to
an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Posted by se...@apache.org.
HIVE-16473 : Hive-on-Tez may fail to write to an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1542e39b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1542e39b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1542e39b
Branch: refs/heads/branch-2
Commit: 1542e39b818f7f48e089dabf4bd115e119068d1d
Parents: 3182524
Author: sergey <se...@apache.org>
Authored: Wed Apr 19 12:27:04 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Apr 19 12:57:06 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1542e39b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 6c8bf29..1c84c6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -324,6 +328,14 @@ public class TezTask extends Task<TezWork> {
}
}
+ void checkOutputSpec(BaseWork work, JobConf jc) throws IOException {
+ for (Operator<?> op : work.getAllOperators()) {
+ if (op instanceof FileSinkOperator) {
+ ((FileSinkOperator) op).checkOutputSpecs(null, jc);
+ }
+ }
+ }
+
DAG build(JobConf conf, TezWork work, Path scratchDir,
LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
throws Exception {
@@ -357,7 +369,6 @@ public class TezTask extends Task<TezWork> {
setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf);
for (BaseWork w: ws) {
-
boolean isFinal = work.getLeaves().contains(w);
// translate work to vertex
@@ -379,6 +390,8 @@ public class TezTask extends Task<TezWork> {
children.add(v);
}
}
+ JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+ checkOutputSpec(w, parentConf);
// create VertexGroup
Vertex[] vertexArray = new Vertex[unionWorkItems.size()];
@@ -391,7 +404,7 @@ public class TezTask extends Task<TezWork> {
// For a vertex group, all Outputs use the same Key-class, Val-class and partitioner.
// Pick any one source vertex to figure out the Edge configuration.
- JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+
// now hook up the children
for (BaseWork v: children) {
@@ -404,6 +417,7 @@ public class TezTask extends Task<TezWork> {
} else {
// Regular vertices
JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
+ checkOutputSpec(w, wxConf);
Vertex wx =
utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
work, work.getVertexType(w));
[3/5] hive git commit: HIVE-16473 : Hive-on-Tez may fail to write to
an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Posted by se...@apache.org.
HIVE-16473 : Hive-on-Tez may fail to write to an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/48002e7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/48002e7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/48002e7d
Branch: refs/heads/branch-2.2
Commit: 48002e7d4356657b15e41f5795dc9df9e574932f
Parents: f13de0b
Author: sergey <se...@apache.org>
Authored: Wed Apr 19 12:27:04 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Apr 19 12:32:12 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/48002e7d/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index cd616c6..66015c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -319,6 +323,14 @@ public class TezTask extends Task<TezWork> {
}
}
+ void checkOutputSpec(BaseWork work, JobConf jc) throws IOException {
+ for (Operator<?> op : work.getAllOperators()) {
+ if (op instanceof FileSinkOperator) {
+ ((FileSinkOperator) op).checkOutputSpecs(null, jc);
+ }
+ }
+ }
+
DAG build(JobConf conf, TezWork work, Path scratchDir,
LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
throws Exception {
@@ -352,7 +364,6 @@ public class TezTask extends Task<TezWork> {
setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf);
for (BaseWork w: ws) {
-
boolean isFinal = work.getLeaves().contains(w);
// translate work to vertex
@@ -374,6 +385,8 @@ public class TezTask extends Task<TezWork> {
children.add(v);
}
}
+ JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+ checkOutputSpec(w, parentConf);
// create VertexGroup
Vertex[] vertexArray = new Vertex[unionWorkItems.size()];
@@ -386,7 +399,7 @@ public class TezTask extends Task<TezWork> {
// For a vertex group, all Outputs use the same Key-class, Val-class and partitioner.
// Pick any one source vertex to figure out the Edge configuration.
- JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+
// now hook up the children
for (BaseWork v: children) {
@@ -399,6 +412,7 @@ public class TezTask extends Task<TezWork> {
} else {
// Regular vertices
JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
+ checkOutputSpec(w, wxConf);
Vertex wx =
utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
work, work.getVertexType(w));
[4/5] hive git commit: HIVE-16473 : Hive-on-Tez may fail to write to
an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Posted by se...@apache.org.
HIVE-16473 : Hive-on-Tez may fail to write to an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bf5a718b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bf5a718b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bf5a718b
Branch: refs/heads/branch-1
Commit: bf5a718b352cf3da9679e38898ebb77975c88f5b
Parents: f8db958
Author: sergey <se...@apache.org>
Authored: Wed Apr 19 12:27:04 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Apr 19 12:56:45 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bf5a718b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 3c10169..f6cf8c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+
+import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -273,6 +278,14 @@ public class TezTask extends Task<TezWork> {
}
}
+ void checkOutputSpec(BaseWork work, JobConf jc) throws IOException {
+ for (Operator<?> op : work.getAllOperators()) {
+ if (op instanceof FileSinkOperator) {
+ ((FileSinkOperator) op).checkOutputSpecs(null, jc);
+ }
+ }
+ }
+
DAG build(JobConf conf, TezWork work, Path scratchDir,
LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
throws Exception {
@@ -304,7 +317,6 @@ public class TezTask extends Task<TezWork> {
setAccessControlsForCurrentUser(dag);
for (BaseWork w: ws) {
-
boolean isFinal = work.getLeaves().contains(w);
// translate work to vertex
@@ -326,6 +338,8 @@ public class TezTask extends Task<TezWork> {
children.add(v);
}
}
+ JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+ checkOutputSpec(w, parentConf);
// create VertexGroup
Vertex[] vertexArray = new Vertex[unionWorkItems.size()];
@@ -338,7 +352,7 @@ public class TezTask extends Task<TezWork> {
// For a vertex group, all Outputs use the same Key-class, Val-class and partitioner.
// Pick any one source vertex to figure out the Edge configuration.
- JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+
// now hook up the children
for (BaseWork v: children) {
@@ -351,6 +365,7 @@ public class TezTask extends Task<TezWork> {
} else {
// Regular vertices
JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
+ checkOutputSpec(w, wxConf);
Vertex wx =
utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
work, work.getVertexType(w));