You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2017/01/03 18:39:59 UTC
hive git commit: HIVE-15528: Expose Spark job error in SparkTask
(Zhihai via Xuefu)
Repository: hive
Updated Branches:
refs/heads/master 5d45974e9 -> c928ad3d3
HIVE-15528: Expose Spark job error in SparkTask (Zhihai via Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c928ad3d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c928ad3d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c928ad3d
Branch: refs/heads/master
Commit: c928ad3d3f958d1e2e109b689fc5c6e9ee3e619b
Parents: 5d45974
Author: Xuefu Zhang <xu...@uber.com>
Authored: Tue Jan 3 10:39:39 2017 -0800
Committer: Xuefu Zhang <xu...@uber.com>
Committed: Tue Jan 3 10:39:39 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/exec/spark/SparkTask.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c928ad3d/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
index f836065..87d80a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
@@ -114,7 +114,7 @@ public class SparkTask extends Task<SparkWork> {
this.jobID = jobRef.getSparkJobStatus().getAppID();
rc = jobRef.monitorJob();
SparkJobStatus sparkJobStatus = jobRef.getSparkJobStatus();
- getSparkJobInfo(sparkJobStatus);
+ getSparkJobInfo(sparkJobStatus, rc);
if (rc == 0) {
sparkStatistics = sparkJobStatus.getSparkStatistics();
if (LOG.isInfoEnabled() && sparkStatistics != null) {
@@ -139,6 +139,7 @@ public class SparkTask extends Task<SparkWork> {
// org.apache.commons.lang.StringUtils
console.printError(msg, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
LOG.error(msg, e);
+ setException(e);
rc = 1;
} finally {
startTime = perfLogger.getEndTime(PerfLogger.SPARK_SUBMIT_TO_RUNNING);
@@ -196,6 +197,7 @@ public class SparkTask extends Task<SparkWork> {
String mesg = "Job Commit failed with exception '"
+ Utilities.getNameMessage(e) + "'";
console.printError(mesg, "\n" + StringUtils.stringifyException(e));
+ setException(e);
}
}
return rc;
@@ -330,7 +332,7 @@ public class SparkTask extends Task<SparkWork> {
return counters;
}
- private void getSparkJobInfo(SparkJobStatus sparkJobStatus) {
+ private void getSparkJobInfo(SparkJobStatus sparkJobStatus, int rc) {
try {
stageIds = new ArrayList<Integer>();
int[] ids = sparkJobStatus.getStageIds();
@@ -355,6 +357,12 @@ public class SparkTask extends Task<SparkWork> {
succeededTaskCount = sumComplete;
totalTaskCount = sumTotal;
failedTaskCount = sumFailed;
+ if (rc != 0) {
+ Throwable error = sparkJobStatus.getError();
+ if (error != null) {
+ setException(error);
+ }
+ }
} catch (Exception e) {
LOG.error("Failed to get Spark job information", e);
}