You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by st...@apache.org on 2018/02/15 23:20:37 UTC
hive git commit: HIVE-18672: Printed state in RemoteSparkJobMonitor
is ambiguous (Sahil Takiar, reviewed by Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master 217811254 -> fd561935d
HIVE-18672: Printed state in RemoteSparkJobMonitor is ambiguous (Sahil Takiar, reviewed by Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fd561935
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fd561935
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fd561935
Branch: refs/heads/master
Commit: fd561935d4d1e34f25bc4c851a8acf37ec7879a8
Parents: 2178112
Author: Sahil Takiar <ta...@gmail.com>
Authored: Thu Feb 15 15:01:36 2018 -0800
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Thu Feb 15 15:01:36 2018 -0800
----------------------------------------------------------------------
.../exec/spark/status/RemoteSparkJobMonitor.java | 18 +++++++++---------
.../ql/exec/spark/status/SparkJobMonitor.java | 2 +-
2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fd561935/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
index adb65a5..3467ae4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java
@@ -62,10 +62,11 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING);
startTime = System.currentTimeMillis();
+ JobHandle.State state = null;
while (true) {
try {
- JobHandle.State state = sparkJobStatus.getRemoteJobState();
+ state = sparkJobStatus.getRemoteJobState();
Preconditions.checkState(sparkJobStatus.isRemoteActive(), "Connection to remote Spark driver was lost");
switch (state) {
@@ -76,14 +77,13 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
HiveException he = new HiveException(ErrorMsg.SPARK_JOB_MONITOR_TIMEOUT,
Long.toString(timeCount));
console.printError(he.getMessage());
- console.printError("Status: " + state);
sparkJobStatus.setError(he);
running = false;
done = true;
rc = 2;
}
if (LOG.isDebugEnabled()) {
- console.printInfo("state = " + state);
+ console.printInfo("Spark job state = " + state );
}
break;
case STARTED:
@@ -98,8 +98,7 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
console.printInfo("\nQuery Hive on Spark job[" + sparkJobStatus.getJobId() +
"] stages: " + Arrays.toString(sparkJobStatus.getStageIds()));
- console.printInfo("\nStatus: Running (Hive on Spark job["
- + sparkJobStatus.getJobId() + "])");
+ console.printInfo("Spark job[" + sparkJobStatus.getJobId() + "] status = RUNNING");
running = true;
String format = "Job Progress Format\nCurrentTime StageId_StageAttemptId: "
@@ -142,8 +141,8 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
printStatus(progressMap, lastProgressMap);
lastProgressMap = progressMap;
double duration = (System.currentTimeMillis() - startTime) / 1000.0;
- console.printInfo("Status: Finished successfully in "
- + String.format("%.2f seconds", duration));
+ console.printInfo("Spark job[" + sparkJobStatus.getJobId() + "] finished successfully in "
+ + String.format("%.2f second(s)", duration));
running = false;
done = true;
break;
@@ -176,7 +175,7 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
rc = 3;
break;
case CANCELLED:
- console.printInfo("Status: Cancelled");
+ console.printInfo("Spark job[" + sparkJobStatus.getJobId() + " was cancelled");
running = false;
done = true;
rc = 3;
@@ -193,7 +192,8 @@ public class RemoteSparkJobMonitor extends SparkJobMonitor {
finalException = new HiveException(e, ErrorMsg.SPARK_JOB_INTERRUPTED);
LOG.warn("Interrupted while monitoring the Hive on Spark application, exiting");
} else {
- String msg = " with exception '" + Utilities.getNameMessage(e) + "'";
+ String msg = " with exception '" + Utilities.getNameMessage(e) + "' Last known state = " +
+ (state != null ? state.name() : "UNKNOWN");
msg = "Failed to monitor Job[" + sparkJobStatus.getJobId() + "]" + msg;
// Has to use full name to make sure it does not conflict with
http://git-wip-us.apache.org/repos/asf/hive/blob/fd561935/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
index b3e49fe..7afd886 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java
@@ -82,7 +82,7 @@ abstract class SparkJobMonitor {
monitorTimeoutInterval = hiveConf.getTimeVar(
HiveConf.ConfVars.SPARK_JOB_MONITOR_TIMEOUT, TimeUnit.SECONDS);
inPlaceUpdate = InPlaceUpdate.canRenderInPlace(hiveConf) && !SessionState.getConsole().getIsSilent();
- console = SessionState.getConsole();
+ console = new SessionState.LogHelper(LOG);
out = SessionState.LogHelper.getInfoStream();
}