You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by su...@apache.org on 2017/01/24 18:04:16 UTC
hive git commit: HIVE-15662: check startTime in SparkTask to make
sure startTime is not less than submitTime (Zhihai Xu, reviewed by Chao Sun)
Repository: hive
Updated Branches:
refs/heads/master 719125cf1 -> e37500a88
HIVE-15662: check startTime in SparkTask to make sure startTime is not less than submitTime (Zhihai Xu, reviewed by Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e37500a8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e37500a8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e37500a8
Branch: refs/heads/master
Commit: e37500a88ddc4988c09fe0b5f942279714d077e5
Parents: 719125c
Author: Zhihai Xu <zh...@gmail.com>
Authored: Tue Jan 24 10:03:27 2017 -0800
Committer: Chao Sun <su...@apache.org>
Committed: Tue Jan 24 10:03:27 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e37500a8/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
index 2a0aeda..4c01329 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java
@@ -144,6 +144,13 @@ public class SparkTask extends Task<SparkWork> {
rc = 1;
} finally {
startTime = perfLogger.getEndTime(PerfLogger.SPARK_SUBMIT_TO_RUNNING);
+ // The startTime may not be set if the sparkTask finished too fast,
+ // because SparkJobMonitor will sleep for 1 second then check the state,
+ // right after sleep, the spark job may be already completed.
+ // In this case, set startTime the same as submitTime.
+ if (startTime < submitTime) {
+ startTime = submitTime;
+ }
finishTime = perfLogger.getEndTime(PerfLogger.SPARK_RUN_JOB);
Utilities.clearWork(conf);
if (sparkSession != null && sparkSessionManager != null) {