You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ka...@apache.org on 2015/07/21 03:09:08 UTC
spark git commit: [SPARK-9187] [WEBUI] Timeline view may show
negative value for running tasks
Repository: spark
Updated Branches:
refs/heads/master ff3c72dba -> 66bb8003b
[SPARK-9187] [WEBUI] Timeline view may show negative value for running tasks
For running tasks, the executorRunTime metrics is 0 which causes negative executorComputingTime in the timeline. It also causes an incorrect SchedulerDelay time.
Author: Carson Wang <ca...@intel.com>
Closes #7526 from carsonwang/timeline-negValue and squashes the following commits:
7b17db2 [Carson Wang] Fix negative value in timeline view
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/66bb8003
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/66bb8003
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/66bb8003
Branch: refs/heads/master
Commit: 66bb8003b949860b8652542e1232bc48665448c2
Parents: ff3c72d
Author: Carson Wang <ca...@intel.com>
Authored: Mon Jul 20 18:08:59 2015 -0700
Committer: Kay Ousterhout <ka...@gmail.com>
Committed: Mon Jul 20 18:08:59 2015 -0700
----------------------------------------------------------------------
.../org/apache/spark/ui/jobs/StagePage.scala | 23 +++++++++++++-------
1 file changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/66bb8003/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 27b82aa..6e077bf 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -537,20 +537,27 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
(metricsOpt.flatMap(_.shuffleWriteMetrics
.map(_.shuffleWriteTime)).getOrElse(0L) / 1e6).toLong
val shuffleWriteTimeProportion = toProportion(shuffleWriteTime)
- val executorComputingTime = metricsOpt.map(_.executorRunTime).getOrElse(0L) -
- shuffleReadTime - shuffleWriteTime
- val executorComputingTimeProportion = toProportion(executorComputingTime)
+
val serializationTime = metricsOpt.map(_.resultSerializationTime).getOrElse(0L)
val serializationTimeProportion = toProportion(serializationTime)
val deserializationTime = metricsOpt.map(_.executorDeserializeTime).getOrElse(0L)
val deserializationTimeProportion = toProportion(deserializationTime)
val gettingResultTime = getGettingResultTime(taskUIData.taskInfo, currentTime)
val gettingResultTimeProportion = toProportion(gettingResultTime)
- val schedulerDelay = totalExecutionTime -
- (executorComputingTime + shuffleReadTime + shuffleWriteTime +
- serializationTime + deserializationTime + gettingResultTime)
- val schedulerDelayProportion =
- (100 - executorComputingTimeProportion - shuffleReadTimeProportion -
+ val schedulerDelay =
+ metricsOpt.map(getSchedulerDelay(taskInfo, _, currentTime)).getOrElse(0L)
+ val schedulerDelayProportion = toProportion(schedulerDelay)
+
+ val executorOverhead = serializationTime + deserializationTime
+ val executorRunTime = if (taskInfo.running) {
+ totalExecutionTime - executorOverhead - gettingResultTime
+ } else {
+ metricsOpt.map(_.executorRunTime).getOrElse(
+ totalExecutionTime - executorOverhead - gettingResultTime)
+ }
+ val executorComputingTime = executorRunTime - shuffleReadTime - shuffleWriteTime
+ val executorComputingTimeProportion =
+ (100 - schedulerDelayProportion - shuffleReadTimeProportion -
shuffleWriteTimeProportion - serializationTimeProportion -
deserializationTimeProportion - gettingResultTimeProportion)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org