You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/06/30 22:57:34 UTC
spark git commit: [SPARK-8560] [UI] The Executors page will have
negative if having resubmitted tasks
Repository: spark
Updated Branches:
refs/heads/master 61d7b533d -> 79f0b371a
[SPARK-8560] [UI] The Executors page will have negative if having resubmitted tasks
when the ```taskEnd.reason``` is ```Resubmitted```, it shouldn't do statistics. Because this tasks has a ```SUCCESS``` taskEnd before.
Author: xutingjun <xu...@huawei.com>
Closes #6950 from XuTingjun/pageError and squashes the following commits:
af35dc3 [xutingjun] When taskEnd is Resubmitted, don't do statistics
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/79f0b371
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/79f0b371
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/79f0b371
Branch: refs/heads/master
Commit: 79f0b371a36560a009c1b0943c928adc5a1bdd8f
Parents: 61d7b53
Author: xutingjun <xu...@huawei.com>
Authored: Tue Jun 30 13:56:59 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Tue Jun 30 13:57:27 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/ui/exec/ExecutorsTab.scala | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/79f0b371/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
index 39583af..a88fc4c 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
@@ -19,7 +19,7 @@ package org.apache.spark.ui.exec
import scala.collection.mutable.HashMap
-import org.apache.spark.{ExceptionFailure, SparkContext}
+import org.apache.spark.{Resubmitted, ExceptionFailure, SparkContext}
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.scheduler._
import org.apache.spark.storage.{StorageStatus, StorageStatusListener}
@@ -92,15 +92,22 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp
val info = taskEnd.taskInfo
if (info != null) {
val eid = info.executorId
- executorToTasksActive(eid) = executorToTasksActive.getOrElse(eid, 1) - 1
- executorToDuration(eid) = executorToDuration.getOrElse(eid, 0L) + info.duration
taskEnd.reason match {
+ case Resubmitted =>
+ // Note: For resubmitted tasks, we continue to use the metrics that belong to the
+ // first attempt of this task. This may not be 100% accurate because the first attempt
+ // could have failed half-way through. The correct fix would be to keep track of the
+ // metrics added by each attempt, but this is much more complicated.
+ return
case e: ExceptionFailure =>
executorToTasksFailed(eid) = executorToTasksFailed.getOrElse(eid, 0) + 1
case _ =>
executorToTasksComplete(eid) = executorToTasksComplete.getOrElse(eid, 0) + 1
}
+ executorToTasksActive(eid) = executorToTasksActive.getOrElse(eid, 1) - 1
+ executorToDuration(eid) = executorToDuration.getOrElse(eid, 0L) + info.duration
+
// Update shuffle read/write
val metrics = taskEnd.taskMetrics
if (metrics != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org