You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/05/22 12:58:46 UTC
spark git commit: [SPARK-20591][WEB UI] Succeeded tasks num not equal
in all jobs page and job detail page on spark web ui when speculative task(s)
exist.
Repository: spark
Updated Branches:
refs/heads/master be846db48 -> 190d8b0b6
[SPARK-20591][WEB UI] Succeeded tasks num not equal in all jobs page and job detail page on spark web ui when speculative task(s) exist.
## What changes were proposed in this pull request?
Modified succeeded num in job detail page from "completed = stageData.completedIndices.size" to "completed = stageData.numCompleteTasks",which making succeeded tasks num in all jobs page and job detail page look more consistent, and more easily to find which stages the speculative task(s) were in.
## How was this patch tested?
manual tests
Author: fjh100456 <fu...@zte.com.cn>
Closes #17923 from fjh100456/master.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/190d8b0b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/190d8b0b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/190d8b0b
Branch: refs/heads/master
Commit: 190d8b0b6393290349ab834457f9e916fdaef530
Parents: be846db
Author: fjh100456 <fu...@zte.com.cn>
Authored: Mon May 22 13:58:42 2017 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Mon May 22 13:58:42 2017 +0100
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala | 3 ++-
.../scala/org/apache/spark/ui/jobs/JobProgressListener.scala | 1 +
core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala | 1 +
core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala | 7 +------
4 files changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
index a0fd29c..cce7a76 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
@@ -631,7 +631,8 @@ private[ui] class JobPagedTable(
{if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"}
</td>
<td class="progress-cell">
- {UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks,
+ {UIUtils.makeProgressBar(started = job.numActiveTasks,
+ completed = job.completedIndices.size,
failed = job.numFailedTasks, skipped = job.numSkippedTasks,
reasonToNumKilled = job.reasonToNumKilled, total = job.numTasks - job.numSkippedTasks)}
</td>
http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index 7370f9f..1b10feb 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -423,6 +423,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
jobData.numActiveTasks -= 1
taskEnd.reason match {
case Success =>
+ jobData.completedIndices.add((taskEnd.stageId, info.index))
jobData.numCompletedTasks += 1
case kill: TaskKilled =>
jobData.reasonToNumKilled = jobData.reasonToNumKilled.updated(
http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
index 8d280bc..048c4ad 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
@@ -62,6 +62,7 @@ private[spark] object UIData {
var numTasks: Int = 0,
var numActiveTasks: Int = 0,
var numCompletedTasks: Int = 0,
+ var completedIndices: OpenHashSet[(Int, Int)] = new OpenHashSet[(Int, Int)](),
var numSkippedTasks: Int = 0,
var numFailedTasks: Int = 0,
var reasonToNumKilled: Map[String, Int] = Map.empty,
http://git-wip-us.apache.org/repos/asf/spark/blob/190d8b0b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index bdd1488..267c8dc 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -320,12 +320,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
eventually(timeout(5 seconds), interval(50 milliseconds)) {
goToUi(sc, "/jobs")
find(cssSelector(".stage-progress-cell")).get.text should be ("2/2 (1 failed)")
- // Ideally, the following test would pass, but currently we overcount completed tasks
- // if task recomputations occur:
- // find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)")
- // Instead, we guarantee that the total number of tasks is always correct, while the number
- // of completed tasks may be higher:
- find(cssSelector(".progress-cell .progress")).get.text should be ("3/2 (1 failed)")
+ find(cssSelector(".progress-cell .progress")).get.text should be ("2/2 (1 failed)")
}
val jobJson = getJson(sc.ui.get, "jobs")
(jobJson \ "numTasks").extract[Int]should be (2)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org