You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/02/24 02:27:41 UTC
spark git commit: [SPARK-23475][UI][BACKPORT-2.3] Show also skipped
stages
Repository: spark
Updated Branches:
refs/heads/branch-2.3 285b841ff -> 578607b30
[SPARK-23475][UI][BACKPORT-2.3] Show also skipped stages
## What changes were proposed in this pull request?
SPARK-20648 introduced the status `SKIPPED` for the stages. On the UI, previously, skipped stages were shown as `PENDING`; after this change, they are not shown on the UI.
The PR introduce a new section in order to show also `SKIPPED` stages in a proper table.
Manual backport from to branch-2.3.
## How was this patch tested?
added UT
Author: Marco Gaido <ma...@gmail.com>
Closes #20662 from mgaido91/SPARK-23475_2.3.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/578607b3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/578607b3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/578607b3
Branch: refs/heads/branch-2.3
Commit: 578607b30a65d101380acbe9c9740dc267a5d55c
Parents: 285b841
Author: Marco Gaido <ma...@gmail.com>
Authored: Fri Feb 23 18:27:33 2018 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Fri Feb 23 18:27:33 2018 -0800
----------------------------------------------------------------------
.../apache/spark/ui/jobs/AllStagesPage.scala | 27 ++++++++++++++++++++
.../org/apache/spark/ui/UISeleniumSuite.scala | 17 ++++++++++++
2 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/578607b3/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
index b1e3434..9325b90 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
@@ -36,6 +36,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val activeStages = allStages.filter(_.status == StageStatus.ACTIVE)
val pendingStages = allStages.filter(_.status == StageStatus.PENDING)
+ val skippedStages = allStages.filter(_.status == StageStatus.SKIPPED)
val completedStages = allStages.filter(_.status == StageStatus.COMPLETE)
val failedStages = allStages.filter(_.status == StageStatus.FAILED).reverse
@@ -51,6 +52,9 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val completedStagesTable =
new StageTableBase(parent.store, request, completedStages, "completed", "completedStage",
parent.basePath, subPath, parent.isFairScheduler, false, false)
+ val skippedStagesTable =
+ new StageTableBase(parent.store, request, skippedStages, "skipped", "skippedStage",
+ parent.basePath, subPath, parent.isFairScheduler, false, false)
val failedStagesTable =
new StageTableBase(parent.store, request, failedStages, "failed", "failedStage",
parent.basePath, subPath, parent.isFairScheduler, false, true)
@@ -66,6 +70,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val shouldShowActiveStages = activeStages.nonEmpty
val shouldShowPendingStages = pendingStages.nonEmpty
val shouldShowCompletedStages = completedStages.nonEmpty
+ val shouldShowSkippedStages = skippedStages.nonEmpty
val shouldShowFailedStages = failedStages.nonEmpty
val appSummary = parent.store.appSummary()
@@ -103,6 +108,14 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
}
}
{
+ if (shouldShowSkippedStages) {
+ <li>
+ <a href="#skipped"><strong>Skipped Stages:</strong></a>
+ {skippedStages.size}
+ </li>
+ }
+ }
+ {
if (shouldShowFailedStages) {
<li>
<a href="#failed"><strong>Failed Stages:</strong></a>
@@ -133,6 +146,20 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
content ++= <h4 id="completed">Completed Stages ({completedStageNumStr})</h4> ++
completedStagesTable.toNodeSeq
}
+ if (shouldShowSkippedStages) {
+ content ++=
+ <span id="skipped" class="collapse-aggregated-allSkippedStages collapse-table"
+ onClick="collapseTable('collapse-aggregated-allSkippedStages',
+ 'aggregated-allSkippedStages')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a>Skipped Stages ({skippedStages.size})</a>
+ </h4>
+ </span> ++
+ <div class="aggregated-allSkippedStages collapsible-table">
+ {skippedStagesTable.toNodeSeq}
+ </div>
+ }
if (shouldShowFailedStages) {
content ++= <h4 id ="failed">Failed Stages ({numFailedStages})</h4> ++
failedStagesTable.toNodeSeq
http://git-wip-us.apache.org/repos/asf/spark/blob/578607b3/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 3265467..0f20eea 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -706,6 +706,23 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
}
}
+ test("stages page should show skipped stages") {
+ withSpark(newSparkContext()) { sc =>
+ val rdd = sc.parallelize(0 to 100, 100).repartition(10).cache()
+ rdd.count()
+ rdd.count()
+
+ eventually(timeout(5 seconds), interval(50 milliseconds)) {
+ goToUi(sc, "/stages")
+ find(id("skipped")).get.text should be("Skipped Stages (1)")
+ }
+ val stagesJson = getJson(sc.ui.get, "stages")
+ stagesJson.children.size should be (4)
+ val stagesStatus = stagesJson.children.map(_ \ "status")
+ stagesStatus.count(_ == JString(StageStatus.SKIPPED.name())) should be (1)
+ }
+ }
+
def goToUi(sc: SparkContext, path: String): Unit = {
goToUi(sc.ui.get, path)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org