You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/02/22 19:00:19 UTC
spark git commit: [SPARK-23475][UI] Show also skipped stages
Repository: spark
Updated Branches:
refs/heads/master 45cf714ee -> 87293c746
[SPARK-23475][UI] Show also skipped stages
## What changes were proposed in this pull request?
SPARK-20648 introduced the status `SKIPPED` for the stages. On the UI, previously, skipped stages were shown as `PENDING`; after this change, they are not shown on the UI.
The PR introduce a new section in order to show also `SKIPPED` stages in a proper table.
## How was this patch tested?
manual tests
Author: Marco Gaido <ma...@gmail.com>
Closes #20651 from mgaido91/SPARK-23475.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/87293c74
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/87293c74
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/87293c74
Branch: refs/heads/master
Commit: 87293c746e19d66f475d506d0adb43421f496843
Parents: 45cf714
Author: Marco Gaido <ma...@gmail.com>
Authored: Thu Feb 22 11:00:12 2018 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Thu Feb 22 11:00:12 2018 -0800
----------------------------------------------------------------------
.../org/apache/spark/ui/static/webui.js | 1 +
.../apache/spark/ui/jobs/AllStagesPage.scala | 27 ++++++++++++++++++++
.../org/apache/spark/ui/UISeleniumSuite.scala | 17 ++++++++++++
3 files changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/87293c74/core/src/main/resources/org/apache/spark/ui/static/webui.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index 83009df..f01c567 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -72,6 +72,7 @@ $(function() {
collapseTablePageLoad('collapse-aggregated-allActiveStages','aggregated-allActiveStages');
collapseTablePageLoad('collapse-aggregated-allPendingStages','aggregated-allPendingStages');
collapseTablePageLoad('collapse-aggregated-allCompletedStages','aggregated-allCompletedStages');
+ collapseTablePageLoad('collapse-aggregated-allSkippedStages','aggregated-allSkippedStages');
collapseTablePageLoad('collapse-aggregated-allFailedStages','aggregated-allFailedStages');
collapseTablePageLoad('collapse-aggregated-activeStages','aggregated-activeStages');
collapseTablePageLoad('collapse-aggregated-pendingOrSkippedStages','aggregated-pendingOrSkippedStages');
http://git-wip-us.apache.org/repos/asf/spark/blob/87293c74/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
index 606dc1e..38450b9 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
@@ -36,6 +36,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val activeStages = allStages.filter(_.status == StageStatus.ACTIVE)
val pendingStages = allStages.filter(_.status == StageStatus.PENDING)
+ val skippedStages = allStages.filter(_.status == StageStatus.SKIPPED)
val completedStages = allStages.filter(_.status == StageStatus.COMPLETE)
val failedStages = allStages.filter(_.status == StageStatus.FAILED).reverse
@@ -51,6 +52,9 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val completedStagesTable =
new StageTableBase(parent.store, request, completedStages, "completed", "completedStage",
parent.basePath, subPath, parent.isFairScheduler, false, false)
+ val skippedStagesTable =
+ new StageTableBase(parent.store, request, skippedStages, "skipped", "skippedStage",
+ parent.basePath, subPath, parent.isFairScheduler, false, false)
val failedStagesTable =
new StageTableBase(parent.store, request, failedStages, "failed", "failedStage",
parent.basePath, subPath, parent.isFairScheduler, false, true)
@@ -66,6 +70,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val shouldShowActiveStages = activeStages.nonEmpty
val shouldShowPendingStages = pendingStages.nonEmpty
val shouldShowCompletedStages = completedStages.nonEmpty
+ val shouldShowSkippedStages = skippedStages.nonEmpty
val shouldShowFailedStages = failedStages.nonEmpty
val appSummary = parent.store.appSummary()
@@ -103,6 +108,14 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
}
}
{
+ if (shouldShowSkippedStages) {
+ <li id="completed-summary">
+ <a href="#skipped"><strong>Skipped Stages:</strong></a>
+ {skippedStages.size}
+ </li>
+ }
+ }
+ {
if (shouldShowFailedStages) {
<li>
<a href="#failed"><strong>Failed Stages:</strong></a>
@@ -172,6 +185,20 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
{completedStagesTable.toNodeSeq}
</div>
}
+ if (shouldShowSkippedStages) {
+ content ++=
+ <span id="skipped" class="collapse-aggregated-allSkippedStages collapse-table"
+ onClick="collapseTable('collapse-aggregated-allSkippedStages',
+ 'aggregated-allSkippedStages')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a>Skipped Stages ({skippedStages.size})</a>
+ </h4>
+ </span> ++
+ <div class="aggregated-allSkippedStages collapsible-table">
+ {skippedStagesTable.toNodeSeq}
+ </div>
+ }
if (shouldShowFailedStages) {
content ++=
<span id ="failed" class="collapse-aggregated-allFailedStages collapse-table"
http://git-wip-us.apache.org/repos/asf/spark/blob/87293c74/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index ed51fc4..e86cadf 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -707,6 +707,23 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
}
}
+ test("stages page should show skipped stages") {
+ withSpark(newSparkContext()) { sc =>
+ val rdd = sc.parallelize(0 to 100, 100).repartition(10).cache()
+ rdd.count()
+ rdd.count()
+
+ eventually(timeout(5 seconds), interval(50 milliseconds)) {
+ goToUi(sc, "/stages")
+ find(id("skipped")).get.text should be("Skipped Stages (1)")
+ }
+ val stagesJson = getJson(sc.ui.get, "stages")
+ stagesJson.children.size should be (4)
+ val stagesStatus = stagesJson.children.map(_ \ "status")
+ stagesStatus.count(_ == JString(StageStatus.SKIPPED.name())) should be (1)
+ }
+ }
+
def goToUi(sc: SparkContext, path: String): Unit = {
goToUi(sc.ui.get, path)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org