You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ka...@apache.org on 2014/11/25 03:03:22 UTC
spark git commit: [SPARK-4266] [Web-UI] Reduce stage page load time.
Repository: spark
Updated Branches:
refs/heads/master 6cf507685 -> d24d5bf06
[SPARK-4266] [Web-UI] Reduce stage page load time.
The commit changes the java script used to show/hide additional
metrics in order to reduce page load time. SPARK-4016 significantly
increased page load time for the stage page when stages had a lot
(thousands or tens of thousands) of tasks, due to the additional
Javascript to hide some metrics by default and stripe the tables.
This commit reduces page load time in two ways:
(1) Now, all of the metrics that are hidden by default are
hidden by setting "display: none;" using CSS for the page,
rather than hiding them using javascript after the page loads.
Without this change, for stages with thousands of tasks, there
was a few second delay after page load, where first the additional
metrics were shown, and then after a delay were hidden once the
relevant JS finished running.
(2) CSS is used to stripe all of the tables except for the summary
table. The summary table needs javascript to do the striping because
some rows are hidden, but the javascript striping is slower, which
again resulted in a delay when it was used for the task table (where
for a few seconds after page load, all of the rows in the task table
would be white, while the browser finished running the JS to stripe
the table).
cc pwendell
This change is intended to be backported to 1.2 to avoid a regression in
UI performance when users run large jobs.
Author: Kay Ousterhout <ka...@gmail.com>
Closes #3328 from kayousterhout/SPARK-4266 and squashes the following commits:
f964091 [Kay Ousterhout] [SPARK-4266] [Web-UI] Reduce stage page load time.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d24d5bf0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d24d5bf0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d24d5bf0
Branch: refs/heads/master
Commit: d24d5bf064572a2319627736b1fbf112b4a78edf
Parents: 6cf5076
Author: Kay Ousterhout <ka...@gmail.com>
Authored: Mon Nov 24 18:03:10 2014 -0800
Committer: Kay Ousterhout <ka...@gmail.com>
Committed: Mon Nov 24 18:03:10 2014 -0800
----------------------------------------------------------------------
.../spark/ui/static/additional-metrics.js | 9 ++------
.../org/apache/spark/ui/static/table.js | 24 +++++++++++---------
.../org/apache/spark/ui/static/webui.css | 6 +++++
.../scala/org/apache/spark/ui/UIUtils.scala | 11 ++++-----
.../apache/spark/ui/exec/ExecutorsPage.scala | 2 +-
.../apache/spark/ui/jobs/ExecutorTable.scala | 2 +-
.../org/apache/spark/ui/jobs/StagePage.scala | 9 +++++++-
.../spark/ui/jobs/TaskDetailsClassNames.scala | 3 +++
8 files changed, 39 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js b/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js
index d33c5c7..14ba37d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js
@@ -28,17 +28,12 @@ $(function() {
$(this).find('.expand-additional-metrics-arrow').toggleClass('arrow-closed');
});
- $("input:checkbox:not(:checked)").each(function() {
- var column = "table ." + $(this).attr("name");
- $(column).hide();
- });
- // Stripe table rows after rows have been hidden to ensure correct striping.
- stripeTables();
+ stripeSummaryTable();
$("input:checkbox").click(function() {
var column = "table ." + $(this).attr("name");
$(column).toggle();
- stripeTables();
+ stripeSummaryTable();
});
$("#select-all-metrics").click(function() {
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/resources/org/apache/spark/ui/static/table.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/table.js b/core/src/main/resources/org/apache/spark/ui/static/table.js
index 6bb0301..656147e 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/table.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/table.js
@@ -15,16 +15,18 @@
* limitations under the License.
*/
-/* Adds background colors to stripe table rows. This is necessary (instead of using css or the
- * table striping provided by bootstrap) to appropriately stripe tables with hidden rows. */
-function stripeTables() {
- $("table.table-striped-custom").each(function() {
- $(this).find("tr:not(:hidden)").each(function (index) {
- if (index % 2 == 1) {
- $(this).css("background-color", "#f9f9f9");
- } else {
- $(this).css("background-color", "#ffffff");
- }
- });
+/* Adds background colors to stripe table rows in the summary table (on the stage page). This is
+ * necessary (instead of using css or the table striping provided by bootstrap) because the summary
+ * table has hidden rows.
+ *
+ * An ID selector (rather than a class selector) is used to ensure this runs quickly even on pages
+ * with thousands of task rows (ID selectors are much faster than class selectors). */
+function stripeSummaryTable() {
+ $("#task-summary-table").find("tr:not(:hidden)").each(function (index) {
+ if (index % 2 == 1) {
+ $(this).css("background-color", "#f9f9f9");
+ } else {
+ $(this).css("background-color", "#ffffff");
+ }
});
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/resources/org/apache/spark/ui/static/webui.css
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css
index db57712..cdf85bf 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css
@@ -168,3 +168,9 @@ span.additional-metric-title {
border-left: 5px solid black;
display: inline-block;
}
+
+/* Hide all additional metrics by default. This is done here rather than using JavaScript to
+ * avoid slow page loads for stage pages with large numbers (e.g., thousands) of tasks. */
+.scheduler_delay, .gc_time, .deserialization_time, .serialization_time, .getting_result_time {
+ display: none;
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 0c418be..09079bb 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -26,7 +26,8 @@ import org.apache.spark.Logging
/** Utility functions for generating XML pages with spark content. */
private[spark] object UIUtils extends Logging {
- val TABLE_CLASS = "table table-bordered table-striped-custom table-condensed sortable"
+ val TABLE_CLASS_NOT_STRIPED = "table table-bordered table-condensed sortable"
+ val TABLE_CLASS_STRIPED = TABLE_CLASS_NOT_STRIPED + " table-striped"
// SimpleDateFormat is not thread-safe. Don't expose it to avoid improper use.
private val dateFormat = new ThreadLocal[SimpleDateFormat]() {
@@ -248,12 +249,10 @@ private[spark] object UIUtils extends Logging {
data: Iterable[T],
fixedWidth: Boolean = false,
id: Option[String] = None,
- headerClasses: Seq[String] = Seq.empty): Seq[Node] = {
+ headerClasses: Seq[String] = Seq.empty,
+ stripeRowsWithCss: Boolean = true): Seq[Node] = {
- var listingTableClass = TABLE_CLASS
- if (fixedWidth) {
- listingTableClass += " table-fixed"
- }
+ val listingTableClass = if (stripeRowsWithCss) TABLE_CLASS_STRIPED else TABLE_CLASS_NOT_STRIPED
val colWidth = 100.toDouble / headers.size
val colWidthAttr = if (fixedWidth) colWidth + "%" else ""
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
index 71b59b1..363cb96 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
@@ -57,7 +57,7 @@ private[ui] class ExecutorsPage(
val execInfoSorted = execInfo.sortBy(_.id)
val execTable =
- <table class={UIUtils.TABLE_CLASS}>
+ <table class={UIUtils.TABLE_CLASS_STRIPED}>
<thead>
<th>Executor ID</th>
<th>Address</th>
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
index 35bbe8b..9836d11 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
@@ -36,7 +36,7 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage
/** Special table which merges two header cells. */
private def executorTable[T](): Seq[Node] = {
- <table class={UIUtils.TABLE_CLASS}>
+ <table class={UIUtils.TABLE_CLASS_STRIPED}>
<thead>
<th>Executor ID</th>
<th>Address</th>
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 40e05f8..bfa54f8 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -322,8 +322,15 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val quantileHeaders = Seq("Metric", "Min", "25th percentile",
"Median", "75th percentile", "Max")
+ // The summary table does not use CSS to stripe rows, which doesn't work with hidden
+ // rows (instead, JavaScript in table.js is used to stripe the non-hidden rows).
Some(UIUtils.listingTable(
- quantileHeaders, identity[Seq[Node]], listings, fixedWidth = true))
+ quantileHeaders,
+ identity[Seq[Node]],
+ listings,
+ fixedWidth = true,
+ id = Some("task-summary-table"),
+ stripeRowsWithCss = false))
}
val executorTable = new ExecutorTable(stageId, stageAttemptId, parent)
http://git-wip-us.apache.org/repos/asf/spark/blob/d24d5bf0/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala b/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala
index eb371bd..ca942c4 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/TaskDetailsClassNames.scala
@@ -20,6 +20,9 @@ package org.apache.spark.ui.jobs
/**
* Names of the CSS classes corresponding to each type of task detail. Used to allow users
* to optionally show/hide columns.
+ *
+ * If new optional metrics are added here, they should also be added to the end of webui.css
+ * to have the style set to "display: none;" by default.
*/
private object TaskDetailsClassNames {
val SCHEDULER_DELAY = "scheduler_delay"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org