You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2015/05/01 10:37:47 UTC
[5/5] spark git commit: [SPARK-3468] [WEBUI] Timeline-View feature
[SPARK-3468] [WEBUI] Timeline-View feature
I sometimes trouble-shoot and analyse the cause of long time spending job.
At the time, I find the stages which spends long time or fails, then I find the tasks which spends long time or fails, next I analyse the proportion of each phase in a task.
Another case, I find executors which spends long time for running a task and analyse the details of a task.
In such situation, I think it's helpful to visualize timeline for each application, job, task and the details of proportion of activity for each task.
I added 3 timeline features to existing Web UI.
[Application Timeline View]
This view shows following things.
* When each executor was added/removed and the reason why it's removed.
* When each job was started/finished.
* Status of each job.
[Stage Timeline View]
Similar to Application Timeline View, this view shows following things.
* When each executor was added/removed and the reason why it's removed.
* When each job was started/finished.
* Status of each stage.
[Task Assignment Timeline View]
This view shows following things.
* When each task started/finished
* How long each task spent and the proportion.
* Status of each task.
* Where each task ran on.
All the view above is zoomable by mouse wheel action and scrollable by drag action.
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Closes #2342 from sarutak/timeline-viewer-feature and squashes the following commits:
11fe67d [Kousuke Saruta] Fixed conflict
79ac03d [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
a91abd3 [Kousuke Saruta] Merge branch 'master' of https://github.com/apache/spark into timeline-viewer-feature
ef34a5b [Kousuke Saruta] Implement tooltip using bootstrap
b09d0c5 [Kousuke Saruta] Move `stroke` and `fill` attribute of rect elements to css
d3c63c8 [Kousuke Saruta] Fixed a little bit bugs
a36291b [Kousuke Saruta] Merge branch 'master' of https://github.com/apache/spark into timeline-viewer-feature
28714b6 [Kousuke Saruta] Fixed highlight issue
0dc4278 [Kousuke Saruta] Addressed most of Patrics's feedbacks
8110acf [Kousuke Saruta] Added scroll limit to Job timeline
974a64a [Kousuke Saruta] Removed unused function
ee7a7f0 [Kousuke Saruta] Refactored
6a91872 [Kousuke Saruta] Temporary commit
6693f34 [Kousuke Saruta] Added link to job/stage box in the timeline in order to move to corresponding row when we click
8f88222 [Kousuke Saruta] Added job/stage description
aeed4b1 [Kousuke Saruta] Removed stage timeline
fc1696c [Kousuke Saruta] Merge branch 'timeline-viewer-feature' of github.com:sarutak/spark into timeline-viewer-feature
999ccd4 [Kousuke Saruta] Improved scalability
0fc6a31 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
19815ae [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
68b7540 [Kousuke Saruta] Merge branch 'timeline-viewer-feature' of github.com:sarutak/spark into timeline-viewer-feature
52b5f0b [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
dec85db [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
fcdab7d [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
dab7cc1 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
09cce97 [Kousuke Saruta] Cleanuped
16f82cf [Kousuke Saruta] Cleanuped
9fb522e [Kousuke Saruta] Cleanuped
d05f2c2 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into timeline-viewer-feature
e85e9aa [Kousuke Saruta] Cleanup: Added TimelineViewUtils.scala
a76e569 [Kousuke Saruta] Removed unused setting in timeline-view.css
5ce1b21 [Kousuke Saruta] Added vis.min.js, vis.min.css and vis.map to .rat-exclude
082f709 [Kousuke Saruta] Added Timeline-View feature for Applications, Jobs and Stages
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7fe0f3f2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7fe0f3f2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7fe0f3f2
Branch: refs/heads/master
Commit: 7fe0f3f2b46c61a5cc4af9166781624409fda8a4
Parents: c24aeb6
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Authored: Fri May 1 01:39:56 2015 -0700
Committer: Patrick Wendell <pa...@databricks.com>
Committed: Fri May 1 01:39:56 2015 -0700
----------------------------------------------------------------------
.rat-excludes | 3 +
.../apache/spark/ui/static/timeline-view.css | 182 +++++++++++++++++
.../org/apache/spark/ui/static/timeline-view.js | 168 ++++++++++++++++
.../org/apache/spark/ui/static/vis.map | 1 +
.../org/apache/spark/ui/static/vis.min.css | 1 +
.../org/apache/spark/ui/static/vis.min.js | 39 ++++
.../org/apache/spark/scheduler/StageInfo.scala | 12 ++
.../scala/org/apache/spark/ui/UIUtils.scala | 5 +
.../org/apache/spark/ui/exec/ExecutorsTab.scala | 11 +
.../org/apache/spark/ui/jobs/AllJobsPage.scala | 200 +++++++++++++++++--
.../org/apache/spark/ui/jobs/JobPage.scala | 174 +++++++++++++++-
.../spark/ui/jobs/JobProgressListener.scala | 7 +
.../org/apache/spark/ui/jobs/JobsTab.scala | 6 +-
.../org/apache/spark/ui/jobs/StageTable.scala | 3 +-
.../scala/org/apache/spark/ui/jobs/UIData.scala | 5 +
15 files changed, 785 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/.rat-excludes
----------------------------------------------------------------------
diff --git a/.rat-excludes b/.rat-excludes
index 8aca5a7..4468da1 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -30,6 +30,9 @@ log4j-defaults.properties
bootstrap-tooltip.js
jquery-1.11.1.min.js
sorttable.js
+vis.min.js
+vis.min.css
+vis.map
.*avsc
.*txt
.*json
http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
new file mode 100644
index 0000000..35ef14e
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+div#application-timeline, div#job-timeline {
+ margin-bottom: 30px;
+}
+
+#application-timeline div.legend-area {
+ margin-top: 5px;
+}
+
+.vis.timeline div.content {
+ width: 100%;
+}
+
+.vis.timeline .item.stage {
+ cursor: pointer;
+}
+
+.vis.timeline .item.stage.succeeded {
+ background-color: #D5DDF6;
+}
+
+.vis.timeline .item.stage.succeeded.selected {
+ background-color: #D5DDF6;
+ border-color: #97B0F8;
+ z-index: auto;
+}
+
+.legend-area rect.completed-stage-legend {
+ fill: #D5DDF6;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.stage.failed {
+ background-color: #FF5475;
+}
+
+.vis.timeline .item.stage.failed.selected {
+ background-color: #FF5475;
+ border-color: #97B0F8;
+ z-index: auto;
+}
+
+.legend-area rect.failed-stage-legend {
+ fill: #FF5475;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.stage.running {
+ background-color: #FDFFCA;
+}
+
+.vis.timeline .item.stage.running.selected {
+ background-color: #FDFFCA;
+ border-color: #97B0F8;
+ z-index: auto;
+}
+
+.legend-area rect.active-stage-legend {
+ fill: #FDFFCA;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.job {
+ cursor: pointer;
+}
+
+.vis.timeline .item.job.succeeded {
+ background-color: #D5DDF6;
+}
+
+.vis.timeline .item.job.succeeded.selected {
+ background-color: #D5DDF6;
+ border-color: #97B0F8;
+ z-index: auto;
+}
+
+.legend-area rect.succeeded-job-legend {
+ fill: #D5DDF6;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.job.failed {
+ background-color: #FF5475;
+}
+
+.vis.timeline .item.job.failed.selected {
+ background-color: #FF5475;
+ border-color: #97B0F8;
+ z-index: auto;
+}
+
+.legend-area rect.failed-job-legend {
+ fill: #FF5475;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.job.running {
+ background-color: #FDFFCA;
+}
+
+.vis.timeline .item.job.running.selected {
+ background-color: #FDFFCA;
+ border-color: #97B0F8;
+ z-index: auto;
+}
+
+.legend-area rect.running-job-legend {
+ fill: #FDFFCA;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.added {
+ background-color: #D5DDF6;
+}
+
+.legend-area rect.executor-added-legend {
+ fill: #D5DDF6;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.removed {
+ background-color: #EBCA59;
+}
+
+.legend-area rect.executor-removed-legend {
+ fill: #EBCA59;
+ stroke: #97B0F8;
+}
+
+.vis.timeline .item.executor.selected {
+ border-color: #FFC200;
+ background-color: #FFF785;
+ z-index: 2;
+}
+
+tr.corresponding-item-hover>td, tr.corresponding-item-hover>th {
+ background-color: #FFE1FA !important;
+}
+
+#application-timeline.collapsed {
+ display: none;
+}
+
+#job-timeline.collapsed {
+ display: none;
+}
+
+.control-panel {
+ margin-bottom: 5px;
+}
+
+span.expand-application-timeline, span.expand-job-timeline {
+ cursor: pointer;
+}
+
+.control-panel input+span {
+ cursor: pointer;
+}
+
+.vis.timeline .item.range .content {
+ position: unset;
+}
+
+.vis.timeline .item .tooltip-inner {
+ max-width: unset !important;
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/7fe0f3f2/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
new file mode 100644
index 0000000..e4a891d
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function drawApplicationTimeline(groupArray, eventObjArray, startTime) {
+ var groups = new vis.DataSet(groupArray);
+ var items = new vis.DataSet(eventObjArray);
+ var container = $("#application-timeline")[0];
+ var options = {
+ groupOrder: function(a, b) {
+ return a.value - b.value
+ },
+ editable: false,
+ showCurrentTime: false,
+ min: startTime,
+ zoomable: false
+ };
+
+ var applicationTimeline = new vis.Timeline(container);
+ applicationTimeline.setOptions(options);
+ applicationTimeline.setGroups(groups);
+ applicationTimeline.setItems(items);
+
+ setupZoomable("#application-timeline-zoom-lock", applicationTimeline);
+ setupExecutorEventAction();
+
+ function setupJobEventAction() {
+ $(".item.range.job.application-timeline-object").each(function() {
+ var getJobId = function(baseElem) {
+ var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
+ var jobId = jobIdText.match("\\(Job (\\d+)\\)")[1];
+ return jobId;
+ };
+
+ $(this).click(function() {
+ window.location.href = "job/?id=" + getJobId(this);
+ });
+
+ $(this).hover(
+ function() {
+ $("#job-" + getJobId(this)).addClass("corresponding-item-hover");
+ $($(this).find("div.application-timeline-content")[0]).tooltip("show");
+ },
+ function() {
+ $("#job-" + getJobId(this)).removeClass("corresponding-item-hover");
+ $($(this).find("div.application-timeline-content")[0]).tooltip("hide");
+ }
+ );
+ });
+ }
+
+ setupJobEventAction();
+
+ $("span.expand-application-timeline").click(function() {
+ $("#application-timeline").toggleClass('collapsed');
+
+ // Switch the class of the arrow from open to closed.
+ $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-open');
+ $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-closed');
+ });
+}
+
+function drawJobTimeline(groupArray, eventObjArray, startTime) {
+ var groups = new vis.DataSet(groupArray);
+ var items = new vis.DataSet(eventObjArray);
+ var container = $('#job-timeline')[0];
+ var options = {
+ groupOrder: function(a, b) {
+ return a.value - b.value;
+ },
+ editable: false,
+ showCurrentTime: false,
+ min: startTime,
+ zoomable: false,
+ };
+
+ var jobTimeline = new vis.Timeline(container);
+ jobTimeline.setOptions(options);
+ jobTimeline.setGroups(groups);
+ jobTimeline.setItems(items);
+
+ setupZoomable("#job-timeline-zoom-lock", jobTimeline);
+ setupExecutorEventAction();
+
+ function setupStageEventAction() {
+ $(".item.range.stage.job-timeline-object").each(function() {
+ var getStageIdAndAttempt = function(baseElem) {
+ var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
+ var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)")[1].split(".");
+ return stageIdAndAttempt;
+ };
+
+ $(this).click(function() {
+ var idAndAttempt = getStageIdAndAttempt(this);
+ var id = idAndAttempt[0];
+ var attempt = idAndAttempt[1];
+ window.location.href = "../../stages/stage/?id=" + id + "&attempt=" + attempt;
+ });
+
+ $(this).hover(
+ function() {
+ var idAndAttempt = getStageIdAndAttempt(this);
+ var id = idAndAttempt[0];
+ var attempt = idAndAttempt[1];
+ $("#stage-" + id + "-" + attempt).addClass("corresponding-item-hover");
+ $($(this).find("div.job-timeline-content")[0]).tooltip("show");
+ },
+ function() {
+ var idAndAttempt = getStageIdAndAttempt(this);
+ var id = idAndAttempt[0];
+ var attempt = idAndAttempt[1];
+ $("#stage-" + id + "-" + attempt).removeClass("corresponding-item-hover");
+ $($(this).find("div.job-timeline-content")[0]).tooltip("hide");
+ }
+ );
+ });
+ }
+
+ setupStageEventAction();
+
+ $("span.expand-job-timeline").click(function() {
+ $("#job-timeline").toggleClass('collapsed');
+
+ // Switch the class of the arrow from open to closed.
+ $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-open');
+ $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-closed');
+ });
+}
+
+function setupExecutorEventAction() {
+ $(".item.box.executor").each(function () {
+ $(this).hover(
+ function() {
+ $($(this).find(".executor-event-content")[0]).tooltip("show");
+ },
+ function() {
+ $($(this).find(".executor-event-content")[0]).tooltip("hide");
+ }
+ );
+ });
+}
+
+function setupZoomable(id, timeline) {
+ $(id + '>input[type="checkbox"]').click(function() {
+ if (this.checked) {
+ timeline.setOptions({zoomable: false});
+ } else {
+ timeline.setOptions({zoomable: true});
+ }
+ });
+
+ $(id + ">span").click(function() {
+ $(this).parent().find('input:checkbox').trigger('click');
+ });
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org