You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2016/04/20 14:24:05 UTC
spark git commit: [SPARK-8171][WEB UI] Javascript based infinite
scrolling for the log page
Repository: spark
Updated Branches:
refs/heads/master ed9d80385 -> 834277884
[SPARK-8171][WEB UI] Javascript based infinite scrolling for the log page
Updated the log page by replacing the current pagination with a javascript-based infinite scroll solution
Author: Alex Bozarth <aj...@us.ibm.com>
Closes #10910 from ajbozarth/spark8171.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/83427788
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/83427788
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/83427788
Branch: refs/heads/master
Commit: 834277884fcdaab4758604272881ffb2369e25f0
Parents: ed9d803
Author: Alex Bozarth <aj...@us.ibm.com>
Authored: Wed Apr 20 21:24:11 2016 +0900
Committer: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Committed: Wed Apr 20 21:24:11 2016 +0900
----------------------------------------------------------------------
.../org/apache/spark/ui/static/log-view.js | 129 +++++++++++++++++++
.../org/apache/spark/ui/static/webui.css | 10 ++
.../apache/spark/deploy/worker/ui/LogPage.scala | 75 +++++------
.../scala/org/apache/spark/ui/JettyUtils.scala | 4 +-
.../scala/org/apache/spark/ui/UIUtils.scala | 1 +
5 files changed, 175 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/83427788/core/src/main/resources/org/apache/spark/ui/static/log-view.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
new file mode 100644
index 0000000..1782b4f
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var baseParams;
+
+var curLogLength;
+var startByte;
+var endByte;
+var totalLogLength;
+
+var byteLength;
+
+function setLogScroll(oldHeight) {
+ var logContent = $(".log-content");
+ logContent.scrollTop(logContent[0].scrollHeight - oldHeight);
+}
+
+function tailLog() {
+ var logContent = $(".log-content");
+ logContent.scrollTop(logContent[0].scrollHeight);
+}
+
+function setLogData() {
+ $('#log-data').html("Showing " + curLogLength + " Bytes: " + startByte
+ + " - " + endByte + " of " + totalLogLength);
+}
+
+function disableMoreButton() {
+ var moreBtn = $(".log-more-btn");
+ moreBtn.attr("disabled", "disabled");
+ moreBtn.html("Top of Log");
+}
+
+function noNewAlert() {
+ var alert = $(".no-new-alert");
+ alert.css("display", "block");
+ window.setTimeout(function () {alert.css("display", "none");}, 4000);
+}
+
+function loadMore() {
+ var offset = Math.max(startByte - byteLength, 0);
+ var moreByteLength = Math.min(byteLength, startByte);
+
+ $.ajax({
+ type: "GET",
+ url: "/log" + baseParams + "&offset=" + offset + "&byteLength=" + moreByteLength,
+ success: function (data) {
+ var oldHeight = $(".log-content")[0].scrollHeight;
+ var newlineIndex = data.indexOf('\n');
+ var dataInfo = data.substring(0, newlineIndex).match(/\d+/g);
+ var retStartByte = dataInfo[0];
+ var retLogLength = dataInfo[2];
+
+ var cleanData = data.substring(newlineIndex + 1);
+ if (retStartByte == 0) {
+ disableMoreButton();
+ }
+ $("pre", ".log-content").prepend(cleanData);
+
+ curLogLength = curLogLength + (startByte - retStartByte);
+ startByte = retStartByte;
+ totalLogLength = retLogLength;
+ setLogScroll(oldHeight);
+ setLogData();
+ }
+ });
+}
+
+function loadNew() {
+ $.ajax({
+ type: "GET",
+ url: "/log" + baseParams + "&byteLength=0",
+ success: function (data) {
+ var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g);
+ var newDataLen = dataInfo[2] - totalLogLength;
+ if (newDataLen != 0) {
+ $.ajax({
+ type: "GET",
+ url: "/log" + baseParams + "&byteLength=" + newDataLen,
+ success: function (data) {
+ var newlineIndex = data.indexOf('\n');
+ var dataInfo = data.substring(0, newlineIndex).match(/\d+/g);
+ var retStartByte = dataInfo[0];
+ var retEndByte = dataInfo[1];
+ var retLogLength = dataInfo[2];
+
+ var cleanData = data.substring(newlineIndex + 1);
+ $("pre", ".log-content").append(cleanData);
+
+ curLogLength = curLogLength + (retEndByte - retStartByte);
+ endByte = retEndByte;
+ totalLogLength = retLogLength;
+ tailLog();
+ setLogData();
+ }
+ });
+ } else {
+ noNewAlert();
+ }
+ }
+ });
+}
+
+function initLogPage(params, logLen, start, end, totLogLen, defaultLen) {
+ baseParams = params;
+ curLogLength = logLen;
+ startByte = start;
+ endByte = end;
+ totalLogLength = totLogLen;
+ byteLength = defaultLen;
+ tailLog();
+ if (startByte == 0) {
+ disableMoreButton();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/spark/blob/83427788/core/src/main/resources/org/apache/spark/ui/static/webui.css
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css
index 47dd916..595e80a 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css
@@ -237,3 +237,13 @@ a.expandbutton {
color: #333;
text-decoration: none;
}
+
+.log-more-btn, .log-new-btn {
+ width: 100%
+}
+
+.no-new-alert {
+ text-align: center;
+ margin: 0;
+ padding: 4px 0;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/spark/blob/83427788/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
index e75c0ce..3473c41 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
@@ -20,7 +20,7 @@ package org.apache.spark.deploy.worker.ui
import java.io.File
import javax.servlet.http.HttpServletRequest
-import scala.xml.Node
+import scala.xml.{Node, Unparsed}
import org.apache.spark.internal.Logging
import org.apache.spark.ui.{UIUtils, WebUIPage}
@@ -31,10 +31,9 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
private val worker = parent.worker
private val workDir = new File(parent.workDir.toURI.normalize().getPath)
private val supportedLogTypes = Set("stderr", "stdout")
+ private val defaultBytes = 100 * 1024
def renderLog(request: HttpServletRequest): String = {
- val defaultBytes = 100 * 1024
-
val appId = Option(request.getParameter("appId"))
val executorId = Option(request.getParameter("executorId"))
val driverId = Option(request.getParameter("driverId"))
@@ -44,9 +43,9 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
val logDir = (appId, executorId, driverId) match {
case (Some(a), Some(e), None) =>
- s"${workDir.getPath}/$appId/$executorId/"
+ s"${workDir.getPath}/$a/$e/"
case (None, None, Some(d)) =>
- s"${workDir.getPath}/$driverId/"
+ s"${workDir.getPath}/$d/"
case _ =>
throw new Exception("Request must specify either application or driver identifiers")
}
@@ -57,7 +56,6 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
}
def render(request: HttpServletRequest): Seq[Node] = {
- val defaultBytes = 100 * 1024
val appId = Option(request.getParameter("appId"))
val executorId = Option(request.getParameter("executorId"))
val driverId = Option(request.getParameter("driverId"))
@@ -76,49 +74,44 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with
val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength)
val linkToMaster = <p><a href={worker.activeMasterWebUiUrl}>Back to Master</a></p>
- val range = <span>Bytes {startByte.toString} - {endByte.toString} of {logLength}</span>
-
- val backButton =
- if (startByte > 0) {
- <a href={"?%s&logType=%s&offset=%s&byteLength=%s"
- .format(params, logType, math.max(startByte - byteLength, 0), byteLength)}>
- <button type="button" class="btn btn-default">
- Previous {Utils.bytesToString(math.min(byteLength, startByte))}
- </button>
- </a>
- } else {
- <button type="button" class="btn btn-default" disabled="disabled">
- Previous 0 B
- </button>
- }
+ val curLogLength = endByte - startByte
+ val range =
+ <span id="log-data">
+ Showing {curLogLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength}
+ </span>
+
+ val moreButton =
+ <button type="button" onclick={"loadMore()"} class="log-more-btn btn btn-default">
+ Load More
+ </button>
+
+ val newButton =
+ <button type="button" onclick={"loadNew()"} class="log-new-btn btn btn-default">
+ Load New
+ </button>
+
+ val alert =
+ <div class="no-new-alert alert alert-info" style="display: none;">
+ End of Log
+ </div>
- val nextButton =
- if (endByte < logLength) {
- <a href={"?%s&logType=%s&offset=%s&byteLength=%s".
- format(params, logType, endByte, byteLength)}>
- <button type="button" class="btn btn-default">
- Next {Utils.bytesToString(math.min(byteLength, logLength - endByte))}
- </button>
- </a>
- } else {
- <button type="button" class="btn btn-default" disabled="disabled">
- Next 0 B
- </button>
- }
+ val logParams = "?%s&logType=%s".format(params, logType)
+ val jsOnload = "window.onload = " +
+ s"initLogPage('$logParams', $curLogLength, $startByte, $endByte, $logLength, $byteLength);"
val content =
<div>
{linkToMaster}
- <div>
- <div style="float:left; margin-right:10px">{backButton}</div>
- <div style="float:left;">{range}</div>
- <div style="float:right; margin-left:10px">{nextButton}</div>
- </div>
- <br />
- <div style="height:500px; overflow:auto; padding:5px;">
+ {range}
+ <div class="log-content" style="height:80vh; overflow:auto; padding:5px;">
+ <div>{moreButton}</div>
<pre>{logText}</pre>
+ {alert}
+ <div>{newButton}</div>
</div>
+ <script>{Unparsed(jsOnload)}</script>
</div>
+
UIUtils.basicSparkPage(content, logType + " log page for " + pageName)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/83427788/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 119165f..db24f03 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -84,9 +84,7 @@ private[spark] object JettyUtils extends Logging {
val result = servletParams.responder(request)
response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate")
response.setHeader("X-Frame-Options", xFrameOptionsValue)
- // scalastyle:off println
- response.getWriter.println(servletParams.extractFn(result))
- // scalastyle:on println
+ response.getWriter.print(servletParams.extractFn(result))
} else {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED)
response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate")
http://git-wip-us.apache.org/repos/asf/spark/blob/83427788/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 28d277d..6241593 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -168,6 +168,7 @@ private[spark] object UIUtils extends Logging {
<script src={prependBaseUri("/static/table.js")}></script>
<script src={prependBaseUri("/static/additional-metrics.js")}></script>
<script src={prependBaseUri("/static/timeline-view.js")}></script>
+ <script src={prependBaseUri("/static/log-view.js")}></script>
}
def vizHeaderNodes: Seq[Node] = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org