You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2017/08/01 13:41:23 UTC
incubator-livy git commit: [LIVY-344][WEB UI] Create Session Log Page
Repository: incubator-livy
Updated Branches:
refs/heads/master 60ee047e9 -> 69300a2fe
[LIVY-344][WEB UI] Create Session Log Page
[LIVY-344](https://issues.cloudera.org/browse/LIVY-344)
Created a Session Log page for viewing the session log.
Updated All Sessions Page and Session Page with links to the Log Page.
Log Page exists for both Interactive Sessions and Batches.
Tested Manually, screenshots below
Author: Alex Bozarth <aj...@us.ibm.com>
Closes #25 from ajbozarth/log-ui.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/69300a2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/69300a2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/69300a2f
Branch: refs/heads/master
Commit: 69300a2fe4c81d711369e4bf23db0847cc15a178
Parents: 60ee047
Author: Alex Bozarth <aj...@us.ibm.com>
Authored: Tue Aug 1 21:41:17 2017 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Tue Aug 1 21:41:17 2017 +0800
----------------------------------------------------------------------
.../livy/server/ui/static/css/livy-ui.css | 4 +-
.../livy/server/ui/static/js/all-sessions.js | 4 +-
.../apache/livy/server/ui/static/js/livy-ui.js | 12 +++++-
.../livy/server/ui/static/js/session-log.js | 43 ++++++++++++++++++++
.../apache/livy/server/ui/static/js/session.js | 2 +-
.../org/apache/livy/server/SessionServlet.scala | 5 ++-
.../org/apache/livy/server/ui/UIServlet.scala | 32 ++++++++++++++-
.../org/apache/livy/utils/SparkYarnApp.scala | 2 +-
8 files changed, 94 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/resources/org/apache/livy/server/ui/static/css/livy-ui.css
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/css/livy-ui.css b/server/src/main/resources/org/apache/livy/server/ui/static/css/livy-ui.css
index bb8b416..9fa9556 100644
--- a/server/src/main/resources/org/apache/livy/server/ui/static/css/livy-ui.css
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/css/livy-ui.css
@@ -16,7 +16,7 @@
*/
body {
- padding-top: 15px;
+ padding: 15px 0;
}
pre {
@@ -32,6 +32,6 @@ td .progress {
margin: 0;
}
-.session-summary {
+#session-summary {
margin: 20px 0;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js b/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js
index 440c8ef..28b08a8 100644
--- a/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/js/all-sessions.js
@@ -25,7 +25,7 @@ function loadSessionsTable(sessions) {
tdWrap(session.proxyUser) +
tdWrap(session.kind) +
tdWrap(session.state) +
- tdWrap(driverLogLink(session)) +
+ tdWrap(logLinks(session, "session")) +
"</tr>"
);
});
@@ -38,7 +38,7 @@ function loadBatchesTable(sessions) {
tdWrap(session.id) +
tdWrap(appIdLink(session)) +
tdWrap(session.state) +
- tdWrap(driverLogLink(session)) +
+ tdWrap(logLinks(session, "batch")) +
"</tr>"
);
});
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/resources/org/apache/livy/server/ui/static/js/livy-ui.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/js/livy-ui.js b/server/src/main/resources/org/apache/livy/server/ui/static/js/livy-ui.js
index 3ed2c32..6eef2de 100644
--- a/server/src/main/resources/org/apache/livy/server/ui/static/js/livy-ui.js
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/js/livy-ui.js
@@ -46,10 +46,16 @@ function driverLogLink(session) {
if (driverLogUrl != null) {
return anchorLink(driverLogUrl, "driver");
} else {
- return;
+ return "";
}
}
+function logLinks(session, kind) {
+ var sessionLog = divWrap(uiLink(kind + "/" + session.id + "/log", "session"));
+ var driverLog = divWrap(driverLogLink(session));
+ return sessionLog + driverLog;
+}
+
function appIdLink(session) {
var appUiUrl = session.appInfo.sparkUiUrl;
if (appUiUrl != null) {
@@ -71,6 +77,10 @@ function preWrap(inner) {
return "<pre>" + escapeHtml(inner) + "</pre>";
}
+function divWrap(inner) {
+ return "<div>" + inner + "</div>";
+}
+
function progressBar(double) {
var cent = +(double * 100).toFixed(3);
return '<div class="progress"><div class="progress-bar" style="width:'
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/resources/org/apache/livy/server/ui/static/js/session-log.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/js/session-log.js b/server/src/main/resources/org/apache/livy/server/ui/static/js/session-log.js
new file mode 100644
index 0000000..b236938
--- /dev/null
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/js/session-log.js
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+function getLogPath(type, id) {
+ if (type == "session") {
+ return "/sessions/" + id + "/log";
+ } else if (type == "batch") {
+ return "/batches/" + id + "/log";
+ } else {
+ return "";
+ }
+}
+
+function parseLog(logLines) {
+ // TODO: [LIVY-387]: Separate out stdout, stderr, and YARN Diagnostics into different viewers
+ return preWrap(logLines.join("\n"));
+}
+
+$(document).ready(function () {
+ var pathArr = getPathArray();
+ var type = pathArr.shift();
+ var id = pathArr.shift();
+
+ $.getJSON(location.origin + getLogPath(type, id), {size: -1}, function(response) {
+ if (response) {
+ $("#session-log").append(parseLog(response.log));
+ }
+ });
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/resources/org/apache/livy/server/ui/static/js/session.js
----------------------------------------------------------------------
diff --git a/server/src/main/resources/org/apache/livy/server/ui/static/js/session.js b/server/src/main/resources/org/apache/livy/server/ui/static/js/session.js
index a956eaa..7b85546 100644
--- a/server/src/main/resources/org/apache/livy/server/ui/static/js/session.js
+++ b/server/src/main/resources/org/apache/livy/server/ui/static/js/session.js
@@ -57,7 +57,7 @@ function appendSummary(session) {
sumWrap("Proxy User", session.proxyUser) +
sumWrap("Session Kind", session.kind) +
sumWrap("State", session.state) +
- sumWrap("Logs", driverLogLink(session)) +
+ sumWrap("Logs", logLinks(session, "session")) +
"</ul>"
);
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/SessionServlet.scala b/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
index 9a354c0..76b5afa 100644
--- a/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
+++ b/server/src/main/scala/org/apache/livy/server/SessionServlet.scala
@@ -230,8 +230,11 @@ abstract class SessionServlet[S <: Session, R <: RecoveryMetadata](
private def serializeLogs(session: S, fromOpt: Option[Int], sizeOpt: Option[Int]) = {
val lines = session.logLines()
- val size = sizeOpt.getOrElse(100)
+ var size = sizeOpt.getOrElse(100)
var from = fromOpt.getOrElse(-1)
+ if (size < 0) {
+ size = lines.length
+ }
if (from < 0) {
from = math.max(0, lines.length - size)
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala b/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
index 0be94bd..0b0d56a 100644
--- a/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
+++ b/server/src/main/scala/org/apache/livy/server/ui/UIServlet.scala
@@ -30,6 +30,10 @@ class UIServlet extends ScalatraServlet {
private case class SessionPage(id: Int) extends Page {
val name: String = "Session " + id
}
+ private case class LogPage(sessionType: String, id: Int) extends Page {
+ val sessionName: String = sessionType + " " + id
+ val name: String = sessionName + " Log"
+ }
private def getHeader(pageName: String): Seq[Node] =
<head>
@@ -63,9 +67,15 @@ class UIServlet extends ScalatraServlet {
private def getNavBar(page: Page): Seq[Node] = {
val tabs: Seq[Node] = page match {
case _: AllSessionsPage => <li class="active"><a href="#">Sessions</a></li>
- case pageInfo: SessionPage => {
+ case sessionPage: SessionPage => {
+ <li><a href="/ui">Sessions</a></li> ++
+ <li class="active"><a href="#">{sessionPage.name}</a></li>
+ }
+ case logPage: LogPage => {
+ val sessionLink = if (logPage.sessionType == "Session") "/ui/session/" + logPage.id else "#"
<li><a href="/ui">Sessions</a></li> ++
- <li class="active"><a href="#">{pageInfo.name}</a></li>
+ <li><a href={sessionLink}>{logPage.sessionName}</a></li> ++
+ <li class="active"><a href="#">Log</a></li>
}
case _ => Seq.empty
}
@@ -108,4 +118,22 @@ class UIServlet extends ScalatraServlet {
createPage(SessionPage(params("id").toInt), content)
}
+
+ private def getLogPage(page: LogPage): Seq[Node] = {
+ val content =
+ <div id="log-page">
+ <div id="session-log"></div>
+ <script src="/static/js/session-log.js"></script>
+ </div>
+
+ createPage(page, content)
+ }
+
+ get("/session/:id/log") {
+ getLogPage(LogPage("Session", params("id").toInt))
+ }
+
+ get("/batch/:id/log") {
+ getLogPage(LogPage("Batch", params("id").toInt))
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/69300a2f/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala b/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
index b2a828f..0ab6c44 100644
--- a/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
+++ b/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
@@ -192,7 +192,7 @@ class SparkYarnApp private[utils] (
private def getYarnDiagnostics(appReport: ApplicationReport): IndexedSeq[String] = {
Option(appReport.getDiagnostics)
.filter(_.nonEmpty)
- .map[IndexedSeq[String]]("YARN Diagnostics:" +: _.split("\n"))
+ .map[IndexedSeq[String]](_.split("\n"))
.getOrElse(IndexedSeq.empty)
}