You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2016/11/09 18:40:29 UTC
spark git commit: [SPARK-16808][CORE] History Server main page does
not honor APPLICATION_WEB_PROXY_BASE
Repository: spark
Updated Branches:
refs/heads/master 205e6d586 -> 06a13ecca
[SPARK-16808][CORE] History Server main page does not honor APPLICATION_WEB_PROXY_BASE
## What changes were proposed in this pull request?
Application links generated on the history server UI no longer (regression from 1.6) contain the configured spark.ui.proxyBase in the links. To address this, made the uiRoot available globally to all javascripts for Web UI. Updated the mustache template (historypage-template.html) to include the uiroot for rendering links to the applications.
The existing test was not sufficient to verify the scenario where ajax call is used to populate the application listing template, so added a new selenium test case to cover this scenario.
## How was this patch tested?
Existing tests and a new unit test.
No visual changes to the UI.
Author: Vinayak <vi...@in.ibm.com>
Closes #15742 from vijoshi/SPARK-16808_master.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06a13ecc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06a13ecc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06a13ecc
Branch: refs/heads/master
Commit: 06a13ecca728e431c66fafb333b3bcff808e1afd
Parents: 205e6d5
Author: Vinayak <vi...@in.ibm.com>
Authored: Wed Nov 9 10:40:14 2016 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Wed Nov 9 10:40:14 2016 -0800
----------------------------------------------------------------------
.../spark/ui/static/historypage-template.html | 6 +-
.../org/apache/spark/ui/static/historypage.js | 6 +-
.../org/apache/spark/ui/static/webui.js | 6 ++
.../scala/org/apache/spark/ui/UIUtils.scala | 1 +
.../deploy/history/HistoryServerSuite.scala | 80 +++++++++++++++++++-
5 files changed, 92 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
index 1fd6ef4..42e2d9a 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
@@ -68,16 +68,16 @@
<tbody>
{{#applications}}
<tr>
- <td class="rowGroupColumn"><span title="{{id}}"><a href="/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td>
+ <td class="rowGroupColumn"><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td>
<td class="rowGroupColumn">{{name}}</td>
{{#attempts}}
- <td class="attemptIDSpan"><a href="/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td>
+ <td class="attemptIDSpan"><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td>
<td>{{startTime}}</td>
<td>{{endTime}}</td>
<td><span title="{{duration}}" class="durationClass">{{duration}}</span></td>
<td>{{sparkUser}}</td>
<td>{{lastUpdated}}</td>
- <td><a href="/api/v1/applications/{{id}}/{{num}}/logs" class="btn btn-info btn-mini">Download</a></td>
+ <td><a href="{{uiroot}}/api/v1/applications/{{id}}/{{num}}/logs" class="btn btn-info btn-mini">Download</a></td>
{{/attempts}}
</tr>
{{/applications}}
http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/resources/org/apache/spark/ui/static/historypage.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index 2a32e18..6c0ec8d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -119,7 +119,11 @@ $(document).ready(function() {
}
}
- var data = {"applications": array}
+ var data = {
+ "uiroot": uiRoot,
+ "applications": array
+ }
+
$.get("static/historypage-template.html", function(template) {
historySummary.append(Mustache.render($(template).filter("#history-summary-template").html(),data));
var selector = "#history-summary-table";
http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/resources/org/apache/spark/ui/static/webui.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index e37307a..0fa1fcf 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -15,6 +15,12 @@
* limitations under the License.
*/
+var uiRoot = "";
+
+function setUIRoot(val) {
+ uiRoot = val;
+}
+
function collapseTablePageLoad(name, table){
if (window.localStorage.getItem(name) == "true") {
// Set it to false so that the click function can revert it
http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 66b097a..57f6f2f 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -171,6 +171,7 @@ private[spark] object UIUtils extends Logging {
<script src={prependBaseUri("/static/timeline-view.js")}></script>
<script src={prependBaseUri("/static/log-view.js")}></script>
<script src={prependBaseUri("/static/webui.js")}></script>
+ <script>setUIRoot('{UIUtils.uiRoot}')</script>
}
def vizHeaderNodes: Seq[Node] = {
http://git-wip-us.apache.org/repos/asf/spark/blob/06a13ecc/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index a595bc1..715811a 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -29,6 +29,8 @@ import com.codahale.metrics.Counter
import com.google.common.io.{ByteStreams, Files}
import org.apache.commons.io.{FileUtils, IOUtils}
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
+import org.eclipse.jetty.proxy.ProxyServlet
+import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
import org.json4s.JsonAST._
import org.json4s.jackson.JsonMethods
import org.json4s.jackson.JsonMethods._
@@ -258,8 +260,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
getContentAndCode("foobar")._1 should be (HttpServletResponse.SC_NOT_FOUND)
}
- test("relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
- val proxyBaseBeforeTest = System.getProperty("spark.ui.proxyBase")
+ test("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
val uiRoot = Option(System.getenv("APPLICATION_WEB_PROXY_BASE")).getOrElse("/testwebproxybase")
val page = new HistoryPage(server)
val request = mock[HttpServletRequest]
@@ -267,7 +268,6 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
// when
System.setProperty("spark.ui.proxyBase", uiRoot)
val response = page.render(request)
- System.setProperty("spark.ui.proxyBase", Option(proxyBaseBeforeTest).getOrElse(""))
// then
val urls = response \\ "@href" map (_.toString)
@@ -275,6 +275,80 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
all (siteRelativeLinks) should startWith (uiRoot)
}
+ test("ajax rendered relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
+ val uiRoot = "/testwebproxybase"
+ System.setProperty("spark.ui.proxyBase", uiRoot)
+
+ server.stop()
+
+ val conf = new SparkConf()
+ .set("spark.history.fs.logDirectory", logDir)
+ .set("spark.history.fs.update.interval", "0")
+ .set("spark.testing", "true")
+
+ provider = new FsHistoryProvider(conf)
+ provider.checkForLogs()
+ val securityManager = new SecurityManager(conf)
+
+ server = new HistoryServer(conf, provider, securityManager, 18080)
+ server.initialize()
+ server.bind()
+
+ val port = server.boundPort
+
+ val servlet = new ProxyServlet {
+ override def rewriteTarget(request: HttpServletRequest): String = {
+ // servlet acts like a proxy that redirects calls made on
+ // spark.ui.proxyBase context path to the normal servlet handlers operating off "/"
+ val sb = request.getRequestURL()
+
+ if (request.getQueryString() != null) {
+ sb.append(s"?${request.getQueryString()}")
+ }
+
+ val proxyidx = sb.indexOf(uiRoot)
+ sb.delete(proxyidx, proxyidx + uiRoot.length).toString
+ }
+ }
+
+ val contextHandler = new ServletContextHandler
+ val holder = new ServletHolder(servlet)
+ contextHandler.setContextPath(uiRoot)
+ contextHandler.addServlet(holder, "/")
+ server.attachHandler(contextHandler)
+
+ implicit val webDriver: WebDriver = new HtmlUnitDriver(true) {
+ getWebClient.getOptions.setThrowExceptionOnScriptError(false)
+ }
+
+ try {
+ val url = s"http://localhost:$port"
+
+ go to s"$url$uiRoot"
+
+ // expect the ajax call to finish in 5 seconds
+ implicitlyWait(org.scalatest.time.Span(5, org.scalatest.time.Seconds))
+
+ // once this findAll call returns, we know the ajax load of the table completed
+ findAll(ClassNameQuery("odd"))
+
+ val links = findAll(TagNameQuery("a"))
+ .map(_.attribute("href"))
+ .filter(_.isDefined)
+ .map(_.get)
+ .filter(_.startsWith(url)).toList
+
+ // there are atleast some URL links that were generated via javascript,
+ // and they all contain the spark.ui.proxyBase (uiRoot)
+ links.length should be > 4
+ all(links) should startWith(url + uiRoot)
+ } finally {
+ contextHandler.stop()
+ quit()
+ }
+
+ }
+
test("incomplete apps get refreshed") {
implicit val webDriver: WebDriver = new HtmlUnitDriver
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org