You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/05/23 01:05:17 UTC
spark git commit: [SPARK-7760] add /json back into master & worker
pages; add test
Repository: spark
Updated Branches:
refs/heads/master 126d7235d -> 821254fb9
[SPARK-7760] add /json back into master & worker pages; add test
Author: Imran Rashid <ir...@cloudera.com>
Closes #6284 from squito/SPARK-7760 and squashes the following commits:
5e02d8a [Imran Rashid] style; increase timeout
9987399 [Imran Rashid] comment
8c7ed63 [Imran Rashid] add /json back into master & worker pages; add test
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/821254fb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/821254fb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/821254fb
Branch: refs/heads/master
Commit: 821254fb945c3e19540eb57fff1f656737ef484b
Parents: 126d723
Author: Imran Rashid <ir...@cloudera.com>
Authored: Fri May 22 16:05:07 2015 -0700
Committer: Josh Rosen <jo...@databricks.com>
Committed: Fri May 22 16:05:07 2015 -0700
----------------------------------------------------------------------
.../apache/spark/deploy/LocalSparkCluster.scala | 6 +++-
.../main/scala/org/apache/spark/ui/WebUI.scala | 3 ++
.../spark/deploy/master/MasterSuite.scala | 31 ++++++++++++++++++--
3 files changed, 37 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/821254fb/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
index 860e1a2..0550f00 100644
--- a/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/LocalSparkCluster.scala
@@ -43,6 +43,8 @@ class LocalSparkCluster(
private val localHostname = Utils.localHostName()
private val masterActorSystems = ArrayBuffer[ActorSystem]()
private val workerActorSystems = ArrayBuffer[ActorSystem]()
+ // exposed for testing
+ var masterWebUIPort = -1
def start(): Array[String] = {
logInfo("Starting a local Spark cluster with " + numWorkers + " workers.")
@@ -53,7 +55,9 @@ class LocalSparkCluster(
.set("spark.shuffle.service.enabled", "false")
/* Start the Master */
- val (masterSystem, masterPort, _, _) = Master.startSystemAndActor(localHostname, 0, 0, _conf)
+ val (masterSystem, masterPort, webUiPort, _) =
+ Master.startSystemAndActor(localHostname, 0, 0, _conf)
+ masterWebUIPort = webUiPort
masterActorSystems += masterSystem
val masterUrl = "spark://" + Utils.localHostNameForURI() + ":" + masterPort
val masters = Array(masterUrl)
http://git-wip-us.apache.org/repos/asf/spark/blob/821254fb/core/src/main/scala/org/apache/spark/ui/WebUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
index 1df9cd0..594df15 100644
--- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala
@@ -77,7 +77,10 @@ private[spark] abstract class WebUI(
val pagePath = "/" + page.prefix
val renderHandler = createServletHandler(pagePath,
(request: HttpServletRequest) => page.render(request), securityManager, basePath)
+ val renderJsonHandler = createServletHandler(pagePath.stripSuffix("/") + "/json",
+ (request: HttpServletRequest) => page.renderJson(request), securityManager, basePath)
attachHandler(renderHandler)
+ attachHandler(renderJsonHandler)
pageToHandlers.getOrElseUpdate(page, ArrayBuffer[ServletContextHandler]())
.append(renderHandler)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/821254fb/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index 0faa8f6..f97e5ff 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -21,16 +21,20 @@ import java.util.Date
import scala.concurrent.Await
import scala.concurrent.duration._
+import scala.io.Source
import scala.language.postfixOps
import akka.actor.Address
+import org.json4s._
+import org.json4s.jackson.JsonMethods._
import org.scalatest.{FunSuite, Matchers}
+import org.scalatest.concurrent.Eventually
import other.supplier.{CustomPersistenceEngine, CustomRecoveryModeFactory}
-import org.apache.spark.deploy._
import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.deploy._
-class MasterSuite extends FunSuite with Matchers {
+class MasterSuite extends FunSuite with Matchers with Eventually {
test("toAkkaUrl") {
val conf = new SparkConf(loadDefaults = false)
@@ -157,4 +161,27 @@ class MasterSuite extends FunSuite with Matchers {
CustomRecoveryModeFactory.instantiationAttempts should be > instantiationAttempts
}
+ test("Master & worker web ui available") {
+ implicit val formats = org.json4s.DefaultFormats
+ val conf = new SparkConf()
+ val localCluster = new LocalSparkCluster(2, 2, 512, conf)
+ localCluster.start()
+ try {
+ eventually(timeout(5 seconds), interval(100 milliseconds)) {
+ val json = Source.fromURL(s"http://localhost:${localCluster.masterWebUIPort}/json")
+ .getLines().mkString("\n")
+ val JArray(workers) = (parse(json) \ "workers")
+ workers.size should be (2)
+ workers.foreach { workerSummaryJson =>
+ val JString(workerWebUi) = workerSummaryJson \ "webuiaddress"
+ val workerResponse = parse(Source.fromURL(s"${workerWebUi}/json")
+ .getLines().mkString("\n"))
+ (workerResponse \ "cores").extract[Int] should be (2)
+ }
+ }
+ } finally {
+ localCluster.stop()
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org