You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/07/17 02:31:08 UTC
[11/50] [abbrv] git commit: Add `Environment` tab to SparkUI.
Add `Environment` tab to SparkUI.
This adds a tab which displays system property and classpath information. This
can be useful in debugging various types of issues such as:
1. Extra/incorrect Hadoop jars being included in the classpath
2. Spark launching with a different JRE version than intended
3. Spark system properties not being set to intended values
4. User added jars that conflict with Spark jars
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/ddb97f0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/ddb97f0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/ddb97f0f
Branch: refs/heads/master
Commit: ddb97f0fdf16442afaa9cab656376267e4044510
Parents: 37abe84
Author: Patrick Wendell <pw...@gmail.com>
Authored: Sat Jul 13 16:05:21 2013 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Sat Jul 13 16:14:40 2013 -0700
----------------------------------------------------------------------
core/src/main/scala/spark/ui/Page.scala | 2 +-
core/src/main/scala/spark/ui/SparkUI.scala | 7 +--
core/src/main/scala/spark/ui/UIUtils.scala | 5 ++
.../main/scala/spark/ui/env/EnvironmentUI.scala | 58 ++++++++++++++++++++
4 files changed, 67 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ddb97f0f/core/src/main/scala/spark/ui/Page.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/spark/ui/Page.scala b/core/src/main/scala/spark/ui/Page.scala
index c853b44..ed8f918 100644
--- a/core/src/main/scala/spark/ui/Page.scala
+++ b/core/src/main/scala/spark/ui/Page.scala
@@ -1,3 +1,3 @@
package spark.ui
-private[spark] object Page extends Enumeration { val Storage, Jobs = Value }
\ No newline at end of file
+private[spark] object Page extends Enumeration { val Storage, Jobs, Environment = Value }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ddb97f0f/core/src/main/scala/spark/ui/SparkUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/spark/ui/SparkUI.scala b/core/src/main/scala/spark/ui/SparkUI.scala
index b3bdc2c..874e5ba 100644
--- a/core/src/main/scala/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/spark/ui/SparkUI.scala
@@ -5,15 +5,13 @@ import javax.servlet.http.HttpServletRequest
import org.eclipse.jetty.server.{Handler, Server}
import spark.{Logging, SparkContext, Utils}
+import spark.ui.env.EnvironmentUI
import spark.ui.storage.BlockManagerUI
import spark.ui.jobs.JobProgressUI
-import spark.ui.UIUtils._
import spark.ui.JettyUtils._
/** Top level user interface for Spark */
private[spark] class SparkUI(sc: SparkContext) extends Logging {
- // TODO(pwendell): It would be nice to add a view that prints out environment information
-
val host = Utils.localHostName()
val port = Option(System.getProperty("spark.ui.port")).getOrElse(SparkUI.DEFAULT_PORT).toInt
var boundPort: Option[Int] = None
@@ -25,7 +23,8 @@ private[spark] class SparkUI(sc: SparkContext) extends Logging {
)
val storage = new BlockManagerUI(sc)
val jobs = new JobProgressUI(sc)
- val allHandlers = storage.getHandlers ++ jobs.getHandlers ++ handlers
+ val env = new EnvironmentUI(sc)
+ val allHandlers = storage.getHandlers ++ jobs.getHandlers ++ env.getHandlers ++ handlers
/** Bind the HTTP server which backs this web interface */
def bind() {
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ddb97f0f/core/src/main/scala/spark/ui/UIUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/spark/ui/UIUtils.scala b/core/src/main/scala/spark/ui/UIUtils.scala
index 7b79290..36d9c47 100644
--- a/core/src/main/scala/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/spark/ui/UIUtils.scala
@@ -19,6 +19,10 @@ private[spark] object UIUtils {
case Jobs => <li class="active"><a href="/stages">Jobs</a></li>
case _ => <li><a href="/stages">Jobs</a></li>
}
+ val environment = page match {
+ case Environment => <li class="active"><a href="/environment">Environment</a></li>
+ case _ => <li><a href="/environment">Environment</a></li>
+ }
<html>
<head>
@@ -44,6 +48,7 @@ private[spark] object UIUtils {
<ul class="nav">
{storage}
{jobs}
+ {environment}
</ul>
<ul id="infolist">
<li>Application: <strong>{sc.appName}</strong></li>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ddb97f0f/core/src/main/scala/spark/ui/env/EnvironmentUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/spark/ui/env/EnvironmentUI.scala b/core/src/main/scala/spark/ui/env/EnvironmentUI.scala
new file mode 100644
index 0000000..8a3b330
--- /dev/null
+++ b/core/src/main/scala/spark/ui/env/EnvironmentUI.scala
@@ -0,0 +1,58 @@
+package spark.ui.env
+
+import javax.servlet.http.HttpServletRequest
+
+import org.eclipse.jetty.server.Handler
+
+import scala.collection.JavaConversions._
+
+import spark.ui.JettyUtils._
+import spark.ui.UIUtils.headerSparkPage
+import spark.ui.Page.Environment
+import spark.SparkContext
+import spark.ui.UIUtils
+
+import scala.xml.Node
+
+private[spark] class EnvironmentUI(sc: SparkContext) {
+
+ def getHandlers = Seq[(String, Handler)](
+ ("/environment", (request: HttpServletRequest) => envDetails(request))
+ )
+
+ def envDetails(request: HttpServletRequest): Seq[Node] = {
+ val properties = System.getProperties.iterator.toSeq
+
+ val classPathProperty = properties
+ .filter{case (k, v) => k.contains("java.class.path")}
+ .headOption
+ .getOrElse("", "")
+ val sparkProperties = properties.filter(_._1.contains("spark"))
+ val otherProperties = properties.diff(sparkProperties :+ classPathProperty)
+
+ val propertyHeaders = Seq("Name", "Value")
+ def propertyRow(kv: (String, String)) = <tr><td>{kv._1}</td><td>{kv._2}</td></tr>
+ val propertyTable = UIUtils.listingTable(
+ propertyHeaders, propertyRow, sparkProperties ++ otherProperties)
+
+ val classPathEntries = classPathProperty._2
+ .split(System.getProperty("path.separator", ":"))
+ .filterNot(e => e.isEmpty)
+ .map(e => (e, "System Classpath"))
+ val addedJars = sc.addedJars.iterator.toSeq.map{case (path, time) => (path, "Added By User")}
+ val addedFiles = sc.addedFiles.iterator.toSeq.map{case (path, time) => (path, "Added By User")}
+ val classPath = addedJars ++ addedFiles ++ classPathEntries
+
+ val classPathHeaders = Seq("Resource", "Source")
+ def classPathRow(data: (String, String)) = <tr><td>{data._1}</td><td>{data._2}</td></tr>
+ val classPathTable = UIUtils.listingTable(classPathHeaders, classPathRow, classPath)
+
+ val content =
+ <span>
+ <h2>System Properties</h2> {propertyTable}
+ <h2>Classpath Entries</h2> {classPathTable}
+ </span>
+
+ headerSparkPage(content, sc, "Environment", Environment)
+ }
+}