You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/10 03:38:41 UTC
[36/37] git commit: Some usability improvements
Some usability improvements
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/67b9a336
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/67b9a336
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/67b9a336
Branch: refs/heads/master
Commit: 67b9a33628b9934804c36620d8cbc73ef70106ce
Parents: 0f9d2ac
Author: Patrick Wendell <pw...@gmail.com>
Authored: Thu Jan 9 12:42:37 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Thu Jan 9 12:42:37 2014 -0800
----------------------------------------------------------------------
.../apache/spark/deploy/ClientArguments.scala | 14 +++++++-
.../spark/deploy/master/ui/IndexPage.scala | 35 ++++++++++++------
.../spark/deploy/worker/ui/IndexPage.scala | 38 +++++++++++++-------
3 files changed, 62 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/67b9a336/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala b/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala
index 50b92e1..db67c6d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala
@@ -17,6 +17,8 @@
package org.apache.spark.deploy
+import java.net.URL
+
import scala.collection.mutable.ListBuffer
import org.apache.log4j.Level
@@ -68,8 +70,18 @@ private[spark] class ClientArguments(args: Array[String]) {
case "launch" :: _master :: _jarUrl :: _mainClass :: tail =>
cmd = "launch"
- master = _master
+
+ try {
+ new URL(_jarUrl)
+ } catch {
+ case e: Exception =>
+ println(s"Jar url '${_jarUrl}' is not a valid URL.")
+ println(s"Jar must be in URL format (e.g. hdfs://XX, file://XX)")
+ printUsageAndExit(-1)
+ }
+
jarUrl = _jarUrl
+ master = _master
mainClass = _mainClass
_driverOptions ++= tail
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/67b9a336/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
index db1dde5..a9af8df 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/IndexPage.scala
@@ -63,6 +63,10 @@ private[spark] class IndexPage(parent: MasterWebUI) {
val completedDrivers = state.completedDrivers.sortBy(_.startTime).reverse
val completedDriversTable = UIUtils.listingTable(driverHeaders, driverRow, completedDrivers)
+ // For now we only show driver information if the user has submitted drivers to the cluster.
+ // This is until we integrate the notion of drivers and applications in the UI.
+ def hasDrivers = activeDrivers.length > 0 || completedDrivers.length > 0
+
val content =
<div class="row-fluid">
<div class="span12">
@@ -98,26 +102,35 @@ private[spark] class IndexPage(parent: MasterWebUI) {
</div>
</div>
- <div class="row-fluid">
- <div class="span12">
- <h4> Completed Applications </h4>
- {completedAppsTable}
+ <div>
+ {if (hasDrivers)
+ <div class="row-fluid">
+ <div class="span12">
+ <h4> Running Drivers </h4>
+ {activeDriversTable}
+ </div>
</div>
+ }
</div>
<div class="row-fluid">
<div class="span12">
- <h4> Active Drivers </h4>
- {activeDriversTable}
+ <h4> Completed Applications </h4>
+ {completedAppsTable}
</div>
</div>
- <div class="row-fluid">
- <div class="span12">
- <h4> Completed Drivers </h4>
- {completedDriversTable}
+ <div>
+ {if (hasDrivers)
+ <div class="row-fluid">
+ <div class="span12">
+ <h4> Completed Drivers </h4>
+ {completedDriversTable}
+ </div>
</div>
+ }
</div>;
+
UIUtils.basicSparkPage(content, "Spark Master at " + state.uri)
}
@@ -170,7 +183,7 @@ private[spark] class IndexPage(parent: MasterWebUI) {
<td sorttable_customkey={driver.desc.mem.toString}>
{Utils.megabytesToString(driver.desc.mem.toLong)}
</td>
- <td>{driver.desc.command.mainClass}</td>
+ <td>{driver.desc.command.arguments(1)}</td>
</tr>
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/67b9a336/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala
index 0a7f56e..925c6fb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/IndexPage.scala
@@ -58,6 +58,10 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
val finishedDrivers = workerState.finishedDrivers.sortBy(_.driverId).reverse
def finishedDriverTable = UIUtils.listingTable(driverHeaders, driverRow, finishedDrivers)
+ // For now we only show driver information if the user has submitted drivers to the cluster.
+ // This is until we integrate the notion of drivers and applications in the UI.
+ def hasDrivers = runningDrivers.length > 0 || finishedDrivers.length > 0
+
val content =
<div class="row-fluid"> <!-- Worker Details -->
<div class="span12">
@@ -81,6 +85,17 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
</div>
</div>
+ <div>
+ {if (hasDrivers)
+ <div class="row-fluid"> <!-- Running Drivers -->
+ <div class="span12">
+ <h4> Running Drivers {workerState.drivers.size} </h4>
+ {runningDriverTable}
+ </div>
+ </div>
+ }
+ </div>
+
<div class="row-fluid"> <!-- Finished Executors -->
<div class="span12">
<h4> Finished Executors </h4>
@@ -88,18 +103,15 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
</div>
</div>
- <div class="row-fluid"> <!-- Running Drivers -->
- <div class="span12">
- <h4> Running Drivers {workerState.drivers.size} </h4>
- {runningDriverTable}
- </div>
- </div>
-
- <div class="row-fluid"> <!-- Finished Drivers -->
- <div class="span12">
- <h4> Finished Drivers </h4>
- {finishedDriverTable}
- </div>
+ <div>
+ {if (hasDrivers)
+ <div class="row-fluid"> <!-- Finished Drivers -->
+ <div class="span12">
+ <h4> Finished Drivers </h4>
+ {finishedDriverTable}
+ </div>
+ </div>
+ }
</div>;
UIUtils.basicSparkPage(content, "Spark Worker at %s:%s".format(
@@ -133,7 +145,7 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
def driverRow(driver: DriverRunner): Seq[Node] = {
<tr>
<td>{driver.driverId}</td>
- <td>{driver.driverDesc.command.mainClass}</td>
+ <td>{driver.driverDesc.command.arguments(1)}</td>
<td>{driver.finalState.getOrElse(DriverState.RUNNING)}</td>
<td sorttable_customkey={driver.driverDesc.cores.toString}>
{driver.driverDesc.cores.toString}