You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/10 03:38:26 UTC
[21/37] git commit: TODO clean-up
TODO clean-up
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/8da1012f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/8da1012f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/8da1012f
Branch: refs/heads/master
Commit: 8da1012f9bb9431123d3aae26d4bb78f8e5374bc
Parents: faefea3
Author: Patrick Wendell <pw...@gmail.com>
Authored: Sun Dec 29 11:38:12 2013 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Sun Dec 29 11:38:12 2013 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 1 -
.../scala/org/apache/spark/deploy/client/DriverClient.scala | 5 +++--
.../scala/org/apache/spark/deploy/worker/DriverRunner.scala | 2 +-
.../scala/org/apache/spark/deploy/worker/ExecutorRunner.scala | 2 +-
.../scala/org/apache/spark/deploy/worker/WorkerWatcher.scala | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/8da1012f/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 41f810d..ad3337d 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -622,7 +622,6 @@ class SparkContext(
} else {
val uri = new URI(path)
key = uri.getScheme match {
- // TODO: Have this load jars that are available on the driver
// A JAR file which exists only on the driver node
case null | "file" =>
if (SparkHadoopUtil.get.isYarnMode()) {
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/8da1012f/core/src/main/scala/org/apache/spark/deploy/client/DriverClient.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/client/DriverClient.scala b/core/src/main/scala/org/apache/spark/deploy/client/DriverClient.scala
index a5f5763..8a4cdf0 100644
--- a/core/src/main/scala/org/apache/spark/deploy/client/DriverClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/client/DriverClient.scala
@@ -70,8 +70,9 @@ object DriverClient extends Logging {
driverArgs.cmd match {
case "launch" =>
- // TODO: Could modify env here to pass a flag indicating Spark is in deploy-driver mode
- // then use that to load jars locally (e.g. truncate the filesystem path)
+ // TODO: We could add an env variable here and intercept it in `sc.addJar` that would
+ // truncate filesystem paths similar to what YARN does. For now, we just require
+ // people call `addJar` assuming the jar is in the same directory.
val env = Map[String, String]()
System.getenv().foreach{case (k, v) => env(k) = v}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/8da1012f/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index 41500bb..51baa35 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -58,7 +58,7 @@ private[spark] class DriverRunner(
val localJarFilename = downloadUserJar(driverDir)
// Make sure user application jar is on the classpath
- // TODO: This could eventually exploit ability for driver to add jars
+ // TODO: If we add ability to submit multiple jars they should also be added here
val env = Map(driverDesc.command.environment.toSeq: _*)
env("SPARK_CLASSPATH") = env.getOrElse("SPARK_CLASSPATH", "") + s":$localJarFilename"
val newCommand = Command(driverDesc.command.mainClass,
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/8da1012f/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index 2e61d39..fdc9a34 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -50,7 +50,7 @@ private[spark] class ExecutorRunner(
var workerThread: Thread = null
var process: Process = null
- // NOTE: This is now redundant with the automated shut-down enforced by the Executor. It mike
+ // NOTE: This is now redundant with the automated shut-down enforced by the Executor. It might
// make sense to remove this in the future.
var shutdownHook: Thread = null
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/8da1012f/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
index e4352f1..f4184bc 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
@@ -19,7 +19,7 @@ private[spark] class WorkerWatcher(workerUrl: String) extends Actor with Logging
worker ! SendHeartbeat // need to send a message here to initiate connection
}
- // Lets us filter events only from the worker actor
+ // Lets us filter events only from the worker's actor system
private val expectedHostPort = AddressFromURIString(workerUrl).hostPort
private def isWorker(address: Address) = address.hostPort == expectedHostPort