You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/05/07 09:11:12 UTC
git commit: SPARK-1746: Support setting SPARK_JAVA_OPTS on executors
for backwards compatibility
Repository: spark
Updated Branches:
refs/heads/master fdae095de -> 913a0a9c0
SPARK-1746: Support setting SPARK_JAVA_OPTS on executors for backwards compatibility
Author: Patrick Wendell <pw...@gmail.com>
Closes #676 from pwendell/worker-opts and squashes the following commits:
54456c4 [Patrick Wendell] SPARK-1746: Support setting SPARK_JAVA_OPTS on executors for backwards compatibility
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/913a0a9c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/913a0a9c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/913a0a9c
Branch: refs/heads/master
Commit: 913a0a9c0a87e164723ebf9616b883b6329bac71
Parents: fdae095
Author: Patrick Wendell <pw...@gmail.com>
Authored: Wed May 7 00:11:05 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed May 7 00:11:05 2014 -0700
----------------------------------------------------------------------
.../org/apache/spark/deploy/worker/CommandUtils.scala | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/913a0a9c/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
index 3e615e7..c7f0f24 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
@@ -47,9 +47,16 @@ object CommandUtils extends Logging {
*/
def buildJavaOpts(command: Command, memory: Int, sparkHome: String): Seq[String] = {
val memoryOpts = Seq(s"-Xms${memory}M", s"-Xmx${memory}M")
- // Note, this will coalesce multiple options into a single command component
val extraOpts = command.extraJavaOptions.map(Utils.splitCommandString).getOrElse(Seq())
+ // Exists for backwards compatibility with older Spark versions
+ val workerLocalOpts = Option(getenv("SPARK_JAVA_OPTS")).map(Utils.splitCommandString)
+ .getOrElse(Nil)
+ if (workerLocalOpts.length > 0) {
+ logWarning("SPARK_JAVA_OPTS was set on the worker. It is deprecated in Spark 1.0.")
+ logWarning("Set SPARK_LOCAL_DIRS for node-specific storage locations.")
+ }
+
val libraryOpts =
if (command.libraryPathEntries.size > 0) {
val joined = command.libraryPathEntries.mkString(File.pathSeparator)
@@ -66,7 +73,7 @@ object CommandUtils extends Logging {
val userClassPath = command.classPathEntries ++ Seq(classPath)
Seq("-cp", userClassPath.filterNot(_.isEmpty).mkString(File.pathSeparator)) ++
- libraryOpts ++ extraOpts ++ memoryOpts
+ libraryOpts ++ extraOpts ++ workerLocalOpts ++ memoryOpts
}
/** Spawn a thread that will redirect a given stream to a file */