You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/05/13 02:09:22 UTC
git commit: Rename testExecutorEnvs --> executorEnvs.
Repository: spark
Updated Branches:
refs/heads/master 8586bf564 -> 3ce526b16
Rename testExecutorEnvs --> executorEnvs.
This was changed, but in fact, it's used for things other than tests.
So I've changed it back.
Author: Patrick Wendell <pw...@gmail.com>
Closes #747 from pwendell/executor-env and squashes the following commits:
36a60a5 [Patrick Wendell] Rename testExecutorEnvs --> executorEnvs.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3ce526b1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3ce526b1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3ce526b1
Branch: refs/heads/master
Commit: 3ce526b168050c572a1feee8e0121e1426f7d9ee
Parents: 8586bf5
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon May 12 17:09:13 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon May 12 17:09:13 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 11 +++++------
.../scheduler/cluster/SparkDeploySchedulerBackend.scala | 2 +-
.../cluster/mesos/CoarseMesosSchedulerBackend.scala | 2 +-
.../scheduler/cluster/mesos/MesosSchedulerBackend.scala | 2 +-
4 files changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3ce526b1/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index e6121a7..35beff0 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -276,19 +276,18 @@ class SparkContext(config: SparkConf) extends Logging {
.getOrElse(512)
// Environment variables to pass to our executors.
- // NOTE: This should only be used for test related settings.
- private[spark] val testExecutorEnvs = HashMap[String, String]()
+ private[spark] val executorEnvs = HashMap[String, String]()
// Convert java options to env vars as a work around
// since we can't set env vars directly in sbt.
for { (envKey, propKey) <- Seq(("SPARK_TESTING", "spark.testing"))
value <- Option(System.getenv(envKey)).orElse(Option(System.getProperty(propKey)))} {
- testExecutorEnvs(envKey) = value
+ executorEnvs(envKey) = value
}
// The Mesos scheduler backend relies on this environment variable to set executor memory.
// TODO: Set this only in the Mesos scheduler.
- testExecutorEnvs("SPARK_EXECUTOR_MEMORY") = executorMemory + "m"
- testExecutorEnvs ++= conf.getExecutorEnv
+ executorEnvs("SPARK_EXECUTOR_MEMORY") = executorMemory + "m"
+ executorEnvs ++= conf.getExecutorEnv
// Set SPARK_USER for user who is running SparkContext.
val sparkUser = Option {
@@ -296,7 +295,7 @@ class SparkContext(config: SparkConf) extends Logging {
}.getOrElse {
SparkContext.SPARK_UNKNOWN_USER
}
- testExecutorEnvs("SPARK_USER") = sparkUser
+ executorEnvs("SPARK_USER") = sparkUser
// Create and start the scheduler
private[spark] var taskScheduler = SparkContext.createTaskScheduler(this, master)
http://git-wip-us.apache.org/repos/asf/spark/blob/3ce526b1/core/src/main/scala/org/apache/spark/scheduler/cluster/SparkDeploySchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/SparkDeploySchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/SparkDeploySchedulerBackend.scala
index cefa417..933f6e0 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/SparkDeploySchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/SparkDeploySchedulerBackend.scala
@@ -54,7 +54,7 @@ private[spark] class SparkDeploySchedulerBackend(
}
val command = Command(
- "org.apache.spark.executor.CoarseGrainedExecutorBackend", args, sc.testExecutorEnvs,
+ "org.apache.spark.executor.CoarseGrainedExecutorBackend", args, sc.executorEnvs,
classPathEntries, libraryPathEntries, extraJavaOpts)
val sparkHome = sc.getSparkHome()
val appDesc = new ApplicationDescription(sc.appName, maxCores, sc.executorMemory, command,
http://git-wip-us.apache.org/repos/asf/spark/blob/3ce526b1/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
index 2cd9d6c..cbe9bb0 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
@@ -122,7 +122,7 @@ private[spark] class CoarseMesosSchedulerBackend(
val extraLibraryPath = conf.getOption(libraryPathOption).map(p => s"-Djava.library.path=$p")
val extraOpts = Seq(extraJavaOpts, extraLibraryPath).flatten.mkString(" ")
- sc.testExecutorEnvs.foreach { case (key, value) =>
+ sc.executorEnvs.foreach { case (key, value) =>
environment.addVariables(Environment.Variable.newBuilder()
.setName(key)
.setValue(value)
http://git-wip-us.apache.org/repos/asf/spark/blob/3ce526b1/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
index 2cea1c8..ff8356f 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
@@ -90,7 +90,7 @@ private[spark] class MesosSchedulerBackend(
"Spark home is not set; set it through the spark.home system " +
"property, the SPARK_HOME environment variable or the SparkContext constructor"))
val environment = Environment.newBuilder()
- sc.testExecutorEnvs.foreach { case (key, value) =>
+ sc.executorEnvs.foreach { case (key, value) =>
environment.addVariables(Environment.Variable.newBuilder()
.setName(key)
.setValue(value)