You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/09/03 22:53:26 UTC
spark git commit: [SPARK-9672] [MESOS] Don’t include SPARK_ENV_LOADED when passing env vars
Repository: spark
Updated Branches:
refs/heads/master 754f853b0 -> e62f4a46f
[SPARK-9672] [MESOS] Don’t include SPARK_ENV_LOADED when passing env vars
This contribution is my original work and I license the work to the project under the project's open source license.
Author: Pat Shields <ye...@gmail.com>
Closes #7979 from pashields/env-loading-on-driver.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e62f4a46
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e62f4a46
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e62f4a46
Branch: refs/heads/master
Commit: e62f4a46f4396ae1e064e3d2ebfa2434f549b090
Parents: 754f853
Author: Pat Shields <ye...@gmail.com>
Authored: Thu Sep 3 13:52:47 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Thu Sep 3 13:53:18 2015 -0700
----------------------------------------------------------------------
.../spark/deploy/rest/RestSubmissionClient.scala | 17 +++++++++++++----
.../deploy/rest/StandaloneRestSubmitSuite.scala | 12 ++++++++++++
2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e62f4a46/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
index 1fe9563..957a928 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
@@ -392,15 +392,14 @@ private[spark] object RestSubmissionClient {
mainClass: String,
appArgs: Array[String],
conf: SparkConf,
- env: Map[String, String] = sys.env): SubmitRestProtocolResponse = {
+ env: Map[String, String] = Map()): SubmitRestProtocolResponse = {
val master = conf.getOption("spark.master").getOrElse {
throw new IllegalArgumentException("'spark.master' must be set.")
}
val sparkProperties = conf.getAll.toMap
- val environmentVariables = env.filter { case (k, _) => k.startsWith("SPARK_") }
val client = new RestSubmissionClient(master)
val submitRequest = client.constructSubmitRequest(
- appResource, mainClass, appArgs, sparkProperties, environmentVariables)
+ appResource, mainClass, appArgs, sparkProperties, env)
client.createSubmission(submitRequest)
}
@@ -413,6 +412,16 @@ private[spark] object RestSubmissionClient {
val mainClass = args(1)
val appArgs = args.slice(2, args.size)
val conf = new SparkConf
- run(appResource, mainClass, appArgs, conf)
+ val env = filterSystemEnvironment(sys.env)
+ run(appResource, mainClass, appArgs, conf, env)
+ }
+
+ /**
+ * Filter non-spark environment variables from any environment.
+ */
+ private[rest] def filterSystemEnvironment(env: Map[String, String]): Map[String, String] = {
+ env.filter { case (k, _) =>
+ (k.startsWith("SPARK_") && k != "SPARK_ENV_LOADED") || k.startsWith("MESOS_")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/e62f4a46/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
index 96e456d..9693e32 100644
--- a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala
@@ -366,6 +366,18 @@ class StandaloneRestSubmitSuite extends SparkFunSuite with BeforeAndAfterEach {
assert(conn3.getResponseCode === HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
}
+ test("client does not send 'SPARK_ENV_LOADED' env var by default") {
+ val environmentVariables = Map("SPARK_VAR" -> "1", "SPARK_ENV_LOADED" -> "1")
+ val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
+ assert(filteredVariables == Map("SPARK_VAR" -> "1"))
+ }
+
+ test("client includes mesos env vars") {
+ val environmentVariables = Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1", "OTHER_VAR" -> "1")
+ val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
+ assert(filteredVariables == Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1"))
+ }
+
/* --------------------- *
| Helper methods |
* --------------------- */
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org