You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2014/12/23 23:28:46 UTC
spark git commit: [SPARK-4730][YARN] Warn against deprecated YARN
settings
Repository: spark
Updated Branches:
refs/heads/master 395b771fe -> 27c5399f4
[SPARK-4730][YARN] Warn against deprecated YARN settings
See https://issues.apache.org/jira/browse/SPARK-4730.
Author: Andrew Or <an...@databricks.com>
Closes #3590 from andrewor14/yarn-settings and squashes the following commits:
36e0753 [Andrew Or] Merge branch 'master' of github.com:apache/spark into yarn-settings
dcd1316 [Andrew Or] Warn against deprecated YARN settings
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/27c5399f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/27c5399f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/27c5399f
Branch: refs/heads/master
Commit: 27c5399f4dd542e36ea579956b8cb0613de25c6d
Parents: 395b771
Author: Andrew Or <an...@databricks.com>
Authored: Tue Dec 23 14:28:36 2014 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Tue Dec 23 14:28:36 2014 -0800
----------------------------------------------------------------------
.../apache/spark/deploy/SparkSubmitArguments.scala | 5 ++++-
.../cluster/YarnClientSchedulerBackend.scala | 17 ++++++++++++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/27c5399f/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 607b4df..f174bc1 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -214,7 +214,10 @@ private[spark] class SparkSubmitArguments(args: Seq[String], env: Map[String, St
""".stripMargin
}
- /** Fill in values by parsing user options. */
+ /**
+ * Fill in values by parsing user options.
+ * NOTE: Any changes here must be reflected in YarnClientSchedulerBackend.
+ */
private def parseOpts(opts: Seq[String]): Unit = {
val EQ_SEPARATED_OPT="""(--[^=]+)=(.+)""".r
http://git-wip-us.apache.org/repos/asf/spark/blob/27c5399f/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
index 2923e67..09597bd 100644
--- a/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
+++ b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
@@ -65,7 +65,8 @@ private[spark] class YarnClientSchedulerBackend(
*/
private def getExtraClientArguments: Seq[String] = {
val extraArgs = new ArrayBuffer[String]
- val optionTuples = // List of (target Client argument, environment variable, Spark property)
+ // List of (target Client argument, environment variable, Spark property)
+ val optionTuples =
List(
("--driver-memory", "SPARK_MASTER_MEMORY", "spark.master.memory"),
("--driver-memory", "SPARK_DRIVER_MEMORY", "spark.driver.memory"),
@@ -78,11 +79,25 @@ private[spark] class YarnClientSchedulerBackend(
("--queue", "SPARK_YARN_QUEUE", "spark.yarn.queue"),
("--name", "SPARK_YARN_APP_NAME", "spark.app.name")
)
+ // Warn against the following deprecated environment variables: env var -> suggestion
+ val deprecatedEnvVars = Map(
+ "SPARK_MASTER_MEMORY" -> "SPARK_DRIVER_MEMORY or --driver-memory through spark-submit",
+ "SPARK_WORKER_INSTANCES" -> "SPARK_WORKER_INSTANCES or --num-executors through spark-submit",
+ "SPARK_WORKER_MEMORY" -> "SPARK_EXECUTOR_MEMORY or --executor-memory through spark-submit",
+ "SPARK_WORKER_CORES" -> "SPARK_EXECUTOR_CORES or --executor-cores through spark-submit")
+ // Do the same for deprecated properties: property -> suggestion
+ val deprecatedProps = Map("spark.master.memory" -> "--driver-memory through spark-submit")
optionTuples.foreach { case (optionName, envVar, sparkProp) =>
if (System.getenv(envVar) != null) {
extraArgs += (optionName, System.getenv(envVar))
+ if (deprecatedEnvVars.contains(envVar)) {
+ logWarning(s"NOTE: $envVar is deprecated. Use ${deprecatedEnvVars(envVar)} instead.")
+ }
} else if (sc.getConf.contains(sparkProp)) {
extraArgs += (optionName, sc.getConf.get(sparkProp))
+ if (deprecatedProps.contains(sparkProp)) {
+ logWarning(s"NOTE: $sparkProp is deprecated. Use ${deprecatedProps(sparkProp)} instead.")
+ }
}
}
extraArgs
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org