You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2014/06/26 15:20:38 UTC
git commit: Remove use of spark.worker.instances
Repository: spark
Updated Branches:
refs/heads/master e4899a253 -> 48a82a827
Remove use of spark.worker.instances
spark.worker.instances was added as part of this commit: https://github.com/apache/spark/commit/1617816090e7b20124a512a43860a21232ebf511
My understanding is that SPARK_WORKER_INSTANCES is supported for backwards compatibility,
but spark.worker.instances is never used (SparkSubmit.scala sets spark.executor.instances) so should
not have been added.
@sryza @pwendell @tgravescs LMK if I'm understanding this correctly
Author: Kay Ousterhout <ka...@gmail.com>
Closes #1214 from kayousterhout/yarn_config and squashes the following commits:
3d7c491 [Kay Ousterhout] Remove use of spark.worker.instances
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/48a82a82
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/48a82a82
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/48a82a82
Branch: refs/heads/master
Commit: 48a82a827c99526b165c78d7e88faec43568a37a
Parents: e4899a2
Author: Kay Ousterhout <ka...@gmail.com>
Authored: Thu Jun 26 08:20:27 2014 -0500
Committer: Thomas Graves <tg...@apache.org>
Committed: Thu Jun 26 08:20:27 2014 -0500
----------------------------------------------------------------------
.../spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/48a82a82/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
index 412dfe3..fd2694f 100644
--- a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
@@ -63,7 +63,7 @@ private[spark] class YarnClientSchedulerBackend(
// variables.
List(("--driver-memory", "SPARK_MASTER_MEMORY", "spark.master.memory"),
("--driver-memory", "SPARK_DRIVER_MEMORY", "spark.driver.memory"),
- ("--num-executors", "SPARK_WORKER_INSTANCES", "spark.worker.instances"),
+ ("--num-executors", "SPARK_WORKER_INSTANCES", "spark.executor.instances"),
("--num-executors", "SPARK_EXECUTOR_INSTANCES", "spark.executor.instances"),
("--executor-memory", "SPARK_WORKER_MEMORY", "spark.executor.memory"),
("--executor-memory", "SPARK_EXECUTOR_MEMORY", "spark.executor.memory"),