You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/06/25 04:45:12 UTC
git commit: SPARK-2248: spark.default.parallelism does not apply in
local mode
Repository: spark
Updated Branches:
refs/heads/master 2714968e1 -> b6b44853c
SPARK-2248: spark.default.parallelism does not apply in local mode
Author: witgo <wi...@qq.com>
Closes #1194 from witgo/SPARK-2248 and squashes the following commits:
6ac950b [witgo] spark.default.parallelism does not apply in local mode
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b6b44853
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b6b44853
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b6b44853
Branch: refs/heads/master
Commit: b6b44853cd61660f2917b99d87c157e2b4430e5c
Parents: 2714968
Author: witgo <wi...@qq.com>
Authored: Tue Jun 24 19:44:37 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Jun 24 19:45:03 2014 -0700
----------------------------------------------------------------------
.../apache/spark/scheduler/local/LocalBackend.scala | 3 ++-
.../spark/SparkContextSchedulerCreationSuite.scala | 16 ++++++++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b6b44853/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala
index 43f0e18..9b95ccc 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala
@@ -97,7 +97,8 @@ private[spark] class LocalBackend(scheduler: TaskSchedulerImpl, val totalCores:
localActor ! ReviveOffers
}
- override def defaultParallelism() = totalCores
+ override def defaultParallelism() =
+ scheduler.conf.getInt("spark.default.parallelism", totalCores)
override def killTask(taskId: Long, executorId: String, interruptThread: Boolean) {
localActor ! KillTask(taskId, interruptThread)
http://git-wip-us.apache.org/repos/asf/spark/blob/b6b44853/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
index 94fba10..67e3be2 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala
@@ -77,6 +77,22 @@ class SparkContextSchedulerCreationSuite
}
}
+ test("local-default-parallelism") {
+ val defaultParallelism = System.getProperty("spark.default.parallelism")
+ System.setProperty("spark.default.parallelism", "16")
+ val sched = createTaskScheduler("local")
+
+ sched.backend match {
+ case s: LocalBackend => assert(s.defaultParallelism() === 16)
+ case _ => fail()
+ }
+
+ Option(defaultParallelism) match {
+ case Some(v) => System.setProperty("spark.default.parallelism", v)
+ case _ => System.clearProperty("spark.default.parallelism")
+ }
+ }
+
test("simr") {
createTaskScheduler("simr://uri").backend match {
case s: SimrSchedulerBackend => // OK