You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by db...@apache.org on 2017/04/27 18:06:26 UTC
spark git commit: [SPARK-20483] Mesos Coarse mode may starve other
Mesos frameworks
Repository: spark
Updated Branches:
refs/heads/master ba7666274 -> 7633933e5
[SPARK-20483] Mesos Coarse mode may starve other Mesos frameworks
## What changes were proposed in this pull request?
Set maxCores to be a multiple of the smallest executor that can be launched. This ensures that we correctly detect the condition where no more executors will be launched when spark.cores.max is not a multiple of spark.executor.cores
## How was this patch tested?
This was manually tested with other sample frameworks measuring their incoming offers to determine if starvation would occur.
dbtsai mgummelt
Author: Davis Shepherd <ds...@netflix.com>
Closes #17786 from dgshep/fix_mesos_max_cores.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7633933e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7633933e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7633933e
Branch: refs/heads/master
Commit: 7633933e54ffb08ab9d959be5f76c26fae29d1d9
Parents: ba76662
Author: Davis Shepherd <ds...@netflix.com>
Authored: Thu Apr 27 18:06:12 2017 +0000
Committer: DB Tsai <db...@dbtsai.com>
Committed: Thu Apr 27 18:06:12 2017 +0000
----------------------------------------------------------------------
.../mesos/MesosCoarseGrainedSchedulerBackend.scala | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7633933e/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index 2a36ec4..8f5b97c 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -60,8 +60,16 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
private val maxCoresOption = conf.getOption("spark.cores.max").map(_.toInt)
+ private val executorCoresOption = conf.getOption("spark.executor.cores").map(_.toInt)
+
+ private val minCoresPerExecutor = executorCoresOption.getOrElse(1)
+
// Maximum number of cores to acquire
- private val maxCores = maxCoresOption.getOrElse(Int.MaxValue)
+ private val maxCores = {
+ val cores = maxCoresOption.getOrElse(Int.MaxValue)
+ // Set maxCores to a multiple of smallest executor we can launch
+ cores - (cores % minCoresPerExecutor)
+ }
private val useFetcherCache = conf.getBoolean("spark.mesos.fetcherCache.enable", false)
@@ -489,8 +497,9 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
}
private def executorCores(offerCPUs: Int): Int = {
- sc.conf.getInt("spark.executor.cores",
- math.min(offerCPUs, maxCores - totalCoresAcquired))
+ executorCoresOption.getOrElse(
+ math.min(offerCPUs, maxCores - totalCoresAcquired)
+ )
}
override def statusUpdate(d: org.apache.mesos.SchedulerDriver, status: TaskStatus) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org