You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/11/20 02:20:27 UTC
spark git commit: [SPARK-4478] Keep totalRegisteredExecutors
up-to-date
Repository: spark
Updated Branches:
refs/heads/master 377b06820 -> 9ccc53c72
[SPARK-4478] Keep totalRegisteredExecutors up-to-date
This rebases PR 3368.
This commit fixes totalRegisteredExecutors update [SPARK-4478], so that we can correctly keep track of number of registered executors.
Author: Akshat Aranya <aa...@quantcast.com>
Closes #3373 from coolfrood/topic/SPARK-4478 and squashes the following commits:
8a4d1e4 [Akshat Aranya] Added comment
150ae93 [Akshat Aranya] [SPARK-4478] Keep totalRegisteredExecutors up-to-date
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9ccc53c7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9ccc53c7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9ccc53c7
Branch: refs/heads/master
Commit: 9ccc53c72c5bcffcc121291710754e1e2d659341
Parents: 377b068
Author: Akshat Aranya <aa...@quantcast.com>
Authored: Wed Nov 19 17:20:20 2014 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Wed Nov 19 17:20:20 2014 -0800
----------------------------------------------------------------------
.../spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9ccc53c7/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 7a6ee56..047fae1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -46,6 +46,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val actorSyste
{
// Use an atomic variable to track total number of cores in the cluster for simplicity and speed
var totalCoreCount = new AtomicInteger(0)
+ // Total number of executors that are currently registered
var totalRegisteredExecutors = new AtomicInteger(0)
val conf = scheduler.sc.conf
private val timeout = AkkaUtils.askTimeout(conf)
@@ -204,6 +205,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val actorSyste
executorsPendingToRemove -= executorId
}
totalCoreCount.addAndGet(-executorInfo.totalCores)
+ totalRegisteredExecutors.addAndGet(-1)
scheduler.executorLost(executorId, SlaveLost(reason))
case None => logError(s"Asked to remove non-existent executor $executorId")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org