You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/11/06 08:14:13 UTC
[1/2] git commit: Ignore a task update status if the executor doesn't
exist anymore.
Updated Branches:
refs/heads/master 9f7b9bb1c -> bf4e6131c
Ignore a task update status if the executor doesn't exist anymore.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/a02eed68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/a02eed68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/a02eed68
Branch: refs/heads/master
Commit: a02eed68110f99c08d8ff379108c96546bbc16b0
Parents: 9f7b9bb
Author: Reynold Xin <rx...@apache.org>
Authored: Tue Nov 5 18:46:38 2013 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Tue Nov 5 18:46:38 2013 -0800
----------------------------------------------------------------------
.../cluster/CoarseGrainedSchedulerBackend.scala | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a02eed68/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 70f3f88..a45bee5 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -87,8 +87,14 @@ class CoarseGrainedSchedulerBackend(scheduler: ClusterScheduler, actorSystem: Ac
case StatusUpdate(executorId, taskId, state, data) =>
scheduler.statusUpdate(taskId, state, data.value)
if (TaskState.isFinished(state)) {
- freeCores(executorId) += 1
- makeOffers(executorId)
+ if (executorActor.contains(executorId)) {
+ freeCores(executorId) += 1
+ makeOffers(executorId)
+ } else {
+ // Ignoring the update since we don't know about the executor.
+ val msg = "Ignored task status update (%d state %s) from unknown executor %s with ID %s"
+ logWarning(msg.format(taskId, state, sender, executorId))
+ }
}
case ReviveOffers =>
@@ -175,7 +181,9 @@ class CoarseGrainedSchedulerBackend(scheduler: ClusterScheduler, actorSystem: Ac
Props(new DriverActor(properties)), name = CoarseGrainedSchedulerBackend.ACTOR_NAME)
}
- private val timeout = Duration.create(System.getProperty("spark.akka.askTimeout", "10").toLong, "seconds")
+ private val timeout = {
+ Duration.create(System.getProperty("spark.akka.askTimeout", "10").toLong, "seconds")
+ }
def stopExecutors() {
try {
[2/2] git commit: Merge pull request #143 from rxin/scheduler-hang
Posted by rx...@apache.org.
Merge pull request #143 from rxin/scheduler-hang
Ignore a task update status if the executor doesn't exist anymore.
Otherwise if the scheduler receives a task update message when the executor's been removed, the scheduler would hang.
It is pretty hard to add unit tests for these right now because it is hard to mock the cluster scheduler. We should do that once @kayousterhout finishes merging the local scheduler and the cluster scheduler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/bf4e6131
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/bf4e6131
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/bf4e6131
Branch: refs/heads/master
Commit: bf4e6131cceef4fe00fb5693117c0732f181dbd9
Parents: 9f7b9bb a02eed6
Author: Reynold Xin <rx...@apache.org>
Authored: Tue Nov 5 23:14:09 2013 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Tue Nov 5 23:14:09 2013 -0800
----------------------------------------------------------------------
.../cluster/CoarseGrainedSchedulerBackend.scala | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------