You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/02/25 02:03:13 UTC
[2/2] git commit: Fix removal from shuffleToMapStage to search for a
key-value pair with our stage instead of using our shuffleID.
Fix removal from shuffleToMapStage to search for a key-value pair with
our stage instead of using our shuffleID.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0661cdcd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0661cdcd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0661cdcd
Branch: refs/heads/branch-0.9
Commit: 0661cdcdfdc0b4fe165c95d951d74fa228c09a3b
Parents: 5e74b8e
Author: Matei Zaharia <ma...@databricks.com>
Authored: Mon Feb 24 13:14:56 2014 -0800
Committer: Matei Zaharia <ma...@databricks.com>
Committed: Mon Feb 24 17:01:21 2014 -0800
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0661cdcd/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index 21b7ccb..536d84f 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -381,8 +381,8 @@ class DAGScheduler(
running -= stage
}
stageToInfos -= stage
- for (shuffleDep <- stage.shuffleDep) {
- shuffleToMapStage.remove(shuffleDep.shuffleId)
+ for ((k, v) <- shuffleToMapStage.find(_._2 == stage)) {
+ shuffleToMapStage.remove(k)
}
if (pendingTasks.contains(stage) && !pendingTasks(stage).isEmpty) {
logDebug("Removing pending status for stage %d".format(stageId))