You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/04/01 08:01:21 UTC
git commit: [Hot Fix #42] Persisted RDD disappears on storage page if
re-used
Repository: spark
Updated Branches:
refs/heads/master 94fe7fd4f -> ada310a9d
[Hot Fix #42] Persisted RDD disappears on storage page if re-used
If a previously persisted RDD is re-used, its information disappears from the Storage page.
This is because the tasks associated with re-using the RDD do not report the RDD's blocks as updated (which is correct). On stage submit, however, we overwrite any existing information regarding that RDD with a fresh one, whether or not the information for the RDD already exists.
Author: Andrew Or <an...@gmail.com>
Closes #281 from andrewor14/ui-storage-fix and squashes the following commits:
408585a [Andrew Or] Fix storage UI bug
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ada310a9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ada310a9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ada310a9
Branch: refs/heads/master
Commit: ada310a9d3d5419e101b24d9b41398f609da1ad3
Parents: 94fe7fd
Author: Andrew Or <an...@gmail.com>
Authored: Mon Mar 31 23:01:14 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Mar 31 23:01:14 2014 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/ui/storage/BlockManagerUI.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ada310a9/core/src/main/scala/org/apache/spark/ui/storage/BlockManagerUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/BlockManagerUI.scala b/core/src/main/scala/org/apache/spark/ui/storage/BlockManagerUI.scala
index 4d8b01d..a7b24ff 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/BlockManagerUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/BlockManagerUI.scala
@@ -84,7 +84,7 @@ private[ui] class BlockManagerListener(storageStatusListener: StorageStatusListe
override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted) = synchronized {
val rddInfo = stageSubmitted.stageInfo.rddInfo
- _rddInfoMap(rddInfo.id) = rddInfo
+ _rddInfoMap.getOrElseUpdate(rddInfo.id, rddInfo)
}
override def onStageCompleted(stageCompleted: SparkListenerStageCompleted) = synchronized {