You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/06/17 10:28:29 UTC
git commit: [SPARK-2144] ExecutorsPage reports incorrect # of RDD
blocks
Repository: spark
Updated Branches:
refs/heads/master 23a12ce20 -> 09deb3eee
[SPARK-2144] ExecutorsPage reports incorrect # of RDD blocks
This is reproducible whenever we drop a block because of memory pressure.
This is because StorageStatusListener actually never removes anything from the block maps of its StorageStatuses. Instead, when a block is dropped, it sets the block's storage level to `StorageLevel.NONE`, when it should just remove it from the map.
This PR includes this simple fix.
Author: Andrew Or <an...@gmail.com>
Closes #1080 from andrewor14/ui-blocks and squashes the following commits:
fcf9f1a [Andrew Or] Remove BlockStatus if it is no longer cached
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/09deb3ee
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/09deb3ee
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/09deb3ee
Branch: refs/heads/master
Commit: 09deb3eee090eb8ec1d9a0cd90825699748e3ffc
Parents: 23a12ce
Author: Andrew Or <an...@gmail.com>
Authored: Tue Jun 17 01:28:22 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Jun 17 01:28:22 2014 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/storage/StorageStatusListener.scala | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/09deb3ee/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
index a6e6627..c694fc8 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
@@ -37,7 +37,11 @@ class StorageStatusListener extends SparkListener {
val filteredStatus = storageStatusList.find(_.blockManagerId.executorId == execId)
filteredStatus.foreach { storageStatus =>
updatedBlocks.foreach { case (blockId, updatedStatus) =>
- storageStatus.blocks(blockId) = updatedStatus
+ if (updatedStatus.storageLevel == StorageLevel.NONE) {
+ storageStatus.blocks.remove(blockId)
+ } else {
+ storageStatus.blocks(blockId) = updatedStatus
+ }
}
}
}