You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/06/16 08:23:43 UTC
git commit: SPARK-1999: StorageLevel in storage tab and RDD Storage
Info never changes
Repository: spark
Updated Branches:
refs/heads/master ca5d9d43b -> a63aa1adb
SPARK-1999: StorageLevel in storage tab and RDD Storage Info never changes
StorageLevel in 'storage tab' and 'RDD Storage Info' never changes even if you call rdd.unpersist() and then you give the rdd another different storage level.
Author: CrazyJvm <cr...@gmail.com>
Closes #968 from CrazyJvm/ui-storagelevel and squashes the following commits:
62555fa [CrazyJvm] change RDDInfo constructor param 'storageLevel' to var, so there's need to add another variable _storageLevel。
9f1571e [CrazyJvm] JIRA https://issues.apache.org/jira/browse/SPARK-1999 UI : StorageLevel in storage tab and RDD Storage Info never changes
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a63aa1ad
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a63aa1ad
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a63aa1ad
Branch: refs/heads/master
Commit: a63aa1adb2dfb19c8189167932ee8569840f96a0
Parents: ca5d9d4
Author: CrazyJvm <cr...@gmail.com>
Authored: Sun Jun 15 23:23:26 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Sun Jun 15 23:23:26 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/storage/RDDInfo.scala | 6 +++---
.../src/main/scala/org/apache/spark/storage/StorageUtils.scala | 3 +++
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a63aa1ad/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
index 023fd6e..5a72e21 100644
--- a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
+++ b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
@@ -26,7 +26,7 @@ class RDDInfo(
val id: Int,
val name: String,
val numPartitions: Int,
- val storageLevel: StorageLevel)
+ var storageLevel: StorageLevel)
extends Ordered[RDDInfo] {
var numCachedPartitions = 0
@@ -36,8 +36,8 @@ class RDDInfo(
override def toString = {
import Utils.bytesToString
- ("RDD \"%s\" (%d) Storage: %s; CachedPartitions: %d; TotalPartitions: %d; MemorySize: %s; " +
- "TachyonSize: %s; DiskSize: %s").format(
+ ("RDD \"%s\" (%d) StorageLevel: %s; CachedPartitions: %d; TotalPartitions: %d; " +
+ "MemorySize: %s; TachyonSize: %s; DiskSize: %s").format(
name, id, storageLevel.toString, numCachedPartitions, numPartitions,
bytesToString(memSize), bytesToString(tachyonSize), bytesToString(diskSize))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/a63aa1ad/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
index 6f3252a..f3bde1d 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
+++ b/core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
@@ -89,10 +89,13 @@ private[spark] object StorageUtils {
// Add up memory, disk and Tachyon sizes
val persistedBlocks =
blocks.filter { status => status.memSize + status.diskSize + status.tachyonSize > 0 }
+ val _storageLevel =
+ if (persistedBlocks.length > 0) persistedBlocks(0).storageLevel else StorageLevel.NONE
val memSize = persistedBlocks.map(_.memSize).reduceOption(_ + _).getOrElse(0L)
val diskSize = persistedBlocks.map(_.diskSize).reduceOption(_ + _).getOrElse(0L)
val tachyonSize = persistedBlocks.map(_.tachyonSize).reduceOption(_ + _).getOrElse(0L)
rddInfoMap.get(rddId).map { rddInfo =>
+ rddInfo.storageLevel = _storageLevel
rddInfo.numCachedPartitions = persistedBlocks.length
rddInfo.memSize = memSize
rddInfo.diskSize = diskSize