You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/07/17 02:30:59 UTC
[02/50] [abbrv] git commit: Show block locations in Web UI.
Show block locations in Web UI.
This fixes SPARK-769. Support is added for enumerating the locations of blocks
in the UI. There is also some minor cleanup in StorageUtils.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/6855338e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/6855338e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/6855338e
Branch: refs/heads/master
Commit: 6855338e1400638188358a7d7926eb86f668c160
Parents: 018d04c
Author: Patrick Wendell <pw...@gmail.com>
Authored: Fri Jul 12 19:24:16 2013 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Fri Jul 12 19:30:32 2013 -0700
----------------------------------------------------------------------
.../main/scala/spark/storage/StorageUtils.scala | 11 +++++++++--
core/src/main/scala/spark/ui/storage/RDDPage.scala | 17 +++++++++++++----
2 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/6855338e/core/src/main/scala/spark/storage/StorageUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/spark/storage/StorageUtils.scala b/core/src/main/scala/spark/storage/StorageUtils.scala
index 950c0cd..3e7fa28 100644
--- a/core/src/main/scala/spark/storage/StorageUtils.scala
+++ b/core/src/main/scala/spark/storage/StorageUtils.scala
@@ -39,12 +39,19 @@ case class RDDInfo(id: Int, name: String, storageLevel: StorageLevel,
private[spark]
object StorageUtils {
- /* Given the current storage status of the BlockManager, returns information for each RDD */
- def rddInfoFromStorageStatus(storageStatusList: Array[StorageStatus],
+ /* Returns RDD-level information, compiled from a list of StorageStatus objects */
+ def rddInfoFromStorageStatus(storageStatusList: Seq[StorageStatus],
sc: SparkContext) : Array[RDDInfo] = {
rddInfoFromBlockStatusList(storageStatusList.flatMap(_.blocks).toMap, sc)
}
+ /* Returns a map of blocks to their locations, compiled from a list of StorageStatus objects */
+ def blockLocationsFromStorageStatus(storageStatusList: Seq[StorageStatus]) = {
+ val blockLocationPairs = storageStatusList
+ .flatMap(s => s.blocks.map(b => (b._1, s.blockManagerId.hostPort)))
+ blockLocationPairs.groupBy(_._1).map{case (k, v) => (k, v.unzip._2)}.toMap
+ }
+
/* Given a list of BlockStatus objets, returns information for each RDD */
def rddInfoFromBlockStatusList(infos: Map[String, BlockStatus],
sc: SparkContext) : Array[RDDInfo] = {
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/6855338e/core/src/main/scala/spark/ui/storage/RDDPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/spark/ui/storage/RDDPage.scala b/core/src/main/scala/spark/ui/storage/RDDPage.scala
index 0cb1e47..428db6f 100644
--- a/core/src/main/scala/spark/ui/storage/RDDPage.scala
+++ b/core/src/main/scala/spark/ui/storage/RDDPage.scala
@@ -26,8 +26,14 @@ private[spark] class RDDPage(parent: BlockManagerUI) {
val workers = filteredStorageStatusList.map((prefix, _))
val workerTable = listingTable(workerHeaders, workerRow, workers)
- val blockHeaders = Seq("Block Name", "Storage Level", "Size in Memory", "Size on Disk")
- val blocks = filteredStorageStatusList.flatMap(_.blocks).toArray.sortWith(_._1 < _._1)
+ val blockHeaders = Seq("Block Name", "Storage Level", "Size in Memory", "Size on Disk",
+ "Locations")
+
+ val blockStatuses = filteredStorageStatusList.flatMap(_.blocks).toArray.sortWith(_._1 < _._1)
+ val blockLocations = StorageUtils.blockLocationsFromStorageStatus(filteredStorageStatusList)
+ val blocks = blockStatuses.map {
+ case(id, status) => (id, status, blockLocations.get(id).getOrElse(Seq("UNKNOWN")))
+ }
val blockTable = listingTable(blockHeaders, blockRow, blocks)
val content =
@@ -74,8 +80,8 @@ private[spark] class RDDPage(parent: BlockManagerUI) {
headerSparkPage(content, parent.sc, "RDD Info: " + rddInfo.name, Jobs)
}
- def blockRow(blk: (String, BlockStatus)): Seq[Node] = {
- val (id, block) = blk
+ def blockRow(row: (String, BlockStatus, Seq[String])): Seq[Node] = {
+ val (id, block, locations) = row
<tr>
<td>{id}</td>
<td>
@@ -87,6 +93,9 @@ private[spark] class RDDPage(parent: BlockManagerUI) {
<td sorttable_customkey={block.diskSize.toString}>
{Utils.memoryBytesToString(block.diskSize)}
</td>
+ <td>
+ {locations.map(l => <span>{l}<br/></span>)}
+ </td>
</tr>
}