You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/02/14 14:46:05 UTC
spark git commit: [SPARK-23394][UI] In RDD storage page show the
executor addresses instead of the IDs
Repository: spark
Updated Branches:
refs/heads/master 357babde5 -> 140f87533
[SPARK-23394][UI] In RDD storage page show the executor addresses instead of the IDs
## What changes were proposed in this pull request?
Extending RDD storage page to show executor addresses in the block table.
## How was this patch tested?
Manually:
Author: “attilapiros” <pi...@gmail.com>
Closes #20589 from attilapiros/SPARK-23394.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/140f8753
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/140f8753
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/140f8753
Branch: refs/heads/master
Commit: 140f87533a468b1046504fc3ff01fbe1637e41cd
Parents: 357babd
Author: “attilapiros” <pi...@gmail.com>
Authored: Wed Feb 14 06:45:54 2018 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Wed Feb 14 06:45:54 2018 -0800
----------------------------------------------------------------------
.../org/apache/spark/ui/storage/RDDPage.scala | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/140f8753/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
index 02cee7f..2674b92 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/RDDPage.scala
@@ -23,7 +23,7 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.{Node, Unparsed}
import org.apache.spark.status.AppStatusStore
-import org.apache.spark.status.api.v1.{RDDDataDistribution, RDDPartitionInfo}
+import org.apache.spark.status.api.v1.{ExecutorSummary, RDDDataDistribution, RDDPartitionInfo}
import org.apache.spark.ui._
import org.apache.spark.util.Utils
@@ -76,7 +76,8 @@ private[ui] class RDDPage(parent: SparkUITab, store: AppStatusStore) extends Web
rddStorageInfo.partitions.get,
blockPageSize,
blockSortColumn,
- blockSortDesc)
+ blockSortDesc,
+ store.executorList(true))
_blockTable.table(page)
} catch {
case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
@@ -182,7 +183,8 @@ private[ui] class BlockDataSource(
rddPartitions: Seq[RDDPartitionInfo],
pageSize: Int,
sortColumn: String,
- desc: Boolean) extends PagedDataSource[BlockTableRowData](pageSize) {
+ desc: Boolean,
+ executorIdToAddress: Map[String, String]) extends PagedDataSource[BlockTableRowData](pageSize) {
private val data = rddPartitions.map(blockRow).sorted(ordering(sortColumn, desc))
@@ -198,7 +200,10 @@ private[ui] class BlockDataSource(
rddPartition.storageLevel,
rddPartition.memoryUsed,
rddPartition.diskUsed,
- rddPartition.executors.mkString(" "))
+ rddPartition.executors
+ .map { id => executorIdToAddress.get(id).getOrElse(id) }
+ .sorted
+ .mkString(" "))
}
/**
@@ -226,7 +231,8 @@ private[ui] class BlockPagedTable(
rddPartitions: Seq[RDDPartitionInfo],
pageSize: Int,
sortColumn: String,
- desc: Boolean) extends PagedTable[BlockTableRowData] {
+ desc: Boolean,
+ executorSummaries: Seq[ExecutorSummary]) extends PagedTable[BlockTableRowData] {
override def tableId: String = "rdd-storage-by-block-table"
@@ -243,7 +249,8 @@ private[ui] class BlockPagedTable(
rddPartitions,
pageSize,
sortColumn,
- desc)
+ desc,
+ executorSummaries.map { ex => (ex.id, ex.hostPort) }.toMap)
override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, "UTF-8")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org