You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2016/03/05 00:04:29 UTC
spark git commit: [SPARK-13459][WEB UI] Separate Alive and Dead
Executors in Executor Totals Table
Repository: spark
Updated Branches:
refs/heads/master b7d414742 -> 5f42c28b1
[SPARK-13459][WEB UI] Separate Alive and Dead Executors in Executor Totals Table
## What changes were proposed in this pull request?
Now that dead executors are shown in the executors table (#10058) the totals table is updated to include the separate totals for alive and dead executors as well as the current total, as originally discussed in #10668
## How was this patch tested?
Manually verified by running the Standalone Web UI in the latest Safari and Firefox ESR
Author: Alex Bozarth <aj...@us.ibm.com>
Closes #11381 from ajbozarth/spark13459.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5f42c28b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5f42c28b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5f42c28b
Branch: refs/heads/master
Commit: 5f42c28b119b79c0ea4910c478853d451cd1a967
Parents: b7d4147
Author: Alex Bozarth <aj...@us.ibm.com>
Authored: Fri Mar 4 17:04:09 2016 -0600
Committer: Tom Graves <tg...@yahoo-inc.com>
Committed: Fri Mar 4 17:04:09 2016 -0600
----------------------------------------------------------------------
.../apache/spark/ui/exec/ExecutorsPage.scala | 84 +++++++++++---------
1 file changed, 45 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5f42c28b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
index eba7a31..791dbe5 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
@@ -86,7 +86,7 @@ private[ui] class ExecutorsPage(
<th>Failed Tasks</th>
<th>Complete Tasks</th>
<th>Total Tasks</th>
- <th data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</th>
+ <th><span data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</span></th>
<th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th>
<th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th>
<th>
@@ -109,13 +109,8 @@ private[ui] class ExecutorsPage(
val content =
<div class="row">
<div class="span12">
- <h4>Dead Executors({deadExecutorInfo.size})</h4>
- </div>
- </div>
- <div class="row">
- <div class="span12">
- <h4>Active Executors({activeExecutorInfo.size})</h4>
- {execSummary(activeExecutorInfo)}
+ <h4>Summary</h4>
+ {execSummary(activeExecutorInfo, deadExecutorInfo)}
</div>
</div>
<div class = "row">
@@ -198,7 +193,7 @@ private[ui] class ExecutorsPage(
</tr>
}
- private def execSummary(execInfo: Seq[ExecutorSummary]): Seq[Node] = {
+ private def execSummaryRow(execInfo: Seq[ExecutorSummary], rowName: String): Seq[Node] = {
val maximumMemory = execInfo.map(_.maxMemory).sum
val memoryUsed = execInfo.map(_.memoryUsed).sum
val diskUsed = execInfo.map(_.diskUsed).sum
@@ -207,37 +202,46 @@ private[ui] class ExecutorsPage(
val totalShuffleRead = execInfo.map(_.totalShuffleRead).sum
val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum
- val sumContent =
- <tr>
- <td>{execInfo.map(_.rddBlocks).sum}</td>
- <td sorttable_customkey={memoryUsed.toString}>
- {Utils.bytesToString(memoryUsed)} /
- {Utils.bytesToString(maximumMemory)}
- </td>
- <td sorttable_customkey={diskUsed.toString}>
- {Utils.bytesToString(diskUsed)}
- </td>
- <td>{totalCores}</td>
- {taskData(execInfo.map(_.maxTasks).sum,
- execInfo.map(_.activeTasks).sum,
- execInfo.map(_.failedTasks).sum,
- execInfo.map(_.completedTasks).sum,
- execInfo.map(_.totalTasks).sum,
- execInfo.map(_.totalDuration).sum,
- execInfo.map(_.totalGCTime).sum)}
- <td sorttable_customkey={totalInputBytes.toString}>
- {Utils.bytesToString(totalInputBytes)}
- </td>
- <td sorttable_customkey={totalShuffleRead.toString}>
- {Utils.bytesToString(totalShuffleRead)}
- </td>
- <td sorttable_customkey={totalShuffleWrite.toString}>
- {Utils.bytesToString(totalShuffleWrite)}
- </td>
- </tr>;
+ <tr>
+ <td><b>{rowName}({execInfo.size})</b></td>
+ <td>{execInfo.map(_.rddBlocks).sum}</td>
+ <td sorttable_customkey={memoryUsed.toString}>
+ {Utils.bytesToString(memoryUsed)} /
+ {Utils.bytesToString(maximumMemory)}
+ </td>
+ <td sorttable_customkey={diskUsed.toString}>
+ {Utils.bytesToString(diskUsed)}
+ </td>
+ <td>{totalCores}</td>
+ {taskData(execInfo.map(_.maxTasks).sum,
+ execInfo.map(_.activeTasks).sum,
+ execInfo.map(_.failedTasks).sum,
+ execInfo.map(_.completedTasks).sum,
+ execInfo.map(_.totalTasks).sum,
+ execInfo.map(_.totalDuration).sum,
+ execInfo.map(_.totalGCTime).sum)}
+ <td sorttable_customkey={totalInputBytes.toString}>
+ {Utils.bytesToString(totalInputBytes)}
+ </td>
+ <td sorttable_customkey={totalShuffleRead.toString}>
+ {Utils.bytesToString(totalShuffleRead)}
+ </td>
+ <td sorttable_customkey={totalShuffleWrite.toString}>
+ {Utils.bytesToString(totalShuffleWrite)}
+ </td>
+ </tr>
+ }
+
+ private def execSummary(activeExecInfo: Seq[ExecutorSummary], deadExecInfo: Seq[ExecutorSummary]):
+ Seq[Node] = {
+ val totalExecInfo = activeExecInfo ++ deadExecInfo
+ val activeRow = execSummaryRow(activeExecInfo, "Active");
+ val deadRow = execSummaryRow(deadExecInfo, "Dead");
+ val totalRow = execSummaryRow(totalExecInfo, "Total");
<table class={UIUtils.TABLE_CLASS_STRIPED}>
<thead>
+ <th></th>
<th>RDD Blocks</th>
<th><span data-toggle="tooltip" title={ToolTips.STORAGE_MEMORY}>Storage Memory</span></th>
<th>Disk Used</th>
@@ -246,7 +250,7 @@ private[ui] class ExecutorsPage(
<th>Failed Tasks</th>
<th>Complete Tasks</th>
<th>Total Tasks</th>
- <th data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</th>
+ <th><span data-toggle="tooltip" title={ToolTips.TASK_TIME}>Task Time (GC Time)</span></th>
<th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th>
<th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th>
<th>
@@ -256,7 +260,9 @@ private[ui] class ExecutorsPage(
</th>
</thead>
<tbody>
- {sumContent}
+ {activeRow}
+ {deadRow}
+ {totalRow}
</tbody>
</table>
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org