You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/11/13 01:20:43 UTC
[1/4] git commit: Fix bug JIRA-923
Updated Branches:
refs/heads/master dfd1ebc2d -> b8bf04a08
Fix bug JIRA-923
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/b3208063
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/b3208063
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/b3208063
Branch: refs/heads/master
Commit: b3208063afe7190efd6e54a41b7af28d15c46358
Parents: 714fdab
Author: Andrew xia <ju...@intel.com>
Authored: Mon Nov 11 15:39:10 2013 +0800
Committer: Andrew xia <ju...@intel.com>
Committed: Mon Nov 11 15:39:10 2013 +0800
----------------------------------------------------------------------
.../apache/spark/ui/UIWorkloadGenerator.scala | 2 +-
.../org/apache/spark/ui/jobs/StagePage.scala | 24 ++++++++++++++++----
.../org/apache/spark/ui/jobs/StageTable.scala | 11 +++++----
3 files changed, 28 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b3208063/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala b/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
index 453394d..fcd1b51 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
@@ -35,7 +35,7 @@ private[spark] object UIWorkloadGenerator {
def main(args: Array[String]) {
if (args.length < 2) {
- println("usage: ./spark-class spark.ui.UIWorkloadGenerator [master] [FIFO|FAIR]")
+ println("usage: ./spark-class org.apache.spark.ui.UIWorkloadGenerator [master] [FIFO|FAIR]")
System.exit(1)
}
val master = args(0)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b3208063/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 163a374..9316465 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -152,6 +152,20 @@ private[spark] class StagePage(parent: JobProgressUI) {
else metrics.map(m => parent.formatDuration(m.executorRunTime)).getOrElse("")
val gcTime = metrics.map(m => m.jvmGCTime).getOrElse(0L)
+ var shuffleReadSortable: String = ""
+ var shuffleReadReadable: String = ""
+ if (shuffleRead) {
+ shuffleReadSortable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s => s.remoteBytesRead}.toString()
+ shuffleReadReadable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")
+ }
+
+ var shuffleWriteSortable: String = ""
+ var shuffleWriteReadable: String = ""
+ if (shuffleWrite) {
+ shuffleWriteSortable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => s.shuffleBytesWritten}.toString()
+ shuffleWriteReadable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("")
+ }
+
<tr>
<td>{info.taskId}</td>
<td>{info.status}</td>
@@ -165,12 +179,14 @@ private[spark] class StagePage(parent: JobProgressUI) {
{if (gcTime > 0) parent.formatDuration(gcTime) else ""}
</td>
{if (shuffleRead) {
- <td>{metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>
- Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")}</td>
+ <td sorttable_customkey={shuffleReadSortable}>
+ {shuffleReadReadable}
+ </td>
}}
{if (shuffleWrite) {
- <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>
- Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("")}</td>
+ <td sorttable_customkey={shuffleWriteSortable}>
+ {shuffleWriteReadable}
+ </td>
}}
<td>{exception.map(e =>
<span>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b3208063/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index 07db862..c9ee66f 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -79,11 +79,14 @@ private[spark] class StageTable(val stages: Seq[Stage], val parent: JobProgressU
case None => "Unknown"
}
- val shuffleRead = listener.stageToShuffleRead.getOrElse(s.id, 0L) match {
+ val shuffleReadSortable = listener.stageToShuffleRead.getOrElse(s.id, 0L)
+ val shuffleRead = shuffleReadSortable match {
case 0 => ""
case b => Utils.bytesToString(b)
}
- val shuffleWrite = listener.stageToShuffleWrite.getOrElse(s.id, 0L) match {
+
+ val shuffleWriteSortable = listener.stageToShuffleWrite.getOrElse(s.id, 0L)
+ val shuffleWrite = shuffleWriteSortable match {
case 0 => ""
case b => Utils.bytesToString(b)
}
@@ -119,8 +122,8 @@ private[spark] class StageTable(val stages: Seq[Stage], val parent: JobProgressU
<td class="progress-cell">
{makeProgressBar(startedTasks, completedTasks, failedTasks, totalTasks)}
</td>
- <td>{shuffleRead}</td>
- <td>{shuffleWrite}</td>
+ <td sorttable_customekey={shuffleReadSortable.toString}>{shuffleRead}</td>
+ <td sorttable_customekey={shuffleWriteSortable.toString}>{shuffleWrite}</td>
</tr>
}
}
[4/4] git commit: Merge pull request #160 from xiajunluan/JIRA-923
Posted by ma...@apache.org.
Merge pull request #160 from xiajunluan/JIRA-923
Fix bug JIRA-923
Fix column sort issue in UI for JIRA-923.
https://spark-project.atlassian.net/browse/SPARK-923
Conflicts:
core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/b8bf04a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/b8bf04a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/b8bf04a0
Branch: refs/heads/master
Commit: b8bf04a085162478a64ca5d7be15d1af2f6a930e
Parents: dfd1ebc e13da05
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Tue Nov 12 16:19:50 2013 -0800
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Tue Nov 12 16:19:50 2013 -0800
----------------------------------------------------------------------
.../org/apache/spark/ui/jobs/StagePage.scala | 31 ++++++++++++++++----
.../org/apache/spark/ui/jobs/StageTable.scala | 11 ++++---
2 files changed, 32 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b8bf04a0/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --cc core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 35b5d5f,6c67151..c1c7aa7
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@@ -152,8 -152,23 +152,24 @@@ private[spark] class StagePage(parent:
else metrics.map(m => parent.formatDuration(m.executorRunTime)).getOrElse("")
val gcTime = metrics.map(m => m.jvmGCTime).getOrElse(0L)
+ var shuffleReadSortable: String = ""
+ var shuffleReadReadable: String = ""
+ if (shuffleRead) {
+ shuffleReadSortable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s => s.remoteBytesRead}.toString()
+ shuffleReadReadable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>
+ Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")
+ }
+
+ var shuffleWriteSortable: String = ""
+ var shuffleWriteReadable: String = ""
+ if (shuffleWrite) {
+ shuffleWriteSortable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => s.shuffleBytesWritten}.toString()
+ shuffleWriteReadable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>
+ Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("")
+ }
+
<tr>
+ <td>{info.index}</td>
<td>{info.taskId}</td>
<td>{info.status}</td>
<td>{info.taskLocality}</td>
@@@ -166,14 -181,14 +182,17 @@@
{if (gcTime > 0) parent.formatDuration(gcTime) else ""}
</td>
{if (shuffleRead) {
- <td>{metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>
- Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")}</td>
+ <td sorttable_customkey={shuffleReadSortable}>
+ {shuffleReadReadable}
+ </td>
}}
{if (shuffleWrite) {
- <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>
- parent.formatDuration(s.shuffleWriteTime / (1000 * 1000))}.getOrElse("")}</td>
- <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>
- Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("")}</td>
++ <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>
++ parent.formatDuration(s.shuffleWriteTime / (1000 * 1000))}.getOrElse("")}
++ </td>
+ <td sorttable_customkey={shuffleWriteSortable}>
+ {shuffleWriteReadable}
+ </td>
}}
<td>{exception.map(e =>
<span>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b8bf04a0/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
----------------------------------------------------------------------
diff --cc core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index d7d0441,c9ee66f..9ad6de3
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@@ -79,11 -79,14 +79,14 @@@ private[spark] class StageTable(val sta
case None => "Unknown"
}
- val shuffleRead = listener.stageIdToShuffleRead.getOrElse(s.stageId, 0L) match {
- val shuffleReadSortable = listener.stageToShuffleRead.getOrElse(s.id, 0L)
++ val shuffleReadSortable = listener.stageIdToShuffleRead.getOrElse(s.stageId, 0L)
+ val shuffleRead = shuffleReadSortable match {
case 0 => ""
case b => Utils.bytesToString(b)
}
- val shuffleWrite = listener.stageIdToShuffleWrite.getOrElse(s.stageId, 0L) match {
+
- val shuffleWriteSortable = listener.stageToShuffleWrite.getOrElse(s.id, 0L)
++ val shuffleWriteSortable = listener.stageIdToShuffleWrite.getOrElse(s.stageId, 0L)
+ val shuffleWrite = shuffleWriteSortable match {
case 0 => ""
case b => Utils.bytesToString(b)
}
[3/4] git commit: fix format error
Posted by ma...@apache.org.
fix format error
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/e13da054
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/e13da054
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/e13da054
Branch: refs/heads/master
Commit: e13da05424866785dcbe37754113a2a8aa63f706
Parents: 37d2f37
Author: Andrew xia <ju...@intel.com>
Authored: Mon Nov 11 19:15:45 2013 +0800
Committer: Andrew xia <ju...@intel.com>
Committed: Mon Nov 11 19:15:45 2013 +0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/ui/jobs/StagePage.scala | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/e13da054/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 964843e..6c67151 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -181,14 +181,14 @@ private[spark] class StagePage(parent: JobProgressUI) {
{if (gcTime > 0) parent.formatDuration(gcTime) else ""}
</td>
{if (shuffleRead) {
- <td sorttable_customkey={shuffleReadSortable}>
- {shuffleReadReadable}
- </td>
+ <td sorttable_customkey={shuffleReadSortable}>
+ {shuffleReadReadable}
+ </td>
}}
{if (shuffleWrite) {
- <td sorttable_customkey={shuffleWriteSortable}>
- {shuffleWriteReadable}
- </td>
+ <td sorttable_customkey={shuffleWriteSortable}>
+ {shuffleWriteReadable}
+ </td>
}}
<td>{exception.map(e =>
<span>
[2/4] git commit: cut lines to less than 100
Posted by ma...@apache.org.
cut lines to less than 100
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/37d2f374
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/37d2f374
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/37d2f374
Branch: refs/heads/master
Commit: 37d2f3749eec583acb23b6b1e0a5b453170fc965
Parents: b320806
Author: Andrew xia <ju...@intel.com>
Authored: Mon Nov 11 15:49:32 2013 +0800
Committer: Andrew xia <ju...@intel.com>
Committed: Mon Nov 11 15:49:32 2013 +0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/37d2f374/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 9316465..964843e 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -156,14 +156,16 @@ private[spark] class StagePage(parent: JobProgressUI) {
var shuffleReadReadable: String = ""
if (shuffleRead) {
shuffleReadSortable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s => s.remoteBytesRead}.toString()
- shuffleReadReadable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")
+ shuffleReadReadable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>
+ Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")
}
var shuffleWriteSortable: String = ""
var shuffleWriteReadable: String = ""
if (shuffleWrite) {
shuffleWriteSortable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => s.shuffleBytesWritten}.toString()
- shuffleWriteReadable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("")
+ shuffleWriteReadable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s =>
+ Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("")
}
<tr>