You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/11/20 01:11:42 UTC
[1/4] git commit: correct number of tasks in ExecutorsUI
Updated Branches:
refs/heads/master aa638ed9c -> f568912f8
correct number of tasks in ExecutorsUI
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/b60839e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/b60839e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/b60839e5
Branch: refs/heads/master
Commit: b60839e56a335d0e30578d1c4cad5b0319d565df
Parents: 96e0fb4
Author: BlackNiuza <ze...@gmail.com>
Authored: Sun Nov 17 21:38:57 2013 +0800
Committer: shiyun.wxm <sh...@taobao.com>
Committed: Sun Nov 17 21:38:57 2013 +0800
----------------------------------------------------------------------
.../org/apache/spark/ui/exec/ExecutorsUI.scala | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b60839e5/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
index 42e9be6..ba198b2 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
@@ -100,15 +100,16 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
}
def getExecInfo(a: Int): Seq[String] = {
- val execId = sc.getExecutorStorageStatus(a).blockManagerId.executorId
- val hostPort = sc.getExecutorStorageStatus(a).blockManagerId.hostPort
- val rddBlocks = sc.getExecutorStorageStatus(a).blocks.size.toString
- val memUsed = sc.getExecutorStorageStatus(a).memUsed().toString
- val maxMem = sc.getExecutorStorageStatus(a).maxMem.toString
- val diskUsed = sc.getExecutorStorageStatus(a).diskUsed().toString
- val activeTasks = listener.executorToTasksActive.get(a.toString).map(l => l.size).getOrElse(0)
- val failedTasks = listener.executorToTasksFailed.getOrElse(a.toString, 0)
- val completedTasks = listener.executorToTasksComplete.getOrElse(a.toString, 0)
+ val status = sc.getExecutorStorageStatus(a)
+ val execId = status.blockManagerId.executorId
+ val hostPort = status.blockManagerId.hostPort
+ val rddBlocks = status.blocks.size.toString
+ val memUsed = status.memUsed().toString
+ val maxMem = status.maxMem.toString
+ val diskUsed = status.diskUsed().toString
+ val activeTasks = listener.executorToTasksActive.getOrElse(execId, Seq[Long]()).size
+ val failedTasks = listener.executorToTasksFailed.getOrElse(execId, 0)
+ val completedTasks = listener.executorToTasksComplete.getOrElse(execId, 0)
val totalTasks = activeTasks + failedTasks + completedTasks
Seq(
[2/4] git commit: rename "a" to "statusId"
Posted by ma...@apache.org.
rename "a" to "statusId"
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/ecfbaf24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/ecfbaf24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/ecfbaf24
Branch: refs/heads/master
Commit: ecfbaf24426948a9c09225190e71bc1148a9944b
Parents: b60839e
Author: BlackNiuza <ze...@gmail.com>
Authored: Mon Nov 18 09:51:40 2013 +0800
Committer: shiyun.wxm <sh...@taobao.com>
Committed: Mon Nov 18 09:51:40 2013 +0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ecfbaf24/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
index ba198b2..26245a6 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
@@ -76,7 +76,7 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
</tr>
}
- val execInfo = for (b <- 0 until storageStatusList.size) yield getExecInfo(b)
+ val execInfo = for (statusId <- 0 until storageStatusList.size) yield getExecInfo(statusId)
val execTable = UIUtils.listingTable(execHead, execRow, execInfo)
val content =
@@ -99,8 +99,8 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
UIUtils.headerSparkPage(content, sc, "Executors (" + execInfo.size + ")", Executors)
}
- def getExecInfo(a: Int): Seq[String] = {
- val status = sc.getExecutorStorageStatus(a)
+ def getExecInfo(statusId: Int): Seq[String] = {
+ val status = sc.getExecutorStorageStatus(statusId)
val execId = status.blockManagerId.executorId
val hostPort = status.blockManagerId.hostPort
val rddBlocks = status.blocks.size.toString
[3/4] git commit: use HashSet.empty[Long] instead of Seq[Long]
Posted by ma...@apache.org.
use HashSet.empty[Long] instead of Seq[Long]
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/eda05fa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/eda05fa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/eda05fa4
Branch: refs/heads/master
Commit: eda05fa43953f601d14853f3416e99e012a1bbba
Parents: ecfbaf2
Author: shiyun.wxm <sh...@taobao.com>
Authored: Mon Nov 18 13:31:14 2013 +0800
Committer: shiyun.wxm <sh...@taobao.com>
Committed: Mon Nov 18 13:31:14 2013 +0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/eda05fa4/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
index 26245a6..e596690 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsUI.scala
@@ -107,7 +107,7 @@ private[spark] class ExecutorsUI(val sc: SparkContext) {
val memUsed = status.memUsed().toString
val maxMem = status.maxMem.toString
val diskUsed = status.diskUsed().toString
- val activeTasks = listener.executorToTasksActive.getOrElse(execId, Seq[Long]()).size
+ val activeTasks = listener.executorToTasksActive.getOrElse(execId, HashSet.empty[Long]).size
val failedTasks = listener.executorToTasksFailed.getOrElse(execId, 0)
val completedTasks = listener.executorToTasksComplete.getOrElse(execId, 0)
val totalTasks = activeTasks + failedTasks + completedTasks
[4/4] git commit: Merge pull request #181 from
BlackNiuza/fix_tasks_number
Posted by ma...@apache.org.
Merge pull request #181 from BlackNiuza/fix_tasks_number
correct number of tasks in ExecutorsUI
Index `a` is not `execId` here
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/f568912f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/f568912f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/f568912f
Branch: refs/heads/master
Commit: f568912f85f58ae152db90f199c1f3a002f270c1
Parents: aa638ed eda05fa
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Tue Nov 19 16:11:31 2013 -0800
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Tue Nov 19 16:11:31 2013 -0800
----------------------------------------------------------------------
.../org/apache/spark/ui/exec/ExecutorsUI.scala | 23 ++++++++++----------
1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------