You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2013/12/20 03:12:46 UTC
[2/8] git commit: UI to display serialisation time of a stage.
UI to display serialisation time of a stage.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/882d5448
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/882d5448
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/882d5448
Branch: refs/heads/master
Commit: 882d544856c61573cdd6124e921f700d580d170d
Parents: 8a397a9
Author: Tor Myklebust <tm...@csclub.uwaterloo.ca>
Authored: Mon Dec 16 13:27:03 2013 -0500
Committer: Tor Myklebust <tm...@csclub.uwaterloo.ca>
Committed: Mon Dec 16 13:27:03 2013 -0500
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/882d5448/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 69f9446..81651bd 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -101,6 +101,11 @@ private[spark] class StagePage(parent: JobProgressUI) {
None
}
else {
+ val serializationTimes = validTasks.map{case (info, metrics, exception) =>
+ metrics.get.serializationTime.toDouble}
+ val serializationQuantiles = "Serialization Time" +: Distribution(serializationTimes).get.getQuantiles().map(
+ ms => parent.formatDuration(ms.toLong))
+
val serviceTimes = validTasks.map{case (info, metrics, exception) =>
metrics.get.executorRunTime.toDouble}
val serviceQuantiles = "Duration" +: Distribution(serviceTimes).get.getQuantiles().map(
@@ -149,6 +154,7 @@ private[spark] class StagePage(parent: JobProgressUI) {
val shuffleWriteQuantiles = "Shuffle Write" +: getQuantileCols(shuffleWriteSizes)
val listings: Seq[Seq[String]] = Seq(
+ serializationQuantiles,
serviceQuantiles,
gettingResultQuantiles,
schedulerDelayQuantiles,