You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2015/06/29 17:47:10 UTC
spark git commit: [SPARK-8702] [WEBUI] Avoid massive concating
strings in Javascript
Repository: spark
Updated Branches:
refs/heads/master 660c6cec7 -> 630bd5fd8
[SPARK-8702] [WEBUI] Avoid massive concating strings in Javascript
When there are massive tasks, such as `sc.parallelize(1 to 100000, 10000).count()`, the generated JS codes have a lot of string concatenations in the stage page, nearly 40 string concatenations for one task.
We can generate the whole string for a task instead of execution string concatenations in the browser.
Before this patch, the load time of the page is about 21 seconds.
After this patch, it reduces to about 17 seconds.
One disadvantage is that the generated JS codes become hard to read.
Author: zsxwing <zs...@gmail.com>
Closes #7082 from zsxwing/js-string and squashes the following commits:
b29231d [zsxwing] Avoid massive concating strings in Javascript
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/630bd5fd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/630bd5fd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/630bd5fd
Branch: refs/heads/master
Commit: 630bd5fd80193ab6dc6ad0e7bcc13ee0dadabd38
Parents: 660c6ce
Author: zsxwing <zs...@gmail.com>
Authored: Tue Jun 30 00:46:55 2015 +0900
Committer: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Committed: Tue Jun 30 00:46:55 2015 +0900
----------------------------------------------------------------------
.../org/apache/spark/ui/jobs/StagePage.scala | 88 ++++++++++----------
1 file changed, 44 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/630bd5fd/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index b83a49f..e96bf49 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -572,55 +572,55 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val attempt = taskInfo.attempt
val timelineObject =
s"""
- {
- 'className': 'task task-assignment-timeline-object',
- 'group': '$executorId',
- 'content': '<div class="task-assignment-timeline-content"' +
- 'data-toggle="tooltip" data-placement="top"' +
- 'data-html="true" data-container="body"' +
- 'data-title="${s"Task " + index + " (attempt " + attempt + ")"}<br>' +
- 'Status: ${taskInfo.status}<br>' +
- 'Launch Time: ${UIUtils.formatDate(new Date(launchTime))}' +
- '${
+ |{
+ |'className': 'task task-assignment-timeline-object',
+ |'group': '$executorId',
+ |'content': '<div class="task-assignment-timeline-content"
+ |data-toggle="tooltip" data-placement="top"
+ |data-html="true" data-container="body"
+ |data-title="${s"Task " + index + " (attempt " + attempt + ")"}<br>
+ |Status: ${taskInfo.status}<br>
+ |Launch Time: ${UIUtils.formatDate(new Date(launchTime))}
+ |${
if (!taskInfo.running) {
s"""<br>Finish Time: ${UIUtils.formatDate(new Date(finishTime))}"""
} else {
""
}
- }' +
- '<br>Scheduler Delay: $schedulerDelay ms' +
- '<br>Task Deserialization Time: ${UIUtils.formatDuration(deserializationTime)}' +
- '<br>Shuffle Read Time: ${UIUtils.formatDuration(shuffleReadTime)}' +
- '<br>Executor Computing Time: ${UIUtils.formatDuration(executorComputingTime)}' +
- '<br>Shuffle Write Time: ${UIUtils.formatDuration(shuffleWriteTime)}' +
- '<br>Result Serialization Time: ${UIUtils.formatDuration(serializationTime)}' +
- '<br>Getting Result Time: ${UIUtils.formatDuration(gettingResultTime)}">' +
- '<svg class="task-assignment-timeline-duration-bar">' +
- '<rect class="scheduler-delay-proportion" ' +
- 'x="$schedulerDelayProportionPos%" y="0px" height="26px"' +
- 'width="$schedulerDelayProportion%""></rect>' +
- '<rect class="deserialization-time-proportion" '+
- 'x="$deserializationTimeProportionPos%" y="0px" height="26px"' +
- 'width="$deserializationTimeProportion%"></rect>' +
- '<rect class="shuffle-read-time-proportion" ' +
- 'x="$shuffleReadTimeProportionPos%" y="0px" height="26px"' +
- 'width="$shuffleReadTimeProportion%"></rect>' +
- '<rect class="executor-runtime-proportion" ' +
- 'x="$executorRuntimeProportionPos%" y="0px" height="26px"' +
- 'width="$executorComputingTimeProportion%"></rect>' +
- '<rect class="shuffle-write-time-proportion" ' +
- 'x="$shuffleWriteTimeProportionPos%" y="0px" height="26px"' +
- 'width="$shuffleWriteTimeProportion%"></rect>' +
- '<rect class="serialization-time-proportion" ' +
- 'x="$serializationTimeProportionPos%" y="0px" height="26px"' +
- 'width="$serializationTimeProportion%"></rect>' +
- '<rect class="getting-result-time-proportion" ' +
- 'x="$gettingResultTimeProportionPos%" y="0px" height="26px"' +
- 'width="$gettingResultTimeProportion%"></rect></svg>',
- 'start': new Date($launchTime),
- 'end': new Date($finishTime)
- }
- """
+ }
+ |<br>Scheduler Delay: $schedulerDelay ms
+ |<br>Task Deserialization Time: ${UIUtils.formatDuration(deserializationTime)}
+ |<br>Shuffle Read Time: ${UIUtils.formatDuration(shuffleReadTime)}
+ |<br>Executor Computing Time: ${UIUtils.formatDuration(executorComputingTime)}
+ |<br>Shuffle Write Time: ${UIUtils.formatDuration(shuffleWriteTime)}
+ |<br>Result Serialization Time: ${UIUtils.formatDuration(serializationTime)}
+ |<br>Getting Result Time: ${UIUtils.formatDuration(gettingResultTime)}">
+ |<svg class="task-assignment-timeline-duration-bar">
+ |<rect class="scheduler-delay-proportion"
+ |x="$schedulerDelayProportionPos%" y="0px" height="26px"
+ |width="$schedulerDelayProportion%""></rect>
+ |<rect class="deserialization-time-proportion"
+ |x="$deserializationTimeProportionPos%" y="0px" height="26px"
+ |width="$deserializationTimeProportion%"></rect>
+ |<rect class="shuffle-read-time-proportion"
+ |x="$shuffleReadTimeProportionPos%" y="0px" height="26px"
+ |width="$shuffleReadTimeProportion%"></rect>
+ |<rect class="executor-runtime-proportion"
+ |x="$executorRuntimeProportionPos%" y="0px" height="26px"
+ |width="$executorComputingTimeProportion%"></rect>
+ |<rect class="shuffle-write-time-proportion"
+ |x="$shuffleWriteTimeProportionPos%" y="0px" height="26px"
+ |width="$shuffleWriteTimeProportion%"></rect>
+ |<rect class="serialization-time-proportion"
+ |x="$serializationTimeProportionPos%" y="0px" height="26px"
+ |width="$serializationTimeProportion%"></rect>
+ |<rect class="getting-result-time-proportion"
+ |x="$gettingResultTimeProportionPos%" y="0px" height="26px"
+ |width="$gettingResultTimeProportion%"></rect></svg>',
+ |'start': new Date($launchTime),
+ |'end': new Date($finishTime)
+ |}
+ |""".stripMargin.replaceAll("\n", " ")
timelineObject
}.mkString("[", ",", "]")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org