You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sh...@apache.org on 2014/08/10 21:44:31 UTC
git commit: [SPARK-2950] Add gc time and shuffle write time to
JobLogger
Repository: spark
Updated Branches:
refs/heads/master 3570119c3 -> 1d03a26a4
[SPARK-2950] Add gc time and shuffle write time to JobLogger
The JobLogger is very useful for performing offline performance profiling of Spark jobs. GC Time and Shuffle Write time are available in TaskMetrics but are currently missed from the JobLogger output. This patch adds these two fields.
~~Since this is a small change, I didn't create a JIRA. Let me know if I should do that.~~
cc kayousterhout
Author: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Closes #1869 from shivaram/job-logger and squashes the following commits:
1b709fc [Shivaram Venkataraman] Add a space before GC_TIME
c418105 [Shivaram Venkataraman] Add gc time and shuffle write time to JobLogger
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1d03a26a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1d03a26a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1d03a26a
Branch: refs/heads/master
Commit: 1d03a26a4895c24ebfab1a3cf6656af75cb53003
Parents: 3570119
Author: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Authored: Sun Aug 10 12:44:17 2014 -0700
Committer: Shivaram Venkataraman <sh...@cs.berkeley.edu>
Committed: Sun Aug 10 12:44:17 2014 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/scheduler/JobLogger.scala | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1d03a26a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala
index 47dd112..4d6b5c8 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala
@@ -162,6 +162,7 @@ class JobLogger(val user: String, val logDirName: String) extends SparkListener
" START_TIME=" + taskInfo.launchTime + " FINISH_TIME=" + taskInfo.finishTime +
" EXECUTOR_ID=" + taskInfo.executorId + " HOST=" + taskMetrics.hostname
val executorRunTime = " EXECUTOR_RUN_TIME=" + taskMetrics.executorRunTime
+ val gcTime = " GC_TIME=" + taskMetrics.jvmGCTime
val inputMetrics = taskMetrics.inputMetrics match {
case Some(metrics) =>
" READ_METHOD=" + metrics.readMethod.toString +
@@ -179,11 +180,13 @@ class JobLogger(val user: String, val logDirName: String) extends SparkListener
case None => ""
}
val writeMetrics = taskMetrics.shuffleWriteMetrics match {
- case Some(metrics) => " SHUFFLE_BYTES_WRITTEN=" + metrics.shuffleBytesWritten
+ case Some(metrics) =>
+ " SHUFFLE_BYTES_WRITTEN=" + metrics.shuffleBytesWritten +
+ " SHUFFLE_WRITE_TIME=" + metrics.shuffleWriteTime
case None => ""
}
- stageLogInfo(stageId, status + info + executorRunTime + inputMetrics + shuffleReadMetrics +
- writeMetrics)
+ stageLogInfo(stageId, status + info + executorRunTime + gcTime + inputMetrics +
+ shuffleReadMetrics + writeMetrics)
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org