You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/02/02 01:23:21 UTC
spark git commit: [SPARK-12637][CORE] Print stage info of finished
stages properly
Repository: spark
Updated Branches:
refs/heads/master be7a2fc07 -> 715a19d56
[SPARK-12637][CORE] Print stage info of finished stages properly
Improve printing of StageInfo in onStageCompleted
See also https://github.com/apache/spark/pull/10585
Author: Sean Owen <so...@cloudera.com>
Closes #10922 from srowen/SPARK-12637.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/715a19d5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/715a19d5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/715a19d5
Branch: refs/heads/master
Commit: 715a19d56fc934d4aec5025739ff650daf4580b7
Parents: be7a2fc
Author: Sean Owen <so...@cloudera.com>
Authored: Mon Feb 1 16:23:17 2016 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Mon Feb 1 16:23:17 2016 -0800
----------------------------------------------------------------------
.../org/apache/spark/scheduler/SparkListener.scala | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/715a19d5/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
index ed3adbd..7b09c2e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
@@ -270,7 +270,7 @@ class StatsReportListener extends SparkListener with Logging {
override def onStageCompleted(stageCompleted: SparkListenerStageCompleted) {
implicit val sc = stageCompleted
- this.logInfo("Finished stage: " + stageCompleted.stageInfo)
+ this.logInfo(s"Finished stage: ${getStatusDetail(stageCompleted.stageInfo)}")
showMillisDistribution("task runtime:", (info, _) => Some(info.duration), taskInfoMetrics)
// Shuffle write
@@ -297,6 +297,17 @@ class StatsReportListener extends SparkListener with Logging {
taskInfoMetrics.clear()
}
+ private def getStatusDetail(info: StageInfo): String = {
+ val failureReason = info.failureReason.map("(" + _ + ")").getOrElse("")
+ val timeTaken = info.submissionTime.map(
+ x => info.completionTime.getOrElse(System.currentTimeMillis()) - x
+ ).getOrElse("-")
+
+ s"Stage(${info.stageId}, ${info.attemptId}); Name: '${info.name}'; " +
+ s"Status: ${info.getStatusString}$failureReason; numTasks: ${info.numTasks}; " +
+ s"Took: $timeTaken msec"
+ }
+
}
private[spark] object StatsReportListener extends Logging {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org