You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/11/25 00:50:59 UTC
[1/2] git commit: Fix 'timeWriting' stat for shuffle files
Updated Branches:
refs/heads/master 718cc803f -> 972171b9d
Fix 'timeWriting' stat for shuffle files
Due to concurrent git branches, changes from shuffle file consolidation patch
caused the shuffle write timing patch to no longer actually measure the time,
since it requires time be measured after the stream has been closed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/ccea38b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/ccea38b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/ccea38b7
Branch: refs/heads/master
Commit: ccea38b759c81abea27bc0a51157a31d369839b5
Parents: e2ebc3a
Author: Aaron Davidson <aa...@databricks.com>
Authored: Thu Nov 21 21:36:08 2013 -0800
Committer: Aaron Davidson <aa...@databricks.com>
Committed: Thu Nov 21 21:36:08 2013 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/scheduler/ShuffleMapTask.scala | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ccea38b7/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
index 1dc71a0..0f2deb4 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
@@ -167,6 +167,7 @@ private[spark] class ShuffleMapTask(
var totalTime = 0L
val compressedSizes: Array[Byte] = shuffle.writers.map { writer: BlockObjectWriter =>
writer.commit()
+ writer.close()
val size = writer.fileSegment().length
totalBytes += size
totalTime += writer.timeWriting()
@@ -184,14 +185,16 @@ private[spark] class ShuffleMapTask(
} catch { case e: Exception =>
// If there is an exception from running the task, revert the partial writes
// and throw the exception upstream to Spark.
- if (shuffle != null) {
- shuffle.writers.foreach(_.revertPartialWrites())
+ if (shuffle != null && shuffle.writers != null) {
+ for (writer <- shuffle.writers) {
+ writer.revertPartialWrites()
+ writer.close()
+ }
}
throw e
} finally {
// Release the writers back to the shuffle block manager.
if (shuffle != null && shuffle.writers != null) {
- shuffle.writers.foreach(_.close())
shuffle.releaseWriters(success)
}
// Execute the callbacks on task completion.
[2/2] git commit: Merge pull request #197 from aarondav/patrick-fix
Posted by rx...@apache.org.
Merge pull request #197 from aarondav/patrick-fix
Fix 'timeWriting' stat for shuffle files
Due to concurrent git branches, changes from shuffle file consolidation patch
caused the shuffle write timing patch to no longer actually measure the time,
since it requires time be measured after the stream has been closed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/972171b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/972171b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/972171b9
Branch: refs/heads/master
Commit: 972171b9d93b07e8511a2da3a33f897ba033484b
Parents: 718cc80 ccea38b
Author: Reynold Xin <rx...@apache.org>
Authored: Mon Nov 25 07:50:46 2013 +0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Mon Nov 25 07:50:46 2013 +0800
----------------------------------------------------------------------
.../scala/org/apache/spark/scheduler/ShuffleMapTask.scala | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------