You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/11/19 05:15:05 UTC
spark git commit: [SPARK-4380] Log more precise number of bytes
spilled (1.1)
Repository: spark
Updated Branches:
refs/heads/branch-1.1 f9739b9c8 -> e22a75923
[SPARK-4380] Log more precise number of bytes spilled (1.1)
This is the branch-1.1 version of #3243.
Author: Andrew Or <an...@databricks.com>
Closes #3355 from andrewor14/spill-log-bytes-1.1 and squashes the following commits:
36ec152 [Andrew Or] Log more precise representation of bytes in spilling code
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e22a7592
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e22a7592
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e22a7592
Branch: refs/heads/branch-1.1
Commit: e22a75923e508e17b924f341d02cd5cd679210ca
Parents: f9739b9
Author: Andrew Or <an...@databricks.com>
Authored: Tue Nov 18 20:15:00 2014 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Tue Nov 18 20:15:00 2014 -0800
----------------------------------------------------------------------
.../apache/spark/util/collection/ExternalAppendOnlyMap.scala | 5 +++--
.../scala/org/apache/spark/util/collection/ExternalSorter.scala | 5 +++--
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e22a7592/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index 3e09c25..96697d2 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -174,8 +174,9 @@ class ExternalAppendOnlyMap[K, V, C](
private def spill(mapSize: Long): Unit = {
spillCount += 1
val threadId = Thread.currentThread().getId
- logInfo("Thread %d spilling in-memory map of %d MB to disk (%d time%s so far)"
- .format(threadId, mapSize / (1024 * 1024), spillCount, if (spillCount > 1) "s" else ""))
+ logInfo("Thread %d spilling in-memory batch of %s to disk (%d times%s so far)"
+ .format(threadId, org.apache.spark.util.Utils.bytesToString(mapSize),
+ spillCount, if (spillCount > 1) "s" else ""))
val (blockId, file) = diskBlockManager.createTempLocalBlock()
curWriteMetrics = new ShuffleWriteMetrics()
var writer = blockManager.getDiskWriter(blockId, file, serializer, fileBufferSize,
http://git-wip-us.apache.org/repos/asf/spark/blob/e22a7592/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
index 97ddd96..d414ce3 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
@@ -268,8 +268,9 @@ private[spark] class ExternalSorter[K, V, C](
spillCount += 1
val threadId = Thread.currentThread().getId
- logInfo("Thread %d spilling in-memory batch of %d MB to disk (%d spill%s so far)"
- .format(threadId, memorySize / (1024 * 1024), spillCount, if (spillCount > 1) "s" else ""))
+ logInfo("Thread %d spilling in-memory batch of %s to disk (%d spill%s so far)"
+ .format(threadId, org.apache.spark.util.Utils.bytesToString(memorySize),
+ spillCount, if (spillCount > 1) "s" else ""))
if (bypassMergeSort) {
spillToPartitionFiles(collection)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org