You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/14 08:26:54 UTC
[1/2] git commit: Enable compression by default for spills
Updated Branches:
refs/heads/master fdaabdc67 -> 055be5c69
Enable compression by default for spills
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0984647a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0984647a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0984647a
Branch: refs/heads/master
Commit: 0984647aaefffcecf85ebfbdb45e41ecd1e49a8c
Parents: fdaabdc
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Jan 13 23:25:25 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Jan 13 23:25:25 2014 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/storage/BlockManager.scala | 2 +-
docs/configuration.md | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0984647a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
index 6461dee..ed53558 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManager.scala
@@ -81,7 +81,7 @@ private[spark] class BlockManager(
// Whether to compress RDD partitions that are stored serialized
val compressRdds = conf.getBoolean("spark.rdd.compress", false)
// Whether to compress shuffle output temporarily spilled to disk
- val compressShuffleSpill = conf.getBoolean("spark.shuffle.spill.compress", false)
+ val compressShuffleSpill = conf.getBoolean("spark.shuffle.spill.compress", true)
val heartBeatFrequency = BlockManager.getHeartBeatFrequency(conf)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0984647a/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index be06bd1..da70cab 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -156,7 +156,7 @@ Apart from these, the following properties are also available, and may be useful
</tr>
<tr>
<td>spark.shuffle.spill.compress</td>
- <td>false</td>
+ <td>true</td>
<td>
Whether to compress data spilled during shuffles.
</td>
[2/2] git commit: Merge pull request #415 from
pwendell/shuffle-compress
Posted by pw...@apache.org.
Merge pull request #415 from pwendell/shuffle-compress
Enable compression by default for spills
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/055be5c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/055be5c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/055be5c6
Branch: refs/heads/master
Commit: 055be5c6940a82e2d7fa39f968a434643a1eb1e2
Parents: fdaabdc 0984647
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Jan 13 23:26:44 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Jan 13 23:26:44 2014 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/storage/BlockManager.scala | 2 +-
docs/configuration.md | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------