You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/09/03 03:38:39 UTC
[07/19] git commit: Fix spark.io.compression.codec and change default
codec to LZF
Fix spark.io.compression.codec and change default codec to LZF
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/9329a7d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/9329a7d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/9329a7d4
Branch: refs/heads/master
Commit: 9329a7d4cd10a080dad451e4173c85f2a04809d6
Parents: d9a53b9
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Mon Sep 2 10:15:22 2013 -0700
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Mon Sep 2 10:15:22 2013 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/io/CompressionCodec.scala | 12 ++++--------
docs/configuration.md | 2 +-
2 files changed, 5 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/9329a7d4/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 90a0420..570a979 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -39,17 +39,13 @@ trait CompressionCodec {
private[spark] object CompressionCodec {
def createCodec(): CompressionCodec = {
- // Set the default codec to Snappy since the LZF implementation initializes a pretty large
- // buffer for every stream, which results in a lot of memory overhead when the number of
- // shuffle reduce buckets are large.
- createCodec(classOf[SnappyCompressionCodec].getName)
+ createCodec(System.getProperty(
+ "spark.io.compression.codec", classOf[LZFCompressionCodec].getName))
}
def createCodec(codecName: String): CompressionCodec = {
- Class.forName(
- System.getProperty("spark.io.compression.codec", codecName),
- true,
- Thread.currentThread.getContextClassLoader).newInstance().asInstanceOf[CompressionCodec]
+ Class.forName(codecName, true, Thread.currentThread.getContextClassLoader)
+ .newInstance().asInstanceOf[CompressionCodec]
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/9329a7d4/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index aaf85ed..310e78a 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -147,7 +147,7 @@ Apart from these, the following properties are also available, and may be useful
</tr>
<tr>
<td>spark.io.compression.codec</td>
- <td>org.apache.spark.io.<br />SnappyCompressionCodec</td>
+ <td>org.apache.spark.io.<br />LZFCompressionCodec</td>
<td>
The compression codec class to use for various compressions. By default, Spark provides two
codecs: <code>org.apache.spark.io.LZFCompressionCodec</code> and <code>org.apache.spark.io.SnappyCompressionCodec</code>.