You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/07/16 03:47:56 UTC
git commit: [SPARK-2469] Use Snappy (instead of LZF) for default
shuffle compression codec
Repository: spark
Updated Branches:
refs/heads/master c2048a516 -> 4576d80a5
[SPARK-2469] Use Snappy (instead of LZF) for default shuffle compression codec
This reduces shuffle compression memory usage by 3x.
Author: Reynold Xin <rx...@apache.org>
Closes #1415 from rxin/snappy and squashes the following commits:
06c1a01 [Reynold Xin] SPARK-2469: Use Snappy (instead of LZF) for default shuffle compression codec.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4576d80a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4576d80a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4576d80a
Branch: refs/heads/master
Commit: 4576d80a5155c9fbfebe9c36cca06c208bca5bd3
Parents: c2048a5
Author: Reynold Xin <rx...@apache.org>
Authored: Tue Jul 15 18:47:39 2014 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Tue Jul 15 18:47:39 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/io/CompressionCodec.scala | 4 ++--
.../test/scala/org/apache/spark/io/CompressionCodecSuite.scala | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4576d80a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 33402c9..1b66218 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -56,7 +56,7 @@ private[spark] object CompressionCodec {
ctor.newInstance(conf).asInstanceOf[CompressionCodec]
}
- val DEFAULT_COMPRESSION_CODEC = classOf[LZFCompressionCodec].getName
+ val DEFAULT_COMPRESSION_CODEC = classOf[SnappyCompressionCodec].getName
}
@@ -103,7 +103,7 @@ class LZFCompressionCodec(conf: SparkConf) extends CompressionCodec {
/**
* :: DeveloperApi ::
* Snappy implementation of [[org.apache.spark.io.CompressionCodec]].
- * Block size can be configured by spark.io.compression.snappy.block.size.
+ * Block size can be configured by `spark.io.compression.snappy.block.size`.
*
* Note: The wire protocol for this codec is not guaranteed to be compatible across versions
* of Spark. This is intended for use as an internal compression utility within a single Spark
http://git-wip-us.apache.org/repos/asf/spark/blob/4576d80a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala
index 42fc395..3f882a7 100644
--- a/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala
+++ b/core/src/test/scala/org/apache/spark/io/CompressionCodecSuite.scala
@@ -46,7 +46,7 @@ class CompressionCodecSuite extends FunSuite {
test("default compression codec") {
val codec = CompressionCodec.createCodec(conf)
- assert(codec.getClass === classOf[LZFCompressionCodec])
+ assert(codec.getClass === classOf[SnappyCompressionCodec])
testCodec(codec)
}