You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/01/26 12:36:07 UTC
spark git commit: [SPARK-12961][CORE] Prevent snappy-java memory leak
Repository: spark
Updated Branches:
refs/heads/master 6743de3a9 -> 5936bf9fa
[SPARK-12961][CORE] Prevent snappy-java memory leak
JIRA: https://issues.apache.org/jira/browse/SPARK-12961
To prevent memory leak in snappy-java, just call the method once and cache the result. After the library releases new version, we can remove this object.
JoshRosen
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #10875 from viirya/prevent-snappy-memory-leak.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5936bf9f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5936bf9f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5936bf9f
Branch: refs/heads/master
Commit: 5936bf9fa85ccf7f0216145356140161c2801682
Parents: 6743de3
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Tue Jan 26 11:36:00 2016 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue Jan 26 11:36:00 2016 +0000
----------------------------------------------------------------------
.../org/apache/spark/io/CompressionCodec.scala | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5936bf9f/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
index 7178046..ae014be 100644
--- a/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
+++ b/core/src/main/scala/org/apache/spark/io/CompressionCodec.scala
@@ -149,12 +149,7 @@ class LZFCompressionCodec(conf: SparkConf) extends CompressionCodec {
*/
@DeveloperApi
class SnappyCompressionCodec(conf: SparkConf) extends CompressionCodec {
-
- try {
- Snappy.getNativeLibraryVersion
- } catch {
- case e: Error => throw new IllegalArgumentException(e)
- }
+ val version = SnappyCompressionCodec.version
override def compressedOutputStream(s: OutputStream): OutputStream = {
val blockSize = conf.getSizeAsBytes("spark.io.compression.snappy.blockSize", "32k").toInt
@@ -165,6 +160,19 @@ class SnappyCompressionCodec(conf: SparkConf) extends CompressionCodec {
}
/**
+ * Object guards against memory leak bug in snappy-java library:
+ * (https://github.com/xerial/snappy-java/issues/131).
+ * Before a new version of the library, we only call the method once and cache the result.
+ */
+private final object SnappyCompressionCodec {
+ private lazy val version: String = try {
+ Snappy.getNativeLibraryVersion
+ } catch {
+ case e: Error => throw new IllegalArgumentException(e)
+ }
+}
+
+/**
* Wrapper over [[SnappyOutputStream]] which guards against write-after-close and double-close
* issues. See SPARK-7660 for more details. This wrapping can be removed if we upgrade to a version
* of snappy-java that contains the fix for https://github.com/xerial/snappy-java/issues/107.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org