You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2012/10/03 02:10:22 UTC
svn commit: r1393244 - in
/hadoop/common/branches/branch-2/hadoop-common-project: ./ hadoop-auth/
hadoop-common/ hadoop-common/src/ hadoop-common/src/main/docs/
hadoop-common/src/main/java/
hadoop-common/src/main/java/org/apache/hadoop/io/compress/ had...
Author: eli
Date: Wed Oct 3 00:10:22 2012
New Revision: 1393244
URL: http://svn.apache.org/viewvc?rev=1393244&view=rev
Log:
HADOOP-8756. Fix SEGV when libsnappy is in java.library.path but not LD_LIBRARY_PATH. Contributed by Colin Patrick McCabe
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
- copied unchanged from r1393243, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
Removed:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/LoadSnappy.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project:r1393243
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-auth/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-auth:r1393243
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common:r1393243
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 3 00:10:22 2012
@@ -53,6 +53,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8616. ViewFS configuration requires a trailing slash. (Sandy Ryza
via atm)
+ HADOOP-8756. Fix SEGV when libsnappy is in java.library.path but
+ not LD_LIBRARY_PATH. (Colin Patrick McCabe via eli)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1393243
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt Wed Oct 3 00:10:22 2012
@@ -123,6 +123,7 @@ add_dual_library(hadoop
${D}/security/JniBasedUnixGroupsMapping.c
${D}/security/JniBasedUnixGroupsNetgroupMapping.c
${D}/security/getGroup.c
+ ${D}/util/NativeCodeLoader.c
${D}/util/NativeCrc32.c
${D}/util/bulk_crc32.c
)
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/config.h.cmake Wed Oct 3 00:10:22 2012
@@ -2,7 +2,6 @@
#define CONFIG_H
#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
-#cmakedefine HADOOP_RUNAS_HOME "@HADOOP_RUNAS_HOME@"
#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
#cmakedefine HAVE_SYNC_FILE_RANGE
#cmakedefine HAVE_POSIX_FADVISE
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1393243
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1393243
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java Wed Oct 3 00:10:22 2012
@@ -24,7 +24,6 @@ import java.io.OutputStream;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.compress.snappy.LoadSnappy;
import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -34,11 +33,6 @@ import org.apache.hadoop.util.NativeCode
* This class creates snappy compressors/decompressors.
*/
public class SnappyCodec implements Configurable, CompressionCodec {
-
- static {
- LoadSnappy.isLoaded();
- }
-
Configuration conf;
/**
@@ -63,11 +57,26 @@ public class SnappyCodec implements Conf
/**
* Are the native snappy libraries loaded & initialized?
- *
- * @return true if loaded & initialized, otherwise false
*/
+ public static void checkNativeCodeLoaded() {
+ if (!NativeCodeLoader.buildSupportsSnappy()) {
+ throw new RuntimeException("native snappy library not available: " +
+ "this version of libhadoop was built without " +
+ "snappy support.");
+ }
+ if (!SnappyCompressor.isNativeCodeLoaded()) {
+ throw new RuntimeException("native snappy library not available: " +
+ "SnappyCompressor has not been loaded.");
+ }
+ if (!SnappyDecompressor.isNativeCodeLoaded()) {
+ throw new RuntimeException("native snappy library not available: " +
+ "SnappyDecompressor has not been loaded.");
+ }
+ }
+
public static boolean isNativeCodeLoaded() {
- return LoadSnappy.isLoaded() && NativeCodeLoader.isNativeCodeLoaded();
+ return SnappyCompressor.isNativeCodeLoaded() &&
+ SnappyDecompressor.isNativeCodeLoaded();
}
/**
@@ -97,9 +106,7 @@ public class SnappyCodec implements Conf
public CompressionOutputStream createOutputStream(OutputStream out,
Compressor compressor)
throws IOException {
- if (!isNativeCodeLoaded()) {
- throw new RuntimeException("native snappy library not available");
- }
+ checkNativeCodeLoaded();
int bufferSize = conf.getInt(
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
@@ -117,10 +124,7 @@ public class SnappyCodec implements Conf
*/
@Override
public Class<? extends Compressor> getCompressorType() {
- if (!isNativeCodeLoaded()) {
- throw new RuntimeException("native snappy library not available");
- }
-
+ checkNativeCodeLoaded();
return SnappyCompressor.class;
}
@@ -131,9 +135,7 @@ public class SnappyCodec implements Conf
*/
@Override
public Compressor createCompressor() {
- if (!isNativeCodeLoaded()) {
- throw new RuntimeException("native snappy library not available");
- }
+ checkNativeCodeLoaded();
int bufferSize = conf.getInt(
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
@@ -167,10 +169,7 @@ public class SnappyCodec implements Conf
public CompressionInputStream createInputStream(InputStream in,
Decompressor decompressor)
throws IOException {
- if (!isNativeCodeLoaded()) {
- throw new RuntimeException("native snappy library not available");
- }
-
+ checkNativeCodeLoaded();
return new BlockDecompressorStream(in, decompressor, conf.getInt(
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT));
@@ -183,10 +182,7 @@ public class SnappyCodec implements Conf
*/
@Override
public Class<? extends Decompressor> getDecompressorType() {
- if (!isNativeCodeLoaded()) {
- throw new RuntimeException("native snappy library not available");
- }
-
+ checkNativeCodeLoaded();
return SnappyDecompressor.class;
}
@@ -197,9 +193,7 @@ public class SnappyCodec implements Conf
*/
@Override
public Decompressor createDecompressor() {
- if (!isNativeCodeLoaded()) {
- throw new RuntimeException("native snappy library not available");
- }
+ checkNativeCodeLoaded();
int bufferSize = conf.getInt(
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY,
CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT);
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java Wed Oct 3 00:10:22 2012
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.Compressor;
+import org.apache.hadoop.util.NativeCodeLoader;
/**
* A {@link Compressor} based on the snappy compression algorithm.
@@ -51,22 +52,24 @@ public class SnappyCompressor implements
private long bytesRead = 0L;
private long bytesWritten = 0L;
-
+ private static boolean nativeSnappyLoaded = false;
+
static {
- if (LoadSnappy.isLoaded()) {
- // Initialize the native library
+ if (NativeCodeLoader.isNativeCodeLoaded() &&
+ NativeCodeLoader.buildSupportsSnappy()) {
try {
initIDs();
+ nativeSnappyLoaded = true;
} catch (Throwable t) {
- // Ignore failure to load/initialize snappy
- LOG.warn(t.toString());
+ LOG.error("failed to load SnappyCompressor", t);
}
- } else {
- LOG.error("Cannot load " + SnappyCompressor.class.getName() +
- " without snappy library!");
}
}
-
+
+ public static boolean isNativeCodeLoaded() {
+ return nativeSnappyLoaded;
+ }
+
/**
* Creates a new compressor.
*
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java Wed Oct 3 00:10:22 2012
@@ -25,6 +25,7 @@ import java.nio.ByteBuffer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.util.NativeCodeLoader;
/**
* A {@link Decompressor} based on the snappy compression algorithm.
@@ -47,21 +48,24 @@ public class SnappyDecompressor implemen
private int userBufOff = 0, userBufLen = 0;
private boolean finished;
+ private static boolean nativeSnappyLoaded = false;
+
static {
- if (LoadSnappy.isLoaded()) {
- // Initialize the native library
+ if (NativeCodeLoader.isNativeCodeLoaded() &&
+ NativeCodeLoader.buildSupportsSnappy()) {
try {
initIDs();
+ nativeSnappyLoaded = true;
} catch (Throwable t) {
- // Ignore failure to load/initialize snappy
- LOG.warn(t.toString());
+ LOG.error("failed to load SnappyDecompressor", t);
}
- } else {
- LOG.error("Cannot load " + SnappyDecompressor.class.getName() +
- " without snappy library!");
}
}
-
+
+ public static boolean isNativeCodeLoaded() {
+ return nativeSnappyLoaded;
+ }
+
/**
* Creates a new compressor.
*
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java Wed Oct 3 00:10:22 2012
@@ -75,6 +75,11 @@ public class NativeCodeLoader {
}
/**
+ * Returns true only if this build was compiled with support for snappy.
+ */
+ public static native boolean buildSupportsSnappy();
+
+ /**
* Return if native hadoop libraries, if present, can be used for this job.
* @param conf configuration
*
Propchange: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1393243
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java?rev=1393244&r1=1393243&r2=1393244&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java Wed Oct 3 00:10:22 2012
@@ -52,7 +52,6 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.snappy.LoadSnappy;
import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor;
import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater;
import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater;
@@ -101,14 +100,9 @@ public class TestCodec {
@Test
public void testSnappyCodec() throws IOException {
- if (LoadSnappy.isAvailable()) {
- if (LoadSnappy.isLoaded()) {
- codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec");
- codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec");
- }
- else {
- Assert.fail("Snappy native available but Hadoop native not");
- }
+ if (SnappyCodec.isNativeCodeLoaded()) {
+ codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec");
+ codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec");
}
}