You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2012/03/02 23:51:07 UTC
svn commit: r1296506 - in /hadoop/common/branches/branch-1.0: ./ src/core/
src/core/org/apache/hadoop/fs/ src/core/org/apache/hadoop/io/compress/
src/core/org/apache/hadoop/io/compress/snappy/ src/native/
src/native/src/org/apache/hadoop/io/compress/sn...
Author: vinodkv
Date: Fri Mar 2 22:51:06 2012
New Revision: 1296506
URL: http://svn.apache.org/viewvc?rev=1296506&view=rev
Log:
HADOOP-7206. Support for Snappy compression. Contributed by Issei Yoshida and Alejandro Abdelnur. Backported from trunk/0.23.
svn merge --ignore-ancestry -c 1296505 ../branch-1
Added:
hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/io/compress/SnappyCodec.java
- copied unchanged from r1296505, hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/SnappyCodec.java
hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/io/compress/snappy/
- copied from r1296505, hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/snappy/
hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/io/compress/snappy/LoadSnappy.java
- copied unchanged from r1296505, hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/snappy/LoadSnappy.java
hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
- copied unchanged from r1296505, hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
- copied unchanged from r1296505, hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
hadoop/common/branches/branch-1.0/src/native/src/org/apache/hadoop/io/compress/snappy/
- copied from r1296505, hadoop/common/branches/branch-1/src/native/src/org/apache/hadoop/io/compress/snappy/
hadoop/common/branches/branch-1.0/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
- copied unchanged from r1296505, hadoop/common/branches/branch-1/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
hadoop/common/branches/branch-1.0/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
- copied unchanged from r1296505, hadoop/common/branches/branch-1/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
hadoop/common/branches/branch-1.0/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
- copied unchanged from r1296505, hadoop/common/branches/branch-1/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
Modified:
hadoop/common/branches/branch-1.0/CHANGES.txt
hadoop/common/branches/branch-1.0/build.xml
hadoop/common/branches/branch-1.0/src/core/core-default.xml
hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/branches/branch-1.0/src/native/Makefile.am
hadoop/common/branches/branch-1.0/src/native/configure.ac
hadoop/common/branches/branch-1.0/src/native/packageNativeHadoop.sh
hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/io/compress/TestCodec.java
Modified: hadoop/common/branches/branch-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/CHANGES.txt?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1.0/CHANGES.txt Fri Mar 2 22:51:06 2012
@@ -4,6 +4,9 @@ Release 1.0.2 - unreleased
NEW FEATURES
+ HADOOP-7206. Support Snappy compression. (Issei Yoshida and
+ Alejandro Abdelnur via vinodkv).
+
IMPROVEMENTS
BUG FIXES
Modified: hadoop/common/branches/branch-1.0/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/build.xml?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/build.xml (original)
+++ hadoop/common/branches/branch-1.0/build.xml Fri Mar 2 22:51:06 2012
@@ -207,6 +207,14 @@
<property name="package.buildroot" value="/tmp/hadoop_package_build_${user.name}"/>
<property name="package.build.dir" value="/tmp/hadoop_package_build_${user.name}/BUILD"/>
+ <!-- Indicate is Snappy native library should be bundled with Hadoop or not -->
+ <property name="bundle.snappy" value="false"/>
+
+ <!-- Snappy native library location -->
+ <property name="snappy.prefix" value="/usr/local"/>
+ <property name="snappy.lib" value="${snappy.prefix}/lib"/>
+ <property name="snappy.include" value="${snappy.prefix}/include"/>
+
<!-- IVY properteis set here -->
<property name="ivy.dir" location="ivy" />
<loadproperties srcfile="${ivy.dir}/libraries.properties"/>
@@ -298,6 +306,9 @@
<property name="build.dir.eclipse-test-resources" value="${build.dir.eclipse}/test-resources/"/>
<property name="build.dir.eclipse-test-resources-webapps" value="${build.dir.eclipse}/test-resources/webapps"/>
+ <!-- Use environment -->
+ <property environment="env" />
+
<!-- check if clover reports should be generated -->
<condition property="clover.enabled">
<and>
@@ -603,6 +614,7 @@
<mkdir dir="${build.native}/lib"/>
<mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/zlib"/>
+ <mkdir dir="${build.native}/src/org/apache/hadoop/io/compress/snappy"/>
<mkdir dir="${build.native}/src/org/apache/hadoop/io/nativeio"/>
<mkdir dir="${build.native}/src/org/apache/hadoop/security"/>
@@ -616,6 +628,16 @@
<class name="org.apache.hadoop.io.compress.zlib.ZlibDecompressor" />
</javah>
+ <javah
+ classpath="${build.classes}"
+ destdir="${build.native}/src/org/apache/hadoop/io/compress/snappy"
+ force="yes"
+ verbose="yes"
+ >
+ <class name="org.apache.hadoop.io.compress.snappy.SnappyCompressor"/>
+ <class name="org.apache.hadoop.io.compress.snappy.SnappyDecompressor"/>
+ </javah>
+
<javah
classpath="${build.classes}"
destdir="${build.native}/src/org/apache/hadoop/io/nativeio"
@@ -647,7 +669,7 @@
<env key="OS_ARCH" value="${os.arch}"/>
<env key="JVM_DATA_MODEL" value="${sun.arch.data.model}"/>
<env key="HADOOP_NATIVE_SRCDIR" value="${native.src.dir}"/>
- <arg line="${native.src.dir}/configure"/>
+ <arg line="${native.src.dir}/configure"/>
</exec>
<exec dir="${build.native}" executable="${make.cmd}" failonerror="true">
@@ -1043,7 +1065,7 @@
value="@{test.krb5.conf.filename}"/>
<sysproperty key="hadoop.policy.file" value="hadoop-policy.xml" />
<sysproperty key="java.library.path"
- value="${build.native}/lib:${lib.dir}/native/${build.platform}:${lib.file.path}" />
+ value="${build.native}/lib:${lib.dir}/native/${build.platform}:${lib.file.path}:${snappy.lib}" />
<sysproperty key="install.c++.examples"
value="${install.c++.examples}" />
<sysproperty key="testjar"
@@ -1495,6 +1517,8 @@
<env key="BASE_NATIVE_LIB_DIR" value="${lib.dir}/native"/>
<env key="BUILD_NATIVE_DIR" value="${build.dir}/native"/>
<env key="DIST_LIB_DIR" value="${dist.dir}/lib/native"/>
+ <env key="BUNDLE_SNAPPY_LIB" value="${bundle.snappy}"/>
+ <env key="SNAPPY_LIB_DIR" value="${snappy.prefix}/lib"/>
<arg line="${native.src.dir}/packageNativeHadoop.sh"/>
</exec>
Modified: hadoop/common/branches/branch-1.0/src/core/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/core/core-default.xml?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/core/core-default.xml (original)
+++ hadoop/common/branches/branch-1.0/src/core/core-default.xml Fri Mar 2 22:51:06 2012
@@ -114,7 +114,7 @@
<property>
<name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec</value>
+ <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec</value>
<description>A list of the compression codec classes that can be used
for compression/decompression.</description>
</property>
Modified: hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/branches/branch-1.0/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java Fri Mar 2 22:51:06 2012
@@ -52,5 +52,17 @@ public class CommonConfigurationKeys {
"ipc.server.read.threadpool.size";
public static final int IPC_SERVER_RPC_READ_THREADS_DEFAULT = 1;
+ public static final String IO_NATIVE_LIB_AVAILABLE_KEY =
+ "hadoop.native.lib";
+ /** Default value for IO_NATIVE_LIB_AVAILABLE_KEY */
+ public static final boolean IO_NATIVE_LIB_AVAILABLE_DEFAULT = true;
+
+ /** Internal buffer size for Snappy compressor/decompressors */
+ public static final String IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY =
+ "io.compression.codec.snappy.buffersize";
+
+ /** Default value for IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY */
+ public static final int IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT =
+ 256 * 1024;
}
Modified: hadoop/common/branches/branch-1.0/src/native/Makefile.am
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/native/Makefile.am?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/native/Makefile.am (original)
+++ hadoop/common/branches/branch-1.0/src/native/Makefile.am Fri Mar 2 22:51:06 2012
@@ -33,6 +33,7 @@ export PLATFORM = $(shell echo $$OS_NAME
AM_CPPFLAGS = @JNI_CPPFLAGS@ -I$(HADOOP_NATIVE_SRCDIR)/src \
-Isrc/org/apache/hadoop/io/compress/zlib \
+ -Isrc/org/apache/hadoop/io/compress/snappy \
-Isrc/org/apache/hadoop/io/nativeio \
-Isrc/org/apache/hadoop/security
AM_LDFLAGS = @JNI_LDFLAGS@ -m$(JVM_DATA_MODEL)
@@ -41,6 +42,8 @@ AM_CFLAGS = -g -Wall -fPIC -O2 -m$(JVM_D
lib_LTLIBRARIES = libhadoop.la
libhadoop_la_SOURCES = src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c \
src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c \
+ src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c \
+ src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c \
src/org/apache/hadoop/security/getGroup.c \
src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c \
src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c \
Modified: hadoop/common/branches/branch-1.0/src/native/configure.ac
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/native/configure.ac?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/native/configure.ac (original)
+++ hadoop/common/branches/branch-1.0/src/native/configure.ac Fri Mar 2 22:51:06 2012
@@ -87,6 +87,9 @@ AC_SUBST([JNI_CPPFLAGS])
dnl Check for zlib headers
AC_CHECK_HEADERS([zlib.h zconf.h], AC_COMPUTE_NEEDED_DSO(z,HADOOP_ZLIB_LIBRARY), AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.))
+dnl Check for snappy headers
+AC_CHECK_HEADERS([snappy-c.h], AC_COMPUTE_NEEDED_DSO(snappy,HADOOP_SNAPPY_LIBRARY), AC_MSG_WARN(Snappy headers were not found... building without snappy.))
+
dnl Check for headers needed by the native Group resolution implementation
AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.))
Modified: hadoop/common/branches/branch-1.0/src/native/packageNativeHadoop.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/native/packageNativeHadoop.sh?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/native/packageNativeHadoop.sh (original)
+++ hadoop/common/branches/branch-1.0/src/native/packageNativeHadoop.sh Fri Mar 2 22:51:06 2012
@@ -62,4 +62,17 @@ then
done
fi
+if [ "${BUNDLE_SNAPPY_LIB}" = "true" ]
+then
+ if [ -d ${SNAPPY_LIB_DIR} ]
+ then
+ echo "Copying Snappy library in ${SNAPPY_LIB_DIR} to $DIST_LIB_DIR/"
+ cd ${SNAPPY_LIB_DIR}
+ $TAR . | (cd $DIST_LIB_DIR/; $UNTAR)
+ else
+ echo "Snappy lib directory ${SNAPPY_LIB_DIR} does not exist"
+ exit 1
+ fi
+fi
+
#vim: ts=2: sw=2: et
Modified: hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/io/compress/TestCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/io/compress/TestCodec.java?rev=1296506&r1=1296505&r2=1296506&view=diff
==============================================================================
--- hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/io/compress/TestCodec.java (original)
+++ hadoop/common/branches/branch-1.0/src/test/org/apache/hadoop/io/compress/TestCodec.java Fri Mar 2 22:51:06 2012
@@ -42,7 +42,6 @@ import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
@@ -50,18 +49,17 @@ import org.apache.hadoop.io.DataOutputBu
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.RandomDatum;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
-import org.apache.hadoop.io.compress.CompressorStream;
-import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.snappy.LoadSnappy;
import org.apache.hadoop.io.compress.zlib.BuiltInGzipDecompressor;
import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater;
import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater;
import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionLevel;
import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionStrategy;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
+import org.apache.hadoop.util.ReflectionUtils;
public class TestCodec extends TestCase {
@@ -86,6 +84,19 @@ public class TestCodec extends TestCase
codecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec");
}
+
+ public void testSnappyCodec() throws IOException {
+ if (LoadSnappy.isAvailable()) {
+ if (LoadSnappy.isLoaded()) {
+ codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec");
+ codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec");
+ }
+ else {
+ fail("Snappy native available but Hadoop native not");
+ }
+ }
+ }
+
public void testGzipCodecWithParam() throws IOException {
Configuration conf = new Configuration(this.conf);
ZlibFactory.setCompressionLevel(conf, CompressionLevel.BEST_COMPRESSION);