You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/08/11 08:27:20 UTC
svn commit: r1512878 - in /hadoop/common/branches/branch-1-win: ./ bin/
src/native/ src/native/src/org/apache/hadoop/io/compress/snappy/
Author: cnauroth
Date: Sun Aug 11 06:27:20 2013
New Revision: 1512878
URL: http://svn.apache.org/r1512878
Log:
HADOOP-9802. Support Snappy codec on Windows. Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd
hadoop/common/branches/branch-1-win/build.xml
hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj
hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Sun Aug 11 06:27:20 2013
@@ -328,6 +328,8 @@ Branch-hadoop-1-win (branched from branc
HADOOP-9551. Backport common utils introduced with HADOOP-9413
to branch-1-win. (ivanmi)
+ HADOOP-9802. Support Snappy codec on Windows. (cnauroth)
+
Merged from branch-1
HDFS-385. Backport: Add support for an experimental API that allows a
Modified: hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd (original)
+++ hadoop/common/branches/branch-1-win/bin/hadoop-config.cmd Sun Aug 11 06:27:20 2013
@@ -183,7 +183,7 @@ if exist %HADOOP_CORE_HOME%\build\native
@rem For the disto case, check the lib\native folder
if exist %HADOOP_CORE_HOME%\lib\native (
- set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_CORE_HOME%\lib\native\%JAVA_PLATFORM%
+ set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_CORE_HOME%\lib\native\%JAVA_PLATFORM%;%HADOOP_CORE_HOME%\lib\native
)
set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR%
Modified: hadoop/common/branches/branch-1-win/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/build.xml?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/build.xml (original)
+++ hadoop/common/branches/branch-1-win/build.xml Sun Aug 11 06:27:20 2013
@@ -702,10 +702,21 @@
<mkdir dir="${build.native}/lib"/>
<mkdir dir="${build.native.win}/src/org/apache/hadoop/io/compress/zlib"/>
+ <mkdir dir="${build.native.win}/src/org/apache/hadoop/io/compress/snappy"/>
<mkdir dir="${build.native.win}/src/org/apache/hadoop/io/nativeio"/>
<javah
classpath="${build.classes}"
+ destdir="${build.native.win}/src/org/apache/hadoop/io/compress/snappy"
+ force="yes"
+ verbose="yes"
+ >
+ <class name="org.apache.hadoop.io.compress.snappy.SnappyCompressor"/>
+ <class name="org.apache.hadoop.io.compress.snappy.SnappyDecompressor"/>
+ </javah>
+
+ <javah
+ classpath="${build.classes}"
destdir="${build.native.win}/src/org/apache/hadoop/io/compress/zlib"
force="yes"
verbose="yes"
Modified: hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj (original)
+++ hadoop/common/branches/branch-1-win/src/native/hadoop.vcxproj Sun Aug 11 06:27:20 2013
@@ -63,7 +63,7 @@
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
- <IncludePath>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;src;..\winutils\include;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\nativeio;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib;$(ZLIB_HOME);$(IncludePath)</IncludePath>
+ <IncludePath>$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;src;..\winutils\include;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\nativeio;..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib;$(ZLIB_HOME);..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\snappy;$(SNAPPY_HOME);$(IncludePath)</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
@@ -115,6 +115,8 @@
</ProjectReference>
</ItemGroup>
<ItemGroup>
+ <ClCompile Include="src\org\apache\hadoop\io\compress\snappy\SnappyCompressor.c" Condition="Exists('$(SNAPPY_HOME)')" />
+ <ClCompile Include="src\org\apache\hadoop\io\compress\snappy\SnappyDecompressor.c" Condition="Exists('$(SNAPPY_HOME)')" />
<ClCompile Include="src\org\apache\hadoop\io\compress\zlib\ZlibCompressor.c" Condition="Exists('$(ZLIB_HOME)')" />
<ClCompile Include="src\org\apache\hadoop\io\compress\zlib\ZlibDecompressor.c" Condition="Exists('$(ZLIB_HOME)')" />
<ClCompile Include="src\org\apache\hadoop\io\nativeio\file_descriptor.c" />
@@ -128,6 +130,7 @@
<ClInclude Include="..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib\org_apache_hadoop_io_compress_zlib_ZlibCompressor.h" />
<ClInclude Include="..\..\build\native\$(OS)-$(Platform)\src\org\apache\hadoop\io\compress\zlib\org_apache_hadoop_io_compress_zlib_ZlibDecompressor.h" />
<ClInclude Include="src\org\apache\hadoop\io\compress\zlib\org_apache_hadoop_io_compress_zlib.h" />
+ <ClInclude Include="src\org\apache\hadoop\io\compress\snappy\org_apache_hadoop_io_compress_snappy.h" />
<ClInclude Include="src\org\apache\hadoop\io\nativeio\file_descriptor.h" />
<ClInclude Include="src\org_apache_hadoop.h" />
</ItemGroup>
Modified: hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c (original)
+++ hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c Sun Aug 11 06:27:20 2013
@@ -20,8 +20,12 @@
#include <config.h>
#endif
+#include "org_apache_hadoop_io_compress_snappy.h"
+#include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h"
+
#if defined HADOOP_SNAPPY_LIBRARY
+#ifdef UNIX
#if defined HAVE_STDIO_H
#include <stdio.h>
#else
@@ -45,9 +49,7 @@
#else
#error 'dlfcn.h not found'
#endif
-
-#include "org_apache_hadoop_io_compress_snappy.h"
-#include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h"
+#endif
static jfieldID SnappyCompressor_clazz;
static jfieldID SnappyCompressor_uncompressedDirectBuf;
@@ -55,11 +57,18 @@ static jfieldID SnappyCompressor_uncompr
static jfieldID SnappyCompressor_compressedDirectBuf;
static jfieldID SnappyCompressor_directBufferSize;
+#ifdef UNIX
static snappy_status (*dlsym_snappy_compress)(const char*, size_t, char*, size_t*);
+#endif
+
+#ifdef WINDOWS
+typedef snappy_status (__cdecl *__dlsym_snappy_compress)(const char*, size_t, char*, size_t*);
+static __dlsym_snappy_compress dlsym_snappy_compress;
+#endif
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_initIDs
(JNIEnv *env, jclass clazz){
-
+#ifdef UNIX
// Load libsnappy.so
void *libsnappy = dlopen(HADOOP_SNAPPY_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
if (!libsnappy) {
@@ -68,10 +77,25 @@ JNIEXPORT void JNICALL Java_org_apache_h
THROW(env, "java/lang/UnsatisfiedLinkError", msg);
return;
}
+#endif
+
+#ifdef WINDOWS
+ HMODULE libsnappy = LoadLibrary(HADOOP_SNAPPY_LIBRARY);
+ if (!libsnappy) {
+ THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load snappy.dll");
+ return;
+ }
+#endif
// Locate the requisite symbols from libsnappy.so
+#ifdef UNIX
dlerror(); // Clear any existing error
LOAD_DYNAMIC_SYMBOL(dlsym_snappy_compress, env, libsnappy, "snappy_compress");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_snappy_compress, dlsym_snappy_compress, env, libsnappy, "snappy_compress");
+#endif
SnappyCompressor_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz",
"Ljava/lang/Class;");
@@ -89,6 +113,9 @@ JNIEXPORT void JNICALL Java_org_apache_h
JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_compressBytesDirect
(JNIEnv *env, jobject thisj){
+ const char* uncompressed_bytes;
+ char* compressed_bytes;
+ snappy_status ret;
// Get members of SnappyCompressor
jobject clazz = (*env)->GetStaticObjectField(env, thisj, SnappyCompressor_clazz);
jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, SnappyCompressor_uncompressedDirectBuf);
@@ -98,7 +125,7 @@ JNIEXPORT jint JNICALL Java_org_apache_h
// Get the input direct buffer
LOCK_CLASS(env, clazz, "SnappyCompressor");
- const char* uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+ uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
UNLOCK_CLASS(env, clazz, "SnappyCompressor");
if (uncompressed_bytes == 0) {
@@ -107,14 +134,14 @@ JNIEXPORT jint JNICALL Java_org_apache_h
// Get the output direct buffer
LOCK_CLASS(env, clazz, "SnappyCompressor");
- char* compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+ compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
UNLOCK_CLASS(env, clazz, "SnappyCompressor");
if (compressed_bytes == 0) {
return (jint)0;
}
- snappy_status ret = dlsym_snappy_compress(uncompressed_bytes, uncompressed_direct_buf_len, compressed_bytes, &compressed_direct_buf_len);
+ ret = dlsym_snappy_compress(uncompressed_bytes, uncompressed_direct_buf_len, compressed_bytes, (size_t* ) &compressed_direct_buf_len);
if (ret != SNAPPY_OK){
THROW(env, "java/lang/InternalError", "Could not compress data. Buffer length is too small.");
}
Modified: hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c (original)
+++ hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c Sun Aug 11 06:27:20 2013
@@ -20,8 +20,12 @@
#include <config.h>
#endif
+#include "org_apache_hadoop_io_compress_snappy.h"
+#include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h"
+
#if defined HADOOP_SNAPPY_LIBRARY
+#ifdef UNIX
#if defined HAVE_STDIO_H
#include <stdio.h>
#else
@@ -45,9 +49,7 @@
#else
#error 'dlfcn.h not found'
#endif
-
-#include "org_apache_hadoop_io_compress_snappy.h"
-#include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h"
+#endif
static jfieldID SnappyDecompressor_clazz;
static jfieldID SnappyDecompressor_compressedDirectBuf;
@@ -55,12 +57,20 @@ static jfieldID SnappyDecompressor_compr
static jfieldID SnappyDecompressor_uncompressedDirectBuf;
static jfieldID SnappyDecompressor_directBufferSize;
+#ifdef UNIX
static snappy_status (*dlsym_snappy_uncompress)(const char*, size_t, char*, size_t*);
+#endif
+
+#ifdef WINDOWS
+typedef snappy_status (__cdecl *__dlsym_snappy_uncompress)(const char*, size_t, char*, size_t*);
+static __dlsym_snappy_uncompress dlsym_snappy_uncompress;
+#endif
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompressor_initIDs
(JNIEnv *env, jclass clazz){
// Load libsnappy.so
+#ifdef UNIX
void *libsnappy = dlopen(HADOOP_SNAPPY_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
if (!libsnappy) {
char* msg = (char*)malloc(1000);
@@ -68,11 +78,27 @@ JNIEXPORT void JNICALL Java_org_apache_h
THROW(env, "java/lang/UnsatisfiedLinkError", msg);
return;
}
+#endif
+
+#ifdef WINDOWS
+ HMODULE libsnappy = LoadLibrary(HADOOP_SNAPPY_LIBRARY);
+ if (!libsnappy) {
+ THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load snappy.dll");
+ return;
+ }
+#endif
// Locate the requisite symbols from libsnappy.so
+#ifdef UNIX
dlerror(); // Clear any existing error
LOAD_DYNAMIC_SYMBOL(dlsym_snappy_uncompress, env, libsnappy, "snappy_uncompress");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_snappy_uncompress, dlsym_snappy_uncompress, env, libsnappy, "snappy_uncompress");
+#endif
+
SnappyDecompressor_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz",
"Ljava/lang/Class;");
SnappyDecompressor_compressedDirectBuf = (*env)->GetFieldID(env,clazz,
@@ -89,6 +115,9 @@ JNIEXPORT void JNICALL Java_org_apache_h
JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompressor_decompressBytesDirect
(JNIEnv *env, jobject thisj){
+ const char* compressed_bytes = NULL;
+ char* uncompressed_bytes = NULL;
+ snappy_status ret;
// Get members of SnappyDecompressor
jobject clazz = (*env)->GetStaticObjectField(env,thisj, SnappyDecompressor_clazz);
jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, SnappyDecompressor_compressedDirectBuf);
@@ -98,7 +127,7 @@ JNIEXPORT jint JNICALL Java_org_apache_h
// Get the input direct buffer
LOCK_CLASS(env, clazz, "SnappyDecompressor");
- const char* compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+ compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
UNLOCK_CLASS(env, clazz, "SnappyDecompressor");
if (compressed_bytes == 0) {
@@ -107,14 +136,14 @@ JNIEXPORT jint JNICALL Java_org_apache_h
// Get the output direct buffer
LOCK_CLASS(env, clazz, "SnappyDecompressor");
- char* uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+ uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
UNLOCK_CLASS(env, clazz, "SnappyDecompressor");
if (uncompressed_bytes == 0) {
return (jint)0;
}
- snappy_status ret = dlsym_snappy_uncompress(compressed_bytes, compressed_direct_buf_len, uncompressed_bytes, &uncompressed_direct_buf_len);
+ ret = dlsym_snappy_uncompress(compressed_bytes, compressed_direct_buf_len, uncompressed_bytes, &uncompressed_direct_buf_len);
if (ret == SNAPPY_BUFFER_TOO_SMALL){
THROW(env, "java/lang/InternalError", "Could not decompress data. Buffer length is too small.");
} else if (ret == SNAPPY_INVALID_INPUT){
Modified: hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h?rev=1512878&r1=1512877&r2=1512878&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h (original)
+++ hadoop/common/branches/branch-1-win/src/native/src/org/apache/hadoop/io/compress/snappy/org_apache_hadoop_io_compress_snappy.h Sun Aug 11 06:27:20 2013
@@ -20,7 +20,9 @@
#if !defined ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H
#define ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H
+#include "org_apache_hadoop.h"
+#ifdef UNIX
#if defined HAVE_CONFIG_H
#include <config.h>
#endif
@@ -51,8 +53,12 @@
#error 'jni.h not found'
#endif
- #include "org_apache_hadoop.h"
-
#endif //define HADOOP_SNAPPY_LIBRARY
+#endif
+
+#ifdef WINDOWS
+#define HADOOP_SNAPPY_LIBRARY L"snappy.dll"
+#include <snappy-c.h>
+#endif
#endif //ORG_APACHE_HADOOP_IO_COMPRESS_SNAPPY_SNAPPY_H