You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/07/04 01:40:32 UTC
svn commit: r1607768 - in
/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common:
./ src/ src/main/java/org/apache/hadoop/crypto/
src/main/java/org/apache/hadoop/util/
src/main/native/src/org/apache/hadoop/util/ src/test/java/org/apa...
Author: cmccabe
Date: Thu Jul 3 23:40:31 2014
New Revision: 1607768
URL: http://svn.apache.org/r1607768
Log:
HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL (hitliuyi via cmccabe)
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/config.h.cmake
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/pom.xml Thu Jul 3 23:40:31 2014
@@ -483,6 +483,10 @@
<snappy.lib></snappy.lib>
<snappy.include></snappy.include>
<require.snappy>false</require.snappy>
+ <openssl.prefix></openssl.prefix>
+ <openssl.lib></openssl.lib>
+ <openssl.include></openssl.include>
+ <require.openssl>false</require.openssl>
</properties>
<build>
<plugins>
@@ -532,6 +536,7 @@
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+ <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
<javahClassName>org.apache.hadoop.net.unix.DomainSocket</javahClassName>
<javahClassName>org.apache.hadoop.net.unix.DomainSocketWatcher</javahClassName>
@@ -552,7 +557,7 @@
<configuration>
<target>
<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
- <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
+ <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
@@ -596,6 +601,11 @@
<snappy.include></snappy.include>
<require.snappy>false</require.snappy>
<bundle.snappy.in.bin>true</bundle.snappy.in.bin>
+ <openssl.prefix></openssl.prefix>
+ <openssl.lib></openssl.lib>
+ <openssl.include></openssl.include>
+ <require.openssl>false</require.openssl>
+ <bundle.openssl.in.bin>true</bundle.openssl.in.bin>
</properties>
<build>
<plugins>
@@ -641,6 +651,7 @@
<javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
<javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+ <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
<javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
</javahClassNames>
<javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
@@ -685,6 +696,10 @@
<argument>/p:CustomSnappyLib=${snappy.lib}</argument>
<argument>/p:CustomSnappyInclude=${snappy.include}</argument>
<argument>/p:RequireSnappy=${require.snappy}</argument>
+ <argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
+ <argument>/p:CustomOpensslLib=${openssl.lib}</argument>
+ <argument>/p:CustomOpensslInclude=${openssl.include}</argument>
+ <argument>/p:RequireOpenssl=${require.openssl}</argument>
</arguments>
</configuration>
</execution>
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/CMakeLists.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/CMakeLists.txt Thu Jul 3 23:40:31 2014
@@ -145,6 +145,37 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_
ENDIF(REQUIRE_SNAPPY)
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+set_find_shared_library_version("1.0.0")
+SET(OPENSSL_NAME "crypto")
+IF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+ SET(OPENSSL_NAME "eay32")
+ENDIF()
+find_library(OPENSSL_LIBRARY
+ NAMES ${OPENSSL_NAME}
+ PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib
+ ${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH)
+find_library(OPENSSL_LIBRARY
+ NAMES ${OPENSSL_NAME})
+SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
+find_path(OPENSSL_INCLUDE_DIR
+ NAMES openssl/evp.h
+ PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include
+ ${CUSTOM_OPENSSL_INCLUDE} NO_DEFAULT_PATH)
+find_path(OPENSSL_INCLUDE_DIR
+ NAMES openssl/evp.h)
+if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
+ GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
+ SET(OPENSSL_SOURCE_FILES
+ "${D}/crypto/OpensslCipher.c")
+else (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
+ SET(OPENSSL_INCLUDE_DIR "")
+ SET(OPENSSL_SOURCE_FILES "")
+ IF(REQUIRE_OPENSSL)
+ MESSAGE(FATAL_ERROR "Required openssl library could not be found. OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_INCLUDE_DIR=${CUSTOM_OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}")
+ ENDIF(REQUIRE_OPENSSL)
+endif (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
+
include_directories(
${GENERATED_JAVAH}
main/native/src
@@ -155,6 +186,7 @@ include_directories(
${ZLIB_INCLUDE_DIRS}
${BZIP2_INCLUDE_DIR}
${SNAPPY_INCLUDE_DIR}
+ ${OPENSSL_INCLUDE_DIR}
${D}/util
)
CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
@@ -172,6 +204,7 @@ add_dual_library(hadoop
${D}/io/compress/lz4/lz4.c
${D}/io/compress/lz4/lz4hc.c
${SNAPPY_SOURCE_FILES}
+ ${OPENSSL_SOURCE_FILES}
${D}/io/compress/zlib/ZlibCompressor.c
${D}/io/compress/zlib/ZlibDecompressor.c
${BZIP2_SOURCE_FILES}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/config.h.cmake
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/config.h.cmake?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/config.h.cmake (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/config.h.cmake Thu Jul 3 23:40:31 2014
@@ -21,6 +21,7 @@
#cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
+#cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@"
#cmakedefine HAVE_SYNC_FILE_RANGE
#cmakedefine HAVE_POSIX_FADVISE
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java Thu Jul 3 23:40:31 2014
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import com.google.common.base.Preconditions;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class AESCTRCryptoCodec extends CryptoCodec {
-
- protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING;
-
- /**
- * For AES, the algorithm block is fixed size of 128 bits.
- * @see http://en.wikipedia.org/wiki/Advanced_Encryption_Standard
- */
- private static final int AES_BLOCK_SIZE = SUITE.getAlgorithmBlockSize();
- private static final int CTR_OFFSET = 8;
-
- @Override
- public CipherSuite getCipherSuite() {
- return SUITE;
- }
-
- /**
- * The IV is produced by adding the initial IV to the counter. IV length
- * should be the same as {@link #AES_BLOCK_SIZE}
- */
- @Override
- public void calculateIV(byte[] initIV, long counter, byte[] IV) {
- Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE);
- Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE);
-
- System.arraycopy(initIV, 0, IV, 0, CTR_OFFSET);
- long l = (initIV[CTR_OFFSET + 0] << 56)
- + ((initIV[CTR_OFFSET + 1] & 0xFF) << 48)
- + ((initIV[CTR_OFFSET + 2] & 0xFF) << 40)
- + ((initIV[CTR_OFFSET + 3] & 0xFF) << 32)
- + ((initIV[CTR_OFFSET + 4] & 0xFF) << 24)
- + ((initIV[CTR_OFFSET + 5] & 0xFF) << 16)
- + ((initIV[CTR_OFFSET + 6] & 0xFF) << 8)
- + (initIV[CTR_OFFSET + 7] & 0xFF);
- l += counter;
- IV[CTR_OFFSET + 0] = (byte) (l >>> 56);
- IV[CTR_OFFSET + 1] = (byte) (l >>> 48);
- IV[CTR_OFFSET + 2] = (byte) (l >>> 40);
- IV[CTR_OFFSET + 3] = (byte) (l >>> 32);
- IV[CTR_OFFSET + 4] = (byte) (l >>> 24);
- IV[CTR_OFFSET + 5] = (byte) (l >>> 16);
- IV[CTR_OFFSET + 6] = (byte) (l >>> 8);
- IV[CTR_OFFSET + 7] = (byte) (l);
- }
-}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java Thu Jul 3 23:40:31 2014
@@ -35,7 +35,7 @@ public abstract class CryptoCodec implem
public static CryptoCodec getInstance(Configuration conf) {
final Class<? extends CryptoCodec> klass = conf.getClass(
- HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JCEAESCTRCryptoCodec.class,
+ HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JceAesCtrCryptoCodec.class,
CryptoCodec.class);
return ReflectionUtils.newInstance(klass, conf);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java Thu Jul 3 23:40:31 2014
@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.security.GeneralSecurityException;
-import java.security.SecureRandom;
-
-import javax.crypto.Cipher;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Preconditions;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT;
-
-/**
- * Implement the AES-CTR crypto codec using JCE provider.
- */
-@InterfaceAudience.Private
-public class JCEAESCTRCryptoCodec extends AESCTRCryptoCodec {
- private Configuration conf;
- private String provider;
- private SecureRandom random;
-
- public JCEAESCTRCryptoCodec() {
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- provider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY);
- final String secureRandomAlg = conf.get(
- HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY,
- HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT);
- try {
- random = (provider != null) ?
- SecureRandom.getInstance(secureRandomAlg, provider) :
- SecureRandom.getInstance(secureRandomAlg);
- } catch (GeneralSecurityException e) {
- throw new IllegalArgumentException(e);
- }
- }
-
- @Override
- public Encryptor createEncryptor() throws GeneralSecurityException {
- return new JCEAESCTRCipher(Cipher.ENCRYPT_MODE, provider);
- }
-
- @Override
- public Decryptor createDecryptor() throws GeneralSecurityException {
- return new JCEAESCTRCipher(Cipher.DECRYPT_MODE, provider);
- }
-
- @Override
- public void generateSecureRandom(byte[] bytes) {
- random.nextBytes(bytes);
- }
-
- private static class JCEAESCTRCipher implements Encryptor, Decryptor {
- private final Cipher cipher;
- private final int mode;
- private boolean contextReset = false;
-
- public JCEAESCTRCipher(int mode, String provider)
- throws GeneralSecurityException {
- this.mode = mode;
- if (provider == null || provider.isEmpty()) {
- cipher = Cipher.getInstance(SUITE.getName());
- } else {
- cipher = Cipher.getInstance(SUITE.getName(), provider);
- }
- }
-
- @Override
- public void init(byte[] key, byte[] iv) throws IOException {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(iv);
- contextReset = false;
- try {
- cipher.init(mode, new SecretKeySpec(key, "AES"),
- new IvParameterSpec(iv));
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- /**
- * AES-CTR will consume all of the input data. It requires enough space in
- * the destination buffer to encrypt entire input buffer.
- */
- @Override
- public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
- throws IOException {
- process(inBuffer, outBuffer);
- }
-
- /**
- * AES-CTR will consume all of the input data. It requires enough space in
- * the destination buffer to decrypt entire input buffer.
- */
- @Override
- public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
- throws IOException {
- process(inBuffer, outBuffer);
- }
-
- private void process(ByteBuffer inBuffer, ByteBuffer outBuffer)
- throws IOException {
- try {
- int inputSize = inBuffer.remaining();
- // Cipher#update will maintain crypto context.
- int n = cipher.update(inBuffer, outBuffer);
- if (n < inputSize) {
- /**
- * Typically code will not get here. Cipher#update will consume all
- * input data and put result in outBuffer.
- * Cipher#doFinal will reset the crypto context.
- */
- contextReset = true;
- cipher.doFinal(inBuffer, outBuffer);
- }
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public boolean isContextReset() {
- return contextReset;
- }
- }
-}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java Thu Jul 3 23:40:31 2014
@@ -78,6 +78,11 @@ public class NativeCodeLoader {
* Returns true only if this build was compiled with support for snappy.
*/
public static native boolean buildSupportsSnappy();
+
+ /**
+ * Returns true only if this build was compiled with support for openssl.
+ */
+ public static native boolean buildSupportsOpenssl();
public static native String getLibraryName();
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java Thu Jul 3 23:40:31 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.util;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.OpensslCipher;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
@@ -57,12 +58,14 @@ public class NativeLibraryChecker {
boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
boolean zlibLoaded = false;
boolean snappyLoaded = false;
+ boolean opensslLoaded = false;
// lz4 is linked within libhadoop
boolean lz4Loaded = nativeHadoopLoaded;
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
String hadoopLibraryName = "";
String zlibLibraryName = "";
String snappyLibraryName = "";
+ String opensslLibraryName = "";
String lz4LibraryName = "";
String bzip2LibraryName = "";
if (nativeHadoopLoaded) {
@@ -76,6 +79,11 @@ public class NativeLibraryChecker {
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
snappyLibraryName = SnappyCodec.getLibraryName();
}
+ opensslLoaded = NativeCodeLoader.buildSupportsOpenssl() &&
+ OpensslCipher.isNativeCodeLoaded();
+ if (opensslLoaded) {
+ opensslLibraryName = OpensslCipher.getLibraryName();
+ }
if (lz4Loaded) {
lz4LibraryName = Lz4Codec.getLibraryName();
}
@@ -84,11 +92,12 @@ public class NativeLibraryChecker {
}
}
System.out.println("Native library checking:");
- System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
- System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
- System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
- System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
- System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
+ System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
+ System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
+ System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
+ System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
+ System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
+ System.out.printf("openssl: %b %s\n", opensslLoaded, opensslLibraryName);
if ((!nativeHadoopLoaded) ||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
// return 1 to indicated check failed
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c Thu Jul 3 23:40:31 2014
@@ -39,6 +39,16 @@ JNIEXPORT jboolean JNICALL Java_org_apac
#endif
}
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl
+ (JNIEnv *env, jclass clazz)
+{
+#ifdef HADOOP_OPENSSL_LIBRARY
+ return JNI_TRUE;
+#else
+ return JNI_FALSE;
+#endif
+}
+
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
(JNIEnv *env, jclass clazz)
{
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java Thu Jul 3 23:40:31 2014
@@ -17,38 +17,165 @@
*/
package org.apache.hadoop.crypto;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.junit.AfterClass;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.RandomDatum;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Assert;
-import org.junit.BeforeClass;
import org.junit.Test;
public class TestCryptoCodec {
- private static CryptoCodec codec;
+ private static final Log LOG= LogFactory.getLog(TestCryptoCodec.class);
+ private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+ 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
+ private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+ 0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
+ private static final int bufferSize = 4096;
- @BeforeClass
- public static void init() throws Exception {
- Configuration conf = new Configuration();
- codec = CryptoCodec.getInstance(conf);
+ private Configuration conf = new Configuration();
+ private int count = 10000;
+ private int seed = new Random().nextInt();
+
+ @Test(timeout=120000)
+ public void testJceAesCtrCryptoCodec() throws Exception {
+ cryptoCodecTest(conf, seed, 0,
+ "org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
+ cryptoCodecTest(conf, seed, count,
+ "org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
}
- @AfterClass
- public static void shutdown() throws Exception {
+ @Test(timeout=1200000)
+ public void testOpensslAesCtrCryptoCodec() throws Exception {
+ if (NativeCodeLoader.buildSupportsOpenssl()) {
+ Assert.assertTrue(OpensslCipher.isNativeCodeLoaded());
+ }
+ if (OpensslCipher.isNativeCodeLoaded()) {
+ cryptoCodecTest(conf, seed, 0,
+ "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
+ cryptoCodecTest(conf, seed, count,
+ "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
+ }
}
- @Test(timeout=120000)
- public void testSecureRandom() throws Exception {
- // len = 16
- checkSecureRandom(16);
+ private void cryptoCodecTest(Configuration conf, int seed, int count,
+ String codecClass) throws IOException, GeneralSecurityException {
+ CryptoCodec codec = null;
+ try {
+ codec = (CryptoCodec)ReflectionUtils.newInstance(
+ conf.getClassByName(codecClass), conf);
+ } catch (ClassNotFoundException cnfe) {
+ throw new IOException("Illegal crypto codec!");
+ }
+ LOG.info("Created a Codec object of type: " + codecClass);
- // len = 32
- checkSecureRandom(32);
+ // Generate data
+ DataOutputBuffer data = new DataOutputBuffer();
+ RandomDatum.Generator generator = new RandomDatum.Generator(seed);
+ for(int i = 0; i < count; ++i) {
+ generator.next();
+ RandomDatum key = generator.getKey();
+ RandomDatum value = generator.getValue();
+
+ key.write(data);
+ value.write(data);
+ }
+ LOG.info("Generated " + count + " records");
+ // Encrypt data
+ DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer();
+ CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer,
+ codec, bufferSize, key, iv);
+ out.write(data.getData(), 0, data.getLength());
+ out.flush();
+ out.close();
+ LOG.info("Finished encrypting data");
+
+ // Decrypt data
+ DataInputBuffer decryptedDataBuffer = new DataInputBuffer();
+ decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0,
+ encryptedDataBuffer.getLength());
+ CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer,
+ codec, bufferSize, key, iv);
+ DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in));
+
+ // Check
+ DataInputBuffer originalData = new DataInputBuffer();
+ originalData.reset(data.getData(), 0, data.getLength());
+ DataInputStream originalIn = new DataInputStream(
+ new BufferedInputStream(originalData));
+
+ for(int i=0; i < count; ++i) {
+ RandomDatum k1 = new RandomDatum();
+ RandomDatum v1 = new RandomDatum();
+ k1.readFields(originalIn);
+ v1.readFields(originalIn);
+
+ RandomDatum k2 = new RandomDatum();
+ RandomDatum v2 = new RandomDatum();
+ k2.readFields(dataIn);
+ v2.readFields(dataIn);
+ assertTrue("original and encrypted-then-decrypted-output not equal",
+ k1.equals(k2) && v1.equals(v2));
+
+ // original and encrypted-then-decrypted-output have the same hashCode
+ Map<RandomDatum, String> m = new HashMap<RandomDatum, String>();
+ m.put(k1, k1.toString());
+ m.put(v1, v1.toString());
+ String result = m.get(k2);
+ assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
+ result = m.get(v2);
+ assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
+ }
+
+ // Decrypt data byte-at-a-time
+ originalData.reset(data.getData(), 0, data.getLength());
+ decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0,
+ encryptedDataBuffer.getLength());
+ in = new CryptoInputStream(decryptedDataBuffer,
+ codec, bufferSize, key, iv);
+
+ // Check
+ originalIn = new DataInputStream(new BufferedInputStream(originalData));
+ int expected;
+ do {
+ expected = originalIn.read();
+ assertEquals("Decrypted stream read by byte does not match",
+ expected, in.read());
+ } while (expected != -1);
+
+ LOG.info("SUCCESS! Completed checking " + count + " records");
+
+ // Check secure random generator
+ testSecureRandom(codec);
+ }
+
+ /** Test secure random generator */
+ private void testSecureRandom(CryptoCodec codec) {
+ // len = 16
+ checkSecureRandom(codec, 16);
+ // len = 32
+ checkSecureRandom(codec, 32);
// len = 128
- checkSecureRandom(128);
+ checkSecureRandom(codec, 128);
}
- private void checkSecureRandom(int len) {
+ private void checkSecureRandom(CryptoCodec codec, int len) {
byte[] rand = new byte[len];
byte[] rand1 = new byte[len];
codec.generateSecureRandom(rand);
@@ -56,28 +183,6 @@ public class TestCryptoCodec {
Assert.assertEquals(len, rand.length);
Assert.assertEquals(len, rand1.length);
- Assert.assertFalse(bytesArrayEquals(rand, rand1));
- }
-
- private boolean bytesArrayEquals(byte[] expected, byte[] actual) {
- if ((expected == null && actual != null) ||
- (expected != null && actual == null)) {
- return false;
- }
- if (expected == null && actual == null) {
- return true;
- }
-
- if (expected.length != actual.length) {
- return false;
- }
-
- for (int i = 0; i < expected.length; i++) {
- if (expected[i] != actual[i]) {
- return false;
- }
- }
-
- return true;
+ Assert.assertFalse(Arrays.equals(rand, rand1));
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java?rev=1607768&r1=1607767&r2=1607768&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java Thu Jul 3 23:40:31 2014
@@ -22,6 +22,7 @@ import static org.junit.Assert.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.crypto.OpensslCipher;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
@@ -54,6 +55,9 @@ public class TestNativeCodeLoader {
if (NativeCodeLoader.buildSupportsSnappy()) {
assertFalse(SnappyCodec.getLibraryName().isEmpty());
}
+ if (NativeCodeLoader.buildSupportsOpenssl()) {
+ assertFalse(OpensslCipher.getLibraryName().isEmpty());
+ }
assertFalse(Lz4Codec.getLibraryName().isEmpty());
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
}