You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tu...@apache.org on 2014/08/29 00:07:04 UTC
[09/11] HDFS-6134 and HADOOP-10150 subtasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
new file mode 100644
index 0000000..4ca79b3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.GeneralSecurityException;
+import java.security.SecureRandom;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.crypto.random.OsSecureRandom;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Implement the AES-CTR crypto codec using JNI into OpenSSL.
+ */
+@InterfaceAudience.Private
+public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec {
+ private static final Log LOG =
+ LogFactory.getLog(OpensslAesCtrCryptoCodec.class.getName());
+
+ private Configuration conf;
+ private Random random;
+
+ public OpensslAesCtrCryptoCodec() {
+ String loadingFailureReason = OpensslCipher.getLoadingFailureReason();
+ if (loadingFailureReason != null) {
+ throw new RuntimeException(loadingFailureReason);
+ }
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ final Class<? extends Random> klass = conf.getClass(
+ HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY, OsSecureRandom.class,
+ Random.class);
+ try {
+ random = ReflectionUtils.newInstance(klass, conf);
+ } catch (Exception e) {
+ LOG.info("Unable to use " + klass.getName() + ". Falling back to " +
+ "Java SecureRandom.", e);
+ this.random = new SecureRandom();
+ }
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ Closeable r = (Closeable) this.random;
+ r.close();
+ } catch (ClassCastException e) {
+ }
+ super.finalize();
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public Encryptor createEncryptor() throws GeneralSecurityException {
+ return new OpensslAesCtrCipher(OpensslCipher.ENCRYPT_MODE);
+ }
+
+ @Override
+ public Decryptor createDecryptor() throws GeneralSecurityException {
+ return new OpensslAesCtrCipher(OpensslCipher.DECRYPT_MODE);
+ }
+
+ @Override
+ public void generateSecureRandom(byte[] bytes) {
+ random.nextBytes(bytes);
+ }
+
+ private static class OpensslAesCtrCipher implements Encryptor, Decryptor {
+ private final OpensslCipher cipher;
+ private final int mode;
+ private boolean contextReset = false;
+
+ public OpensslAesCtrCipher(int mode) throws GeneralSecurityException {
+ this.mode = mode;
+ cipher = OpensslCipher.getInstance(SUITE.getName());
+ }
+
+ @Override
+ public void init(byte[] key, byte[] iv) throws IOException {
+ Preconditions.checkNotNull(key);
+ Preconditions.checkNotNull(iv);
+ contextReset = false;
+ cipher.init(mode, key, iv);
+ }
+
+ /**
+ * AES-CTR will consume all of the input data. It requires enough space in
+ * the destination buffer to encrypt entire input buffer.
+ */
+ @Override
+ public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
+ throws IOException {
+ process(inBuffer, outBuffer);
+ }
+
+ /**
+ * AES-CTR will consume all of the input data. It requires enough space in
+ * the destination buffer to decrypt entire input buffer.
+ */
+ @Override
+ public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
+ throws IOException {
+ process(inBuffer, outBuffer);
+ }
+
+ private void process(ByteBuffer inBuffer, ByteBuffer outBuffer)
+ throws IOException {
+ try {
+ int inputSize = inBuffer.remaining();
+ // OpensslCipher#update will maintain crypto context.
+ int n = cipher.update(inBuffer, outBuffer);
+ if (n < inputSize) {
+ /**
+ * Typically code will not get here. OpensslCipher#update will
+ * consume all input data and put result in outBuffer.
+ * OpensslCipher#doFinal will reset the crypto context.
+ */
+ contextReset = true;
+ cipher.doFinal(outBuffer);
+ }
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public boolean isContextReset() {
+ return contextReset;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
new file mode 100644
index 0000000..264652b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto;
+
+import java.nio.ByteBuffer;
+import java.security.NoSuchAlgorithmException;
+import java.util.StringTokenizer;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.ShortBufferException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.NativeCodeLoader;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * OpenSSL cipher using JNI.
+ * Currently only AES-CTR is supported. It's flexible to add
+ * other crypto algorithms/modes.
+ */
+@InterfaceAudience.Private
+public final class OpensslCipher {
+ private static final Log LOG =
+ LogFactory.getLog(OpensslCipher.class.getName());
+ public static final int ENCRYPT_MODE = 1;
+ public static final int DECRYPT_MODE = 0;
+
+ /** Currently only support AES/CTR/NoPadding. */
+ private static enum AlgMode {
+ AES_CTR;
+
+ static int get(String algorithm, String mode)
+ throws NoSuchAlgorithmException {
+ try {
+ return AlgMode.valueOf(algorithm + "_" + mode).ordinal();
+ } catch (Exception e) {
+ throw new NoSuchAlgorithmException("Doesn't support algorithm: " +
+ algorithm + " and mode: " + mode);
+ }
+ }
+ }
+
+ private static enum Padding {
+ NoPadding;
+
+ static int get(String padding) throws NoSuchPaddingException {
+ try {
+ return Padding.valueOf(padding).ordinal();
+ } catch (Exception e) {
+ throw new NoSuchPaddingException("Doesn't support padding: " + padding);
+ }
+ }
+ }
+
+ private long context = 0;
+ private final int alg;
+ private final int padding;
+
+ private static final String loadingFailureReason;
+
+ static {
+ String loadingFailure = null;
+ try {
+ if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ loadingFailure = "build does not support openssl.";
+ } else {
+ initIDs();
+ }
+ } catch (Throwable t) {
+ loadingFailure = t.getMessage();
+ LOG.debug("Failed to load OpenSSL Cipher.", t);
+ } finally {
+ loadingFailureReason = loadingFailure;
+ }
+ }
+
+ public static String getLoadingFailureReason() {
+ return loadingFailureReason;
+ }
+
+ private OpensslCipher(long context, int alg, int padding) {
+ this.context = context;
+ this.alg = alg;
+ this.padding = padding;
+ }
+
+ /**
+ * Return an <code>OpensslCipher<code> object that implements the specified
+ * transformation.
+ *
+ * @param transformation the name of the transformation, e.g.,
+ * AES/CTR/NoPadding.
+ * @return OpensslCipher an <code>OpensslCipher<code> object
+ * @throws NoSuchAlgorithmException if <code>transformation</code> is null,
+ * empty, in an invalid format, or if Openssl doesn't implement the
+ * specified algorithm.
+ * @throws NoSuchPaddingException if <code>transformation</code> contains
+ * a padding scheme that is not available.
+ */
+ public static final OpensslCipher getInstance(String transformation)
+ throws NoSuchAlgorithmException, NoSuchPaddingException {
+ Transform transform = tokenizeTransformation(transformation);
+ int algMode = AlgMode.get(transform.alg, transform.mode);
+ int padding = Padding.get(transform.padding);
+ long context = initContext(algMode, padding);
+ return new OpensslCipher(context, algMode, padding);
+ }
+
+ /** Nested class for algorithm, mode and padding. */
+ private static class Transform {
+ final String alg;
+ final String mode;
+ final String padding;
+
+ public Transform(String alg, String mode, String padding) {
+ this.alg = alg;
+ this.mode = mode;
+ this.padding = padding;
+ }
+ }
+
+ private static Transform tokenizeTransformation(String transformation)
+ throws NoSuchAlgorithmException {
+ if (transformation == null) {
+ throw new NoSuchAlgorithmException("No transformation given.");
+ }
+
+ /*
+ * Array containing the components of a Cipher transformation:
+ *
+ * index 0: algorithm (e.g., AES)
+ * index 1: mode (e.g., CTR)
+ * index 2: padding (e.g., NoPadding)
+ */
+ String[] parts = new String[3];
+ int count = 0;
+ StringTokenizer parser = new StringTokenizer(transformation, "/");
+ while (parser.hasMoreTokens() && count < 3) {
+ parts[count++] = parser.nextToken().trim();
+ }
+ if (count != 3 || parser.hasMoreTokens()) {
+ throw new NoSuchAlgorithmException("Invalid transformation format: " +
+ transformation);
+ }
+ return new Transform(parts[0], parts[1], parts[2]);
+ }
+
+ /**
+ * Initialize this cipher with a key and IV.
+ *
+ * @param mode {@link #ENCRYPT_MODE} or {@link #DECRYPT_MODE}
+ * @param key crypto key
+ * @param iv crypto iv
+ */
+ public void init(int mode, byte[] key, byte[] iv) {
+ context = init(context, mode, alg, padding, key, iv);
+ }
+
+ /**
+ * Continues a multiple-part encryption or decryption operation. The data
+ * is encrypted or decrypted, depending on how this cipher was initialized.
+ * <p/>
+ *
+ * All <code>input.remaining()</code> bytes starting at
+ * <code>input.position()</code> are processed. The result is stored in
+ * the output buffer.
+ * <p/>
+ *
+ * Upon return, the input buffer's position will be equal to its limit;
+ * its limit will not have changed. The output buffer's position will have
+ * advanced by n, when n is the value returned by this method; the output
+ * buffer's limit will not have changed.
+ * <p/>
+ *
+ * If <code>output.remaining()</code> bytes are insufficient to hold the
+ * result, a <code>ShortBufferException</code> is thrown.
+ *
+ * @param input the input ByteBuffer
+ * @param output the output ByteBuffer
+ * @return int number of bytes stored in <code>output</code>
+ * @throws ShortBufferException if there is insufficient space in the
+ * output buffer
+ */
+ public int update(ByteBuffer input, ByteBuffer output)
+ throws ShortBufferException {
+ checkState();
+ Preconditions.checkArgument(input.isDirect() && output.isDirect(),
+ "Direct buffers are required.");
+ int len = update(context, input, input.position(), input.remaining(),
+ output, output.position(), output.remaining());
+ input.position(input.limit());
+ output.position(output.position() + len);
+ return len;
+ }
+
+ /**
+ * Finishes a multiple-part operation. The data is encrypted or decrypted,
+ * depending on how this cipher was initialized.
+ * <p/>
+ *
+ * The result is stored in the output buffer. Upon return, the output buffer's
+ * position will have advanced by n, where n is the value returned by this
+ * method; the output buffer's limit will not have changed.
+ * <p/>
+ *
+ * If <code>output.remaining()</code> bytes are insufficient to hold the result,
+ * a <code>ShortBufferException</code> is thrown.
+ * <p/>
+ *
+ * Upon finishing, this method resets this cipher object to the state it was
+ * in when previously initialized. That is, the object is available to encrypt
+ * or decrypt more data.
+ * <p/>
+ *
+ * If any exception is thrown, this cipher object need to be reset before it
+ * can be used again.
+ *
+ * @param output the output ByteBuffer
+ * @return int number of bytes stored in <code>output</code>
+ * @throws ShortBufferException
+ * @throws IllegalBlockSizeException
+ * @throws BadPaddingException
+ */
+ public int doFinal(ByteBuffer output) throws ShortBufferException,
+ IllegalBlockSizeException, BadPaddingException {
+ checkState();
+ Preconditions.checkArgument(output.isDirect(), "Direct buffer is required.");
+ int len = doFinal(context, output, output.position(), output.remaining());
+ output.position(output.position() + len);
+ return len;
+ }
+
+ /** Forcibly clean the context. */
+ public void clean() {
+ if (context != 0) {
+ clean(context);
+ context = 0;
+ }
+ }
+
+ /** Check whether context is initialized. */
+ private void checkState() {
+ Preconditions.checkState(context != 0);
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ clean();
+ }
+
+ private native static void initIDs();
+
+ private native static long initContext(int alg, int padding);
+
+ private native long init(long context, int mode, int alg, int padding,
+ byte[] key, byte[] iv);
+
+ private native int update(long context, ByteBuffer input, int inputOffset,
+ int inputLength, ByteBuffer output, int outputOffset, int maxOutputLength);
+
+ private native int doFinal(long context, ByteBuffer output, int offset,
+ int maxOutputLength);
+
+ private native void clean(long context);
+
+ public native static String getLibraryName();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
new file mode 100644
index 0000000..b1fa988
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.random;
+
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.NativeCodeLoader;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * OpenSSL secure random using JNI.
+ * This implementation is thread-safe.
+ * <p/>
+ *
+ * If using an Intel chipset with RDRAND, the high-performance hardware
+ * random number generator will be used and it's much faster than
+ * {@link java.security.SecureRandom}. If RDRAND is unavailable, default
+ * OpenSSL secure random generator will be used. It's still faster
+ * and can generate strong random bytes.
+ * <p/>
+ * @see https://wiki.openssl.org/index.php/Random_Numbers
+ * @see http://en.wikipedia.org/wiki/RdRand
+ */
+@InterfaceAudience.Private
+public class OpensslSecureRandom extends Random {
+ private static final long serialVersionUID = -7828193502768789584L;
+ private static final Log LOG =
+ LogFactory.getLog(OpensslSecureRandom.class.getName());
+
+ /** If native SecureRandom unavailable, use java SecureRandom */
+ private java.security.SecureRandom fallback = null;
+ private static boolean nativeEnabled = false;
+ static {
+ if (NativeCodeLoader.isNativeCodeLoaded() &&
+ NativeCodeLoader.buildSupportsOpenssl()) {
+ try {
+ initSR();
+ nativeEnabled = true;
+ } catch (Throwable t) {
+ LOG.error("Failed to load Openssl SecureRandom", t);
+ }
+ }
+ }
+
+ public static boolean isNativeCodeLoaded() {
+ return nativeEnabled;
+ }
+
+ public OpensslSecureRandom() {
+ if (!nativeEnabled) {
+ fallback = new java.security.SecureRandom();
+ }
+ }
+
+ /**
+ * Generates a user-specified number of random bytes.
+ * It's thread-safe.
+ *
+ * @param bytes the array to be filled in with random bytes.
+ */
+ @Override
+ public void nextBytes(byte[] bytes) {
+ if (!nativeEnabled || !nextRandBytes(bytes)) {
+ fallback.nextBytes(bytes);
+ }
+ }
+
+ @Override
+ public void setSeed(long seed) {
+ // Self-seeding.
+ }
+
+ /**
+ * Generates an integer containing the user-specified number of
+ * random bits (right justified, with leading zeros).
+ *
+ * @param numBits number of random bits to be generated, where
+ * 0 <= <code>numBits</code> <= 32.
+ *
+ * @return int an <code>int</code> containing the user-specified number
+ * of random bits (right justified, with leading zeros).
+ */
+ @Override
+ final protected int next(int numBits) {
+ Preconditions.checkArgument(numBits >= 0 && numBits <= 32);
+ int numBytes = (numBits + 7) / 8;
+ byte b[] = new byte[numBytes];
+ int next = 0;
+
+ nextBytes(b);
+ for (int i = 0; i < numBytes; i++) {
+ next = (next << 8) + (b[i] & 0xFF);
+ }
+
+ return next >>> (numBytes * 8 - numBits);
+ }
+
+ private native static void initSR();
+ private native boolean nextRandBytes(byte[] bytes);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java
new file mode 100644
index 0000000..c6cb0a8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OsSecureRandom.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.random;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT;
+
+/**
+ * A Random implementation that uses random bytes sourced from the
+ * operating system.
+ */
+@InterfaceAudience.Private
+public class OsSecureRandom extends Random implements Closeable, Configurable {
+ private static final long serialVersionUID = 6391500337172057900L;
+
+ private transient Configuration conf;
+
+ private final int RESERVOIR_LENGTH = 8192;
+
+ private String randomDevPath;
+
+ private transient FileInputStream stream;
+
+ private final byte[] reservoir = new byte[RESERVOIR_LENGTH];
+
+ private int pos = reservoir.length;
+
+ private void fillReservoir(int min) {
+ if (pos >= reservoir.length - min) {
+ try {
+ IOUtils.readFully(stream, reservoir, 0, reservoir.length);
+ } catch (IOException e) {
+ throw new RuntimeException("failed to fill reservoir", e);
+ }
+ pos = 0;
+ }
+ }
+
+ public OsSecureRandom() {
+ }
+
+ @Override
+ synchronized public void setConf(Configuration conf) {
+ this.conf = conf;
+ this.randomDevPath = conf.get(
+ HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY,
+ HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT);
+ File randomDevFile = new File(randomDevPath);
+ try {
+ this.stream = new FileInputStream(randomDevFile);
+ fillReservoir(0);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ synchronized public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ synchronized public void nextBytes(byte[] bytes) {
+ int off = 0;
+ int n = 0;
+ while (off < bytes.length) {
+ fillReservoir(0);
+ n = Math.min(bytes.length - off, reservoir.length - pos);
+ System.arraycopy(reservoir, pos, bytes, off, n);
+ off += n;
+ pos += n;
+ }
+ }
+
+ @Override
+ synchronized protected int next(int nbits) {
+ fillReservoir(4);
+ int n = 0;
+ for (int i = 0; i < 4; i++) {
+ n = ((n << 8) | (reservoir[pos++] & 0xff));
+ }
+ return n & (0xffffffff >> (32 - nbits));
+ }
+
+ @Override
+ synchronized public void close() throws IOException {
+ stream.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index b4aedb3..c1101c5f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -283,5 +283,4 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final String NFS_EXPORTS_ALLOWED_HOSTS_SEPARATOR = ";";
public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY = "nfs.exports.allowed.hosts";
public static final String NFS_EXPORTS_ALLOWED_HOSTS_KEY_DEFAULT = "* rw";
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index abcb878..8309e2e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -296,6 +296,21 @@ public class CommonConfigurationKeysPublic {
/** Class to override Sasl Properties for a connection */
public static final String HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS =
"hadoop.security.saslproperties.resolver.class";
+ public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX =
+ "hadoop.security.crypto.codec.classes";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY =
+ "hadoop.security.crypto.cipher.suite";
+ public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT =
+ "AES/CTR/NoPadding";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY =
+ "hadoop.security.crypto.jce.provider";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY =
+ "hadoop.security.crypto.buffer.size";
+ /** Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY */
+ public static final int HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT = 8192;
/** Class to override Impersonation provider */
public static final String HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS =
"hadoop.security.impersonation.provider.class";
@@ -326,5 +341,20 @@ public class CommonConfigurationKeysPublic {
"hadoop.security.kms.client.encrypted.key.cache.expiry";
/** Default value for KMS_CLIENT_ENC_KEY_CACHE_EXPIRY (12 hrs)*/
public static final int KMS_CLIENT_ENC_KEY_CACHE_EXPIRY_DEFAULT = 43200000;
+
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY =
+ "hadoop.security.java.secure.random.algorithm";
+ /** Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY */
+ public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT =
+ "SHA1PRNG";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY =
+ "hadoop.security.secure.random.impl";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY =
+ "hadoop.security.random.device.file.path";
+ public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT =
+ "/dev/urandom";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
index ec3654b..9a59b1d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
@@ -107,7 +107,7 @@ public class FSDataOutputStream extends DataOutputStream
}
/**
- * Get a reference to the wrapped output stream. Used by unit tests.
+ * Get a reference to the wrapped output stream.
*
* @return the underlying output stream
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
new file mode 100644
index 0000000..f960233
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.crypto.CipherSuite;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * FileEncryptionInfo encapsulates all the encryption-related information for
+ * an encrypted file.
+ */
+@InterfaceAudience.Private
+public class FileEncryptionInfo {
+
+ private final CipherSuite cipherSuite;
+ private final byte[] edek;
+ private final byte[] iv;
+ private final String ezKeyVersionName;
+
+ /**
+ * Create a FileEncryptionInfo.
+ *
+ * @param suite CipherSuite used to encrypt the file
+ * @param edek encrypted data encryption key (EDEK) of the file
+ * @param iv initialization vector (IV) used to encrypt the file
+ * @param ezKeyVersionName name of the KeyVersion used to encrypt the
+ * encrypted data encryption key.
+ */
+ public FileEncryptionInfo(final CipherSuite suite, final byte[] edek,
+ final byte[] iv, final String ezKeyVersionName) {
+ checkNotNull(suite);
+ checkNotNull(edek);
+ checkNotNull(iv);
+ checkNotNull(ezKeyVersionName);
+ checkArgument(edek.length == suite.getAlgorithmBlockSize(),
+ "Unexpected key length");
+ checkArgument(iv.length == suite.getAlgorithmBlockSize(),
+ "Unexpected IV length");
+ this.cipherSuite = suite;
+ this.edek = edek;
+ this.iv = iv;
+ this.ezKeyVersionName = ezKeyVersionName;
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.crypto.CipherSuite} used to encrypt
+ * the file.
+ */
+ public CipherSuite getCipherSuite() {
+ return cipherSuite;
+ }
+
+ /**
+ * @return encrypted data encryption key (EDEK) for the file
+ */
+ public byte[] getEncryptedDataEncryptionKey() {
+ return edek;
+ }
+
+ /**
+ * @return initialization vector (IV) for the cipher used to encrypt the file
+ */
+ public byte[] getIV() {
+ return iv;
+ }
+
+ /**
+ * @return name of the encryption zone KeyVersion used to encrypt the
+ * encrypted data encryption key (EDEK).
+ */
+ public String getEzKeyVersionName() { return ezKeyVersionName; }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder("{");
+ builder.append("cipherSuite: " + cipherSuite);
+ builder.append(", edek: " + Hex.encodeHexString(edek));
+ builder.append(", iv: " + Hex.encodeHexString(iv));
+ builder.append(", ezKeyVersionName: " + ezKeyVersionName);
+ builder.append("}");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataInputStream.java
new file mode 100644
index 0000000..8758d28
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataInputStream.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.crypto;
+
+import java.io.IOException;
+
+import org.apache.hadoop.crypto.CryptoCodec;
+import org.apache.hadoop.crypto.CryptoInputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+
+public class CryptoFSDataInputStream extends FSDataInputStream {
+
+ public CryptoFSDataInputStream(FSDataInputStream in, CryptoCodec codec,
+ int bufferSize, byte[] key, byte[] iv) throws IOException {
+ super(new CryptoInputStream(in, codec, bufferSize, key, iv));
+ }
+
+ public CryptoFSDataInputStream(FSDataInputStream in, CryptoCodec codec,
+ byte[] key, byte[] iv) throws IOException {
+ super(new CryptoInputStream(in, codec, key, iv));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataOutputStream.java
new file mode 100644
index 0000000..3beb361
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/crypto/CryptoFSDataOutputStream.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.crypto;
+
+import java.io.IOException;
+
+import org.apache.hadoop.crypto.CryptoCodec;
+import org.apache.hadoop.crypto.CryptoOutputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+
+public class CryptoFSDataOutputStream extends FSDataOutputStream {
+ private final FSDataOutputStream fsOut;
+
+ public CryptoFSDataOutputStream(FSDataOutputStream out, CryptoCodec codec,
+ int bufferSize, byte[] key, byte[] iv) throws IOException {
+ super(new CryptoOutputStream(out, codec, bufferSize, key, iv,
+ out.getPos()), null, out.getPos());
+ this.fsOut = out;
+ }
+
+ public CryptoFSDataOutputStream(FSDataOutputStream out, CryptoCodec codec,
+ byte[] key, byte[] iv) throws IOException {
+ super(new CryptoOutputStream(out, codec, key, iv, out.getPos()),
+ null, out.getPos());
+ this.fsOut = out;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return fsOut.getPos();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index ac3b1e6..da67f1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -58,6 +58,17 @@ abstract class CommandWithDestination extends FsCommand {
private boolean writeChecksum = true;
/**
+ * The name of the raw xattr namespace. It would be nice to use
+ * XAttr.RAW.name() but we can't reference the hadoop-hdfs project.
+ */
+ private static final String RAW = "raw.";
+
+ /**
+ * The name of the reserved raw directory.
+ */
+ private static final String RESERVED_RAW = "/.reserved/raw";
+
+ /**
*
* This method is used to enable the force(-f) option while copying the files.
*
@@ -231,7 +242,7 @@ abstract class CommandWithDestination extends FsCommand {
/**
* Called with a source and target destination pair
* @param src for the operation
- * @param target for the operation
+ * @param dst for the operation
* @throws IOException if anything goes wrong
*/
protected void processPath(PathData src, PathData dst) throws IOException {
@@ -253,6 +264,8 @@ abstract class CommandWithDestination extends FsCommand {
// modify dst as we descend to append the basename of the
// current directory being processed
dst = getTargetPath(src);
+ final boolean preserveRawXattrs =
+ checkPathsForReservedRaw(src.path, dst.path);
if (dst.exists) {
if (!dst.stat.isDirectory()) {
throw new PathIsNotDirectoryException(dst.toString());
@@ -268,7 +281,7 @@ abstract class CommandWithDestination extends FsCommand {
}
super.recursePath(src);
if (dst.stat.isDirectory()) {
- preserveAttributes(src, dst);
+ preserveAttributes(src, dst, preserveRawXattrs);
}
} finally {
dst = savedDst;
@@ -295,19 +308,61 @@ abstract class CommandWithDestination extends FsCommand {
* @param target where to copy the item
* @throws IOException if copy fails
*/
- protected void copyFileToTarget(PathData src, PathData target) throws IOException {
+ protected void copyFileToTarget(PathData src, PathData target)
+ throws IOException {
+ final boolean preserveRawXattrs =
+ checkPathsForReservedRaw(src.path, target.path);
src.fs.setVerifyChecksum(verifyChecksum);
InputStream in = null;
try {
in = src.fs.open(src.path);
copyStreamToTarget(in, target);
- preserveAttributes(src, target);
+ preserveAttributes(src, target, preserveRawXattrs);
} finally {
IOUtils.closeStream(in);
}
}
/**
+ * Check the source and target paths to ensure that they are either both in
+ * /.reserved/raw or neither in /.reserved/raw. If neither src nor target are
+ * in /.reserved/raw, then return false, indicating not to preserve raw.*
+ * xattrs. If both src/target are in /.reserved/raw, then return true,
+ * indicating raw.* xattrs should be preserved. If only one of src/target is
+ * in /.reserved/raw then throw an exception.
+ *
+ * @param src The source path to check. This should be a fully-qualified
+ * path, not relative.
+ * @param target The target path to check. This should be a fully-qualified
+ * path, not relative.
+ * @return true if raw.* xattrs should be preserved.
+ * @throws PathOperationException is only one of src/target are in
+ * /.reserved/raw.
+ */
+ private boolean checkPathsForReservedRaw(Path src, Path target)
+ throws PathOperationException {
+ final boolean srcIsRR = Path.getPathWithoutSchemeAndAuthority(src).
+ toString().startsWith(RESERVED_RAW);
+ final boolean dstIsRR = Path.getPathWithoutSchemeAndAuthority(target).
+ toString().startsWith(RESERVED_RAW);
+ boolean preserveRawXattrs = false;
+ if (srcIsRR && !dstIsRR) {
+ final String s = "' copy from '" + RESERVED_RAW + "' to non '" +
+ RESERVED_RAW + "'. Either both source and target must be in '" +
+ RESERVED_RAW + "' or neither.";
+ throw new PathOperationException("'" + src.toString() + s);
+ } else if (!srcIsRR && dstIsRR) {
+ final String s = "' copy from non '" + RESERVED_RAW +"' to '" +
+ RESERVED_RAW + "'. Either both source and target must be in '" +
+ RESERVED_RAW + "' or neither.";
+ throw new PathOperationException("'" + dst.toString() + s);
+ } else if (srcIsRR && dstIsRR) {
+ preserveRawXattrs = true;
+ }
+ return preserveRawXattrs;
+ }
+
+ /**
* Copies the stream contents to a temporary file. If the copy is
* successful, the temporary file will be renamed to the real path,
* else the temporary file will be deleted.
@@ -337,9 +392,11 @@ abstract class CommandWithDestination extends FsCommand {
* attribute to preserve.
* @param src source to preserve
* @param target where to preserve attributes
+ * @param preserveRawXAttrs true if raw.* xattrs should be preserved
* @throws IOException if fails to preserve attributes
*/
- protected void preserveAttributes(PathData src, PathData target)
+ protected void preserveAttributes(PathData src, PathData target,
+ boolean preserveRawXAttrs)
throws IOException {
if (shouldPreserve(FileAttribute.TIMESTAMPS)) {
target.fs.setTimes(
@@ -369,13 +426,17 @@ abstract class CommandWithDestination extends FsCommand {
target.fs.setAcl(target.path, srcFullEntries);
}
}
- if (shouldPreserve(FileAttribute.XATTR)) {
+ final boolean preserveXAttrs = shouldPreserve(FileAttribute.XATTR);
+ if (preserveXAttrs || preserveRawXAttrs) {
Map<String, byte[]> srcXAttrs = src.fs.getXAttrs(src.path);
if (srcXAttrs != null) {
Iterator<Entry<String, byte[]>> iter = srcXAttrs.entrySet().iterator();
while (iter.hasNext()) {
Entry<String, byte[]> entry = iter.next();
- target.fs.setXAttr(target.path, entry.getKey(), entry.getValue());
+ final String xattrName = entry.getKey();
+ if (xattrName.startsWith(RAW) || preserveXAttrs) {
+ target.fs.setXAttr(target.path, entry.getKey(), entry.getValue());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 4dd2f4a..3fd870c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -143,7 +143,11 @@ class CopyCommands {
"timestamps, ownership, permission. If -pa is specified, " +
"then preserves permission also because ACL is a super-set of " +
"permission. Passing -f overwrites the destination if it " +
- "already exists.\n";
+ "already exists. raw namespace extended attributes are preserved " +
+ "if (1) they are supported (HDFS only) and, (2) all of the source and " +
+ "target pathnames are in the /.reserved/raw hierarchy. raw namespace " +
+ "xattr preservation is determined solely by the presence (or absence) " +
+ "of the /.reserved/raw prefix and not by the -p option.\n";
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
index 5667d98..533fc07 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
@@ -78,6 +78,11 @@ public class NativeCodeLoader {
* Returns true only if this build was compiled with support for snappy.
*/
public static native boolean buildSupportsSnappy();
+
+ /**
+ * Returns true only if this build was compiled with support for openssl.
+ */
+ public static native boolean buildSupportsOpenssl();
public static native String getLibraryName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
index 84117e2..0d87bce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.util;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.OpensslCipher;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
@@ -60,6 +61,8 @@ public class NativeLibraryChecker {
// lz4 is linked within libhadoop
boolean lz4Loaded = nativeHadoopLoaded;
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
+ boolean openSslLoaded = false;
+ String openSslDetail = "";
String hadoopLibraryName = "";
String zlibLibraryName = "";
String snappyLibraryName = "";
@@ -76,6 +79,13 @@ public class NativeLibraryChecker {
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
snappyLibraryName = SnappyCodec.getLibraryName();
}
+ if (OpensslCipher.getLoadingFailureReason() != null) {
+ openSslDetail = OpensslCipher.getLoadingFailureReason();
+ openSslLoaded = false;
+ } else {
+ openSslDetail = OpensslCipher.getLibraryName();
+ openSslLoaded = true;
+ }
if (lz4Loaded) {
lz4LibraryName = Lz4Codec.getLibraryName();
}
@@ -84,11 +94,12 @@ public class NativeLibraryChecker {
}
}
System.out.println("Native library checking:");
- System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
- System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
- System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
- System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
- System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
+ System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
+ System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
+ System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
+ System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
+ System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
+ System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail);
if ((!nativeHadoopLoaded) ||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
// return 1 to indicated check failed
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
new file mode 100644
index 0000000..5cb5bba
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
@@ -0,0 +1,382 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "org_apache_hadoop_crypto.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "org_apache_hadoop_crypto_OpensslCipher.h"
+
+#ifdef UNIX
+static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
+static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
+static int (*dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
+static void (*dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
+static int (*dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
+static int (*dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, const EVP_CIPHER *, \
+ ENGINE *, const unsigned char *, const unsigned char *, int);
+static int (*dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, unsigned char *, \
+ int *, const unsigned char *, int);
+static int (*dlsym_EVP_CipherFinal_ex)(EVP_CIPHER_CTX *, unsigned char *, int *);
+static EVP_CIPHER * (*dlsym_EVP_aes_256_ctr)(void);
+static EVP_CIPHER * (*dlsym_EVP_aes_128_ctr)(void);
+static void *openssl;
+#endif
+
+#ifdef WINDOWS
+typedef EVP_CIPHER_CTX * (__cdecl *__dlsym_EVP_CIPHER_CTX_new)(void);
+typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
+typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_cleanup)(EVP_CIPHER_CTX *);
+typedef void (__cdecl *__dlsym_EVP_CIPHER_CTX_init)(EVP_CIPHER_CTX *);
+typedef int (__cdecl *__dlsym_EVP_CIPHER_CTX_set_padding)(EVP_CIPHER_CTX *, int);
+typedef int (__cdecl *__dlsym_EVP_CipherInit_ex)(EVP_CIPHER_CTX *, \
+ const EVP_CIPHER *, ENGINE *, const unsigned char *, \
+ const unsigned char *, int);
+typedef int (__cdecl *__dlsym_EVP_CipherUpdate)(EVP_CIPHER_CTX *, \
+ unsigned char *, int *, const unsigned char *, int);
+typedef int (__cdecl *__dlsym_EVP_CipherFinal_ex)(EVP_CIPHER_CTX *, \
+ unsigned char *, int *);
+typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_aes_256_ctr)(void);
+typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_aes_128_ctr)(void);
+static __dlsym_EVP_CIPHER_CTX_new dlsym_EVP_CIPHER_CTX_new;
+static __dlsym_EVP_CIPHER_CTX_free dlsym_EVP_CIPHER_CTX_free;
+static __dlsym_EVP_CIPHER_CTX_cleanup dlsym_EVP_CIPHER_CTX_cleanup;
+static __dlsym_EVP_CIPHER_CTX_init dlsym_EVP_CIPHER_CTX_init;
+static __dlsym_EVP_CIPHER_CTX_set_padding dlsym_EVP_CIPHER_CTX_set_padding;
+static __dlsym_EVP_CipherInit_ex dlsym_EVP_CipherInit_ex;
+static __dlsym_EVP_CipherUpdate dlsym_EVP_CipherUpdate;
+static __dlsym_EVP_CipherFinal_ex dlsym_EVP_CipherFinal_ex;
+static __dlsym_EVP_aes_256_ctr dlsym_EVP_aes_256_ctr;
+static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr;
+static HMODULE openssl;
+#endif
+
+static void loadAesCtr(JNIEnv *env)
+{
+#ifdef UNIX
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_aes_256_ctr, env, openssl, "EVP_aes_256_ctr");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_aes_128_ctr, env, openssl, "EVP_aes_128_ctr");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_aes_256_ctr, dlsym_EVP_aes_256_ctr, \
+ env, openssl, "EVP_aes_256_ctr");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_aes_128_ctr, dlsym_EVP_aes_128_ctr, \
+ env, openssl, "EVP_aes_128_ctr");
+#endif
+}
+
+JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
+ (JNIEnv *env, jclass clazz)
+{
+ char msg[1000];
+#ifdef UNIX
+ openssl = dlopen(HADOOP_OPENSSL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
+#endif
+
+#ifdef WINDOWS
+ openssl = LoadLibrary(HADOOP_OPENSSL_LIBRARY);
+#endif
+
+ if (!openssl) {
+ snprintf(msg, sizeof(msg), "Cannot load %s (%s)!", HADOOP_OPENSSL_LIBRARY, \
+ dlerror());
+ THROW(env, "java/lang/UnsatisfiedLinkError", msg);
+ return;
+ }
+
+#ifdef UNIX
+ dlerror(); // Clear any existing error
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_new, env, openssl, \
+ "EVP_CIPHER_CTX_new");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_free, env, openssl, \
+ "EVP_CIPHER_CTX_free");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_cleanup, env, openssl, \
+ "EVP_CIPHER_CTX_cleanup");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_init, env, openssl, \
+ "EVP_CIPHER_CTX_init");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_set_padding, env, openssl, \
+ "EVP_CIPHER_CTX_set_padding");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \
+ "EVP_CipherInit_ex");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherUpdate, env, openssl, \
+ "EVP_CipherUpdate");
+ LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherFinal_ex, env, openssl, \
+ "EVP_CipherFinal_ex");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_new, dlsym_EVP_CIPHER_CTX_new, \
+ env, openssl, "EVP_CIPHER_CTX_new");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_free, dlsym_EVP_CIPHER_CTX_free, \
+ env, openssl, "EVP_CIPHER_CTX_free");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_cleanup, \
+ dlsym_EVP_CIPHER_CTX_cleanup, env,
+ openssl, "EVP_CIPHER_CTX_cleanup");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_init, dlsym_EVP_CIPHER_CTX_init, \
+ env, openssl, "EVP_CIPHER_CTX_init");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_set_padding, \
+ dlsym_EVP_CIPHER_CTX_set_padding, env, \
+ openssl, "EVP_CIPHER_CTX_set_padding");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \
+ env, openssl, "EVP_CipherInit_ex");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherUpdate, dlsym_EVP_CipherUpdate, \
+ env, openssl, "EVP_CipherUpdate");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherFinal_ex, dlsym_EVP_CipherFinal_ex, \
+ env, openssl, "EVP_CipherFinal_ex");
+#endif
+
+ loadAesCtr(env);
+ jthrowable jthr = (*env)->ExceptionOccurred(env);
+ if (jthr) {
+ (*env)->DeleteLocalRef(env, jthr);
+ THROW(env, "java/lang/UnsatisfiedLinkError", \
+ "Cannot find AES-CTR support, is your version of Openssl new enough?");
+ return;
+ }
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initContext
+ (JNIEnv *env, jclass clazz, jint alg, jint padding)
+{
+ if (alg != AES_CTR) {
+ THROW(env, "java/security/NoSuchAlgorithmException", NULL);
+ return (jlong)0;
+ }
+ if (padding != NOPADDING) {
+ THROW(env, "javax/crypto/NoSuchPaddingException", NULL);
+ return (jlong)0;
+ }
+
+ if (dlsym_EVP_aes_256_ctr == NULL || dlsym_EVP_aes_128_ctr == NULL) {
+ THROW(env, "java/security/NoSuchAlgorithmException", \
+ "Doesn't support AES CTR.");
+ return (jlong)0;
+ }
+
+ // Create and initialize a EVP_CIPHER_CTX
+ EVP_CIPHER_CTX *context = dlsym_EVP_CIPHER_CTX_new();
+ if (!context) {
+ THROW(env, "java/lang/OutOfMemoryError", NULL);
+ return (jlong)0;
+ }
+
+ return JLONG(context);
+}
+
+// Only supports AES-CTR currently
+static EVP_CIPHER * getEvpCipher(int alg, int keyLen)
+{
+ EVP_CIPHER *cipher = NULL;
+ if (alg == AES_CTR) {
+ if (keyLen == KEY_LENGTH_256) {
+ cipher = dlsym_EVP_aes_256_ctr();
+ } else if (keyLen == KEY_LENGTH_128) {
+ cipher = dlsym_EVP_aes_128_ctr();
+ }
+ }
+ return cipher;
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_init
+ (JNIEnv *env, jobject object, jlong ctx, jint mode, jint alg, jint padding,
+ jbyteArray key, jbyteArray iv)
+{
+ int jKeyLen = (*env)->GetArrayLength(env, key);
+ int jIvLen = (*env)->GetArrayLength(env, iv);
+ if (jKeyLen != KEY_LENGTH_128 && jKeyLen != KEY_LENGTH_256) {
+ THROW(env, "java/lang/IllegalArgumentException", "Invalid key length.");
+ return (jlong)0;
+ }
+ if (jIvLen != IV_LENGTH) {
+ THROW(env, "java/lang/IllegalArgumentException", "Invalid iv length.");
+ return (jlong)0;
+ }
+
+ EVP_CIPHER_CTX *context = CONTEXT(ctx);
+ if (context == 0) {
+ // Create and initialize a EVP_CIPHER_CTX
+ context = dlsym_EVP_CIPHER_CTX_new();
+ if (!context) {
+ THROW(env, "java/lang/OutOfMemoryError", NULL);
+ return (jlong)0;
+ }
+ }
+
+ jbyte *jKey = (*env)->GetByteArrayElements(env, key, NULL);
+ if (jKey == NULL) {
+ THROW(env, "java/lang/InternalError", "Cannot get bytes array for key.");
+ return (jlong)0;
+ }
+ jbyte *jIv = (*env)->GetByteArrayElements(env, iv, NULL);
+ if (jIv == NULL) {
+ (*env)->ReleaseByteArrayElements(env, key, jKey, 0);
+ THROW(env, "java/lang/InternalError", "Cannot get bytes array for iv.");
+ return (jlong)0;
+ }
+
+ int rc = dlsym_EVP_CipherInit_ex(context, getEvpCipher(alg, jKeyLen), \
+ NULL, (unsigned char *)jKey, (unsigned char *)jIv, mode == ENCRYPT_MODE);
+ (*env)->ReleaseByteArrayElements(env, key, jKey, 0);
+ (*env)->ReleaseByteArrayElements(env, iv, jIv, 0);
+ if (rc == 0) {
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherInit_ex.");
+ return (jlong)0;
+ }
+
+ if (padding == NOPADDING) {
+ dlsym_EVP_CIPHER_CTX_set_padding(context, 0);
+ }
+
+ return JLONG(context);
+}
+
+// https://www.openssl.org/docs/crypto/EVP_EncryptInit.html
+static int check_update_max_output_len(EVP_CIPHER_CTX *context, int input_len,
+ int max_output_len)
+{
+ if (context->flags & EVP_CIPH_NO_PADDING) {
+ if (max_output_len >= input_len) {
+ return 1;
+ }
+ return 0;
+ } else {
+ int b = context->cipher->block_size;
+ if (context->encrypt) {
+ if (max_output_len >= input_len + b - 1) {
+ return 1;
+ }
+ } else {
+ if (max_output_len >= input_len + b) {
+ return 1;
+ }
+ }
+
+ return 0;
+ }
+}
+
+JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_update
+ (JNIEnv *env, jobject object, jlong ctx, jobject input, jint input_offset,
+ jint input_len, jobject output, jint output_offset, jint max_output_len)
+{
+ EVP_CIPHER_CTX *context = CONTEXT(ctx);
+ if (!check_update_max_output_len(context, input_len, max_output_len)) {
+ THROW(env, "javax/crypto/ShortBufferException", \
+ "Output buffer is not sufficient.");
+ return 0;
+ }
+ unsigned char *input_bytes = (*env)->GetDirectBufferAddress(env, input);
+ unsigned char *output_bytes = (*env)->GetDirectBufferAddress(env, output);
+ if (input_bytes == NULL || output_bytes == NULL) {
+ THROW(env, "java/lang/InternalError", "Cannot get buffer address.");
+ return 0;
+ }
+ input_bytes = input_bytes + input_offset;
+ output_bytes = output_bytes + output_offset;
+
+ int output_len = 0;
+ if (!dlsym_EVP_CipherUpdate(context, output_bytes, &output_len, \
+ input_bytes, input_len)) {
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherUpdate.");
+ return 0;
+ }
+ return output_len;
+}
+
+// https://www.openssl.org/docs/crypto/EVP_EncryptInit.html
+static int check_doFinal_max_output_len(EVP_CIPHER_CTX *context,
+ int max_output_len)
+{
+ if (context->flags & EVP_CIPH_NO_PADDING) {
+ return 1;
+ } else {
+ int b = context->cipher->block_size;
+ if (max_output_len >= b) {
+ return 1;
+ }
+
+ return 0;
+ }
+}
+
+JNIEXPORT jint JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_doFinal
+ (JNIEnv *env, jobject object, jlong ctx, jobject output, jint offset,
+ jint max_output_len)
+{
+ EVP_CIPHER_CTX *context = CONTEXT(ctx);
+ if (!check_doFinal_max_output_len(context, max_output_len)) {
+ THROW(env, "javax/crypto/ShortBufferException", \
+ "Output buffer is not sufficient.");
+ return 0;
+ }
+ unsigned char *output_bytes = (*env)->GetDirectBufferAddress(env, output);
+ if (output_bytes == NULL) {
+ THROW(env, "java/lang/InternalError", "Cannot get buffer address.");
+ return 0;
+ }
+ output_bytes = output_bytes + offset;
+
+ int output_len = 0;
+ if (!dlsym_EVP_CipherFinal_ex(context, output_bytes, &output_len)) {
+ dlsym_EVP_CIPHER_CTX_cleanup(context);
+ THROW(env, "java/lang/InternalError", "Error in EVP_CipherFinal_ex.");
+ return 0;
+ }
+ return output_len;
+}
+
+JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_clean
+ (JNIEnv *env, jobject object, jlong ctx)
+{
+ EVP_CIPHER_CTX *context = CONTEXT(ctx);
+ if (context) {
+ dlsym_EVP_CIPHER_CTX_free(context);
+ }
+}
+
+JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibraryName
+ (JNIEnv *env, jclass clazz)
+{
+#ifdef UNIX
+ if (dlsym_EVP_CIPHER_CTX_init) {
+ Dl_info dl_info;
+ if(dladdr(
+ dlsym_EVP_CIPHER_CTX_init,
+ &dl_info)) {
+ return (*env)->NewStringUTF(env, dl_info.dli_fname);
+ }
+ }
+
+ return (*env)->NewStringUTF(env, HADOOP_OPENSSL_LIBRARY);
+#endif
+
+#ifdef WINDOWS
+ LPWSTR filename = NULL;
+ GetLibraryName(dlsym_EVP_CIPHER_CTX_init, &filename);
+ if (filename != NULL) {
+ return (*env)->NewString(env, filename, (jsize) wcslen(filename));
+ } else {
+ return (*env)->NewStringUTF(env, "Unavailable");
+ }
+#endif
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h
new file mode 100644
index 0000000..0afab02
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/org_apache_hadoop_crypto.h
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ORG_APACHE_HADOOP_CRYPTO_H
+#define ORG_APACHE_HADOOP_CRYPTO_H
+
+#include "org_apache_hadoop.h"
+
+#ifdef UNIX
+#include <dlfcn.h>
+#include "config.h"
+#endif
+
+#ifdef WINDOWS
+#include "winutils.h"
+#endif
+
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
+
+/**
+ * A helper macro to convert the java 'context-handle'
+ * to a EVP_CIPHER_CTX pointer.
+ */
+#define CONTEXT(context) ((EVP_CIPHER_CTX*)((ptrdiff_t)(context)))
+
+/**
+ * A helper macro to convert the EVP_CIPHER_CTX pointer to the
+ * java 'context-handle'.
+ */
+#define JLONG(context) ((jlong)((ptrdiff_t)(context)))
+
+#define KEY_LENGTH_128 16
+#define KEY_LENGTH_256 32
+#define IV_LENGTH 16
+
+#define ENCRYPT_MODE 1
+#define DECRYPT_MODE 0
+
+/** Currently only support AES/CTR/NoPadding. */
+#define AES_CTR 0
+#define NOPADDING 0
+#define PKCSPADDING 1
+
+#endif //ORG_APACHE_HADOOP_CRYPTO_H
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
new file mode 100644
index 0000000..6c31d10
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "org_apache_hadoop_crypto_random.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef UNIX
+#include <pthread.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#endif
+
+#ifdef WINDOWS
+#include <windows.h>
+#endif
+
+#include "org_apache_hadoop_crypto_random_OpensslSecureRandom.h"
+
+#ifdef UNIX
+static void * (*dlsym_CRYPTO_malloc) (int, const char *, int);
+static void (*dlsym_CRYPTO_free) (void *);
+static int (*dlsym_CRYPTO_num_locks) (void);
+static void (*dlsym_CRYPTO_set_locking_callback) (void (*)());
+static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)());
+static void (*dlsym_ENGINE_load_rdrand) (void);
+static ENGINE * (*dlsym_ENGINE_by_id) (const char *);
+static int (*dlsym_ENGINE_init) (ENGINE *);
+static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int);
+static int (*dlsym_ENGINE_finish) (ENGINE *);
+static int (*dlsym_ENGINE_free) (ENGINE *);
+static void (*dlsym_ENGINE_cleanup) (void);
+static int (*dlsym_RAND_bytes) (unsigned char *, int);
+static unsigned long (*dlsym_ERR_get_error) (void);
+#endif
+
+#ifdef WINDOWS
+typedef void * (__cdecl *__dlsym_CRYPTO_malloc) (int, const char *, int);
+typedef void (__cdecl *__dlsym_CRYPTO_free) (void *);
+typedef int (__cdecl *__dlsym_CRYPTO_num_locks) (void);
+typedef void (__cdecl *__dlsym_CRYPTO_set_locking_callback) \
+ (void (*)(int, int, char *, int);
+typedef void (__cdecl *__dlsym_ENGINE_load_rdrand) (void);
+typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id) (const char *);
+typedef int (__cdecl *__dlsym_ENGINE_init) (ENGINE *);
+typedef int (__cdecl *__dlsym_ENGINE_set_default) (ENGINE *, unsigned int);
+typedef int (__cdecl *__dlsym_ENGINE_finish) (ENGINE *);
+typedef int (__cdecl *__dlsym_ENGINE_free) (ENGINE *);
+typedef void (__cdecl *__dlsym_ENGINE_cleanup) (void);
+typedef int (__cdecl *__dlsym_RAND_bytes) (unsigned char *, int);
+typedef unsigned long (__cdecl *__dlsym_ERR_get_error) (void);
+static __dlsym_CRYPTO_malloc dlsym_CRYPTO_malloc;
+static __dlsym_CRYPTO_free dlsym_CRYPTO_free;
+static __dlsym_CRYPTO_num_locks dlsym_CRYPTO_num_locks;
+static __dlsym_CRYPTO_set_locking_callback dlsym_CRYPTO_set_locking_callback;
+static __dlsym_ENGINE_load_rdrand dlsym_ENGINE_load_rdrand;
+static __dlsym_ENGINE_by_id dlsym_ENGINE_by_id;
+static __dlsym_ENGINE_init dlsym_ENGINE_init;
+static __dlsym_ENGINE_set_default dlsym_ENGINE_set_default;
+static __dlsym_ENGINE_finish dlsym_ENGINE_finish;
+static __dlsym_ENGINE_free dlsym_ENGINE_free;
+static __dlsym_ENGINE_cleanup dlsym_ENGINE_cleanup;
+static __dlsym_RAND_bytes dlsym_RAND_bytes;
+static __dlsym_ERR_get_error dlsym_ERR_get_error;
+#endif
+
+static ENGINE * openssl_rand_init(void);
+static void openssl_rand_clean(ENGINE *eng, int clean_locks);
+static int openssl_rand_bytes(unsigned char *buf, int num);
+
+JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR
+ (JNIEnv *env, jclass clazz)
+{
+ char msg[1000];
+#ifdef UNIX
+ void *openssl = dlopen(HADOOP_OPENSSL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
+#endif
+
+#ifdef WINDOWS
+ HMODULE openssl = LoadLibrary(HADOOP_OPENSSL_LIBRARY);
+#endif
+
+ if (!openssl) {
+ snprintf(msg, sizeof(msg), "Cannot load %s (%s)!", HADOOP_OPENSSL_LIBRARY, \
+ dlerror());
+ THROW(env, "java/lang/UnsatisfiedLinkError", msg);
+ return;
+ }
+
+#ifdef UNIX
+ dlerror(); // Clear any existing error
+ LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_malloc, env, openssl, "CRYPTO_malloc");
+ LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_free, env, openssl, "CRYPTO_free");
+ LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_num_locks, env, openssl, "CRYPTO_num_locks");
+ LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_set_locking_callback, \
+ env, openssl, "CRYPTO_set_locking_callback");
+ LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_set_id_callback, env, \
+ openssl, "CRYPTO_set_id_callback");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_load_rdrand, env, \
+ openssl, "ENGINE_load_rdrand");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \
+ openssl, "ENGINE_set_default");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup");
+ LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_malloc, dlsym_CRYPTO_malloc, \
+ env, openssl, "CRYPTO_malloc");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_free, dlsym_CRYPTO_free, \
+ env, openssl, "CRYPTO_free");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_num_locks, dlsym_CRYPTO_num_locks, \
+ env, openssl, "CRYPTO_num_locks");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_CRYPTO_set_locking_callback, \
+ dlsym_CRYPTO_set_locking_callback, \
+ env, openssl, "CRYPTO_set_locking_callback");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_load_rdrand, dlsym_ENGINE_load_rdrand, \
+ env, openssl, "ENGINE_load_rdrand");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_by_id, dlsym_ENGINE_by_id, \
+ env, openssl, "ENGINE_by_id");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_init, dlsym_ENGINE_init, \
+ env, openssl, "ENGINE_init");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_set_default, dlsym_ENGINE_set_default, \
+ env, openssl, "ENGINE_set_default");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_finish, dlsym_ENGINE_finish, \
+ env, openssl, "ENGINE_finish");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_free, dlsym_ENGINE_free, \
+ env, openssl, "ENGINE_free");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_cleanup, dlsym_ENGINE_cleanup, \
+ env, openssl, "ENGINE_cleanup");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_RAND_bytes, dlsym_RAND_bytes, \
+ env, openssl, "RAND_bytes");
+ LOAD_DYNAMIC_SYMBOL(__dlsym_ERR_get_error, dlsym_ERR_get_error, \
+ env, openssl, "ERR_get_error");
+#endif
+
+ openssl_rand_init();
+}
+
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B
+ (JNIEnv *env, jobject object, jbyteArray bytes)
+{
+ if (NULL == bytes) {
+ THROW(env, "java/lang/NullPointerException", "Buffer cannot be null.");
+ return JNI_FALSE;
+ }
+ jbyte *b = (*env)->GetByteArrayElements(env, bytes, NULL);
+ if (NULL == b) {
+ THROW(env, "java/lang/InternalError", "Cannot get bytes array.");
+ return JNI_FALSE;
+ }
+ int b_len = (*env)->GetArrayLength(env, bytes);
+ int ret = openssl_rand_bytes((unsigned char *)b, b_len);
+ (*env)->ReleaseByteArrayElements(env, bytes, b, 0);
+
+ if (1 != ret) {
+ return JNI_FALSE;
+ }
+ return JNI_TRUE;
+}
+
+/**
+ * To ensure thread safety for random number generators, we need to call
+ * CRYPTO_set_locking_callback.
+ * http://wiki.openssl.org/index.php/Random_Numbers
+ * Example: crypto/threads/mttest.c
+ */
+
+#ifdef WINDOWS
+static void windows_locking_callback(int mode, int type, char *file, int line);
+static HANDLE *lock_cs;
+
+static void locks_setup(void)
+{
+ int i;
+ lock_cs = dlsym_CRYPTO_malloc(dlsym_CRYPTO_num_locks() * sizeof(HANDLE), \
+ __FILE__, __LINE__);
+
+ for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
+ lock_cs[i] = CreateMutex(NULL, FALSE, NULL);
+ }
+ dlsym_CRYPTO_set_locking_callback((void (*)(int, int, char *, int)) \
+ windows_locking_callback);
+ /* id callback defined */
+}
+
+static void locks_cleanup(void)
+{
+ int i;
+ dlsym_CRYPTO_set_locking_callback(NULL);
+
+ for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
+ CloseHandle(lock_cs[i]);
+ }
+ dlsym_CRYPTO_free(lock_cs);
+}
+
+static void windows_locking_callback(int mode, int type, char *file, int line)
+{
+ UNUSED(file), UNUSED(line);
+
+ if (mode & CRYPTO_LOCK) {
+ WaitForSingleObject(lock_cs[type], INFINITE);
+ } else {
+ ReleaseMutex(lock_cs[type]);
+ }
+}
+#endif /* WINDOWS */
+
+#ifdef UNIX
+static void pthreads_locking_callback(int mode, int type, char *file, int line);
+static unsigned long pthreads_thread_id(void);
+static pthread_mutex_t *lock_cs;
+
+static void locks_setup(void)
+{
+ int i;
+ lock_cs = dlsym_CRYPTO_malloc(dlsym_CRYPTO_num_locks() * \
+ sizeof(pthread_mutex_t), __FILE__, __LINE__);
+
+ for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
+ pthread_mutex_init(&(lock_cs[i]), NULL);
+ }
+
+ dlsym_CRYPTO_set_id_callback((unsigned long (*)())pthreads_thread_id);
+ dlsym_CRYPTO_set_locking_callback((void (*)())pthreads_locking_callback);
+}
+
+static void locks_cleanup(void)
+{
+ int i;
+ dlsym_CRYPTO_set_locking_callback(NULL);
+
+ for (i = 0; i < dlsym_CRYPTO_num_locks(); i++) {
+ pthread_mutex_destroy(&(lock_cs[i]));
+ }
+
+ dlsym_CRYPTO_free(lock_cs);
+}
+
+static void pthreads_locking_callback(int mode, int type, char *file, int line)
+{
+ UNUSED(file), UNUSED(line);
+
+ if (mode & CRYPTO_LOCK) {
+ pthread_mutex_lock(&(lock_cs[type]));
+ } else {
+ pthread_mutex_unlock(&(lock_cs[type]));
+ }
+}
+
+static unsigned long pthreads_thread_id(void)
+{
+ return (unsigned long)syscall(SYS_gettid);
+}
+
+#endif /* UNIX */
+
+/**
+ * If using an Intel chipset with RDRAND, the high-performance hardware
+ * random number generator will be used.
+ */
+static ENGINE * openssl_rand_init(void)
+{
+ locks_setup();
+
+ dlsym_ENGINE_load_rdrand();
+ ENGINE *eng = dlsym_ENGINE_by_id("rdrand");
+
+ int ret = -1;
+ do {
+ if (NULL == eng) {
+ break;
+ }
+
+ int rc = dlsym_ENGINE_init(eng);
+ if (0 == rc) {
+ break;
+ }
+
+ rc = dlsym_ENGINE_set_default(eng, ENGINE_METHOD_RAND);
+ if (0 == rc) {
+ break;
+ }
+
+ ret = 0;
+ } while(0);
+
+ if (ret == -1) {
+ openssl_rand_clean(eng, 0);
+ }
+
+ return eng;
+}
+
+static void openssl_rand_clean(ENGINE *eng, int clean_locks)
+{
+ if (NULL != eng) {
+ dlsym_ENGINE_finish(eng);
+ dlsym_ENGINE_free(eng);
+ }
+
+ dlsym_ENGINE_cleanup();
+ if (clean_locks) {
+ locks_cleanup();
+ }
+}
+
+static int openssl_rand_bytes(unsigned char *buf, int num)
+{
+ return dlsym_RAND_bytes(buf, num);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h
new file mode 100644
index 0000000..1200718
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ORG_APACHE_HADOOP_CRYPTO_RANDOM_H
+#define ORG_APACHE_HADOOP_CRYPTO_RANDOM_H
+
+#include "org_apache_hadoop.h"
+
+#ifdef UNIX
+#include <dlfcn.h>
+#include "config.h"
+#endif
+
+#ifdef WINDOWS
+#include "winutils.h"
+#endif
+
+#define UNUSED(x) ((void)(x))
+
+#include <openssl/crypto.h>
+#include <openssl/engine.h>
+#include <openssl/rand.h>
+#include <openssl/err.h>
+
+#endif //ORG_APACHE_HADOOP_CRYPTO_RANDOM_H
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
index d03050c..3625112 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
@@ -39,6 +39,16 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
#endif
}
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl
+ (JNIEnv *env, jclass clazz)
+{
+#ifdef HADOOP_OPENSSL_LIBRARY
+ return JNI_TRUE;
+#else
+ return JNI_FALSE;
+#endif
+}
+
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
(JNIEnv *env, jclass clazz)
{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 50873f0..ee3cbf0 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1438,6 +1438,74 @@ for ldap providers in the same way as above does.
true.
</description>
</property>
+
+<property>
+ <name>hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE</name>
+ <value></value>
+ <description>
+ The prefix for a given crypto codec, contains a comma-separated
+ list of implementation classes for a given crypto codec (eg EXAMPLECIPHERSUITE).
+ The first implementation will be used if available, others are fallbacks.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.crypto.codec.classes.aes.ctr.nopadding</name>
+ <value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
+ <description>
+ Comma-separated list of crypto codec implementations for AES/CTR/NoPadding.
+ The first implementation will be used if available, others are fallbacks.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.crypto.cipher.suite</name>
+ <value>AES/CTR/NoPadding</value>
+ <description>
+ Cipher suite for crypto codec.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.crypto.jce.provider</name>
+ <value></value>
+ <description>
+ The JCE provider name used in CryptoCodec.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.crypto.buffer.size</name>
+ <value>8192</value>
+ <description>
+ The buffer size used by CryptoInputStream and CryptoOutputStream.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.java.secure.random.algorithm</name>
+ <value>SHA1PRNG</value>
+ <description>
+ The java secure random algorithm.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.secure.random.impl</name>
+ <value></value>
+ <description>
+ Implementation of secure random.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.random.device.file.path</name>
+ <value>/dev/urandom</value>
+ <description>
+ OS security random device file path.
+ </description>
+</property>
+
<property>
<name>fs.har.impl.disable.cache</name>
<value>true</value>
@@ -1476,4 +1544,5 @@ for ldap providers in the same way as above does.
key will be dropped. Default = 12hrs
</description>
</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c77bd85b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
index 48e0b21..53e42cb 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
@@ -168,15 +168,22 @@ cp
Copy files from source to destination. This command allows multiple sources
as well in which case the destination must be a directory.
+ 'raw.*' namespace extended attributes are preserved if (1) the source and
+ destination filesystems support them (HDFS only), and (2) all source and
+ destination pathnames are in the /.reserved/raw hierarchy. Determination of
+ whether raw.* namespace xattrs are preserved is independent of the
+ -p (preserve) flag.
+
Options:
* The -f option will overwrite the destination if it already exists.
- * The -p option will preserve file attributes [topx] (timestamps,
+ * The -p option will preserve file attributes [topx] (timestamps,
ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
then preserves timestamps, ownership, permission. If -pa is specified,
then preserves permission also because ACL is a super-set of
- permission.
+ permission. Determination of whether raw namespace extended attributes
+ are preserved is independent of the -p flag.
Example: