You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2019/05/10 16:32:27 UTC
[hadoop] branch branch-2.8 updated: HADOOP-14524. Make CryptoCodec
Closeable so it can be cleaned up proactively.
This is an automated email from the ASF dual-hosted git repository.
kihwal pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.8 by this push:
new 934d0b3 HADOOP-14524. Make CryptoCodec Closeable so it can be cleaned up proactively.
934d0b3 is described below
commit 934d0b3dd677b873f3dc5ca3bfce280df2905d77
Author: Kihwal Lee <ki...@apache.org>
AuthorDate: Fri May 10 11:32:00 2019 -0500
HADOOP-14524. Make CryptoCodec Closeable so it can be cleaned up proactively.
(cherry picked from commit 2afe9722af2fa4c8e870b0dbf1328e31d1f0fec7)
Conflicts:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
---
.../apache/hadoop/crypto/AesCtrCryptoCodec.java | 6 ++
.../java/org/apache/hadoop/crypto/CryptoCodec.java | 3 +-
.../apache/hadoop/crypto/CryptoInputStream.java | 1 +
.../apache/hadoop/crypto/CryptoOutputStream.java | 1 +
.../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 12 +++-
.../crypto/key/KeyProviderCryptoExtension.java | 76 ++++++++++++----------
...tCryptoStreamsWithOpensslAesCtrCryptoCodec.java | 32 ++++++++-
.../datatransfer/sasl/DataTransferSaslUtil.java | 1 +
.../org/apache/hadoop/mapreduce/CryptoUtils.java | 14 +++-
9 files changed, 105 insertions(+), 41 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java
index 5e286b9..3e52560 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AesCtrCryptoCodec.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import com.google.common.base.Preconditions;
+import java.io.IOException;
+
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class AesCtrCryptoCodec extends CryptoCodec {
@@ -61,4 +63,8 @@ public abstract class AesCtrCryptoCodec extends CryptoCodec {
IV[i] = (byte) sum;
}
}
+
+ @Override
+ public void close() throws IOException {
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
index 493e23d..d9c16bb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.crypto;
+import java.io.Closeable;
import java.security.GeneralSecurityException;
import java.util.List;
@@ -42,7 +43,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public abstract class CryptoCodec implements Configurable {
+public abstract class CryptoCodec implements Configurable, Closeable {
public static Logger LOG = LoggerFactory.getLogger(CryptoCodec.class);
/**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index b7ded92..0be6e34 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -315,6 +315,7 @@ public class CryptoInputStream extends FilterInputStream implements
super.close();
freeBuffers();
+ codec.close();
closed = true;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
index fff0945..f8b05075 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
@@ -222,6 +222,7 @@ public class CryptoOutputStream extends FilterOutputStream implements
}
try {
super.close();
+ codec.close();
freeBuffers();
} finally {
closed = true;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
index 63c382c..0ebbcec 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
@@ -100,7 +100,17 @@ public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec {
public void generateSecureRandom(byte[] bytes) {
random.nextBytes(bytes);
}
-
+
+ @Override
+ public void close() throws IOException {
+ try {
+ Closeable r = (Closeable) this.random;
+ r.close();
+ } catch (ClassCastException e) {
+ }
+ super.close();
+ }
+
private static class OpensslAesCtrCipher implements Encryptor, Decryptor {
private final OpensslCipher cipher;
private final int mode;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index b32366b..d9a064f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -254,26 +254,30 @@ public class KeyProviderCryptoExtension extends
// Generate random bytes for new key and IV
CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
- final byte[] newKey = new byte[encryptionKey.getMaterial().length];
- cc.generateSecureRandom(newKey);
- final byte[] iv = new byte[cc.getCipherSuite().getAlgorithmBlockSize()];
- cc.generateSecureRandom(iv);
- // Encryption key IV is derived from new key's IV
- final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv);
- Encryptor encryptor = cc.createEncryptor();
- encryptor.init(encryptionKey.getMaterial(), encryptionIV);
- int keyLen = newKey.length;
- ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
- ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
- bbIn.put(newKey);
- bbIn.flip();
- encryptor.encrypt(bbIn, bbOut);
- bbOut.flip();
- byte[] encryptedKey = new byte[keyLen];
- bbOut.get(encryptedKey);
- return new EncryptedKeyVersion(encryptionKeyName,
- encryptionKey.getVersionName(), iv,
- new KeyVersion(encryptionKey.getName(), EEK, encryptedKey));
+ try {
+ final byte[] newKey = new byte[encryptionKey.getMaterial().length];
+ cc.generateSecureRandom(newKey);
+ final byte[] iv = new byte[cc.getCipherSuite().getAlgorithmBlockSize()];
+ cc.generateSecureRandom(iv);
+ // Encryption key IV is derived from new key's IV
+ final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv);
+ Encryptor encryptor = cc.createEncryptor();
+ encryptor.init(encryptionKey.getMaterial(), encryptionIV);
+ int keyLen = newKey.length;
+ ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
+ ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
+ bbIn.put(newKey);
+ bbIn.flip();
+ encryptor.encrypt(bbIn, bbOut);
+ bbOut.flip();
+ byte[] encryptedKey = new byte[keyLen];
+ bbOut.get(encryptedKey);
+ return new EncryptedKeyVersion(encryptionKeyName,
+ encryptionKey.getVersionName(), iv,
+ new KeyVersion(encryptionKey.getName(), EEK, encryptedKey));
+ } finally {
+ cc.close();
+ }
}
@Override
@@ -300,20 +304,24 @@ public class KeyProviderCryptoExtension extends
EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv());
CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
- Decryptor decryptor = cc.createDecryptor();
- decryptor.init(encryptionKey.getMaterial(), encryptionIV);
- final KeyVersion encryptedKV =
- encryptedKeyVersion.getEncryptedKeyVersion();
- int keyLen = encryptedKV.getMaterial().length;
- ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
- ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
- bbIn.put(encryptedKV.getMaterial());
- bbIn.flip();
- decryptor.decrypt(bbIn, bbOut);
- bbOut.flip();
- byte[] decryptedKey = new byte[keyLen];
- bbOut.get(decryptedKey);
- return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
+ try {
+ Decryptor decryptor = cc.createDecryptor();
+ decryptor.init(encryptionKey.getMaterial(), encryptionIV);
+ final KeyVersion encryptedKV =
+ encryptedKeyVersion.getEncryptedKeyVersion();
+ int keyLen = encryptedKV.getMaterial().length;
+ ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
+ ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
+ bbIn.put(encryptedKV.getMaterial());
+ bbIn.flip();
+ decryptor.decrypt(bbIn, bbOut);
+ bbOut.flip();
+ byte[] decryptedKey = new byte[keyLen];
+ bbOut.get(decryptedKey);
+ return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
+ } finally {
+ cc.close();
+ }
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index cc02f48..241e876 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -18,12 +18,17 @@
package org.apache.hadoop.crypto;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.random.OsSecureRandom;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
extends TestCryptoStreams {
@@ -32,8 +37,7 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
public static void init() throws Exception {
GenericTestUtils.assumeInNativeProfile();
Configuration conf = new Configuration();
- conf.set(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
+ conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
OpensslAesCtrCryptoCodec.class.getName());
codec = CryptoCodec.getInstance(conf);
assertNotNull("Unable to instantiate codec " +
@@ -42,4 +46,28 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(),
codec.getClass().getCanonicalName());
}
+
+ @Test
+ public void testCodecClosesRandom() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
+ Configuration conf = new Configuration();
+ conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
+ OpensslAesCtrCryptoCodec.class.getName());
+ conf.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY,
+ OsSecureRandom.class.getName());
+ CryptoCodec codecWithRandom = CryptoCodec.getInstance(conf);
+ assertNotNull(
+ "Unable to instantiate codec " + OpensslAesCtrCryptoCodec.class
+ .getName() + ", is the required " + "version of OpenSSL installed?",
+ codecWithRandom);
+ OsSecureRandom random =
+ (OsSecureRandom) Whitebox.getInternalState(codecWithRandom, "random");
+ // trigger the OsSecureRandom to create an internal FileInputStream
+ random.nextBytes(new byte[10]);
+ assertNotNull(Whitebox.getInternalState(random, "stream"));
+ // verify closing the codec closes the codec's random's stream.
+ codecWithRandom.close();
+ assertNull(Whitebox.getInternalState(random, "stream"));
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java
index 5e07550..f4651eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java
@@ -286,6 +286,7 @@ public final class DataTransferSaslUtil {
codec.generateSecureRandom(inIv);
codec.generateSecureRandom(outKey);
codec.generateSecureRandom(outIv);
+ codec.close();
return new CipherOption(suite, inKey, inIv, outKey, outIv);
}
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/CryptoUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/CryptoUtils.java
index c4130b1..ff2badc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/CryptoUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/CryptoUtils.java
@@ -66,16 +66,24 @@ public class CryptoUtils {
if (isEncryptedSpillEnabled(conf)) {
byte[] iv = new byte[cryptoCodec.getCipherSuite().getAlgorithmBlockSize()];
cryptoCodec.generateSecureRandom(iv);
+ cryptoCodec.close();
return iv;
} else {
return null;
}
}
- public static int cryptoPadding(Configuration conf) {
+ public static int cryptoPadding(Configuration conf) throws IOException {
// Sizeof(IV) + long(start-offset)
- return isEncryptedSpillEnabled(conf) ? CryptoCodec.getInstance(conf)
- .getCipherSuite().getAlgorithmBlockSize() + 8 : 0;
+ if (!isEncryptedSpillEnabled(conf)) {
+ return 0;
+ }
+ final CryptoCodec cryptoCodec = CryptoCodec.getInstance(conf);
+ try {
+ return cryptoCodec.getCipherSuite().getAlgorithmBlockSize() + 8;
+ } finally {
+ cryptoCodec.close();
+ }
}
private static byte[] getEncryptionKey() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org