You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tu...@apache.org on 2014/08/29 23:22:35 UTC
git commit: HADOOP-10994. KeyProviderCryptoExtension should use
CryptoCodec for generation/decryption of keys. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk b03653f9a -> c60da4d3b
HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for generation/decryption of keys. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c60da4d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c60da4d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c60da4d3
Branch: refs/heads/trunk
Commit: c60da4d3b31e5fa0c4b27cf75ab7ed4add56396a
Parents: b03653f
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Fri Aug 29 14:21:58 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Fri Aug 29 14:21:58 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 1 +
.../apache/hadoop/crypto/key/KeyProvider.java | 20 ++++++++
.../crypto/key/KeyProviderCryptoExtension.java | 51 +++++++++++++-------
.../hadoop/crypto/key/KeyProviderExtension.java | 1 +
.../apache/hadoop/crypto/key/UserProvider.java | 5 +-
.../crypto/key/kms/KMSClientProvider.java | 1 +
.../crypto/key/TestCachingKeyProvider.java | 6 +++
.../hadoop/crypto/key/TestKeyProvider.java | 17 ++++++-
...TestKeyProviderDelegationTokenExtension.java | 13 +++--
10 files changed, 94 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1930e5d..2bc3e4b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -476,6 +476,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10814. Update Tomcat version used by HttpFS and KMS to latest
6.x version. (rkanter via tucu)
+ HADOOP-10994. KeyProviderCryptoExtension should use CryptoCodec for
+ generation/decryption of keys. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index 2503151..30583eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -108,6 +108,7 @@ public class JavaKeyStoreProvider extends KeyProvider {
private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
+ super(conf);
this.uri = uri;
path = ProviderUtils.unnestUri(uri);
fs = path.getFileSystem(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index a34ae10..36ccbad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -56,6 +56,8 @@ public abstract class KeyProvider {
"hadoop.security.key.default.bitlength";
public static final int DEFAULT_BITLENGTH = 128;
+ private final Configuration conf;
+
/**
* The combination of both the key version name and the key material.
*/
@@ -354,6 +356,24 @@ public abstract class KeyProvider {
}
/**
+ * Constructor.
+ *
+ * @param conf configuration for the provider
+ */
+ public KeyProvider(Configuration conf) {
+ this.conf = new Configuration(conf);
+ }
+
+ /**
+ * Return the provider configuration.
+ *
+ * @return the provider configuration
+ */
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
* A helper function to create an options object.
* @param conf the configuration to use
* @return a new options object
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 026f285..e2fb5cb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.crypto.key;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
@@ -29,6 +30,9 @@ import javax.crypto.spec.SecretKeySpec;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.crypto.CryptoCodec;
+import org.apache.hadoop.crypto.Decryptor;
+import org.apache.hadoop.crypto.Encryptor;
/**
* A KeyProvider with Cryptographic Extensions specifically for generating
@@ -239,18 +243,25 @@ public class KeyProviderCryptoExtension extends
Preconditions.checkNotNull(encryptionKey,
"No KeyVersion exists for key '%s' ", encryptionKeyName);
// Generate random bytes for new key and IV
- Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
+
+ CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
final byte[] newKey = new byte[encryptionKey.getMaterial().length];
- RANDOM.get().nextBytes(newKey);
- final byte[] iv = new byte[cipher.getBlockSize()];
- RANDOM.get().nextBytes(iv);
+ cc.generateSecureRandom(newKey);
+ final byte[] iv = new byte[cc.getCipherSuite().getAlgorithmBlockSize()];
+ cc.generateSecureRandom(iv);
// Encryption key IV is derived from new key's IV
final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv);
- // Encrypt the new key
- cipher.init(Cipher.ENCRYPT_MODE,
- new SecretKeySpec(encryptionKey.getMaterial(), "AES"),
- new IvParameterSpec(encryptionIV));
- final byte[] encryptedKey = cipher.doFinal(newKey);
+ Encryptor encryptor = cc.createEncryptor();
+ encryptor.init(encryptionKey.getMaterial(), encryptionIV);
+ int keyLen = newKey.length;
+ ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
+ ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
+ bbIn.put(newKey);
+ bbIn.flip();
+ encryptor.encrypt(bbIn, bbOut);
+ bbOut.flip();
+ byte[] encryptedKey = new byte[keyLen];
+ bbOut.get(encryptedKey);
return new EncryptedKeyVersion(encryptionKeyName,
encryptionKey.getVersionName(), iv,
new KeyVersion(encryptionKey.getName(), EEK, encryptedKey));
@@ -274,19 +285,25 @@ public class KeyProviderCryptoExtension extends
KeyProviderCryptoExtension.EEK,
encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
);
- final byte[] encryptionKeyMaterial = encryptionKey.getMaterial();
+
// Encryption key IV is determined from encrypted key's IV
final byte[] encryptionIV =
EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv());
- // Init the cipher with encryption key parameters
- Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
- cipher.init(Cipher.DECRYPT_MODE,
- new SecretKeySpec(encryptionKeyMaterial, "AES"),
- new IvParameterSpec(encryptionIV));
- // Decrypt the encrypted key
+
+ CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
+ Decryptor decryptor = cc.createDecryptor();
+ decryptor.init(encryptionKey.getMaterial(), encryptionIV);
final KeyVersion encryptedKV =
encryptedKeyVersion.getEncryptedKeyVersion();
- final byte[] decryptedKey = cipher.doFinal(encryptedKV.getMaterial());
+ int keyLen = encryptedKV.getMaterial().length;
+ ByteBuffer bbIn = ByteBuffer.allocateDirect(keyLen);
+ ByteBuffer bbOut = ByteBuffer.allocateDirect(keyLen);
+ bbIn.put(encryptedKV.getMaterial());
+ bbIn.flip();
+ decryptor.decrypt(bbIn, bbOut);
+ bbOut.flip();
+ byte[] decryptedKey = new byte[keyLen];
+ bbOut.get(decryptedKey);
return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
index ba048b5..ec4c3b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java
@@ -40,6 +40,7 @@ public abstract class KeyProviderExtension
private E extension;
public KeyProviderExtension(KeyProvider keyProvider, E extensions) {
+ super(keyProvider.getConf());
this.keyProvider = keyProvider;
this.extension = extensions;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
index e09b3f8..bf8f2fed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
@@ -44,7 +44,8 @@ public class UserProvider extends KeyProvider {
private final Credentials credentials;
private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
- private UserProvider() throws IOException {
+ private UserProvider(Configuration conf) throws IOException {
+ super(conf);
user = UserGroupInformation.getCurrentUser();
credentials = user.getCredentials();
}
@@ -145,7 +146,7 @@ public class UserProvider extends KeyProvider {
public KeyProvider createProvider(URI providerName,
Configuration conf) throws IOException {
if (SCHEME_NAME.equals(providerName.getScheme())) {
- return new UserProvider();
+ return new UserProvider(conf);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index bce1eb5..dc9e6cb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -283,6 +283,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
public KMSClientProvider(URI uri, Configuration conf) throws IOException {
+ super(conf);
Path path = ProviderUtils.unnestUri(uri);
URL url = path.toUri().toURL();
kmsUrl = createServiceURL(url);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestCachingKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestCachingKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestCachingKeyProvider.java
index 2eff699..b8d29a6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestCachingKeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestCachingKeyProvider.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.crypto.key;
import java.util.Date;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.junit.Assert;
import org.junit.Test;
@@ -32,6 +33,7 @@ public class TestCachingKeyProvider {
KeyProvider mockProv = Mockito.mock(KeyProvider.class);
Mockito.when(mockProv.getCurrentKey(Mockito.eq("k1"))).thenReturn(mockKey);
Mockito.when(mockProv.getCurrentKey(Mockito.eq("k2"))).thenReturn(null);
+ Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
// asserting caching
@@ -58,6 +60,7 @@ public class TestCachingKeyProvider {
Mockito.when(mockProv.getKeyVersion(Mockito.eq("k1@0")))
.thenReturn(mockKey);
Mockito.when(mockProv.getKeyVersion(Mockito.eq("k2@0"))).thenReturn(null);
+ Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
// asserting caching
@@ -88,6 +91,7 @@ public class TestCachingKeyProvider {
KeyProvider mockProv = Mockito.mock(KeyProvider.class);
Mockito.when(mockProv.getMetadata(Mockito.eq("k1"))).thenReturn(mockMeta);
Mockito.when(mockProv.getMetadata(Mockito.eq("k2"))).thenReturn(null);
+ Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
// asserting caching
@@ -112,6 +116,7 @@ public class TestCachingKeyProvider {
KeyProvider.KeyVersion mockKey = Mockito.mock(KeyProvider.KeyVersion.class);
KeyProvider mockProv = Mockito.mock(KeyProvider.class);
Mockito.when(mockProv.getCurrentKey(Mockito.eq("k1"))).thenReturn(mockKey);
+ Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
Assert.assertEquals(mockKey, cache.getCurrentKey("k1"));
Mockito.verify(mockProv, Mockito.times(1)).getCurrentKey(Mockito.eq("k1"));
@@ -134,6 +139,7 @@ public class TestCachingKeyProvider {
.thenReturn(mockKey);
Mockito.when(mockProv.getMetadata(Mockito.eq("k1"))).thenReturn(
new KMSClientProvider.KMSMetadata("c", 0, "l", null, new Date(), 1));
+ Mockito.when(mockProv.getConf()).thenReturn(new Configuration());
KeyProvider cache = new CachingKeyProvider(mockProv, 100, 100);
Assert.assertEquals(mockKey, cache.getCurrentKey("k1"));
Mockito.verify(mockProv, Mockito.times(1)).getCurrentKey(Mockito.eq("k1"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
index 892cec8..c3335a3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
@@ -159,6 +159,10 @@ public class TestKeyProvider {
private int size;
private byte[] material;
+ public MyKeyProvider(Configuration conf) {
+ super(conf);
+ }
+
@Override
public KeyVersion getKeyVersion(String versionName)
throws IOException {
@@ -216,7 +220,7 @@ public class TestKeyProvider {
@Test
public void testMaterialGeneration() throws Exception {
- MyKeyProvider kp = new MyKeyProvider();
+ MyKeyProvider kp = new MyKeyProvider(new Configuration());
KeyProvider.Options options = new KeyProvider.Options(new Configuration());
options.setCipher(CIPHER);
options.setBitLength(128);
@@ -225,10 +229,19 @@ public class TestKeyProvider {
Assert.assertEquals(CIPHER, kp.algorithm);
Assert.assertNotNull(kp.material);
- kp = new MyKeyProvider();
+ kp = new MyKeyProvider(new Configuration());
kp.rollNewVersion("hello");
Assert.assertEquals(128, kp.size);
Assert.assertEquals(CIPHER, kp.algorithm);
Assert.assertNotNull(kp.material);
}
+
+ @Test
+ public void testConfiguration() throws Exception {
+ Configuration conf = new Configuration(false);
+ conf.set("a", "A");
+ MyKeyProvider kp = new MyKeyProvider(conf);
+ Assert.assertEquals("A", kp.getConf().get("a"));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c60da4d3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java
index 52dedf0..df5d3e8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java
@@ -29,13 +29,18 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestKeyProviderDelegationTokenExtension {
public static abstract class MockKeyProvider extends
KeyProvider implements DelegationTokenExtension {
+
+ public MockKeyProvider() {
+ super(new Configuration(false));
+ }
}
-
+
@Test
public void testCreateExtension() throws Exception {
Configuration conf = new Configuration();
@@ -50,9 +55,11 @@ public class TestKeyProviderDelegationTokenExtension {
Assert.assertNull(kpDTE1.addDelegationTokens("user", credentials));
MockKeyProvider mock = mock(MockKeyProvider.class);
+ Mockito.when(mock.getConf()).thenReturn(new Configuration());
when(mock.addDelegationTokens("renewer", credentials)).thenReturn(
- new Token<?>[] { new Token(null, null, new Text("kind"), new Text(
- "service")) });
+ new Token<?>[]{new Token(null, null, new Text("kind"), new Text(
+ "service"))}
+ );
KeyProviderDelegationTokenExtension kpDTE2 =
KeyProviderDelegationTokenExtension
.createKeyProviderDelegationTokenExtension(mock);