You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2014/07/23 19:30:08 UTC
svn commit: r1612883 - in
/hadoop/common/branches/HDFS-6584/hadoop-common-project: hadoop-common/
hadoop-common/src/main/java/
hadoop-common/src/main/java/org/apache/hadoop/crypto/key/
hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ hadoo...
Author: szetszwo
Date: Wed Jul 23 17:30:06 2014
New Revision: 1612883
URL: http://svn.apache.org/r1612883
Log:
Merge r1609845 through r1612880 from trunk.
Modified:
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt Wed Jul 23 17:30:06 2014
@@ -396,6 +396,12 @@ Trunk (Unreleased)
HADOOP-10840. Fix OutOfMemoryError caused by metrics system in Azure File
System. (Shanyu Zhao via cnauroth)
+ HADOOP-10826. Iteration on KeyProviderFactory.serviceLoader is
+ thread-unsafe. (benoyantony viat tucu)
+
+ HADOOP-10881. Clarify usage of encryption and encrypted encryption
+ key in KeyProviderCryptoExtension. (wang)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -444,6 +450,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-10755. Support negative caching of user-group mapping.
(Lei Xu via wang)
+ HADOOP-10855. Allow Text to be read with a known Length. (todd)
+
OPTIMIZATIONS
BUG FIXES
@@ -794,6 +802,9 @@ Release 2.5.0 - UNRELEASED
HADOOP-10710. hadoop.auth cookie is not properly constructed according to
RFC2109. (Juan Yu via tucu)
+ HADOOP-10864. Tool documentenation is broken. (Akira Ajisaka
+ via Arpit Agarwal)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1612432-1612880
Propchange: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1612432-1612880
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java Wed Jul 23 17:30:06 2014
@@ -21,7 +21,6 @@ package org.apache.hadoop.crypto.key;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
-
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
@@ -30,51 +29,109 @@ import com.google.common.base.Preconditi
import org.apache.hadoop.classification.InterfaceAudience;
/**
- * A KeyProvider with Cytographic Extensions specifically for generating
- * Encrypted Keys as well as decrypting them
+ * A KeyProvider with Cryptographic Extensions specifically for generating
+ * and decrypting encrypted encryption keys.
*
*/
@InterfaceAudience.Private
public class KeyProviderCryptoExtension extends
KeyProviderExtension<KeyProviderCryptoExtension.CryptoExtension> {
+ /**
+ * Designates an encrypted encryption key, or EEK.
+ */
public static final String EEK = "EEK";
+ /**
+ * Designates a decrypted encrypted encryption key, that is, an encryption key
+ * (EK).
+ */
public static final String EK = "EK";
/**
- * This is a holder class whose instance contains the keyVersionName, iv
- * used to generate the encrypted Key and the encrypted KeyVersion
+ * An encrypted encryption key (EEK) and related information. An EEK must be
+ * decrypted using the key's encryption key before it can be used.
*/
public static class EncryptedKeyVersion {
- private String keyName;
- private String keyVersionName;
- private byte[] iv;
- private KeyVersion encryptedKey;
-
- protected EncryptedKeyVersion(String keyName, String keyVersionName,
- byte[] iv, KeyVersion encryptedKey) {
- this.keyName = keyName;
- this.keyVersionName = keyVersionName;
- this.iv = iv;
- this.encryptedKey = encryptedKey;
+ private String encryptionKeyName;
+ private String encryptionKeyVersionName;
+ private byte[] encryptedKeyIv;
+ private KeyVersion encryptedKeyVersion;
+
+ /**
+ * Create a new EncryptedKeyVersion.
+ *
+ * @param keyName Name of the encryption key used to
+ * encrypt the encrypted key.
+ * @param encryptionKeyVersionName Version name of the encryption key used
+ * to encrypt the encrypted key.
+ * @param encryptedKeyIv Initialization vector of the encrypted
+ * key. The IV of the encryption key used to
+ * encrypt the encrypted key is derived from
+ * this IV.
+ * @param encryptedKeyVersion The encrypted encryption key version.
+ */
+ protected EncryptedKeyVersion(String keyName,
+ String encryptionKeyVersionName, byte[] encryptedKeyIv,
+ KeyVersion encryptedKeyVersion) {
+ this.encryptionKeyName = keyName;
+ this.encryptionKeyVersionName = encryptionKeyVersionName;
+ this.encryptedKeyIv = encryptedKeyIv;
+ this.encryptedKeyVersion = encryptedKeyVersion;
}
- public String getKeyName() {
- return keyName;
+ /**
+ * @return Name of the encryption key used to encrypt the encrypted key.
+ */
+ public String getEncryptionKeyName() {
+ return encryptionKeyName;
}
- public String getKeyVersionName() {
- return keyVersionName;
+ /**
+ * @return Version name of the encryption key used to encrypt the encrypted
+ * key.
+ */
+ public String getEncryptionKeyVersionName() {
+ return encryptionKeyVersionName;
}
- public byte[] getIv() {
- return iv;
+ /**
+ * @return Initialization vector of the encrypted key. The IV of the
+ * encryption key used to encrypt the encrypted key is derived from this
+ * IV.
+ */
+ public byte[] getEncryptedKeyIv() {
+ return encryptedKeyIv;
}
- public KeyVersion getEncryptedKey() {
- return encryptedKey;
+ /**
+ * @return The encrypted encryption key version.
+ */
+ public KeyVersion getEncryptedKeyVersion() {
+ return encryptedKeyVersion;
}
+ /**
+ * Derive the initialization vector (IV) for the encryption key from the IV
+ * of the encrypted key. This derived IV is used with the encryption key to
+ * decrypt the encrypted key.
+ * <p/>
+ * The alternative to this is using the same IV for both the encryption key
+ * and the encrypted key. Even a simple symmetric transformation like this
+ * improves security by avoiding IV re-use. IVs will also be fairly unique
+ * among different EEKs.
+ *
+ * @param encryptedKeyIV of the encrypted key (i.e. {@link
+ * #getEncryptedKeyIv()})
+ * @return IV for the encryption key
+ */
+ protected static byte[] deriveIV(byte[] encryptedKeyIV) {
+ byte[] rIv = new byte[encryptedKeyIV.length];
+ // Do a simple XOR transformation to flip all the bits
+ for (int i = 0; i < encryptedKeyIV.length; i++) {
+ rIv[i] = (byte) (encryptedKeyIV[i] ^ 0xff);
+ }
+ return rIv;
+ }
}
/**
@@ -141,53 +198,56 @@ public class KeyProviderCryptoExtension
this.keyProvider = keyProvider;
}
- // the IV used to encrypt a EK typically will be the same IV used to
- // encrypt data with the EK. To avoid any chance of weakening the
- // encryption because the same IV is used, we simply XOR the IV thus we
- // are not using the same IV for 2 different encryptions (even if they
- // are done using different keys)
- private byte[] flipIV(byte[] iv) {
- byte[] rIv = new byte[iv.length];
- for (int i = 0; i < iv.length; i++) {
- rIv[i] = (byte) (iv[i] ^ 0xff);
- }
- return rIv;
- }
-
@Override
public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName)
throws IOException, GeneralSecurityException {
- KeyVersion keyVer = keyProvider.getCurrentKey(encryptionKeyName);
- Preconditions.checkNotNull(keyVer, "No KeyVersion exists for key '%s' ",
- encryptionKeyName);
- byte[] newKey = new byte[keyVer.getMaterial().length];
- SecureRandom.getInstance("SHA1PRNG").nextBytes(newKey);
+ // Fetch the encryption key
+ KeyVersion encryptionKey = keyProvider.getCurrentKey(encryptionKeyName);
+ Preconditions.checkNotNull(encryptionKey,
+ "No KeyVersion exists for key '%s' ", encryptionKeyName);
+ // Generate random bytes for new key and IV
Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
- byte[] iv = SecureRandom.getSeed(cipher.getBlockSize());
- cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(keyVer.getMaterial(),
- "AES"), new IvParameterSpec(flipIV(iv)));
- byte[] ek = cipher.doFinal(newKey);
+ SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
+ final byte[] newKey = new byte[encryptionKey.getMaterial().length];
+ random.nextBytes(newKey);
+ final byte[] iv = random.generateSeed(cipher.getBlockSize());
+ // Encryption key IV is derived from new key's IV
+ final byte[] encryptionIV = EncryptedKeyVersion.deriveIV(iv);
+ // Encrypt the new key
+ cipher.init(Cipher.ENCRYPT_MODE,
+ new SecretKeySpec(encryptionKey.getMaterial(), "AES"),
+ new IvParameterSpec(encryptionIV));
+ final byte[] encryptedKey = cipher.doFinal(newKey);
return new EncryptedKeyVersion(encryptionKeyName,
- keyVer.getVersionName(), iv,
- new KeyVersion(keyVer.getName(), EEK, ek));
+ encryptionKey.getVersionName(), iv,
+ new KeyVersion(encryptionKey.getName(), EEK, encryptedKey));
}
@Override
public KeyVersion decryptEncryptedKey(
EncryptedKeyVersion encryptedKeyVersion) throws IOException,
GeneralSecurityException {
- KeyVersion keyVer =
- keyProvider.getKeyVersion(encryptedKeyVersion.getKeyVersionName());
- Preconditions.checkNotNull(keyVer, "KeyVersion name '%s' does not exist",
- encryptedKeyVersion.getKeyVersionName());
- KeyVersion keyVersion = encryptedKeyVersion.getEncryptedKey();
+ // Fetch the encryption key material
+ final String encryptionKeyVersionName =
+ encryptedKeyVersion.getEncryptionKeyVersionName();
+ final KeyVersion encryptionKey =
+ keyProvider.getKeyVersion(encryptionKeyVersionName);
+ Preconditions.checkNotNull(encryptionKey,
+ "KeyVersion name '%s' does not exist", encryptionKeyVersionName);
+ final byte[] encryptionKeyMaterial = encryptionKey.getMaterial();
+ // Encryption key IV is determined from encrypted key's IV
+ final byte[] encryptionIV =
+ EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv());
+ // Init the cipher with encryption key parameters
Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
cipher.init(Cipher.DECRYPT_MODE,
- new SecretKeySpec(keyVersion.getMaterial(), "AES"),
- new IvParameterSpec(flipIV(encryptedKeyVersion.getIv())));
- byte[] ek =
- cipher.doFinal(encryptedKeyVersion.getEncryptedKey().getMaterial());
- return new KeyVersion(keyVer.getName(), EK, ek);
+ new SecretKeySpec(encryptionKeyMaterial, "AES"),
+ new IvParameterSpec(encryptionIV));
+ // Decrypt the encrypted key
+ final KeyVersion encryptedKV =
+ encryptedKeyVersion.getEncryptedKeyVersion();
+ final byte[] decryptedKey = cipher.doFinal(encryptedKV.getMaterial());
+ return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
}
@Override
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java Wed Jul 23 17:30:06 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.ServiceLoader;
@@ -47,6 +48,15 @@ public abstract class KeyProviderFactory
private static final ServiceLoader<KeyProviderFactory> serviceLoader =
ServiceLoader.load(KeyProviderFactory.class);
+ // Iterate through the serviceLoader to avoid lazy loading.
+ // Lazy loading would require synchronization in concurrent use cases.
+ static {
+ Iterator<KeyProviderFactory> iterServices = serviceLoader.iterator();
+ while (iterServices.hasNext()) {
+ iterServices.next();
+ }
+ }
+
public static List<KeyProvider> getProviders(Configuration conf
) throws IOException {
List<KeyProvider> result = new ArrayList<KeyProvider>();
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java Wed Jul 23 17:30:06 2014
@@ -646,25 +646,28 @@ public class KMSClientProvider extends K
public KeyVersion decryptEncryptedKey(
EncryptedKeyVersion encryptedKeyVersion) throws IOException,
GeneralSecurityException {
- checkNotNull(encryptedKeyVersion.getKeyVersionName(), "versionName");
- checkNotNull(encryptedKeyVersion.getIv(), "iv");
- Preconditions.checkArgument(encryptedKeyVersion.getEncryptedKey()
- .getVersionName().equals(KeyProviderCryptoExtension.EEK),
+ checkNotNull(encryptedKeyVersion.getEncryptionKeyVersionName(),
+ "versionName");
+ checkNotNull(encryptedKeyVersion.getEncryptedKeyIv(), "iv");
+ Preconditions.checkArgument(
+ encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
+ .equals(KeyProviderCryptoExtension.EEK),
"encryptedKey version name must be '%s', is '%s'",
- KeyProviderCryptoExtension.EK, encryptedKeyVersion.getEncryptedKey()
- .getVersionName());
- checkNotNull(encryptedKeyVersion.getEncryptedKey(), "encryptedKey");
+ KeyProviderCryptoExtension.EK,
+ encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
+ );
+ checkNotNull(encryptedKeyVersion.getEncryptedKeyVersion(), "encryptedKey");
Map<String, String> params = new HashMap<String, String>();
params.put(KMSRESTConstants.EEK_OP, KMSRESTConstants.EEK_DECRYPT);
Map<String, Object> jsonPayload = new HashMap<String, Object>();
jsonPayload.put(KMSRESTConstants.NAME_FIELD,
- encryptedKeyVersion.getKeyName());
+ encryptedKeyVersion.getEncryptionKeyName());
jsonPayload.put(KMSRESTConstants.IV_FIELD, Base64.encodeBase64String(
- encryptedKeyVersion.getIv()));
+ encryptedKeyVersion.getEncryptedKeyIv()));
jsonPayload.put(KMSRESTConstants.MATERIAL_FIELD, Base64.encodeBase64String(
- encryptedKeyVersion.getEncryptedKey().getMaterial()));
+ encryptedKeyVersion.getEncryptedKeyVersion().getMaterial()));
URL url = createURL(KMSRESTConstants.KEY_VERSION_RESOURCE,
- encryptedKeyVersion.getKeyVersionName(),
+ encryptedKeyVersion.getEncryptionKeyVersionName(),
KMSRESTConstants.EEK_SUB_RESOURCE, params);
HttpURLConnection conn = createConnection(url, HTTP_POST);
conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java Wed Jul 23 17:30:06 2014
@@ -288,9 +288,7 @@ public class Text extends BinaryComparab
@Override
public void readFields(DataInput in) throws IOException {
int newLength = WritableUtils.readVInt(in);
- setCapacity(newLength, false);
- in.readFully(bytes, 0, newLength);
- length = newLength;
+ readWithKnownLength(in, newLength);
}
public void readFields(DataInput in, int maxLength) throws IOException {
@@ -302,9 +300,7 @@ public class Text extends BinaryComparab
throw new IOException("tried to deserialize " + newLength +
" bytes of data, but maxLength = " + maxLength);
}
- setCapacity(newLength, false);
- in.readFully(bytes, 0, newLength);
- length = newLength;
+ readWithKnownLength(in, newLength);
}
/** Skips over one Text in the input. */
@@ -313,6 +309,17 @@ public class Text extends BinaryComparab
WritableUtils.skipFully(in, length);
}
+ /**
+ * Read a Text object whose length is already known.
+ * This allows creating Text from a stream which uses a different serialization
+ * format.
+ */
+ public void readWithKnownLength(DataInput in, int len) throws IOException {
+ setCapacity(len, false);
+ in.readFully(bytes, 0, len);
+ length = len;
+ }
+
/** serialize
* write this object to out
* length uses zero-compressed encoding
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java Wed Jul 23 17:30:06 2014
@@ -883,8 +883,8 @@ public class NetworkTopology {
* @param seed Used to seed the pseudo-random generator that randomizes the
* set of nodes at each network distance.
*/
- public void sortByDistance(Node reader, Node[] nodes,
- int activeLen, long seed) {
+ public void sortByDistance(Node reader, Node[] nodes, int activeLen,
+ long seed, boolean randomizeBlockLocationsPerBlock) {
/** Sort weights for the nodes array */
int[] weights = new int[activeLen];
for (int i=0; i<activeLen; i++) {
@@ -906,8 +906,11 @@ public class NetworkTopology {
// Seed is normally the block id
// This means we use the same pseudo-random order for each block, for
// potentially better page cache usage.
+ // Seed is not used if we want to randomize block location for every block
Random rand = getRandom();
- rand.setSeed(seed);
+ if (!randomizeBlockLocationsPerBlock) {
+ rand.setSeed(seed);
+ }
int idx = 0;
for (List<Node> list: tree.values()) {
if (list != null) {
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java Wed Jul 23 17:30:06 2014
@@ -279,8 +279,8 @@ public class NetworkTopologyWithNodeGrou
* set of nodes at each network distance.
*/
@Override
- public void sortByDistance( Node reader, Node[] nodes,
- int activeLen, long seed) {
+ public void sortByDistance(Node reader, Node[] nodes, int activeLen,
+ long seed, boolean randomizeBlockLocationsPerBlock) {
// If reader is not a datanode (not in NetworkTopology tree), we need to
// replace this reader with a sibling leaf node in tree.
if (reader != null && !this.contains(reader)) {
@@ -293,7 +293,8 @@ public class NetworkTopologyWithNodeGrou
return;
}
}
- super.sortByDistance(reader, nodes, nodes.length, seed);
+ super.sortByDistance(reader, nodes, nodes.length, seed,
+ randomizeBlockLocationsPerBlock);
}
/** InnerNodeWithNodeGroup represents a switch/router of a data center, rack
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Tool.java Wed Jul 23 17:30:06 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configurab
*
* <p><code>Tool</code>, is the standard for any Map-Reduce tool/application.
* The tool/application should delegate the handling of
- * <a href="{@docRoot}/org/apache/hadoop/util/GenericOptionsParser.html#GenericOptions">
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
* standard command-line options</a> to {@link ToolRunner#run(Tool, String[])}
* and only handle its custom arguments.</p>
*
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java Wed Jul 23 17:30:06 2014
@@ -17,51 +17,112 @@
*/
package org.apache.hadoop.crypto.key;
+import java.net.URI;
+import java.security.SecureRandom;
+import java.util.Arrays;
+
+import javax.crypto.Cipher;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+
import org.apache.hadoop.conf.Configuration;
-import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
-import java.net.URI;
-import java.security.SecureRandom;
+
+import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
public class TestKeyProviderCryptoExtension {
private static final String CIPHER = "AES";
+ private static final String ENCRYPTION_KEY_NAME = "fooKey";
- @Test
- public void testGenerateEncryptedKey() throws Exception {
- Configuration conf = new Configuration();
- KeyProvider kp =
- new UserProvider.Factory().createProvider(new URI("user:///"), conf);
- KeyProvider.Options options = new KeyProvider.Options(conf);
+ private static Configuration conf;
+ private static KeyProvider kp;
+ private static KeyProviderCryptoExtension kpExt;
+ private static KeyProvider.Options options;
+ private static KeyVersion encryptionKey;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ conf = new Configuration();
+ kp = new UserProvider.Factory().createProvider(new URI("user:///"), conf);
+ kpExt = KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
+ options = new KeyProvider.Options(conf);
options.setCipher(CIPHER);
options.setBitLength(128);
- KeyProvider.KeyVersion kv = kp.createKey("foo", SecureRandom.getSeed(16),
- options);
- KeyProviderCryptoExtension kpExt =
- KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
-
+ encryptionKey =
+ kp.createKey(ENCRYPTION_KEY_NAME, SecureRandom.getSeed(16), options);
+ }
+
+ @Test
+ public void testGenerateEncryptedKey() throws Exception {
+ // Generate a new EEK and check it
KeyProviderCryptoExtension.EncryptedKeyVersion ek1 =
- kpExt.generateEncryptedKey(kv.getName());
- Assert.assertEquals(KeyProviderCryptoExtension.EEK,
- ek1.getEncryptedKey().getVersionName());
- Assert.assertEquals("foo", ek1.getKeyName());
- Assert.assertNotNull(ek1.getEncryptedKey().getMaterial());
- Assert.assertEquals(kv.getMaterial().length,
- ek1.getEncryptedKey().getMaterial().length);
- KeyProvider.KeyVersion k1 = kpExt.decryptEncryptedKey(ek1);
- Assert.assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName());
- KeyProvider.KeyVersion k1a = kpExt.decryptEncryptedKey(ek1);
- Assert.assertArrayEquals(k1.getMaterial(), k1a.getMaterial());
- Assert.assertEquals(kv.getMaterial().length, k1.getMaterial().length);
+ kpExt.generateEncryptedKey(encryptionKey.getName());
+ assertEquals("Version name of EEK should be EEK",
+ KeyProviderCryptoExtension.EEK,
+ ek1.getEncryptedKeyVersion().getVersionName());
+ assertEquals("Name of EEK should be encryption key name",
+ ENCRYPTION_KEY_NAME, ek1.getEncryptionKeyName());
+ assertNotNull("Expected encrypted key material",
+ ek1.getEncryptedKeyVersion().getMaterial());
+ assertEquals("Length of encryption key material and EEK material should "
+ + "be the same", encryptionKey.getMaterial().length,
+ ek1.getEncryptedKeyVersion().getMaterial().length
+ );
+
+ // Decrypt EEK into an EK and check it
+ KeyVersion k1 = kpExt.decryptEncryptedKey(ek1);
+ assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName());
+ assertEquals(encryptionKey.getMaterial().length, k1.getMaterial().length);
+ if (Arrays.equals(k1.getMaterial(), encryptionKey.getMaterial())) {
+ fail("Encrypted key material should not equal encryption key material");
+ }
+ if (Arrays.equals(ek1.getEncryptedKeyVersion().getMaterial(),
+ encryptionKey.getMaterial())) {
+ fail("Encrypted key material should not equal decrypted key material");
+ }
+ // Decrypt it again and it should be the same
+ KeyVersion k1a = kpExt.decryptEncryptedKey(ek1);
+ assertArrayEquals(k1.getMaterial(), k1a.getMaterial());
+ // Generate another EEK and make sure it's different from the first
KeyProviderCryptoExtension.EncryptedKeyVersion ek2 =
- kpExt.generateEncryptedKey(kv.getName());
- KeyProvider.KeyVersion k2 = kpExt.decryptEncryptedKey(ek2);
- boolean eq = true;
- for (int i = 0; eq && i < ek2.getEncryptedKey().getMaterial().length; i++) {
- eq = k2.getMaterial()[i] == k1.getMaterial()[i];
+ kpExt.generateEncryptedKey(encryptionKey.getName());
+ KeyVersion k2 = kpExt.decryptEncryptedKey(ek2);
+ if (Arrays.equals(k1.getMaterial(), k2.getMaterial())) {
+ fail("Generated EEKs should have different material!");
}
- Assert.assertFalse(eq);
+ if (Arrays.equals(ek1.getEncryptedKeyIv(), ek2.getEncryptedKeyIv())) {
+ fail("Generated EEKs should have different IVs!");
+ }
+ }
+
+ @Test
+ public void testEncryptDecrypt() throws Exception {
+ // Get an EEK
+ KeyProviderCryptoExtension.EncryptedKeyVersion eek =
+ kpExt.generateEncryptedKey(encryptionKey.getName());
+ final byte[] encryptedKeyIv = eek.getEncryptedKeyIv();
+ final byte[] encryptedKeyMaterial = eek.getEncryptedKeyVersion()
+ .getMaterial();
+ // Decrypt it manually
+ Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding");
+ cipher.init(Cipher.DECRYPT_MODE,
+ new SecretKeySpec(encryptionKey.getMaterial(), "AES"),
+ new IvParameterSpec(KeyProviderCryptoExtension.EncryptedKeyVersion
+ .deriveIV(encryptedKeyIv)));
+ final byte[] manualMaterial = cipher.doFinal(encryptedKeyMaterial);
+ // Decrypt it with the API
+ KeyVersion decryptedKey = kpExt.decryptEncryptedKey(eek);
+ final byte[] apiMaterial = decryptedKey.getMaterial();
+
+ assertArrayEquals("Wrong key material from decryptEncryptedKey",
+ manualMaterial, apiMaterial);
}
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestText.java Wed Jul 23 17:30:06 2014
@@ -24,6 +24,7 @@ import java.nio.BufferUnderflowException
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.util.Random;
+import com.google.common.base.Charsets;
import com.google.common.primitives.Bytes;
/** Unit tests for LargeUTF8. */
@@ -363,6 +364,27 @@ public class TestText extends TestCase {
fail("testReadWriteOperations error !!!");
}
}
+
+ public void testReadWithKnownLength() throws IOException {
+ String line = "hello world";
+ byte[] inputBytes = line.getBytes(Charsets.UTF_8);
+ DataInputBuffer in = new DataInputBuffer();
+ Text text = new Text();
+
+ in.reset(inputBytes, inputBytes.length);
+ text.readWithKnownLength(in, 5);
+ assertEquals("hello", text.toString());
+
+ // Read longer length, make sure it lengthens
+ in.reset(inputBytes, inputBytes.length);
+ text.readWithKnownLength(in, 7);
+ assertEquals("hello w", text.toString());
+
+ // Read shorter length, make sure it shortens
+ in.reset(inputBytes, inputBytes.length);
+ text.readWithKnownLength(in, 2);
+ assertEquals("he", text.toString());
+ }
/**
* test {@code Text.bytesToCodePoint(bytes) }
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java Wed Jul 23 17:30:06 2014
@@ -105,7 +105,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[3];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[2]);
@@ -117,7 +117,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[1];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
@@ -127,7 +127,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(dataNodes[0], testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
@@ -137,7 +137,7 @@ public class TestNetworkTopologyWithNode
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
cluster.sortByDistance(computeNode, testNodes,
- testNodes.length, 0xDEADBEEF);
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java Wed Jul 23 17:30:06 2014
@@ -64,12 +64,12 @@ public class KMSServerJSONUtils {
Map json = new LinkedHashMap();
if (encryptedKeyVersion != null) {
json.put(KMSRESTConstants.VERSION_NAME_FIELD,
- encryptedKeyVersion.getKeyVersionName());
+ encryptedKeyVersion.getEncryptionKeyVersionName());
json.put(KMSRESTConstants.IV_FIELD,
Base64.encodeBase64URLSafeString(
- encryptedKeyVersion.getIv()));
+ encryptedKeyVersion.getEncryptedKeyIv()));
json.put(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD,
- toJSON(encryptedKeyVersion.getEncryptedKey()));
+ toJSON(encryptedKeyVersion.getEncryptedKeyVersion()));
}
return json;
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java?rev=1612883&r1=1612882&r2=1612883&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java Wed Jul 23 17:30:06 2014
@@ -485,10 +485,10 @@ public class TestKMS {
EncryptedKeyVersion ek1 = kpExt.generateEncryptedKey(kv.getName());
Assert.assertEquals(KeyProviderCryptoExtension.EEK,
- ek1.getEncryptedKey().getVersionName());
- Assert.assertNotNull(ek1.getEncryptedKey().getMaterial());
+ ek1.getEncryptedKeyVersion().getVersionName());
+ Assert.assertNotNull(ek1.getEncryptedKeyVersion().getMaterial());
Assert.assertEquals(kv.getMaterial().length,
- ek1.getEncryptedKey().getMaterial().length);
+ ek1.getEncryptedKeyVersion().getMaterial().length);
KeyProvider.KeyVersion k1 = kpExt.decryptEncryptedKey(ek1);
Assert.assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName());
KeyProvider.KeyVersion k1a = kpExt.decryptEncryptedKey(ek1);
@@ -498,8 +498,8 @@ public class TestKMS {
EncryptedKeyVersion ek2 = kpExt.generateEncryptedKey(kv.getName());
KeyProvider.KeyVersion k2 = kpExt.decryptEncryptedKey(ek2);
boolean isEq = true;
- for (int i = 0; isEq && i < ek2.getEncryptedKey().getMaterial().length;
- i++) {
+ for (int i = 0; isEq && i < ek2.getEncryptedKeyVersion()
+ .getMaterial().length; i++) {
isEq = k2.getMaterial()[i] == k1.getMaterial()[i];
}
Assert.assertFalse(isEq);