You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2014/10/09 22:22:13 UTC
git commit: HADOOP-11174. Delegation token for KMS should only be got
once if it already exists. (Yi Liu via wang)
Repository: hadoop
Updated Branches:
refs/heads/trunk 1d4612f5a -> 8d9411449
HADOOP-11174. Delegation token for KMS should only be got once if it already exists. (Yi Liu via wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d941144
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d941144
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d941144
Branch: refs/heads/trunk
Commit: 8d941144952a77a714cd4368cba21b01904926e9
Parents: 1d4612f
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Oct 9 13:21:19 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Oct 9 13:21:19 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../crypto/key/kms/KMSClientProvider.java | 41 +++++++++++++-------
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +++--
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 26 +++++++++++++
4 files changed, 64 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d941144/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a662fb9..257b7b5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -586,6 +586,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11178. Fix findbugs exclude file. (Arun Suresh via wang)
+ HADOOP-11174. Delegation token for KMS should only be got once if it
+ already exists. (Yi Liu via wang)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d941144/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 4f4e843..9b0bd47 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -26,8 +26,10 @@ import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
@@ -48,6 +50,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -773,23 +776,35 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
@Override
public Token<?>[] addDelegationTokens(String renewer,
Credentials credentials) throws IOException {
- Token<?>[] tokens;
- URL url = createURL(null, null, null, null);
- DelegationTokenAuthenticatedURL authUrl =
- new DelegationTokenAuthenticatedURL(configurator);
- try {
- Token<?> token = authUrl.getDelegationToken(url, authToken, renewer);
- if (token != null) {
- credentials.addToken(token.getService(), token);
- tokens = new Token<?>[] { token };
- } else {
- throw new IOException("Got NULL as delegation token");
+ Token<?>[] tokens = null;
+ Text dtService = getDelegationTokenService();
+ Token<?> token = credentials.getToken(dtService);
+ if (token == null) {
+ URL url = createURL(null, null, null, null);
+ DelegationTokenAuthenticatedURL authUrl =
+ new DelegationTokenAuthenticatedURL(configurator);
+ try {
+ token = authUrl.getDelegationToken(url, authToken, renewer);
+ if (token != null) {
+ credentials.addToken(token.getService(), token);
+ tokens = new Token<?>[] { token };
+ } else {
+ throw new IOException("Got NULL as delegation token");
+ }
+ } catch (AuthenticationException ex) {
+ throw new IOException(ex);
}
- } catch (AuthenticationException ex) {
- throw new IOException(ex);
}
return tokens;
}
+
+ private Text getDelegationTokenService() throws IOException {
+ URL url = new URL(kmsUrl);
+ InetSocketAddress addr = new InetSocketAddress(url.getHost(),
+ url.getPort());
+ Text dtService = SecurityUtil.buildTokenService(addr);
+ return dtService;
+ }
/**
* Shutdown valueQueue executor threads
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d941144/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index c384bfb..aeafc72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -118,7 +118,7 @@ public class TestEncryptionZones {
private MiniDFSCluster cluster;
private HdfsAdmin dfsAdmin;
- private DistributedFileSystem fs;
+ protected DistributedFileSystem fs;
private File testRootDir;
private final String TEST_KEY = "testKey";
@@ -149,12 +149,16 @@ public class TestEncryptionZones {
fcWrapper = new FileContextTestWrapper(
FileContext.getFileContext(cluster.getURI(), conf));
dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
+ setProvider();
+ // Create a test key
+ DFSTestUtil.createKey(TEST_KEY, cluster, conf);
+ }
+
+ protected void setProvider() {
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
fs.getClient().provider = cluster.getNameNode().getNamesystem()
.getProvider();
- // Create a test key
- DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d941144/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
index 3a9a89e..51ff96e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
@@ -19,11 +19,16 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Test;
import java.io.File;
+import java.util.Arrays;
import java.util.UUID;
public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
@@ -52,5 +57,26 @@ public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
super.teardown();
miniKMS.stop();
}
+
+ @Override
+ protected void setProvider() {
+ }
+ @Test(timeout = 120000)
+ public void testDelegationToken() throws Exception {
+ final String renewer = "JobTracker";
+ UserGroupInformation.createRemoteUser(renewer);
+
+ Credentials creds = new Credentials();
+ Token<?> tokens[] = fs.addDelegationTokens(renewer, creds);
+ DistributedFileSystem.LOG.debug("Delegation tokens: " +
+ Arrays.asList(tokens));
+ Assert.assertEquals(2, tokens.length);
+ Assert.assertEquals(2, creds.numberOfTokens());
+
+ // If the dt exists, will not get again
+ tokens = fs.addDelegationTokens(renewer, creds);
+ Assert.assertEquals(0, tokens.length);
+ Assert.assertEquals(2, creds.numberOfTokens());
+ }
}