You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2016/06/22 02:03:18 UTC
hadoop git commit: HDFS-7597. DelegationTokenIdentifier should cache
the TokenIdentifier to UGI mapping. Contributed by Daryn Sharp, Bob Hansen,
and Xiao Chen.
Repository: hadoop
Updated Branches:
refs/heads/trunk d8107fcd1 -> d433b16ce
HDFS-7597. DelegationTokenIdentifier should cache the TokenIdentifier to UGI mapping. Contributed by Daryn Sharp, Bob Hansen, and Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d433b16c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d433b16c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d433b16c
Branch: refs/heads/trunk
Commit: d433b16ce6d74f1a44bc29446c74b1cb5f8a10fa
Parents: d8107fc
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Jun 22 11:00:16 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Jun 22 11:00:16 2016 +0900
----------------------------------------------------------------------
.../delegation/DelegationTokenIdentifier.java | 26 ++++++++++++++++
.../web/webhdfs/DataNodeUGIProvider.java | 7 +++++
.../hdfs/security/TestDelegationToken.java | 31 +++++++++++++++++++-
.../hdfs/server/common/TestJspHelper.java | 8 ++++-
.../web/webhdfs/TestDataNodeUGIProvider.java | 1 +
5 files changed, 71 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
index 7bb0748..d753174 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
@@ -21,13 +21,20 @@ package org.apache.hadoop.hdfs.security.token.delegation;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* A delegation token identifier that is specific to HDFS.
*/
@@ -37,6 +44,15 @@ public class DelegationTokenIdentifier
public static final Text HDFS_DELEGATION_KIND =
new Text("HDFS_DELEGATION_TOKEN");
+ @SuppressWarnings("unchecked")
+ private static Map<TokenIdentifier, UserGroupInformation> ugiCache =
+ Collections.synchronizedMap(new LRUMap(64));
+
+ @VisibleForTesting
+ public void clearCache() {
+ ugiCache.clear();
+ }
+
/**
* Create an empty delegation token identifier for reading into.
*/
@@ -59,6 +75,16 @@ public class DelegationTokenIdentifier
}
@Override
+ public UserGroupInformation getUser() {
+ UserGroupInformation ugi = ugiCache.get(this);
+ if (ugi == null) {
+ ugi = super.getUser();
+ ugiCache.put(this, ugi);
+ }
+ return ugi;
+ }
+
+ @Override
public String toString() {
StringBuilder sbld = new StringBuilder();
sbld
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
index 233ba69..fa6f676 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
@@ -61,6 +61,13 @@ public class DataNodeUGIProvider {
}
}
+ @VisibleForTesting
+ void clearCache() throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ params.delegationToken().decodeIdentifier().clearCache();
+ }
+ }
+
UserGroupInformation ugi() throws IOException {
UserGroupInformation ugi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
index 0600536..6112b6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
@@ -239,7 +239,36 @@ public class TestDelegationToken {
}
});
}
-
+
+ @Test
+ public void testDelegationTokenUgi() throws Exception {
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ Token<?>[] tokens = dfs.addDelegationTokens("renewer", null);
+ Assert.assertEquals(1, tokens.length);
+ Token<?> token1 = tokens[0];
+ DelegationTokenIdentifier ident =
+ (DelegationTokenIdentifier) token1.decodeIdentifier();
+ UserGroupInformation expectedUgi = ident.getUser();
+
+ // get 2 new instances (clones) of the identifier, query their ugi
+ // twice each, all ugi instances should be equivalent
+ for (int i=0; i<2; i++) {
+ DelegationTokenIdentifier identClone =
+ (DelegationTokenIdentifier)token1.decodeIdentifier();
+ Assert.assertEquals(ident, identClone);
+ Assert.assertNotSame(ident, identClone);
+ Assert.assertSame(expectedUgi, identClone.getUser());
+ Assert.assertSame(expectedUgi, identClone.getUser());
+ }
+
+ // a new token must decode to a different ugi instance than the first token
+ tokens = dfs.addDelegationTokens("renewer", null);
+ Assert.assertEquals(1, tokens.length);
+ Token<?> token2 = tokens[0];
+ Assert.assertNotEquals(token1, token2);
+ Assert.assertNotSame(expectedUgi, token2.decodeIdentifier().getUser());
+ }
+
/**
* Test that the delegation token secret manager only runs when the
* NN is out of safe mode. This is because the secret manager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
index d0d8d3e..119db8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
@@ -109,6 +109,7 @@ public class TestJspHelper {
//Test attribute name.node.address
//Set the nnaddr url parameter to null.
+ token.decodeIdentifier().clearCache();
when(request.getParameter(JspHelper.NAMENODE_ADDRESS)).thenReturn(null);
InetSocketAddress addr = new InetSocketAddress("localhost", 2222);
when(context.getAttribute(NameNodeHttpServer.NAMENODE_ADDRESS_ATTRIBUTE_KEY))
@@ -116,7 +117,12 @@ public class TestJspHelper {
verifyServiceInToken(context, request, addr.getAddress().getHostAddress()
+ ":2222");
- //Test service already set in the token
+ //Test service already set in the token and DN doesn't change service
+ //when it doesn't know the NN service addr
+ userText = new Text(user+"2");
+ dtId = new DelegationTokenIdentifier(userText, userText, null);
+ token = new Token<DelegationTokenIdentifier>(
+ dtId, new DummySecretManager(0, 0, 0, 0));
token.setService(new Text("3.3.3.3:3333"));
tokenString = token.encodeToUrlString();
//Set the name.node.address attribute in Servlet context to null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d433b16c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
index bce5422..70618b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
@@ -121,6 +121,7 @@ public class TestDataNodeUGIProvider {
"With UGI cache, two UGIs for the different token should not be same",
ugi11, url22);
+ ugiProvider2.clearCache();
awaitCacheEmptyDueToExpiration();
ugi12 = ugiProvider1.ugi();
url22 = ugiProvider2.ugi();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org