You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xi...@apache.org on 2016/07/28 23:37:16 UTC
hadoop git commit: HDFS-9276. Failed to Update HDFS Delegation Token
for long running application in HA mode. Contributed by Liangliang Gu and
John Zhuge
Repository: hadoop
Updated Branches:
refs/heads/trunk ce3d68e9c -> d9aae22fd
HDFS-9276. Failed to Update HDFS Delegation Token for long running application in HA mode. Contributed by Liangliang Gu and John Zhuge
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9aae22f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9aae22f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9aae22f
Branch: refs/heads/trunk
Commit: d9aae22fdf2ab22ae8ce4a9d32ac71b3dde084d3
Parents: ce3d68e
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Jul 28 16:29:22 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Jul 28 16:29:40 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/security/Credentials.java | 22 +++++++++---
.../org/apache/hadoop/security/token/Token.java | 37 +++++++++++++++++---
.../namenode/ha/TestDelegationTokensWithHA.java | 32 +++++++++++++++++
3 files changed, 83 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9aae22f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index 977ccb5..c62a49c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -95,10 +95,24 @@ public class Credentials implements Writable {
* @param t the token object
*/
public void addToken(Text alias, Token<? extends TokenIdentifier> t) {
- if (t != null) {
- tokenMap.put(alias, t);
- } else {
+ if (t == null) {
LOG.warn("Null token ignored for " + alias);
+ } else if (tokenMap.put(alias, t) != null) {
+ // Update private tokens
+ Map<Text, Token<? extends TokenIdentifier>> tokensToAdd =
+ new HashMap<>();
+ for (Map.Entry<Text, Token<? extends TokenIdentifier>> e :
+ tokenMap.entrySet()) {
+ Token<? extends TokenIdentifier> token = e.getValue();
+ if (token instanceof Token.PrivateToken &&
+ ((Token.PrivateToken) token).getPublicService().equals(alias)) {
+ Token<? extends TokenIdentifier> privateToken =
+ new Token.PrivateToken<>(t);
+ privateToken.setService(token.getService());
+ tokensToAdd.put(e.getKey(), privateToken);
+ }
+ }
+ tokenMap.putAll(tokensToAdd);
}
}
@@ -397,7 +411,7 @@ public class Credentials implements Writable {
for(Map.Entry<Text, Token<?>> token: other.tokenMap.entrySet()){
Text key = token.getKey();
if (!tokenMap.containsKey(key) || overwrite) {
- tokenMap.put(key, token.getValue());
+ addToken(key, token.getValue());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9aae22f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
index 4ea686e..784e797 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
@@ -96,10 +96,10 @@ public class Token<T extends TokenIdentifier> implements Writable {
* @param other the token to clone
*/
public Token(Token<T> other) {
- this.identifier = other.identifier;
- this.password = other.password;
- this.kind = other.kind;
- this.service = other.service;
+ this.identifier = other.identifier.clone();
+ this.password = other.password.clone();
+ this.kind = new Text(other.kind);
+ this.service = new Text(other.service);
}
public Token<T> copyToken() {
@@ -230,8 +230,37 @@ public class Token<T extends TokenIdentifier> implements Writable {
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static class PrivateToken<T extends TokenIdentifier> extends Token<T> {
+ final private Text publicService;
+
public PrivateToken(Token<T> token) {
super(token);
+ publicService = new Text(token.getService());
+ }
+
+ public Text getPublicService() {
+ return publicService;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ PrivateToken<?> that = (PrivateToken<?>) o;
+ return publicService.equals(that.publicService);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + publicService.hashCode();
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9aae22f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java
index 16b27d8..632bbf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.*;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -367,6 +368,37 @@ public class TestDelegationTokensWithHA {
token.cancel(conf);
}
+ @Test(timeout = 300000)
+ public void testCancelAndUpdateDelegationTokens() throws Exception {
+ // Create UGI with token1
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser(user);
+
+ ugi1.doAs(new PrivilegedExceptionAction<Void>() {
+ public Void run() throws Exception {
+ final Token<DelegationTokenIdentifier> token1 =
+ getDelegationToken(fs, "JobTracker");
+ UserGroupInformation.getCurrentUser()
+ .addToken(token1.getService(), token1);
+
+ FileSystem fs1 = HATestUtil.configureFailoverFs(cluster, conf);
+
+ // Cancel token1
+ doRenewOrCancel(token1, conf, TokenTestAction.CANCEL);
+
+ // Update UGI with token2
+ final Token<DelegationTokenIdentifier> token2 =
+ getDelegationToken(fs, "JobTracker");
+ UserGroupInformation.getCurrentUser()
+ .addToken(token2.getService(), token2);
+
+ // Check whether token2 works
+ fs1.listFiles(new Path("/"), false);
+ return null;
+ }
+ });
+ }
+
@SuppressWarnings("unchecked")
private Token<DelegationTokenIdentifier> getDelegationToken(FileSystem fs,
String renewer) throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org