You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aa...@apache.org on 2018/05/02 07:15:04 UTC
hadoop git commit: MAPREDUCE-7073. Optimize
TokenCache#obtainTokensForNamenodesInternal
Repository: hadoop
Updated Branches:
refs/heads/trunk 37269261d -> 1a95a4524
MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal
Signed-off-by: Akira Ajisaka <aa...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a95a452
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a95a452
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a95a452
Branch: refs/heads/trunk
Commit: 1a95a4524a8c6c7be601ce8b92640a6a76164a2c
Parents: 3726926
Author: Bibin A Chundatt <bi...@apache.org>
Authored: Wed May 2 16:14:28 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed May 2 16:14:28 2018 +0900
----------------------------------------------------------------------
.../hadoop/mapreduce/security/TokenCache.java | 14 +++++++++-----
.../hadoop/mapreduce/security/TestTokenCache.java | 18 +++++++++---------
2 files changed, 18 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a95a452/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
index 12fced9..1156c67 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -96,8 +97,9 @@ public class TokenCache {
for(Path p: ps) {
fsSet.add(p.getFileSystem(conf));
}
+ String masterPrincipal = Master.getMasterPrincipal(conf);
for (FileSystem fs : fsSet) {
- obtainTokensForNamenodesInternal(fs, credentials, conf);
+ obtainTokensForNamenodesInternal(fs, credentials, conf, masterPrincipal);
}
}
@@ -122,15 +124,17 @@ public class TokenCache {
* @param conf
* @throws IOException
*/
- static void obtainTokensForNamenodesInternal(FileSystem fs,
- Credentials credentials, Configuration conf) throws IOException {
+ static void obtainTokensForNamenodesInternal(FileSystem fs,
+ Credentials credentials, Configuration conf, String renewer)
+ throws IOException {
// RM skips renewing token with empty renewer
String delegTokenRenewer = "";
if (!isTokenRenewalExcluded(fs, conf)) {
- delegTokenRenewer = Master.getMasterPrincipal(conf);
- if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
+ if (StringUtils.isEmpty(renewer)) {
throw new IOException(
"Can't get Master Kerberos principal for use as renewer");
+ } else {
+ delegTokenRenewer = renewer;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a95a452/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
index 127f8ae..a44e533 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
@@ -56,8 +56,8 @@ public class TestTokenCache {
@Test
public void testObtainTokens() throws Exception {
Credentials credentials = new Credentials();
- FileSystem fs = mock(FileSystem.class);
- TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf);
+ FileSystem fs = mock(FileSystem.class);
+ TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf, renewer);
verify(fs).addDelegationTokens(eq(renewer), eq(credentials));
}
@@ -105,23 +105,23 @@ public class TestTokenCache {
checkToken(creds, newerToken1);
// get token for fs1, see that fs2's token was loaded
- TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
+ TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer);
checkToken(creds, newerToken1, token2);
// get token for fs2, nothing should change since already present
- TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf);
+ TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer);
checkToken(creds, newerToken1, token2);
// get token for fs3, should only add token for fs3
- TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf);
+ TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer);
Token<?> token3 = creds.getToken(new Text(fs3.getCanonicalServiceName()));
assertTrue(token3 != null);
checkToken(creds, newerToken1, token2, token3);
// be paranoid, check one last time that nothing changes
- TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
- TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf);
- TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf);
+ TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer);
+ TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer);
+ TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer);
checkToken(creds, newerToken1, token2, token3);
}
@@ -202,7 +202,7 @@ public class TestTokenCache {
// wait to set, else the obtain tokens call above will fail with FNF
conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile);
creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
- TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf);
+ TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer);
String fs_addr = fs1.getCanonicalServiceName();
Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr);
assertNotNull("Token for nn is null", nnt);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org