You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2012/05/08 17:26:12 UTC
svn commit: r1335587 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/te...
Author: bobby
Date: Tue May 8 15:26:11 2012
New Revision: 1335587
URL: http://svn.apache.org/viewvc?rev=1335587&view=rev
Log:
svn merge -c 1335585 FIXES: MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn Sharp via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1335587&r1=1335586&r2=1335587&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue May 8 15:26:11 2012
@@ -216,6 +216,9 @@ Release 0.23.3 - UNRELEASED
OPTIMIZATIONS
+ MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
+ Sharp via bobby)
+
BUG FIXES
MAPREDUCE-4092. commitJob Exception does not fail job (Jon Eagles via
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java?rev=1335587&r1=1335586&r2=1335587&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java Tue May 8 15:26:11 2012
@@ -19,7 +19,9 @@
package org.apache.hadoop.mapreduce.security;
import java.io.IOException;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -92,8 +94,11 @@ public class TokenCache {
static void obtainTokensForNamenodesInternal(Credentials credentials,
Path[] ps, Configuration conf) throws IOException {
+ Set<FileSystem> fsSet = new HashSet<FileSystem>();
for(Path p: ps) {
- FileSystem fs = FileSystem.get(p.toUri(), conf);
+ fsSet.add(p.getFileSystem(conf));
+ }
+ for (FileSystem fs : fsSet) {
obtainTokensForNamenodesInternal(fs, credentials, conf);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1335587&r1=1335586&r2=1335587&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java Tue May 8 15:26:11 2012
@@ -252,6 +252,26 @@ public class TestTokenCache {
}
@Test
+ public void testSingleTokenFetch() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
+ String renewer = Master.getMasterPrincipal(conf);
+ Credentials credentials = new Credentials();
+
+ FileSystem mockFs = mock(FileSystem.class);
+ when(mockFs.getCanonicalServiceName()).thenReturn("host:0");
+ when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0"));
+
+ Path mockPath = mock(Path.class);
+ when(mockPath.getFileSystem(conf)).thenReturn(mockFs);
+
+ Path[] paths = new Path[]{ mockPath, mockPath };
+ when(mockFs.getDelegationTokens("me", credentials)).thenReturn(null);
+ TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf);
+ verify(mockFs, times(1)).getDelegationTokens(renewer, credentials);
+ }
+
+ @Test
public void testCleanUpTokenReferral() throws Exception {
Configuration conf = new Configuration();
conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, "foo");