You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2012/08/23 20:10:29 UTC
svn commit: r1376624 - in
/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/security/
src/test/java/org/apache/hadoop/security/
Author: bobby
Date: Thu Aug 23 18:10:28 2012
New Revision: 1376624
URL: http://svn.apache.org/viewvc?rev=1376624&view=rev
Log:
svn merge -c 1376618 FIXES: HADOOP-8225. DistCp fails when invoked by Oozie (daryn via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 23 18:10:28 2012
@@ -188,6 +188,8 @@ Release 0.23.3 - UNRELEASED
HADOOP-8614. IOUtils#skipFully hangs forever on EOF.
(Colin Patrick McCabe via eli)
+ HADOOP-8225. DistCp fails when invoked by Oozie (daryn via bobby)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java Thu Aug 23 18:10:28 2012
@@ -274,4 +274,10 @@ public class Credentials implements Writ
}
}
}
+
+ public void addTokensToUGI(UserGroupInformation ugi) {
+ for (Map.Entry<Text, Token<?>> token: tokenMap.entrySet()) {
+ ugi.addToken(token.getKey(), token.getValue());
+ }
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java Thu Aug 23 18:10:28 2012
@@ -53,6 +53,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -514,9 +515,7 @@ public class UserGroupInformation {
// user.
Credentials cred = Credentials.readTokenStorageFile(
new Path("file:///" + fileLocation), conf);
- for (Token<?> token: cred.getAllTokens()) {
- loginUser.addToken(token);
- }
+ cred.addTokensToUGI(loginUser);
}
loginUser.spawnAutoRenewalThreadForUserCreds();
} catch (LoginException le) {
@@ -1026,6 +1025,41 @@ public class UserGroupInformation {
public synchronized Set<TokenIdentifier> getTokenIdentifiers() {
return subject.getPublicCredentials(TokenIdentifier.class);
}
+
+ // wrapper to retain the creds key for the token
+ private class NamedToken {
+ Text alias;
+ Token<? extends TokenIdentifier> token;
+ NamedToken(Text alias, Token<? extends TokenIdentifier> token) {
+ this.alias = alias;
+ this.token = token;
+ }
+ @Override
+ public boolean equals(Object o) {
+ boolean equals;
+ if (o == this) {
+ equals = true;
+ } else if (!(o instanceof NamedToken)) {
+ equals = false;
+ } else {
+ Text otherAlias = ((NamedToken)o).alias;
+ if (alias == otherAlias) {
+ equals = true;
+ } else {
+ equals = (otherAlias != null && otherAlias.equals(alias));
+ }
+ }
+ return equals;
+ }
+ @Override
+ public int hashCode() {
+ return (alias != null) ? alias.hashCode() : -1;
+ }
+ @Override
+ public String toString() {
+ return "NamedToken: alias="+alias+" token="+token;
+ }
+ }
/**
* Add a token to this UGI
@@ -1034,7 +1068,22 @@ public class UserGroupInformation {
* @return true on successful add of new token
*/
public synchronized boolean addToken(Token<? extends TokenIdentifier> token) {
- return subject.getPrivateCredentials().add(token);
+ return addToken(token.getService(), token);
+ }
+
+ /**
+ * Add a named token to this UGI
+ *
+ * @param alias Name of the token
+ * @param token Token to be added
+ * @return true on successful add of new token
+ */
+ public synchronized boolean addToken(Text alias,
+ Token<? extends TokenIdentifier> token) {
+ NamedToken namedToken = new NamedToken(alias, token);
+ Collection<Object> ugiCreds = subject.getPrivateCredentials();
+ ugiCreds.remove(namedToken); // allow token to be replaced
+ return ugiCreds.add(new NamedToken(alias, token));
}
/**
@@ -1044,14 +1093,23 @@ public class UserGroupInformation {
*/
public synchronized
Collection<Token<? extends TokenIdentifier>> getTokens() {
- Set<Object> creds = subject.getPrivateCredentials();
- List<Token<?>> result = new ArrayList<Token<?>>(creds.size());
- for(Object o: creds) {
- if (o instanceof Token<?>) {
- result.add((Token<?>) o);
- }
+ return Collections.unmodifiableList(
+ new ArrayList<Token<?>>(getCredentials().getAllTokens()));
+ }
+
+ /**
+ * Obtain the tokens in credentials form associated with this user.
+ *
+ * @return Credentials of tokens associated with this user
+ */
+ public synchronized Credentials getCredentials() {
+ final Credentials credentials = new Credentials();
+ final Set<NamedToken> namedTokens =
+ subject.getPrivateCredentials(NamedToken.class);
+ for (final NamedToken namedToken : namedTokens) {
+ credentials.addToken(namedToken.alias, namedToken.token);
}
- return Collections.unmodifiableList(result);
+ return credentials;
}
/**
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java Thu Aug 23 18:10:28 2012
@@ -213,5 +213,22 @@ public class TestCredentials {
// new token & secret should be added
assertEquals(token[2], creds.getToken(service[2]));
assertEquals(secret[2], new Text(creds.getSecretKey(secret[2])));
- }
+ }
+
+ @Test
+ public void testAddTokensToUGI() {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone");
+ Credentials creds = new Credentials();
+
+ for (int i=0; i < service.length; i++) {
+ creds.addToken(service[i], token[i]);
+ }
+ creds.addTokensToUGI(ugi);
+
+ creds = ugi.getCredentials();
+ for (int i=0; i < service.length; i++) {
+ assertSame(token[i], creds.getToken(service[i]));
+ }
+ assertEquals(service.length, creds.numberOfTokens());
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1376624&r1=1376623&r2=1376624&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java Thu Aug 23 18:10:28 2012
@@ -19,8 +19,7 @@ package org.apache.hadoop.security;
import static org.junit.Assert.*;
import org.junit.*;
-import org.mockito.Mockito;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.*;
import java.io.BufferedReader;
import java.io.IOException;
@@ -35,6 +34,7 @@ import javax.security.auth.login.AppConf
import javax.security.auth.login.LoginContext;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
@@ -193,8 +193,6 @@ public class TestUserGroupInformation {
public void testEqualsWithRealUser() throws Exception {
UserGroupInformation realUgi1 = UserGroupInformation.createUserForTesting(
"RealUser", GROUP_NAMES);
- UserGroupInformation realUgi2 = UserGroupInformation.createUserForTesting(
- "RealUser", GROUP_NAMES);
UserGroupInformation proxyUgi1 = UserGroupInformation.createProxyUser(
USER_NAME, realUgi1);
UserGroupInformation proxyUgi2 =
@@ -212,7 +210,82 @@ public class TestUserGroupInformation {
assertArrayEquals(new String[]{GROUP1_NAME, GROUP2_NAME, GROUP3_NAME},
uugi.getGroupNames());
}
+
+ @SuppressWarnings("unchecked") // from Mockito mocks
+ @Test
+ public <T extends TokenIdentifier> void testAddToken() throws Exception {
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("someone");
+
+ Token<T> t1 = mock(Token.class);
+ Token<T> t2 = mock(Token.class);
+ Token<T> t3 = mock(Token.class);
+
+ // add token to ugi
+ ugi.addToken(t1);
+ checkTokens(ugi, t1);
+
+ // replace token t1 with t2 - with same key (null)
+ ugi.addToken(t2);
+ checkTokens(ugi, t2);
+
+ // change t1 service and add token
+ when(t1.getService()).thenReturn(new Text("t1"));
+ ugi.addToken(t1);
+ checkTokens(ugi, t1, t2);
+ // overwrite t1 token with t3 - same key (!null)
+ when(t3.getService()).thenReturn(new Text("t1"));
+ ugi.addToken(t3);
+ checkTokens(ugi, t2, t3);
+
+ // just try to re-add with new name
+ when(t1.getService()).thenReturn(new Text("t1.1"));
+ ugi.addToken(t1);
+ checkTokens(ugi, t1, t2, t3);
+
+ // just try to re-add with new name again
+ ugi.addToken(t1);
+ checkTokens(ugi, t1, t2, t3);
+ }
+
+ private void checkTokens(UserGroupInformation ugi, Token<?> ... tokens) {
+ // check the ugi's token collection
+ Collection<Token<?>> ugiTokens = ugi.getTokens();
+ for (Token<?> t : tokens) {
+ assertTrue(ugiTokens.contains(t));
+ }
+ assertEquals(tokens.length, ugiTokens.size());
+
+ // check the ugi's credentials
+ Credentials ugiCreds = ugi.getCredentials();
+ for (Token<?> t : tokens) {
+ assertSame(t, ugiCreds.getToken(t.getService()));
+ }
+ assertEquals(tokens.length, ugiCreds.numberOfTokens());
+ }
+
+ @SuppressWarnings("unchecked") // from Mockito mocks
+ @Test
+ public <T extends TokenIdentifier> void testAddNamedToken() throws Exception {
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("someone");
+
+ Token<T> t1 = mock(Token.class);
+ Text service1 = new Text("t1");
+ Text service2 = new Text("t2");
+ when(t1.getService()).thenReturn(service1);
+
+ // add token
+ ugi.addToken(service1, t1);
+ assertSame(t1, ugi.getCredentials().getToken(service1));
+
+ // add token with another name
+ ugi.addToken(service2, t1);
+ assertSame(t1, ugi.getCredentials().getToken(service1));
+ assertSame(t1, ugi.getCredentials().getToken(service2));
+ }
+
@SuppressWarnings("unchecked") // from Mockito mocks
@Test
public <T extends TokenIdentifier> void testUGITokens() throws Exception {
@@ -220,7 +293,9 @@ public class TestUserGroupInformation {
UserGroupInformation.createUserForTesting("TheDoctor",
new String [] { "TheTARDIS"});
Token<T> t1 = mock(Token.class);
+ when(t1.getService()).thenReturn(new Text("t1"));
Token<T> t2 = mock(Token.class);
+ when(t2.getService()).thenReturn(new Text("t2"));
ugi.addToken(t1);
ugi.addToken(t2);