You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/09/04 01:20:48 UTC
svn commit: r1519885 [3/3] - in
/hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/
hadoop-yarn/...
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Tue Sep 3 23:20:45 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -25,12 +26,12 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
+import javax.crypto.SecretKey;
+
import junit.framework.Assert;
import org.apache.commons.logging.Log;
@@ -55,7 +56,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
@@ -198,7 +198,7 @@ public class TestRMStateStore {
ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
String containerIdStr, Token<AMRMTokenIdentifier> appToken,
- Token<ClientToAMTokenIdentifier> clientToAMToken, TestDispatcher dispatcher)
+ SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
throws Exception {
Container container = new ContainerPBImpl();
@@ -207,7 +207,8 @@ public class TestRMStateStore {
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
when(mockAttempt.getMasterContainer()).thenReturn(container);
when(mockAttempt.getAMRMToken()).thenReturn(appToken);
- when(mockAttempt.getClientToAMToken()).thenReturn(clientToAMToken);
+ when(mockAttempt.getClientTokenMasterKey())
+ .thenReturn(clientTokenMasterKey);
dispatcher.attemptId = attemptId;
dispatcher.storedException = null;
store.storeApplicationAttempt(mockAttempt);
@@ -215,7 +216,6 @@ public class TestRMStateStore {
return container.getId();
}
- @SuppressWarnings("unchecked")
void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
long submitTime = System.currentTimeMillis();
Configuration conf = new YarnConfiguration();
@@ -233,33 +233,33 @@ public class TestRMStateStore {
ApplicationId appId1 = attemptId1.getApplicationId();
storeApp(store, appId1, submitTime);
- // create application token1 for attempt1
- List<Token<?>> appAttemptToken1 =
- generateTokens(attemptId1, appTokenMgr, clientToAMTokenMgr, conf);
+ // create application token and client token key for attempt1
+ Token<AMRMTokenIdentifier> appAttemptToken1 =
+ generateAMRMToken(attemptId1, appTokenMgr);
HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
- attemptTokenSet1.addAll(appAttemptToken1);
+ attemptTokenSet1.add(appAttemptToken1);
+ SecretKey clientTokenKey1 =
+ clientToAMTokenMgr.registerApplication(attemptId1);
ContainerId containerId1 = storeAttempt(store, attemptId1,
"container_1352994193343_0001_01_000001",
- (Token<AMRMTokenIdentifier>) (appAttemptToken1.get(0)),
- (Token<ClientToAMTokenIdentifier>)(appAttemptToken1.get(1)),
- dispatcher);
+ appAttemptToken1, clientTokenKey1, dispatcher);
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 =
ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
- // create application token2 for attempt2
- List<Token<?>> appAttemptToken2 =
- generateTokens(attemptId2, appTokenMgr, clientToAMTokenMgr, conf);
+ // create application token and client token key for attempt2
+ Token<AMRMTokenIdentifier> appAttemptToken2 =
+ generateAMRMToken(attemptId2, appTokenMgr);
HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
- attemptTokenSet2.addAll(appAttemptToken2);
+ attemptTokenSet2.add(appAttemptToken2);
+ SecretKey clientTokenKey2 =
+ clientToAMTokenMgr.registerApplication(attemptId2);
ContainerId containerId2 = storeAttempt(store, attemptId2,
"container_1352994193343_0001_02_000001",
- (Token<AMRMTokenIdentifier>) (appAttemptToken2.get(0)),
- (Token<ClientToAMTokenIdentifier>)(appAttemptToken2.get(1)),
- dispatcher);
+ appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
@@ -306,8 +306,12 @@ public class TestRMStateStore {
assertEquals(containerId1, attemptState.getMasterContainer().getId());
// attempt1 applicationToken is loaded correctly
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
assertEquals(attemptTokenSet1, savedTokens);
+ // attempt1 client token master key is loaded correctly
+ assertArrayEquals(clientTokenKey1.getEncoded(),
+ attemptState.getAppAttemptCredentials()
+ .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
attemptState = appState.getAttempt(attemptId2);
// attempt2 is loaded correctly
@@ -317,8 +321,12 @@ public class TestRMStateStore {
assertEquals(containerId2, attemptState.getMasterContainer().getId());
// attempt2 applicationToken is loaded correctly
savedTokens.clear();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
assertEquals(attemptTokenSet2, savedTokens);
+ // attempt2 client token master key is loaded correctly
+ assertArrayEquals(clientTokenKey2.getEncoded(),
+ attemptState.getAppAttemptCredentials()
+ .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
// assert store is in expected state after everything is cleaned
assertTrue(stateStoreHelper.isFinalStateValid());
@@ -357,24 +365,14 @@ public class TestRMStateStore {
Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber());
}
- private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
- AMRMTokenSecretManager appTokenMgr,
- ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, Configuration conf) {
+ private Token<AMRMTokenIdentifier> generateAMRMToken(
+ ApplicationAttemptId attemptId,
+ AMRMTokenSecretManager appTokenMgr) {
AMRMTokenIdentifier appTokenId =
new AMRMTokenIdentifier(attemptId);
Token<AMRMTokenIdentifier> appToken =
new Token<AMRMTokenIdentifier>(appTokenId, appTokenMgr);
appToken.setService(new Text("appToken service"));
-
- ClientToAMTokenIdentifier clientToAMTokenId =
- new ClientToAMTokenIdentifier(attemptId);
- clientToAMTokenMgr.registerApplication(attemptId);
- Token<ClientToAMTokenIdentifier> clientToAMToken =
- new Token<ClientToAMTokenIdentifier>(clientToAMTokenId, clientToAMTokenMgr);
- clientToAMToken.setService(new Text("clientToAMToken service"));
- List<Token<?>> tokenPair = new ArrayList<Token<?>>();
- tokenPair.add(0, appToken);
- tokenPair.add(1, clientToAMToken);
- return tokenPair;
+ return appToken;
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Tue Sep 3 23:20:45 2013
@@ -143,7 +143,8 @@ public class MockRMApp implements RMApp
}
@Override
- public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Tue Sep 3 23:20:45 2013
@@ -726,7 +726,9 @@ public class TestRMAppTransitions {
public void testGetAppReport() {
RMApp app = createNewTestApp(null);
assertAppState(RMAppState.NEW, app);
- ApplicationReport report = app.createAndGetApplicationReport(true);
+ ApplicationReport report = app.createAndGetApplicationReport(null, true);
+ Assert.assertNotNull(report.getApplicationResourceUsageReport());
+ report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java Tue Sep 3 23:20:45 2013
@@ -115,7 +115,6 @@ public class TestClientToAMTokens {
private final byte[] secretKey;
private InetSocketAddress address;
private boolean pinged = false;
- private ClientToAMTokenSecretManager secretManager;
public CustomAM(ApplicationAttemptId appId, byte[] secretKey) {
super("CustomAM");
@@ -132,12 +131,14 @@ public class TestClientToAMTokens {
protected void serviceStart() throws Exception {
Configuration conf = getConfig();
- secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, secretKey);
Server server;
try {
server =
- new RPC.Builder(conf).setProtocol(CustomProtocol.class)
- .setNumHandlers(1).setSecretManager(secretManager)
+ new RPC.Builder(conf)
+ .setProtocol(CustomProtocol.class)
+ .setNumHandlers(1)
+ .setSecretManager(
+ new ClientToAMTokenSecretManager(this.appAttemptId, secretKey))
.setInstance(this).build();
} catch (Exception e) {
throw new YarnRuntimeException(e);
@@ -146,14 +147,10 @@ public class TestClientToAMTokens {
this.address = NetUtils.getConnectAddress(server);
super.serviceStart();
}
-
- public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
- return this.secretManager;
- }
}
@Test
- public void testClientToAMs() throws Exception {
+ public void testClientToAMTokens() throws Exception {
final Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -204,7 +201,7 @@ public class TestClientToAMTokens {
GetApplicationReportResponse reportResponse =
rm.getClientRMService().getApplicationReport(request);
ApplicationReport appReport = reportResponse.getApplicationReport();
- org.apache.hadoop.yarn.api.records.Token clientToAMToken =
+ org.apache.hadoop.yarn.api.records.Token originalClientToAMToken =
appReport.getClientToAMToken();
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
@@ -259,17 +256,47 @@ public class TestClientToAMTokens {
Assert.assertFalse(am.pinged);
}
- // Verify denial for a malicious user
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
Token<ClientToAMTokenIdentifier> token =
- ConverterUtils.convertFromYarn(clientToAMToken, am.address);
+ ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
+
+ // Verify denial for a malicious user with tampered ID
+ verifyTokenWithTamperedID(conf, am, token);
+
+ // Verify denial for a malicious user with tampered user-name
+ verifyTokenWithTamperedUserName(conf, am, token);
+ // Now for an authenticated user
+ verifyValidToken(conf, am, token);
+ }
+
+ private void verifyTokenWithTamperedID(final Configuration conf,
+ final CustomAM am, Token<ClientToAMTokenIdentifier> token)
+ throws IOException {
// Malicious user, messes with appId
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
ClientToAMTokenIdentifier maliciousID =
new ClientToAMTokenIdentifier(BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(app.getApplicationId()
- .getClusterTimestamp(), 42), 43));
+ BuilderUtils.newApplicationId(am.appAttemptId.getApplicationId()
+ .getClusterTimestamp(), 42), 43), UserGroupInformation
+ .getCurrentUser().getShortUserName());
+
+ verifyTamperedToken(conf, am, token, ugi, maliciousID);
+ }
+
+ private void verifyTokenWithTamperedUserName(final Configuration conf,
+ final CustomAM am, Token<ClientToAMTokenIdentifier> token)
+ throws IOException {
+ // Malicious user, messes with appId
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
+ ClientToAMTokenIdentifier maliciousID =
+ new ClientToAMTokenIdentifier(am.appAttemptId, "evilOrc");
+ verifyTamperedToken(conf, am, token, ugi, maliciousID);
+ }
+
+ private void verifyTamperedToken(final Configuration conf, final CustomAM am,
+ Token<ClientToAMTokenIdentifier> token, UserGroupInformation ugi,
+ ClientToAMTokenIdentifier maliciousID) {
Token<ClientToAMTokenIdentifier> maliciousToken =
new Token<ClientToAMTokenIdentifier>(maliciousID.getBytes(),
token.getPassword(), token.getKind(),
@@ -309,8 +336,12 @@ public class TestClientToAMTokens {
+ "Mismatched response."));
Assert.assertFalse(am.pinged);
}
+ }
- // Now for an authenticated user
+ private void verifyValidToken(final Configuration conf, final CustomAM am,
+ Token<ClientToAMTokenIdentifier> token) throws IOException,
+ InterruptedException {
+ UserGroupInformation ugi;
ugi = UserGroupInformation.createRemoteUser("me");
ugi.addToken(token);
@@ -326,5 +357,4 @@ public class TestClientToAMTokens {
}
});
}
-
}