You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by wa...@apache.org on 2013/09/20 01:42:29 UTC
svn commit: r1524865 [7/8] - in
/hadoop/common/branches/HDFS-4949/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protoc...
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Thu Sep 19 23:42:10 2013
@@ -200,15 +200,14 @@ public class MockNodes {
};
private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
- return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null);
+ return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
- NodeState state, String httpAddr, int hostnum, String hostName) {
+ NodeState state, String httpAddr, int hostnum, String hostName, int port) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
- final int port = 123;
if (hostName == null) {
hostName = "host"+ nid;
}
@@ -230,12 +229,17 @@ public class MockNodes {
}
public static RMNode newNodeInfo(int rack, final Resource perNode, int hostnum) {
- return buildRMNode(rack, perNode, null, "localhost:0", hostnum, null);
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, null, 123);
}
public static RMNode newNodeInfo(int rack, final Resource perNode,
int hostnum, String hostName) {
- return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName);
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName, 123);
+ }
+
+ public static RMNode newNodeInfo(int rack, final Resource perNode,
+ int hostnum, String hostName, int port) {
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName, port);
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Thu Sep 19 23:42:10 2013
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
@@ -231,6 +232,14 @@ public class MockRM extends ResourceMana
return nm;
}
+ public MockNM registerNode(String nodeIdStr, int memory, int vCores)
+ throws Exception {
+ MockNM nm =
+ new MockNM(nodeIdStr, memory, vCores, getResourceTrackerService());
+ nm.registerNode();
+ return nm;
+ }
+
public void sendNodeStarted(MockNM nm) throws Exception {
RMNodeImpl node = (RMNodeImpl) getRMContext().getRMNodes().get(
nm.getNodeId());
@@ -387,6 +396,10 @@ public class MockRM extends ResourceMana
return this.rmDTSecretManager;
}
+ public ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager() {
+ return this.clientToAMSecretManager;
+ }
+
@Override
protected void startWepApp() {
// override to disable webapp
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Thu Sep 19 23:42:10 2013
@@ -101,7 +101,7 @@ public class NodeManager implements Cont
request.setNodeId(this.nodeId);
resourceTrackerService.registerNodeManager(request);
this.schedulerNode = new FiCaSchedulerNode(rmContext.getRMNodes().get(
- this.nodeId));
+ this.nodeId), false);
// Sanity check
Assert.assertEquals(capability.getMemory(),
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Thu Sep 19 23:42:10 2013
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -198,6 +199,27 @@ public class TestClientRMService {
}
@Test
+ public void testForceKillApplication() throws YarnException {
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getRMApps()).thenReturn(
+ new ConcurrentHashMap<ApplicationId, RMApp>());
+ ClientRMService rmService = new ClientRMService(rmContext, null, null,
+ null, null);
+ ApplicationId applicationId =
+ BuilderUtils.newApplicationId(System.currentTimeMillis(), 0);
+ KillApplicationRequest request =
+ KillApplicationRequest.newInstance(applicationId);
+ try {
+ rmService.forceKillApplication(request);
+ Assert.fail();
+ } catch (ApplicationNotFoundException ex) {
+ Assert.assertEquals(ex.getMessage(),
+ "Trying to kill an absent " +
+ "application " + request.getApplicationId());
+ }
+ }
+
+ @Test
public void testGetQueueInfo() throws Exception {
YarnScheduler yarnScheduler = mock(YarnScheduler.class);
RMContext rmContext = mock(RMContext.class);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java Thu Sep 19 23:42:10 2013
@@ -260,14 +260,42 @@ public class TestRMNodeTransitions {
@Test
public void testRunningExpire() {
RMNodeImpl node = getRunningNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
+ Assert.assertEquals("Active Nodes", initialActive - 1, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost + 1, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.LOST, node.getState());
}
@Test
public void testUnhealthyExpire() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost + 1, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.LOST, node.getState());
}
@@ -283,32 +311,88 @@ public class TestRMNodeTransitions {
@Test
public void testRunningDecommission() {
RMNodeImpl node = getRunningNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.DECOMMISSION));
+ Assert.assertEquals("Active Nodes", initialActive - 1, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned + 1, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}
@Test
public void testUnhealthyDecommission() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.DECOMMISSION));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned + 1, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}
@Test
public void testRunningRebooting() {
RMNodeImpl node = getRunningNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.REBOOTING));
+ Assert.assertEquals("Active Nodes", initialActive - 1, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted + 1, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
@Test
public void testUnhealthyRebooting() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.REBOOTING));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted + 1, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Thu Sep 19 23:42:10 2013
@@ -18,10 +18,9 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
@@ -35,6 +34,8 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
@@ -63,7 +64,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level;
@@ -77,8 +77,11 @@ public class TestRMRestart {
private YarnConfiguration conf;
+ // Fake rmAddr for token-renewal
+ private static InetSocketAddress rmAddr;
+
@Before
- public void setup() {
+ public void setup() throws UnknownHostException {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
ExitUtil.disableSystemExit();
@@ -86,6 +89,8 @@ public class TestRMRestart {
UserGroupInformation.setConfiguration(conf);
conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+
+ rmAddr = new InetSocketAddress("localhost", 8032);
}
@Test (timeout=180000)
@@ -446,6 +451,7 @@ public class TestRMRestart {
Token<RMDelegationTokenIdentifier> token1 =
new Token<RMDelegationTokenIdentifier>(dtId1,
rm1.getRMDTSecretManager());
+ SecurityUtil.setTokenService(token1, rmAddr);
ts.addToken(userText1, token1);
tokenSet.add(token1);
@@ -456,6 +462,7 @@ public class TestRMRestart {
Token<RMDelegationTokenIdentifier> token2 =
new Token<RMDelegationTokenIdentifier>(dtId2,
rm1.getRMDTSecretManager());
+ SecurityUtil.setTokenService(token2, rmAddr);
ts.addToken(userText2, token2);
tokenSet.add(token2);
@@ -534,16 +541,21 @@ public class TestRMRestart {
Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
attemptState.getMasterContainer().getId());
- // the appToken and clientToAMToken that are generated when RMAppAttempt
- // is created,
+ // the appToken and clientTokenMasterKey that are generated when
+ // RMAppAttempt is created,
HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
tokenSet.add(attempt1.getAMRMToken());
- tokenSet.add(attempt1.getClientToAMToken());
+ byte[] clientTokenMasterKey =
+ attempt1.getClientTokenMasterKey().getEncoded();
- // assert application Token is saved
+ // assert application credentials are saved
+ Credentials savedCredentials = attemptState.getAppAttemptCredentials();
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(savedCredentials.getAllTokens());
Assert.assertEquals(tokenSet, savedTokens);
+ Assert.assertArrayEquals("client token master key not saved",
+ clientTokenMasterKey, savedCredentials.getSecretKey(
+ RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
// start new RM
MockRM rm2 = new TestSecurityMockRM(conf, memStore);
@@ -557,13 +569,18 @@ public class TestRMRestart {
Assert.assertNotNull(loadedAttempt1);
savedTokens.clear();
savedTokens.add(loadedAttempt1.getAMRMToken());
- savedTokens.add(loadedAttempt1.getClientToAMToken());
Assert.assertEquals(tokenSet, savedTokens);
- // assert clientToAMToken is recovered back to api-versioned
- // clientToAMToken
- Assert.assertEquals(attempt1.getClientToAMToken(),
- loadedAttempt1.getClientToAMToken());
+ // assert client token master key is recovered back to api-versioned
+ // client token master key
+ Assert.assertEquals("client token master key not restored",
+ attempt1.getClientTokenMasterKey(),
+ loadedAttempt1.getClientTokenMasterKey());
+
+ // assert secret manager also knows about the key
+ Assert.assertArrayEquals(clientTokenMasterKey,
+ rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
+ .getEncoded());
// Not testing ApplicationTokenSecretManager has the password populated back,
// that is needed in work-preserving restart
@@ -575,6 +592,12 @@ public class TestRMRestart {
@Test
public void testRMDelegationTokenRestoredOnRMRestart() throws Exception {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+
+ conf.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
+ UserGroupInformation.setConfiguration(conf);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
RMState rmState = memStore.getState();
@@ -587,20 +610,23 @@ public class TestRMRestart {
rmState.getRMDTSecretManagerState().getMasterKeyState();
MockRM rm1 = new TestSecurityMockRM(conf, memStore);
+
rm1.start();
// create an empty credential
Credentials ts = new Credentials();
// request a token and add into credential
- GetDelegationTokenRequest request1 = mock(GetDelegationTokenRequest.class);
- when(request1.getRenewer()).thenReturn("renewer1");
+ GetDelegationTokenRequest request1 =
+ GetDelegationTokenRequest.newInstance("renewer1");
+ UserGroupInformation.getCurrentUser().setAuthenticationMethod(
+ AuthMethod.KERBEROS);
GetDelegationTokenResponse response1 =
rm1.getClientRMService().getDelegationToken(request1);
org.apache.hadoop.yarn.api.records.Token delegationToken1 =
response1.getRMDelegationToken();
Token<RMDelegationTokenIdentifier> token1 =
- ConverterUtils.convertFromYarn(delegationToken1, null);
+ ConverterUtils.convertFromYarn(delegationToken1, rmAddr);
RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
HashSet<RMDelegationTokenIdentifier> tokenIdentSet =
@@ -625,21 +651,21 @@ public class TestRMRestart {
rm1.getRMDTSecretManager().getAllTokens();
Assert.assertEquals(tokenIdentSet, allTokensRM1.keySet());
Assert.assertEquals(allTokensRM1, rmDTState);
-
+
// assert sequence number is saved
Assert.assertEquals(
rm1.getRMDTSecretManager().getLatestDTSequenceNumber(),
rmState.getRMDTSecretManagerState().getDTSequenceNumber());
// request one more token
- GetDelegationTokenRequest request2 = mock(GetDelegationTokenRequest.class);
- when(request2.getRenewer()).thenReturn("renewer2");
+ GetDelegationTokenRequest request2 =
+ GetDelegationTokenRequest.newInstance("renewer2");
GetDelegationTokenResponse response2 =
rm1.getClientRMService().getDelegationToken(request2);
org.apache.hadoop.yarn.api.records.Token delegationToken2 =
response2.getRMDelegationToken();
Token<RMDelegationTokenIdentifier> token2 =
- ConverterUtils.convertFromYarn(delegationToken2, null);
+ ConverterUtils.convertFromYarn(delegationToken2, rmAddr);
RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier();
// cancel token2
@@ -663,7 +689,7 @@ public class TestRMRestart {
// assert master keys and tokens are populated back to DTSecretManager
Map<RMDelegationTokenIdentifier, Long> allTokensRM2 =
rm2.getRMDTSecretManager().getAllTokens();
- Assert.assertEquals(allTokensRM1, allTokensRM2);
+ Assert.assertEquals(allTokensRM2.keySet(), allTokensRM1.keySet());
// rm2 has its own master keys when it starts, we use containsAll here
Assert.assertTrue(rm2.getRMDTSecretManager().getAllMasterKeys()
.containsAll(allKeysRM1));
@@ -716,25 +742,24 @@ public class TestRMRestart {
}
@Override
- protected void doSecureLogin() throws IOException {
- // Do nothing.
- }
-
- @Override
- protected DelegationTokenRenewer createDelegationTokenRenewer() {
- return new DelegationTokenRenewer() {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(getRMContext(), getResourceScheduler(),
+ rmAppManager, applicationACLsManager, rmDTSecretManager){
@Override
- protected void renewToken(final DelegationTokenToRenew dttr)
- throws IOException {
- // Do nothing
+ protected void serviceStart() throws Exception {
+ // do nothing
}
@Override
- protected void setTimerForTokenRenewal(DelegationTokenToRenew token)
- throws IOException {
- // Do nothing
+ protected void serviceStop() throws Exception {
+ //do nothing
}
};
}
+
+ @Override
+ protected void doSecureLogin() throws IOException {
+ // Do nothing.
+ }
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java Thu Sep 19 23:42:10 2013
@@ -203,4 +203,21 @@ public class TestResourceManager {
}
}
+ @Test
+ public void testNMExpiryAndHeartbeatIntervalsValidation() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1000);
+ conf.setLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1001);
+ resourceManager = new ResourceManager();;
+ try {
+ resourceManager.init(conf);
+ } catch (YarnRuntimeException e) {
+ // Exception is expected.
+ if (!e.getMessage().startsWith("Nodemanager expiry interval should be no"
+ + " less than heartbeat interval")) {
+ throw e;
+ }
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Thu Sep 19 23:42:10 2013
@@ -115,7 +115,8 @@ public abstract class MockAsm extends Mo
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName,boolean allowAccess) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Thu Sep 19 23:42:10 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -25,12 +26,12 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
+import javax.crypto.SecretKey;
+
import junit.framework.Assert;
import org.apache.commons.logging.Log;
@@ -55,7 +56,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
@@ -129,7 +129,10 @@ public class TestRMStateStore {
class TestFileSystemRMStore extends FileSystemRMStateStore {
TestFileSystemRMStore(Configuration conf) throws Exception {
init(conf);
+ Assert.assertNull(fs);
assertTrue(workingDirPathURI.equals(fsWorkingPath));
+ start();
+ Assert.assertNotNull(fs);
}
}
@@ -195,7 +198,7 @@ public class TestRMStateStore {
ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
String containerIdStr, Token<AMRMTokenIdentifier> appToken,
- Token<ClientToAMTokenIdentifier> clientToAMToken, TestDispatcher dispatcher)
+ SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
throws Exception {
Container container = new ContainerPBImpl();
@@ -204,7 +207,8 @@ public class TestRMStateStore {
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
when(mockAttempt.getMasterContainer()).thenReturn(container);
when(mockAttempt.getAMRMToken()).thenReturn(appToken);
- when(mockAttempt.getClientToAMToken()).thenReturn(clientToAMToken);
+ when(mockAttempt.getClientTokenMasterKey())
+ .thenReturn(clientTokenMasterKey);
dispatcher.attemptId = attemptId;
dispatcher.storedException = null;
store.storeApplicationAttempt(mockAttempt);
@@ -212,13 +216,12 @@ public class TestRMStateStore {
return container.getId();
}
- @SuppressWarnings("unchecked")
void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
long submitTime = System.currentTimeMillis();
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.getRMStateStore();
TestDispatcher dispatcher = new TestDispatcher();
- store.setDispatcher(dispatcher);
+ store.setRMDispatcher(dispatcher);
AMRMTokenSecretManager appTokenMgr =
new AMRMTokenSecretManager(conf);
@@ -230,33 +233,33 @@ public class TestRMStateStore {
ApplicationId appId1 = attemptId1.getApplicationId();
storeApp(store, appId1, submitTime);
- // create application token1 for attempt1
- List<Token<?>> appAttemptToken1 =
- generateTokens(attemptId1, appTokenMgr, clientToAMTokenMgr, conf);
+ // create application token and client token key for attempt1
+ Token<AMRMTokenIdentifier> appAttemptToken1 =
+ generateAMRMToken(attemptId1, appTokenMgr);
HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
- attemptTokenSet1.addAll(appAttemptToken1);
+ attemptTokenSet1.add(appAttemptToken1);
+ SecretKey clientTokenKey1 =
+ clientToAMTokenMgr.registerApplication(attemptId1);
ContainerId containerId1 = storeAttempt(store, attemptId1,
"container_1352994193343_0001_01_000001",
- (Token<AMRMTokenIdentifier>) (appAttemptToken1.get(0)),
- (Token<ClientToAMTokenIdentifier>)(appAttemptToken1.get(1)),
- dispatcher);
+ appAttemptToken1, clientTokenKey1, dispatcher);
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 =
ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
- // create application token2 for attempt2
- List<Token<?>> appAttemptToken2 =
- generateTokens(attemptId2, appTokenMgr, clientToAMTokenMgr, conf);
+ // create application token and client token key for attempt2
+ Token<AMRMTokenIdentifier> appAttemptToken2 =
+ generateAMRMToken(attemptId2, appTokenMgr);
HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
- attemptTokenSet2.addAll(appAttemptToken2);
+ attemptTokenSet2.add(appAttemptToken2);
+ SecretKey clientTokenKey2 =
+ clientToAMTokenMgr.registerApplication(attemptId2);
ContainerId containerId2 = storeAttempt(store, attemptId2,
"container_1352994193343_0001_02_000001",
- (Token<AMRMTokenIdentifier>) (appAttemptToken2.get(0)),
- (Token<ClientToAMTokenIdentifier>)(appAttemptToken2.get(1)),
- dispatcher);
+ appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
@@ -303,8 +306,12 @@ public class TestRMStateStore {
assertEquals(containerId1, attemptState.getMasterContainer().getId());
// attempt1 applicationToken is loaded correctly
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
assertEquals(attemptTokenSet1, savedTokens);
+ // attempt1 client token master key is loaded correctly
+ assertArrayEquals(clientTokenKey1.getEncoded(),
+ attemptState.getAppAttemptCredentials()
+ .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
attemptState = appState.getAttempt(attemptId2);
// attempt2 is loaded correctly
@@ -314,8 +321,12 @@ public class TestRMStateStore {
assertEquals(containerId2, attemptState.getMasterContainer().getId());
// attempt2 applicationToken is loaded correctly
savedTokens.clear();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
assertEquals(attemptTokenSet2, savedTokens);
+ // attempt2 client token master key is loaded correctly
+ assertArrayEquals(clientTokenKey2.getEncoded(),
+ attemptState.getAppAttemptCredentials()
+ .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
// assert store is in expected state after everything is cleaned
assertTrue(stateStoreHelper.isFinalStateValid());
@@ -327,7 +338,7 @@ public class TestRMStateStore {
RMStateStoreHelper stateStoreHelper) throws Exception {
RMStateStore store = stateStoreHelper.getRMStateStore();
TestDispatcher dispatcher = new TestDispatcher();
- store.setDispatcher(dispatcher);
+ store.setRMDispatcher(dispatcher);
// store RM delegation token;
RMDelegationTokenIdentifier dtId1 =
@@ -354,24 +365,14 @@ public class TestRMStateStore {
Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber());
}
- private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
- AMRMTokenSecretManager appTokenMgr,
- ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, Configuration conf) {
+ private Token<AMRMTokenIdentifier> generateAMRMToken(
+ ApplicationAttemptId attemptId,
+ AMRMTokenSecretManager appTokenMgr) {
AMRMTokenIdentifier appTokenId =
new AMRMTokenIdentifier(attemptId);
Token<AMRMTokenIdentifier> appToken =
new Token<AMRMTokenIdentifier>(appTokenId, appTokenMgr);
appToken.setService(new Text("appToken service"));
-
- ClientToAMTokenIdentifier clientToAMTokenId =
- new ClientToAMTokenIdentifier(attemptId);
- clientToAMTokenMgr.registerApplication(attemptId);
- Token<ClientToAMTokenIdentifier> clientToAMToken =
- new Token<ClientToAMTokenIdentifier>(clientToAMTokenId, clientToAMTokenMgr);
- clientToAMToken.setService(new Text("clientToAMToken service"));
- List<Token<?>> tokenPair = new ArrayList<Token<?>>();
- tokenPair.add(0, appToken);
- tokenPair.add(1, clientToAMToken);
- return tokenPair;
+ return appToken;
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Thu Sep 19 23:42:10 2013
@@ -143,7 +143,8 @@ public class MockRMApp implements RMApp
}
@Override
- public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Thu Sep 19 23:42:10 2013
@@ -19,14 +19,20 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import static org.mockito.Mockito.mock;
+import static org.junit.Assume.assumeTrue;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -57,11 +63,16 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+@RunWith(value = Parameterized.class)
public class TestRMAppTransitions {
static final Log LOG = LogFactory.getLog(TestRMAppTransitions.class);
+ private boolean isSecurityEnabled;
+ private Configuration conf;
private RMContext rmContext;
private static int maxAppAttempts =
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS;
@@ -132,10 +143,29 @@ public class TestRMAppTransitions {
public void handle(SchedulerEvent event) {
}
}
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> getTestParameters() {
+ return Arrays.asList(new Object[][] {
+ { Boolean.FALSE },
+ { Boolean.TRUE }
+ });
+ }
+
+ public TestRMAppTransitions(boolean isSecurityEnabled) {
+ this.isSecurityEnabled = isSecurityEnabled;
+ }
@Before
public void setUp() throws Exception {
- Configuration conf = new Configuration();
+ conf = new YarnConfiguration();
+ AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
+ if (isSecurityEnabled) {
+ authMethod = AuthenticationMethod.KERBEROS;
+ }
+ SecurityUtil.setAuthenticationMethod(authMethod, conf);
+ UserGroupInformation.setConfiguration(conf);
+
rmDispatcher = new DrainDispatcher();
ContainerAllocationExpirer containerAllocationExpirer =
mock(ContainerAllocationExpirer.class);
@@ -171,7 +201,6 @@ public class TestRMAppTransitions {
String user = MockApps.newUserName();
String name = MockApps.newAppName();
String queue = MockApps.newQueue();
- Configuration conf = new YarnConfiguration();
// ensure max application attempts set to known value
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts);
YarnScheduler scheduler = mock(YarnScheduler.class);
@@ -191,6 +220,8 @@ public class TestRMAppTransitions {
System.currentTimeMillis(), "YARN");
testAppStartState(applicationId, user, name, queue, application);
+ this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
+ application);
return application;
}
@@ -488,8 +519,6 @@ public class TestRMAppTransitions {
// SUBMITTED => KILLED event RMAppEventType.KILL
RMAppEvent event = new RMAppEvent(application.getApplicationId(),
RMAppEventType.KILL);
- this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
- application);
application.handle(event);
rmDispatcher.await();
assertKilled(application);
@@ -535,8 +564,6 @@ public class TestRMAppTransitions {
// ACCEPTED => KILLED event RMAppEventType.KILL
RMAppEvent event = new RMAppEvent(application.getApplicationId(),
RMAppEventType.KILL);
- this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
- application);
application.handle(event);
rmDispatcher.await();
assertKilled(application);
@@ -726,7 +753,38 @@ public class TestRMAppTransitions {
public void testGetAppReport() {
RMApp app = createNewTestApp(null);
assertAppState(RMAppState.NEW, app);
- ApplicationReport report = app.createAndGetApplicationReport(true);
+ ApplicationReport report = app.createAndGetApplicationReport(null, true);
+ Assert.assertNotNull(report.getApplicationResourceUsageReport());
+ report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
}
+
+ @Test
+ public void testClientTokens() throws Exception {
+ assumeTrue(isSecurityEnabled);
+
+ RMApp app = createNewTestApp(null);
+ assertAppState(RMAppState.NEW, app);
+ ApplicationReport report = app.createAndGetApplicationReport(null, true);
+ Assert.assertNull(report.getClientToAMToken());
+ report = app.createAndGetApplicationReport("clientuser", true);
+ Assert.assertNull(report.getClientToAMToken());
+
+ app = testCreateAppRunning(null);
+ rmDispatcher.await();
+ assertAppState(RMAppState.RUNNING, app);
+ report = app.createAndGetApplicationReport(null, true);
+ Assert.assertNull(report.getClientToAMToken());
+ report = app.createAndGetApplicationReport("clientuser", true);
+ Assert.assertNotNull(report.getClientToAMToken());
+
+ // kill the app attempt and verify client token is unavailable
+ app.handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL));
+ rmDispatcher.await();
+ assertAppAndAttemptKilled(app);
+ report = app.createAndGetApplicationReport(null, true);
+ Assert.assertNull(report.getClientToAMToken());
+ report = app.createAndGetApplicationReport("clientuser", true);
+ Assert.assertNull(report.getClientToAMToken());
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Thu Sep 19 23:42:10 2013
@@ -28,13 +28,19 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -83,7 +89,10 @@ import org.apache.hadoop.yarn.server.res
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+@RunWith(value = Parameterized.class)
public class TestRMAppAttemptTransitions {
private static final Log LOG =
@@ -93,6 +102,7 @@ public class TestRMAppAttemptTransitions
private static final String RM_WEBAPP_ADDR =
YarnConfiguration.getRMWebAppHostAndPort(new Configuration());
+ private boolean isSecurityEnabled;
private RMContext rmContext;
private YarnScheduler scheduler;
private ApplicationMasterService masterService;
@@ -102,6 +112,11 @@ public class TestRMAppAttemptTransitions
private RMApp application;
private RMAppAttempt applicationAttempt;
+
+ private Configuration conf = new Configuration();
+ private AMRMTokenSecretManager amRMTokenManager = spy(new AMRMTokenSecretManager(conf));
+ private ClientToAMTokenSecretManagerInRM clientToAMTokenManager =
+ spy(new ClientToAMTokenSecretManagerInRM());
private final class TestApplicationAttemptEventDispatcher implements
EventHandler<RMAppAttemptEvent> {
@@ -155,22 +170,39 @@ public class TestRMAppAttemptTransitions
private ApplicationSubmissionContext submissionContext = null;
private boolean unmanagedAM;
+ @Parameterized.Parameters
+ public static Collection<Object[]> getTestParameters() {
+ return Arrays.asList(new Object[][] {
+ { Boolean.FALSE },
+ { Boolean.TRUE }
+ });
+ }
+
+ public TestRMAppAttemptTransitions(Boolean isSecurityEnabled) {
+ this.isSecurityEnabled = isSecurityEnabled;
+ }
+
@Before
public void setUp() throws Exception {
+ AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
+ if (isSecurityEnabled) {
+ authMethod = AuthenticationMethod.KERBEROS;
+ }
+ SecurityUtil.setAuthenticationMethod(authMethod, conf);
+ UserGroupInformation.setConfiguration(conf);
InlineDispatcher rmDispatcher = new InlineDispatcher();
ContainerAllocationExpirer containerAllocationExpirer =
mock(ContainerAllocationExpirer.class);
amLivelinessMonitor = mock(AMLivelinessMonitor.class);
amFinishingMonitor = mock(AMLivelinessMonitor.class);
- Configuration conf = new Configuration();
rmContext =
new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, new AMRMTokenSecretManager(conf),
+ null, amRMTokenManager,
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM());
+ clientToAMTokenManager);
RMStateStore store = mock(RMStateStore.class);
((RMContextImpl) rmContext).setStateStore(store);
@@ -261,7 +293,13 @@ public class TestRMAppAttemptTransitions
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
-
+ if (UserGroupInformation.isSecurityEnabled()) {
+ verify(clientToAMTokenManager).registerApplication(
+ applicationAttempt.getAppAttemptId());
+ assertNotNull(applicationAttempt.createClientToken("some client"));
+ }
+ assertNull(applicationAttempt.createClientToken(null));
+ assertNotNull(applicationAttempt.getAMRMToken());
// Check events
verify(masterService).
registerAppAttempt(applicationAttempt.getAppAttemptId());
@@ -288,6 +326,7 @@ public class TestRMAppAttemptTransitions
// this works for unmanaged and managed AM's because this is actually doing
// verify(application).handle(anyObject());
verify(application).handle(any(RMAppRejectedEvent.class));
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
/**
@@ -303,6 +342,7 @@ public class TestRMAppAttemptTransitions
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
/**
@@ -377,6 +417,8 @@ public class TestRMAppAttemptTransitions
// Check events
verify(application, times(2)).handle(any(RMAppFailedAttemptEvent.class));
+
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
/**
@@ -394,16 +436,19 @@ public class TestRMAppAttemptTransitions
* {@link RMAppAttemptState#RUNNING}
*/
private void testAppAttemptRunningState(Container container,
- String host, int rpcPort, String trackingUrl) {
+ String host, int rpcPort, String trackingUrl, boolean unmanagedAM) {
assertEquals(RMAppAttemptState.RUNNING,
applicationAttempt.getAppAttemptState());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(host, applicationAttempt.getHost());
assertEquals(rpcPort, applicationAttempt.getRpcPort());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
- assertEquals(getProxyUrl(applicationAttempt),
- applicationAttempt.getTrackingUrl());
-
+ if (unmanagedAM) {
+ assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl());
+ } else {
+ assertEquals(getProxyUrl(applicationAttempt),
+ applicationAttempt.getTrackingUrl());
+ }
// TODO - need to add more checks relevant to this state
}
@@ -422,6 +467,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getTrackingUrl());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 0);
}
/**
@@ -431,17 +477,23 @@ public class TestRMAppAttemptTransitions
FinalApplicationStatus finalStatus,
String trackingUrl,
String diagnostics,
- int finishedContainerCount) {
+ int finishedContainerCount, boolean unmanagedAM) {
assertEquals(RMAppAttemptState.FINISHED,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
- assertEquals(getProxyUrl(applicationAttempt),
- applicationAttempt.getTrackingUrl());
+ if (unmanagedAM) {
+ assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl());
+
+ } else {
+ assertEquals(getProxyUrl(applicationAttempt),
+ applicationAttempt.getTrackingUrl());
+ }
assertEquals(finishedContainerCount, applicationAttempt
.getJustFinishedContainers().size());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@@ -519,13 +571,14 @@ public class TestRMAppAttemptTransitions
private void runApplicationAttempt(Container container,
String host,
int rpcPort,
- String trackingUrl) {
+ String trackingUrl, boolean unmanagedAM) {
applicationAttempt.handle(
new RMAppAttemptRegistrationEvent(
applicationAttempt.getAppAttemptId(),
host, rpcPort, trackingUrl));
- testAppAttemptRunningState(container, host, rpcPort, trackingUrl);
+ testAppAttemptRunningState(container, host, rpcPort, trackingUrl,
+ unmanagedAM);
}
private void unregisterApplicationAttempt(Container container,
@@ -551,7 +604,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId());
// launch AM
- runApplicationAttempt(null, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true);
// complete a container
applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent(
@@ -565,7 +618,8 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
diagnostics));
- testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1);
+ testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1,
+ true);
}
@Test
@@ -592,6 +646,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.KILL));
testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS);
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test
@@ -666,13 +721,14 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(), cs));
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test
public void testRunningToFailed() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
String containerDiagMsg = "some error";
int exitCode = 123;
ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(),
@@ -695,7 +751,7 @@ public class TestRMAppAttemptTransitions
public void testRunningToKilled() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
applicationAttempt.handle(
new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(),
@@ -709,6 +765,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId().getApplicationId());
assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test(timeout=10000)
@@ -725,13 +782,14 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId().getApplicationId());
assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test(timeout=20000)
public void testRunningExpire() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
applicationAttempt.handle(new RMAppAttemptEvent(
applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
assertEquals(RMAppAttemptState.FAILED,
@@ -742,13 +800,14 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId().getApplicationId());
assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test
public void testUnregisterToKilledFinishing() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
unregisterApplicationAttempt(amContainer,
FinalApplicationStatus.KILLED, "newtrackingurl",
"Killed by user");
@@ -759,14 +818,14 @@ public class TestRMAppAttemptTransitions
public void testNoTrackingUrl() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "");
+ runApplicationAttempt(amContainer, "host", 8042, "", false);
}
@Test
public void testUnregisterToSuccessfulFinishing() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
unregisterApplicationAttempt(amContainer,
FinalApplicationStatus.SUCCEEDED, "mytrackingurl", "Successful");
}
@@ -775,7 +834,7 @@ public class TestRMAppAttemptTransitions
public void testFinishingKill() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.FAILED;
String trackingUrl = "newtrackingurl";
String diagnostics = "Job failed";
@@ -793,7 +852,7 @@ public class TestRMAppAttemptTransitions
public void testFinishingExpire() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
@@ -804,14 +863,14 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.EXPIRE));
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
- diagnostics, 0);
+ diagnostics, 0, false);
}
@Test
public void testFinishingToFinishing() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
@@ -833,7 +892,7 @@ public class TestRMAppAttemptTransitions
public void testSuccessfulFinishingToFinished() {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
@@ -845,7 +904,16 @@ public class TestRMAppAttemptTransitions
BuilderUtils.newContainerStatus(amContainer.getId(),
ContainerState.COMPLETE, "", 0)));
testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
- diagnostics, 0);
+ diagnostics, 0, false);
}
+ private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
+ verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ verify(clientToAMTokenManager, times(count)).unRegisterApplication(appAttemptId);
+ if (count > 0) {
+ assertNull(applicationAttempt.createClientToken("client"));
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java Thu Sep 19 23:42:10 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
@@ -40,6 +41,10 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
@@ -351,4 +356,26 @@ public class TestSchedulerUtils {
RMAppAttemptState.LAUNCHED);
}
+ @Test
+ public void testComparePriorities(){
+ Priority high = Priority.newInstance(1);
+ Priority low = Priority.newInstance(2);
+ assertTrue(high.compareTo(low) > 0);
+ }
+
+ @Test
+ public void testCreateAbnormalContainerStatus() {
+ ContainerStatus cd = SchedulerUtils.createAbnormalContainerStatus(
+ ContainerId.newInstance(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x");
+ Assert.assertEquals(ContainerExitStatus.ABORTED, cd.getExitStatus());
+ }
+
+ @Test
+ public void testCreatePreemptedContainerStatus() {
+ ContainerStatus cd = SchedulerUtils.createPreemptedContainerStatus(
+ ContainerId.newInstance(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1), "x");
+ Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus());
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Thu Sep 19 23:42:10 2013
@@ -19,22 +19,27 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.lang.reflect.Constructor;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -44,19 +49,24 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
@@ -525,4 +535,63 @@ public class TestCapacityScheduler {
assertTrue(appComparator.compare(app2, app3) < 0);
}
+ @Test
+ public void testConcurrentAccessOnApplications() throws Exception {
+ CapacityScheduler cs = new CapacityScheduler();
+ verifyConcurrentAccessOnApplications(
+ cs.applications, FiCaSchedulerApp.class);
+ }
+
+ public static <T extends SchedulerApplication>
+ void verifyConcurrentAccessOnApplications(
+ final Map<ApplicationAttemptId, T> applications, Class<T> clazz)
+ throws Exception {
+ final int size = 10000;
+ final ApplicationId appId = ApplicationId.newInstance(0, 0);
+ final Constructor<T> ctor = clazz.getDeclaredConstructor(
+ ApplicationAttemptId.class, String.class, Queue.class,
+ ActiveUsersManager.class, RMContext.class);
+
+ ApplicationAttemptId appAttemptId0
+ = ApplicationAttemptId.newInstance(appId, 0);
+ applications.put(appAttemptId0, ctor.newInstance(
+ appAttemptId0, null, mock(Queue.class), null, null));
+ assertNotNull(applications.get(appAttemptId0));
+
+ // Imitating the thread of scheduler that will add and remove apps
+ final AtomicBoolean finished = new AtomicBoolean(false);
+ final AtomicBoolean failed = new AtomicBoolean(false);
+ Thread t = new Thread() {
+
+ @Override
+ public void run() {
+ for (int i = 1; i <= size; ++i) {
+ ApplicationAttemptId appAttemptId
+ = ApplicationAttemptId.newInstance(appId, i);
+ try {
+ applications.put(appAttemptId, ctor.newInstance(
+ appAttemptId, null, mock(Queue.class), null, null));
+ } catch (Exception e) {
+ failed.set(true);
+ finished.set(true);
+ return;
+ }
+ }
+ for (int i = 1; i <= size; ++i) {
+ ApplicationAttemptId appAttemptId
+ = ApplicationAttemptId.newInstance(appId, i);
+ applications.remove(appAttemptId);
+ }
+ finished.set(true);
+ }
+ };
+ t.start();
+
+ // Imitating the thread of rmappattempt that will get the app
+ while (!finished.get()) {
+ assertNotNull(applications.get(appAttemptId0));
+ }
+ assertFalse(failed.get());
+ }
+
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java Thu Sep 19 23:42:10 2013
@@ -26,7 +26,6 @@ import static org.mockito.Mockito.doNoth
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.util.HashMap;
@@ -126,7 +125,7 @@ public class TestChildQueueOrder {
throw new Exception();
} catch (Exception e) {
LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() +
- " alloc=" + allocation + " node=" + node.getHostName());
+ " alloc=" + allocation + " node=" + node.getNodeName());
}
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Thu Sep 19 23:42:10 2013
@@ -138,7 +138,7 @@ public class TestParentQueue {
throw new Exception();
} catch (Exception e) {
LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() +
- " alloc=" + allocation + " node=" + node.getHostName());
+ " alloc=" + allocation + " node=" + node.getNodeName());
}
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Thu Sep 19 23:42:10 2013
@@ -160,7 +160,7 @@ public class TestUtils {
when(rmNode.getHostName()).thenReturn(host);
when(rmNode.getRackName()).thenReturn(rack);
- FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode));
+ FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
LOG.info("node = " + host + " avail=" + node.getAvailableResource());
return node;
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu Sep 19 23:42:10 2013
@@ -39,12 +39,10 @@ import javax.xml.parsers.ParserConfigura
import junit.framework.Assert;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -57,11 +55,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -74,6 +72,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
@@ -81,8 +80,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
-import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -132,7 +130,12 @@ public class TestFairScheduler {
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
resourceManager = new ResourceManager();
resourceManager.init(conf);
+
+ // TODO: This test should really be using MockRM. For now starting stuff
+ // that is needed at a bare minimum.
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
+ resourceManager.getRMContext().getStateStore().start();
+
scheduler.reinitialize(conf, resourceManager.getRMContext());
// to initialize the master key
resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
@@ -2145,4 +2148,61 @@ public class TestFairScheduler {
Assert.assertEquals(2, app3.getLiveContainers().size());
Assert.assertEquals(2, app4.getLiveContainers().size());
}
+
+ @Test(timeout = 30000)
+ public void testHostPortNodeName() throws Exception {
+ scheduler.getConf().setBoolean(YarnConfiguration
+ .RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
+ scheduler.reinitialize(scheduler.getConf(),
+ resourceManager.getRMContext());
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024),
+ 1, "127.0.0.1", 1);
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024),
+ 2, "127.0.0.1", 2);
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
+ "user1", 0);
+
+ ResourceRequest nodeRequest = createResourceRequest(1024,
+ node1.getNodeID().getHost() + ":" + node1.getNodeID().getPort(), 1,
+ 1, true);
+ ResourceRequest rackRequest = createResourceRequest(1024,
+ node1.getRackName(), 1, 1, false);
+ ResourceRequest anyRequest = createResourceRequest(1024,
+ ResourceRequest.ANY, 1, 1, false);
+ createSchedulingRequestExistingApplication(nodeRequest, attId1);
+ createSchedulingRequestExistingApplication(rackRequest, attId1);
+ createSchedulingRequestExistingApplication(anyRequest, attId1);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent node1UpdateEvent = new
+ NodeUpdateSchedulerEvent(node1);
+ NodeUpdateSchedulerEvent node2UpdateEvent = new
+ NodeUpdateSchedulerEvent(node2);
+
+ // no matter how many heartbeats, node2 should never get a container
+ FSSchedulerApp app = scheduler.applications.get(attId1);
+ for (int i = 0; i < 10; i++) {
+ scheduler.handle(node2UpdateEvent);
+ assertEquals(0, app.getLiveContainers().size());
+ assertEquals(0, app.getReservedContainers().size());
+ }
+ // then node1 should get the container
+ scheduler.handle(node1UpdateEvent);
+ assertEquals(1, app.getLiveContainers().size());
+ }
+
+ @Test
+ public void testConcurrentAccessOnApplications() throws Exception {
+ FairScheduler fs = new FairScheduler();
+ TestCapacityScheduler.verifyConcurrentAccessOnApplications(
+ fs.applications, FSSchedulerApp.class);
+ }
+
}