You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/09/03 20:30:22 UTC
svn commit: r1519787 [6/6] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apach...
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Tue Sep 3 18:30:05 2013
@@ -200,15 +200,14 @@ public class MockNodes {
};
private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) {
- return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null);
+ return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123);
}
private static RMNode buildRMNode(int rack, final Resource perNode,
- NodeState state, String httpAddr, int hostnum, String hostName) {
+ NodeState state, String httpAddr, int hostnum, String hostName, int port) {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String nodeAddr = hostName + ":" + nid;
- final int port = 123;
if (hostName == null) {
hostName = "host"+ nid;
}
@@ -230,12 +229,17 @@ public class MockNodes {
}
public static RMNode newNodeInfo(int rack, final Resource perNode, int hostnum) {
- return buildRMNode(rack, perNode, null, "localhost:0", hostnum, null);
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, null, 123);
}
public static RMNode newNodeInfo(int rack, final Resource perNode,
int hostnum, String hostName) {
- return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName);
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName, 123);
+ }
+
+ public static RMNode newNodeInfo(int rack, final Resource perNode,
+ int hostnum, String hostName, int port) {
+ return buildRMNode(rack, perNode, null, "localhost:0", hostnum, hostName, port);
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Tue Sep 3 18:30:05 2013
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
@@ -387,6 +388,10 @@ public class MockRM extends ResourceMana
return this.rmDTSecretManager;
}
+ public ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager() {
+ return this.clientToAMSecretManager;
+ }
+
@Override
protected void startWepApp() {
// override to disable webapp
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Tue Sep 3 18:30:05 2013
@@ -101,7 +101,7 @@ public class NodeManager implements Cont
request.setNodeId(this.nodeId);
resourceTrackerService.registerNodeManager(request);
this.schedulerNode = new FiCaSchedulerNode(rmContext.getRMNodes().get(
- this.nodeId));
+ this.nodeId), false);
// Sanity check
Assert.assertEquals(capability.getMemory(),
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Tue Sep 3 18:30:05 2013
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -198,6 +199,27 @@ public class TestClientRMService {
}
@Test
+ public void testForceKillApplication() throws YarnException {
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getRMApps()).thenReturn(
+ new ConcurrentHashMap<ApplicationId, RMApp>());
+ ClientRMService rmService = new ClientRMService(rmContext, null, null,
+ null, null);
+ ApplicationId applicationId =
+ BuilderUtils.newApplicationId(System.currentTimeMillis(), 0);
+ KillApplicationRequest request =
+ KillApplicationRequest.newInstance(applicationId);
+ try {
+ rmService.forceKillApplication(request);
+ Assert.fail();
+ } catch (ApplicationNotFoundException ex) {
+ Assert.assertEquals(ex.getMessage(),
+ "Trying to kill an absent " +
+ "application " + request.getApplicationId());
+ }
+ }
+
+ @Test
public void testGetQueueInfo() throws Exception {
YarnScheduler yarnScheduler = mock(YarnScheduler.class);
RMContext rmContext = mock(RMContext.class);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java Tue Sep 3 18:30:05 2013
@@ -267,7 +267,21 @@ public class TestRMNodeTransitions {
@Test
public void testUnhealthyExpire() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost + 1, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.LOST, node.getState());
}
@@ -291,8 +305,22 @@ public class TestRMNodeTransitions {
@Test
public void testUnhealthyDecommission() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.DECOMMISSION));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned + 1, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}
@@ -307,8 +335,22 @@ public class TestRMNodeTransitions {
@Test
public void testUnhealthyRebooting() {
RMNodeImpl node = getUnhealthyNode();
+ ClusterMetrics cm = ClusterMetrics.getMetrics();
+ int initialActive = cm.getNumActiveNMs();
+ int initialLost = cm.getNumLostNMs();
+ int initialUnhealthy = cm.getUnhealthyNMs();
+ int initialDecommissioned = cm.getNumDecommisionedNMs();
+ int initialRebooted = cm.getNumRebootedNMs();
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.REBOOTING));
+ Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
+ Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
+ Assert.assertEquals("Unhealthy Nodes",
+ initialUnhealthy - 1, cm.getUnhealthyNMs());
+ Assert.assertEquals("Decommissioned Nodes",
+ initialDecommissioned, cm.getNumDecommisionedNMs());
+ Assert.assertEquals("Rebooted Nodes",
+ initialRebooted + 1, cm.getNumRebootedNMs());
Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Tue Sep 3 18:30:05 2013
@@ -18,10 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
@@ -63,7 +64,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level;
@@ -77,8 +77,11 @@ public class TestRMRestart {
private YarnConfiguration conf;
+ // Fake rmAddr for token-renewal
+ private static InetSocketAddress rmAddr;
+
@Before
- public void setup() {
+ public void setup() throws UnknownHostException {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
ExitUtil.disableSystemExit();
@@ -86,6 +89,8 @@ public class TestRMRestart {
UserGroupInformation.setConfiguration(conf);
conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+
+ rmAddr = new InetSocketAddress(InetAddress.getLocalHost(), 123);
}
@Test (timeout=180000)
@@ -446,6 +451,7 @@ public class TestRMRestart {
Token<RMDelegationTokenIdentifier> token1 =
new Token<RMDelegationTokenIdentifier>(dtId1,
rm1.getRMDTSecretManager());
+ SecurityUtil.setTokenService(token1, rmAddr);
ts.addToken(userText1, token1);
tokenSet.add(token1);
@@ -456,6 +462,7 @@ public class TestRMRestart {
Token<RMDelegationTokenIdentifier> token2 =
new Token<RMDelegationTokenIdentifier>(dtId2,
rm1.getRMDTSecretManager());
+ SecurityUtil.setTokenService(token2, rmAddr);
ts.addToken(userText2, token2);
tokenSet.add(token2);
@@ -534,16 +541,21 @@ public class TestRMRestart {
Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
attemptState.getMasterContainer().getId());
- // the appToken and clientToAMToken that are generated when RMAppAttempt
- // is created,
+ // the appToken and clientTokenMasterKey that are generated when
+ // RMAppAttempt is created,
HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
tokenSet.add(attempt1.getAMRMToken());
- tokenSet.add(attempt1.getClientToAMToken());
+ byte[] clientTokenMasterKey =
+ attempt1.getClientTokenMasterKey().getEncoded();
- // assert application Token is saved
+ // assert application credentials are saved
+ Credentials savedCredentials = attemptState.getAppAttemptCredentials();
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(savedCredentials.getAllTokens());
Assert.assertEquals(tokenSet, savedTokens);
+ Assert.assertArrayEquals("client token master key not saved",
+ clientTokenMasterKey, savedCredentials.getSecretKey(
+ RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
// start new RM
MockRM rm2 = new TestSecurityMockRM(conf, memStore);
@@ -557,13 +569,18 @@ public class TestRMRestart {
Assert.assertNotNull(loadedAttempt1);
savedTokens.clear();
savedTokens.add(loadedAttempt1.getAMRMToken());
- savedTokens.add(loadedAttempt1.getClientToAMToken());
Assert.assertEquals(tokenSet, savedTokens);
- // assert clientToAMToken is recovered back to api-versioned
- // clientToAMToken
- Assert.assertEquals(attempt1.getClientToAMToken(),
- loadedAttempt1.getClientToAMToken());
+ // assert client token master key is recovered back to api-versioned
+ // client token master key
+ Assert.assertEquals("client token master key not restored",
+ attempt1.getClientTokenMasterKey(),
+ loadedAttempt1.getClientTokenMasterKey());
+
+ // assert secret manager also knows about the key
+ Assert.assertArrayEquals(clientTokenMasterKey,
+ rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
+ .getEncoded());
// Not testing ApplicationTokenSecretManager has the password populated back,
// that is needed in work-preserving restart
@@ -575,6 +592,7 @@ public class TestRMRestart {
@Test
public void testRMDelegationTokenRestoredOnRMRestart() throws Exception {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
RMState rmState = memStore.getState();
@@ -587,20 +605,21 @@ public class TestRMRestart {
rmState.getRMDTSecretManagerState().getMasterKeyState();
MockRM rm1 = new TestSecurityMockRM(conf, memStore);
+
rm1.start();
// create an empty credential
Credentials ts = new Credentials();
// request a token and add into credential
- GetDelegationTokenRequest request1 = mock(GetDelegationTokenRequest.class);
- when(request1.getRenewer()).thenReturn("renewer1");
+ GetDelegationTokenRequest request1 =
+ GetDelegationTokenRequest.newInstance("renewer1");
GetDelegationTokenResponse response1 =
rm1.getClientRMService().getDelegationToken(request1);
org.apache.hadoop.yarn.api.records.Token delegationToken1 =
response1.getRMDelegationToken();
Token<RMDelegationTokenIdentifier> token1 =
- ConverterUtils.convertFromYarn(delegationToken1, null);
+ ConverterUtils.convertFromYarn(delegationToken1, rmAddr);
RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
HashSet<RMDelegationTokenIdentifier> tokenIdentSet =
@@ -632,14 +651,14 @@ public class TestRMRestart {
rmState.getRMDTSecretManagerState().getDTSequenceNumber());
// request one more token
- GetDelegationTokenRequest request2 = mock(GetDelegationTokenRequest.class);
- when(request2.getRenewer()).thenReturn("renewer2");
+ GetDelegationTokenRequest request2 =
+ GetDelegationTokenRequest.newInstance("renewer2");
GetDelegationTokenResponse response2 =
rm1.getClientRMService().getDelegationToken(request2);
org.apache.hadoop.yarn.api.records.Token delegationToken2 =
response2.getRMDelegationToken();
Token<RMDelegationTokenIdentifier> token2 =
- ConverterUtils.convertFromYarn(delegationToken2, null);
+ ConverterUtils.convertFromYarn(delegationToken2, rmAddr);
RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier();
// cancel token2
@@ -721,20 +740,10 @@ public class TestRMRestart {
}
@Override
- protected DelegationTokenRenewer createDelegationTokenRenewer() {
- return new DelegationTokenRenewer() {
- @Override
- protected void renewToken(final DelegationTokenToRenew dttr)
- throws IOException {
- // Do nothing
- }
-
- @Override
- protected void setTimerForTokenRenewal(DelegationTokenToRenew token)
- throws IOException {
- // Do nothing
- }
- };
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ RMDelegationTokenIdentifier.Renewer.setSecretManager(
+ this.getRMDTSecretManager(), rmAddr);
}
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java Tue Sep 3 18:30:05 2013
@@ -203,4 +203,21 @@ public class TestResourceManager {
}
}
+ @Test
+ public void testNMExpiryAndHeartbeatIntervalsValidation() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1000);
+ conf.setLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1001);
+ resourceManager = new ResourceManager();;
+ try {
+ resourceManager.init(conf);
+ } catch (YarnRuntimeException e) {
+ // Exception is expected.
+ if (!e.getMessage().startsWith("Nodemanager expiry interval should be no"
+ + " less than heartbeat interval")) {
+ throw e;
+ }
+ }
+ }
+
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Tue Sep 3 18:30:05 2013
@@ -115,7 +115,8 @@ public abstract class MockAsm extends Mo
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName,boolean allowAccess) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Tue Sep 3 18:30:05 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -25,12 +26,12 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
+import javax.crypto.SecretKey;
+
import junit.framework.Assert;
import org.apache.commons.logging.Log;
@@ -55,7 +56,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
@@ -129,7 +129,10 @@ public class TestRMStateStore {
class TestFileSystemRMStore extends FileSystemRMStateStore {
TestFileSystemRMStore(Configuration conf) throws Exception {
init(conf);
+ Assert.assertNull(fs);
assertTrue(workingDirPathURI.equals(fsWorkingPath));
+ start();
+ Assert.assertNotNull(fs);
}
}
@@ -195,7 +198,7 @@ public class TestRMStateStore {
ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
String containerIdStr, Token<AMRMTokenIdentifier> appToken,
- Token<ClientToAMTokenIdentifier> clientToAMToken, TestDispatcher dispatcher)
+ SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
throws Exception {
Container container = new ContainerPBImpl();
@@ -204,7 +207,8 @@ public class TestRMStateStore {
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
when(mockAttempt.getMasterContainer()).thenReturn(container);
when(mockAttempt.getAMRMToken()).thenReturn(appToken);
- when(mockAttempt.getClientToAMToken()).thenReturn(clientToAMToken);
+ when(mockAttempt.getClientTokenMasterKey())
+ .thenReturn(clientTokenMasterKey);
dispatcher.attemptId = attemptId;
dispatcher.storedException = null;
store.storeApplicationAttempt(mockAttempt);
@@ -212,13 +216,12 @@ public class TestRMStateStore {
return container.getId();
}
- @SuppressWarnings("unchecked")
void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
long submitTime = System.currentTimeMillis();
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.getRMStateStore();
TestDispatcher dispatcher = new TestDispatcher();
- store.setDispatcher(dispatcher);
+ store.setRMDispatcher(dispatcher);
AMRMTokenSecretManager appTokenMgr =
new AMRMTokenSecretManager(conf);
@@ -230,33 +233,33 @@ public class TestRMStateStore {
ApplicationId appId1 = attemptId1.getApplicationId();
storeApp(store, appId1, submitTime);
- // create application token1 for attempt1
- List<Token<?>> appAttemptToken1 =
- generateTokens(attemptId1, appTokenMgr, clientToAMTokenMgr, conf);
+ // create application token and client token key for attempt1
+ Token<AMRMTokenIdentifier> appAttemptToken1 =
+ generateAMRMToken(attemptId1, appTokenMgr);
HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
- attemptTokenSet1.addAll(appAttemptToken1);
+ attemptTokenSet1.add(appAttemptToken1);
+ SecretKey clientTokenKey1 =
+ clientToAMTokenMgr.registerApplication(attemptId1);
ContainerId containerId1 = storeAttempt(store, attemptId1,
"container_1352994193343_0001_01_000001",
- (Token<AMRMTokenIdentifier>) (appAttemptToken1.get(0)),
- (Token<ClientToAMTokenIdentifier>)(appAttemptToken1.get(1)),
- dispatcher);
+ appAttemptToken1, clientTokenKey1, dispatcher);
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 =
ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
- // create application token2 for attempt2
- List<Token<?>> appAttemptToken2 =
- generateTokens(attemptId2, appTokenMgr, clientToAMTokenMgr, conf);
+ // create application token and client token key for attempt2
+ Token<AMRMTokenIdentifier> appAttemptToken2 =
+ generateAMRMToken(attemptId2, appTokenMgr);
HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
- attemptTokenSet2.addAll(appAttemptToken2);
+ attemptTokenSet2.add(appAttemptToken2);
+ SecretKey clientTokenKey2 =
+ clientToAMTokenMgr.registerApplication(attemptId2);
ContainerId containerId2 = storeAttempt(store, attemptId2,
"container_1352994193343_0001_02_000001",
- (Token<AMRMTokenIdentifier>) (appAttemptToken2.get(0)),
- (Token<ClientToAMTokenIdentifier>)(appAttemptToken2.get(1)),
- dispatcher);
+ appAttemptToken2, clientTokenKey2, dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
@@ -303,8 +306,12 @@ public class TestRMStateStore {
assertEquals(containerId1, attemptState.getMasterContainer().getId());
// attempt1 applicationToken is loaded correctly
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
assertEquals(attemptTokenSet1, savedTokens);
+ // attempt1 client token master key is loaded correctly
+ assertArrayEquals(clientTokenKey1.getEncoded(),
+ attemptState.getAppAttemptCredentials()
+ .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
attemptState = appState.getAttempt(attemptId2);
// attempt2 is loaded correctly
@@ -314,8 +321,12 @@ public class TestRMStateStore {
assertEquals(containerId2, attemptState.getMasterContainer().getId());
// attempt2 applicationToken is loaded correctly
savedTokens.clear();
- savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
assertEquals(attemptTokenSet2, savedTokens);
+ // attempt2 client token master key is loaded correctly
+ assertArrayEquals(clientTokenKey2.getEncoded(),
+ attemptState.getAppAttemptCredentials()
+ .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
// assert store is in expected state after everything is cleaned
assertTrue(stateStoreHelper.isFinalStateValid());
@@ -327,7 +338,7 @@ public class TestRMStateStore {
RMStateStoreHelper stateStoreHelper) throws Exception {
RMStateStore store = stateStoreHelper.getRMStateStore();
TestDispatcher dispatcher = new TestDispatcher();
- store.setDispatcher(dispatcher);
+ store.setRMDispatcher(dispatcher);
// store RM delegation token;
RMDelegationTokenIdentifier dtId1 =
@@ -354,24 +365,14 @@ public class TestRMStateStore {
Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber());
}
- private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
- AMRMTokenSecretManager appTokenMgr,
- ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, Configuration conf) {
+ private Token<AMRMTokenIdentifier> generateAMRMToken(
+ ApplicationAttemptId attemptId,
+ AMRMTokenSecretManager appTokenMgr) {
AMRMTokenIdentifier appTokenId =
new AMRMTokenIdentifier(attemptId);
Token<AMRMTokenIdentifier> appToken =
new Token<AMRMTokenIdentifier>(appTokenId, appTokenMgr);
appToken.setService(new Text("appToken service"));
-
- ClientToAMTokenIdentifier clientToAMTokenId =
- new ClientToAMTokenIdentifier(attemptId);
- clientToAMTokenMgr.registerApplication(attemptId);
- Token<ClientToAMTokenIdentifier> clientToAMToken =
- new Token<ClientToAMTokenIdentifier>(clientToAMTokenId, clientToAMTokenMgr);
- clientToAMToken.setService(new Text("clientToAMToken service"));
- List<Token<?>> tokenPair = new ArrayList<Token<?>>();
- tokenPair.add(0, appToken);
- tokenPair.add(1, clientToAMToken);
- return tokenPair;
+ return appToken;
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Tue Sep 3 18:30:05 2013
@@ -143,7 +143,8 @@ public class MockRMApp implements RMApp
}
@Override
- public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Tue Sep 3 18:30:05 2013
@@ -726,7 +726,9 @@ public class TestRMAppTransitions {
public void testGetAppReport() {
RMApp app = createNewTestApp(null);
assertAppState(RMAppState.NEW, app);
- ApplicationReport report = app.createAndGetApplicationReport(true);
+ ApplicationReport report = app.createAndGetApplicationReport(null, true);
+ Assert.assertNotNull(report.getApplicationResourceUsageReport());
+ report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Tue Sep 3 18:30:05 2013
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
import java.util.Collections;
import java.util.List;
@@ -35,6 +36,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -102,6 +104,11 @@ public class TestRMAppAttemptTransitions
private RMApp application;
private RMAppAttempt applicationAttempt;
+
+ private Configuration conf = new Configuration();
+ private AMRMTokenSecretManager amRMTokenManager = spy(new AMRMTokenSecretManager(conf));
+ private ClientToAMTokenSecretManagerInRM clientToAMTokenManager =
+ spy(new ClientToAMTokenSecretManagerInRM());
private final class TestApplicationAttemptEventDispatcher implements
EventHandler<RMAppAttemptEvent> {
@@ -163,14 +170,13 @@ public class TestRMAppAttemptTransitions
mock(ContainerAllocationExpirer.class);
amLivelinessMonitor = mock(AMLivelinessMonitor.class);
amFinishingMonitor = mock(AMLivelinessMonitor.class);
- Configuration conf = new Configuration();
rmContext =
new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, new AMRMTokenSecretManager(conf),
+ null, amRMTokenManager,
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM());
+ clientToAMTokenManager);
RMStateStore store = mock(RMStateStore.class);
((RMContextImpl) rmContext).setStateStore(store);
@@ -261,7 +267,11 @@ public class TestRMAppAttemptTransitions
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
-
+ if (UserGroupInformation.isSecurityEnabled()) {
+ verify(clientToAMTokenManager).registerApplication(
+ applicationAttempt.getAppAttemptId());
+ }
+ assertNotNull(applicationAttempt.getAMRMToken());
// Check events
verify(masterService).
registerAppAttempt(applicationAttempt.getAppAttemptId());
@@ -288,6 +298,7 @@ public class TestRMAppAttemptTransitions
// this works for unmanaged and managed AM's because this is actually doing
// verify(application).handle(anyObject());
verify(application).handle(any(RMAppRejectedEvent.class));
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
/**
@@ -303,6 +314,7 @@ public class TestRMAppAttemptTransitions
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
/**
@@ -377,6 +389,8 @@ public class TestRMAppAttemptTransitions
// Check events
verify(application, times(2)).handle(any(RMAppFailedAttemptEvent.class));
+
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
/**
@@ -422,6 +436,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getTrackingUrl());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 0);
}
/**
@@ -442,6 +457,7 @@ public class TestRMAppAttemptTransitions
.getJustFinishedContainers().size());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@@ -592,6 +608,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.KILL));
testAppAttemptKilledState(null, EMPTY_DIAGNOSTICS);
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test
@@ -666,6 +683,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId(), cs));
assertEquals(RMAppAttemptState.FAILED,
applicationAttempt.getAppAttemptState());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test
@@ -691,6 +709,27 @@ public class TestRMAppAttemptTransitions
assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
}
+ @Test
+ public void testRunningToKilled() {
+ Container amContainer = allocateApplicationAttempt();
+ launchApplicationAttempt(amContainer);
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ applicationAttempt.handle(
+ new RMAppAttemptEvent(
+ applicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.KILL));
+ assertEquals(RMAppAttemptState.KILLED,
+ applicationAttempt.getAppAttemptState());
+ assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
+ assertEquals(amContainer, applicationAttempt.getMasterContainer());
+ assertEquals(0, applicationAttempt.getRanNodes().size());
+ String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
+ applicationAttempt.getAppAttemptId().getApplicationId());
+ assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
+ assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
+ }
+
@Test(timeout=10000)
public void testLaunchedExpire() {
Container amContainer = allocateApplicationAttempt();
@@ -705,6 +744,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId().getApplicationId());
assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test(timeout=20000)
@@ -722,6 +762,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId().getApplicationId());
assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+ verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
}
@Test
@@ -828,4 +869,10 @@ public class TestRMAppAttemptTransitions
diagnostics, 0);
}
+ private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
+ verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ verify(clientToAMTokenManager, times(count)).unRegisterApplication(appAttemptId);
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java Tue Sep 3 18:30:05 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
@@ -351,4 +352,10 @@ public class TestSchedulerUtils {
RMAppAttemptState.LAUNCHED);
}
+ @Test
+ public void testComparePriorities(){
+ Priority high = Priority.newInstance(1);
+ Priority low = Priority.newInstance(2);
+ assertTrue(high.compareTo(low) > 0);
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java Tue Sep 3 18:30:05 2013
@@ -26,7 +26,6 @@ import static org.mockito.Mockito.doNoth
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.util.HashMap;
@@ -126,7 +125,7 @@ public class TestChildQueueOrder {
throw new Exception();
} catch (Exception e) {
LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() +
- " alloc=" + allocation + " node=" + node.getHostName());
+ " alloc=" + allocation + " node=" + node.getNodeName());
}
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Tue Sep 3 18:30:05 2013
@@ -138,7 +138,7 @@ public class TestParentQueue {
throw new Exception();
} catch (Exception e) {
LOG.info("FOOBAR q.assignContainers q=" + queue.getQueueName() +
- " alloc=" + allocation + " node=" + node.getHostName());
+ " alloc=" + allocation + " node=" + node.getNodeName());
}
final Resource allocatedResource = Resources.createResource(allocation);
if (queue instanceof ParentQueue) {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Tue Sep 3 18:30:05 2013
@@ -160,7 +160,7 @@ public class TestUtils {
when(rmNode.getHostName()).thenReturn(host);
when(rmNode.getRackName()).thenReturn(rack);
- FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode));
+ FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
LOG.info("node = " + host + " avail=" + node.getAvailableResource());
return node;
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Tue Sep 3 18:30:05 2013
@@ -39,12 +39,10 @@ import javax.xml.parsers.ParserConfigura
import junit.framework.Assert;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -57,11 +55,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -81,8 +79,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
-import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -132,7 +128,12 @@ public class TestFairScheduler {
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
resourceManager = new ResourceManager();
resourceManager.init(conf);
+
+ // TODO: This test should really be using MockRM. For now starting stuff
+ // that is needed at a bare minimum.
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
+ resourceManager.getRMContext().getStateStore().start();
+
scheduler.reinitialize(conf, resourceManager.getRMContext());
// to initialize the master key
resourceManager.getRMContainerTokenSecretManager().rollMasterKey();
@@ -1991,6 +1992,7 @@ public class TestFairScheduler {
assertEquals(0, app.getReservedContainers().size());
}
+ @Test
public void testNoMoreCpuOnNode() {
RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1),
1, "127.0.0.1");
@@ -2009,6 +2011,7 @@ public class TestFairScheduler {
assertEquals(1, app.getLiveContainers().size());
}
+ @Test
public void testBasicDRFAssignment() throws Exception {
RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 5));
NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
@@ -2143,4 +2146,54 @@ public class TestFairScheduler {
Assert.assertEquals(2, app3.getLiveContainers().size());
Assert.assertEquals(2, app4.getLiveContainers().size());
}
+
+ @Test(timeout = 30000)
+ public void testHostPortNodeName() throws Exception {
+ scheduler.getConf().setBoolean(YarnConfiguration
+ .RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
+ scheduler.reinitialize(scheduler.getConf(),
+ resourceManager.getRMContext());
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024),
+ 1, "127.0.0.1", 1);
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024),
+ 2, "127.0.0.1", 2);
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
+ "user1", 0);
+
+ ResourceRequest nodeRequest = createResourceRequest(1024,
+ node1.getNodeID().getHost() + ":" + node1.getNodeID().getPort(), 1,
+ 1, true);
+ ResourceRequest rackRequest = createResourceRequest(1024,
+ node1.getRackName(), 1, 1, false);
+ ResourceRequest anyRequest = createResourceRequest(1024,
+ ResourceRequest.ANY, 1, 1, false);
+ createSchedulingRequestExistingApplication(nodeRequest, attId1);
+ createSchedulingRequestExistingApplication(rackRequest, attId1);
+ createSchedulingRequestExistingApplication(anyRequest, attId1);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent node1UpdateEvent = new
+ NodeUpdateSchedulerEvent(node1);
+ NodeUpdateSchedulerEvent node2UpdateEvent = new
+ NodeUpdateSchedulerEvent(node2);
+
+ // no matter how many heartbeats, node2 should never get a container
+ FSSchedulerApp app = scheduler.applications.get(attId1);
+ for (int i = 0; i < 10; i++) {
+ scheduler.handle(node2UpdateEvent);
+ assertEquals(0, app.getLiveContainers().size());
+ assertEquals(0, app.getReservedContainers().size());
+ }
+ // then node1 should get the container
+ scheduler.handle(node1UpdateEvent);
+ assertEquals(1, app.getLiveContainers().size());
+ }
+
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java Tue Sep 3 18:30:05 2013
@@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -46,6 +48,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -80,6 +84,7 @@ public class TestAMRMTokens {
*
* @throws Exception
*/
+ @SuppressWarnings("unchecked")
@Test
public void testTokenExpiry() throws Exception {
@@ -134,6 +139,20 @@ public class TestAMRMTokens {
finishAMRequest.setTrackingUrl("url");
rmClient.finishApplicationMaster(finishAMRequest);
+ // Send RMAppAttemptEventType.CONTAINER_FINISHED to transit RMAppAttempt
+ // from Finishing state to Finished State. Both AMRMToken and
+ // ClientToAMToken will be removed.
+ ContainerStatus containerStatus =
+ BuilderUtils.newContainerStatus(attempt.getMasterContainer().getId(),
+ ContainerState.COMPLETE,
+ "AM Container Finished", 0);
+ rm.getRMContext()
+ .getDispatcher()
+ .getEventHandler()
+ .handle(
+ new RMAppAttemptContainerFinishedEvent(applicationAttemptId,
+ containerStatus));
+
// Now simulate trying to allocate. RPC call itself should throw auth
// exception.
rpc.stopProxy(rmClient, conf); // To avoid using cached client
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java Tue Sep 3 18:30:05 2013
@@ -115,7 +115,6 @@ public class TestClientToAMTokens {
private final byte[] secretKey;
private InetSocketAddress address;
private boolean pinged = false;
- private ClientToAMTokenSecretManager secretManager;
public CustomAM(ApplicationAttemptId appId, byte[] secretKey) {
super("CustomAM");
@@ -132,12 +131,14 @@ public class TestClientToAMTokens {
protected void serviceStart() throws Exception {
Configuration conf = getConfig();
- secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, secretKey);
Server server;
try {
server =
- new RPC.Builder(conf).setProtocol(CustomProtocol.class)
- .setNumHandlers(1).setSecretManager(secretManager)
+ new RPC.Builder(conf)
+ .setProtocol(CustomProtocol.class)
+ .setNumHandlers(1)
+ .setSecretManager(
+ new ClientToAMTokenSecretManager(this.appAttemptId, secretKey))
.setInstance(this).build();
} catch (Exception e) {
throw new YarnRuntimeException(e);
@@ -146,14 +147,10 @@ public class TestClientToAMTokens {
this.address = NetUtils.getConnectAddress(server);
super.serviceStart();
}
-
- public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
- return this.secretManager;
- }
}
@Test
- public void testClientToAMs() throws Exception {
+ public void testClientToAMTokens() throws Exception {
final Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -204,7 +201,7 @@ public class TestClientToAMTokens {
GetApplicationReportResponse reportResponse =
rm.getClientRMService().getApplicationReport(request);
ApplicationReport appReport = reportResponse.getApplicationReport();
- org.apache.hadoop.yarn.api.records.Token clientToAMToken =
+ org.apache.hadoop.yarn.api.records.Token originalClientToAMToken =
appReport.getClientToAMToken();
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
@@ -259,17 +256,47 @@ public class TestClientToAMTokens {
Assert.assertFalse(am.pinged);
}
- // Verify denial for a malicious user
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
Token<ClientToAMTokenIdentifier> token =
- ConverterUtils.convertFromYarn(clientToAMToken, am.address);
+ ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
+
+ // Verify denial for a malicious user with tampered ID
+ verifyTokenWithTamperedID(conf, am, token);
+
+ // Verify denial for a malicious user with tampered user-name
+ verifyTokenWithTamperedUserName(conf, am, token);
+ // Now for an authenticated user
+ verifyValidToken(conf, am, token);
+ }
+
+ private void verifyTokenWithTamperedID(final Configuration conf,
+ final CustomAM am, Token<ClientToAMTokenIdentifier> token)
+ throws IOException {
// Malicious user, messes with appId
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
ClientToAMTokenIdentifier maliciousID =
new ClientToAMTokenIdentifier(BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(app.getApplicationId()
- .getClusterTimestamp(), 42), 43));
+ BuilderUtils.newApplicationId(am.appAttemptId.getApplicationId()
+ .getClusterTimestamp(), 42), 43), UserGroupInformation
+ .getCurrentUser().getShortUserName());
+
+ verifyTamperedToken(conf, am, token, ugi, maliciousID);
+ }
+
+ private void verifyTokenWithTamperedUserName(final Configuration conf,
+ final CustomAM am, Token<ClientToAMTokenIdentifier> token)
+ throws IOException {
+ // Malicious user, messes with appId
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
+ ClientToAMTokenIdentifier maliciousID =
+ new ClientToAMTokenIdentifier(am.appAttemptId, "evilOrc");
+ verifyTamperedToken(conf, am, token, ugi, maliciousID);
+ }
+
+ private void verifyTamperedToken(final Configuration conf, final CustomAM am,
+ Token<ClientToAMTokenIdentifier> token, UserGroupInformation ugi,
+ ClientToAMTokenIdentifier maliciousID) {
Token<ClientToAMTokenIdentifier> maliciousToken =
new Token<ClientToAMTokenIdentifier>(maliciousID.getBytes(),
token.getPassword(), token.getKind(),
@@ -309,8 +336,12 @@ public class TestClientToAMTokens {
+ "Mismatched response."));
Assert.assertFalse(am.pinged);
}
+ }
- // Now for an authenticated user
+ private void verifyValidToken(final Configuration conf, final CustomAM am,
+ Token<ClientToAMTokenIdentifier> token) throws IOException,
+ InterruptedException {
+ UserGroupInformation ugi;
ugi = UserGroupInformation.createRemoteUser("me");
ugi.addToken(token);
@@ -326,5 +357,4 @@ public class TestClientToAMTokens {
}
});
}
-
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Tue Sep 3 18:30:05 2013
@@ -53,6 +53,21 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
+/**
+ * Embedded Yarn minicluster for testcases that need to interact with a cluster.
+ * <p/>
+ * In a real cluster, resource request matching is done using the hostname, and
+ * by default Yarn minicluster works in the exact same way as a real cluster.
+ * <p/>
+ * If a testcase needs to use multiple nodes and exercise resource request
+ * matching to a specific node, then the property
+ * {@YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} should be set
+ * <code>true</code> in the configuration used to initialize the minicluster.
+ * <p/>
+ * With this property set to <code>true</code>, the matching will be done using
+ * the <code>hostname:port</code> of the namenodes. In such case, the AM must
+ * do resource request using <code>hostname:port</code> as the location.
+ */
public class MiniYARNCluster extends CompositeService {
private static final Log LOG = LogFactory.getLog(MiniYARNCluster.class);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Tue Sep 3 18:30:05 2013
@@ -87,8 +87,9 @@ public class WebAppProxy extends Abstrac
@Override
protected void serviceStart() throws Exception {
try {
- proxyServer = new HttpServer("proxy", bindAddress, port,
- port == 0, getConfig(), acl);
+ proxyServer = new HttpServer.Builder().setName("proxy")
+ .setBindAddress(bindAddress).setPort(port).setFindPort(port == 0)
+ .setConf(getConfig()).setACL(acl).build();
proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Tue Sep 3 18:30:05 2013
@@ -25,8 +25,7 @@ Hadoop MapReduce Next Generation - Fair
* {Purpose}
This document describes the <<<FairScheduler>>>, a pluggable scheduler for Hadoop
- which provides a way to share large clusters. <<NOTE:>> The Fair Scheduler
- implementation is currently under development and should be considered experimental.
+ that allows YARN applications to share resources in large clusters fairly.
* {Introduction}
@@ -47,7 +46,7 @@ Hadoop MapReduce Next Generation - Fair
The scheduler organizes apps further into "queues", and shares resources
fairly between these queues. By default, all users share a single queue,
- called âdefaultâ. If an app specifically lists a queue in a container resource
+ named âdefaultâ. If an app specifically lists a queue in a container resource
request, the request is submitted to that queue. It is also possible to assign
queues based on the user name included with the request through
configuration. Within each queue, a scheduling policy is used to share
@@ -85,7 +84,7 @@ Hadoop MapReduce Next Generation - Fair
their parents in the fair scheduler configuration file.
A queue's name starts with the names of its parents, with periods as
- separators. So a queue named "queue1" under the root named, would be referred
+ separators. So a queue named "queue1" under the root queue, would be referred
to as "root.queue1", and a queue named "queue2" under a queue named "parent1"
would be referred to as "root.parent1.queue2". When referring to queues, the
root part of the name is optional, so queue1 could be referred to as just
@@ -118,22 +117,23 @@ Hadoop MapReduce Next Generation - Fair
Customizing the Fair Scheduler typically involves altering two files. First,
scheduler-wide options can be set by adding configuration properties in the
- fair-scheduler.xml file in your existing configuration directory. Second, in
+ yarn-site.xml file in your existing configuration directory. Second, in
most cases users will want to create a manifest file listing which queues
exist and their respective weights and capacities. The location of this file
- is flexible - but it must be declared in fair-scheduler.xml.
+ is flexible - but it must be declared in yarn-site.xml.
* <<<yarn.scheduler.fair.allocation.file>>>
* Path to allocation file. An allocation file is an XML manifest describing
queues and their properties, in addition to certain policy defaults. This file
must be in XML format as described in the next section.
+ Defaults to fair-scheduler.xml in configuration directory.
* <<<yarn.scheduler.fair.user-as-default-queue>>>
* Whether to use the username associated with the allocation as the default
queue name, in the event that a queue name is not specified. If this is set
- to "false" or unset, all jobs have a shared default queue, called "default".
+ to "false" or unset, all jobs have a shared default queue, named "default".
Defaults to true.
* <<<yarn.scheduler.fair.preemption>>>
@@ -158,7 +158,7 @@ Hadoop MapReduce Next Generation - Fair
* If assignmultiple is true, the maximum amount of containers that can be
assigned in one heartbeat. Defaults to -1, which sets no limit.
- * <<<locality.threshold.node>>>
+ * <<<yarn.scheduler.fair.locality.threshold.node>>>
* For applications that request containers on particular nodes, the number of
scheduling opportunities since the last container assignment to wait before
@@ -167,7 +167,7 @@ Hadoop MapReduce Next Generation - Fair
opportunities to pass up. The default value of -1.0 means don't pass up any
scheduling opportunities.
- * <<<locality.threshold.rack>>>
+ * <<<yarn.scheduler.fair.locality.threshold.rack>>>
* For applications that request containers on particular racks, the number of
scheduling opportunities since the last container assignment to wait before
@@ -178,14 +178,15 @@ Hadoop MapReduce Next Generation - Fair
Allocation file format
- The allocation file must be in XML format. The format contains four types of
+ The allocation file must be in XML format. The format contains five types of
elements:
* <<Queue elements>>, which represent queues. Each may contain the following
properties:
* minResources: minimum resources the queue is entitled to, in the form
- "X mb, Y vcores". If a queue's minimum share is not satisfied, it will be
+ "X mb, Y vcores". For the single-resource fairness policy, the vcores
+ value is ignored. If a queue's minimum share is not satisfied, it will be
offered available resources before any other queue under the same parent.
Under the single-resource fairness policy, a queue
is considered unsatisfied if its memory usage is below its minimum memory
@@ -199,7 +200,8 @@ Allocation file format
may be using those resources.
* maxResources: maximum resources a queue is allowed, in the form
- "X mb, Y vcores". A queue will never be assigned a container that would
+ "X mb, Y vcores". For the single-resource fairness policy, the vcores
+ value is ignored. A queue will never be assigned a container that would
put its aggregate usage over this limit.
* maxRunningApps: limit the number of apps from the queue to run at once
@@ -234,19 +236,23 @@ Allocation file format
its fair share before it will try to preempt containers to take resources from
other queues.
+ * <<A defaultQueueSchedulingPolicy element>>, which sets the default scheduling
+ policy for queues; overriden by the schedulingPolicy element in each queue
+ if specified. Defaults to "fair".
+
An example allocation file is given here:
---
<?xml version="1.0"?>
<allocations>
<queue name="sample_queue">
- <minResources>10000 mb</minResources>
- <maxResources>90000 mb</maxResources>
+ <minResources>10000 mb,0vcores</minResources>
+ <maxResources>90000 mb,0vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
<weight>2.0</weight>
<schedulingPolicy>fair</schedulingPolicy>
<queue name="sample_sub_queue">
- <minResources>5000 mb</minResources>
+ <minResources>5000 mb,0vcores</minResources>
</queue>
</queue>
<user name="sample_user">