You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/08/12 23:26:19 UTC
svn commit: r1513258 [9/10] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/conf/ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-...
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Mon Aug 12 21:25:49 2013
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -31,69 +33,101 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+@RunWith(Parameterized.class)
public class TestAMAuthorization {
private static final Log LOG = LogFactory.getLog(TestAMAuthorization.class);
- private static final Configuration confWithSecurityEnabled =
- new Configuration();
- static {
- confWithSecurityEnabled.set(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- UserGroupInformation.setConfiguration(confWithSecurityEnabled);
+ private final Configuration conf;
+ private MockRM rm;
+
+ @Parameters
+ public static Collection<Object[]> configs() {
+ Configuration conf = new Configuration();
+ Configuration confWithSecurity = new Configuration();
+ confWithSecurity.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ UserGroupInformation.AuthenticationMethod.KERBEROS.toString());
+ return Arrays.asList(new Object[][] {{ conf }, { confWithSecurity} });
+ }
+
+ public TestAMAuthorization(Configuration conf) {
+ this.conf = conf;
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ @After
+ public void tearDown() {
+ if (rm != null) {
+ rm.stop();
+ }
}
public static final class MyContainerManager implements ContainerManagementProtocol {
- public ByteBuffer amTokens;
+ public ByteBuffer containerTokens;
public MyContainerManager() {
}
@Override
- public StartContainerResponse
- startContainer(StartContainerRequest request)
+ public StartContainersResponse
+ startContainers(StartContainersRequest request)
throws YarnException {
- amTokens = request.getContainerLaunchContext().getTokens();
- return null;
+ containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
+ return StartContainersResponse.newInstance(null, null, null);
}
@Override
- public StopContainerResponse stopContainer(StopContainerRequest request)
+ public StopContainersResponse stopContainers(StopContainersRequest request)
throws YarnException {
- // TODO Auto-generated method stub
- return null;
+ return StopContainersResponse.newInstance(null, null);
}
@Override
- public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException {
- // TODO Auto-generated method stub
- return null;
+ public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException {
+ return GetContainerStatusesResponse.newInstance(null, null);
+ }
+
+ public Credentials getContainerCredentials() throws IOException {
+ Credentials credentials = new Credentials();
+ DataInputByteBuffer buf = new DataInputByteBuffer();
+ containerTokens.rewind();
+ buf.reset(containerTokens);
+ credentials.readTokenStorageStream(buf);
+ return credentials;
}
}
@@ -112,13 +146,26 @@ public class TestAMAuthorization {
protected ApplicationMasterService createApplicationMasterService() {
return new ApplicationMasterService(getRMContext(), this.scheduler);
}
+
+ @SuppressWarnings("unchecked")
+ public static Token<? extends TokenIdentifier> setupAndReturnAMRMToken(
+ InetSocketAddress rmBindAddress,
+ Collection<Token<? extends TokenIdentifier>> allTokens) {
+ for (Token<? extends TokenIdentifier> token : allTokens) {
+ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
+ SecurityUtil.setTokenService(token, rmBindAddress);
+ return (Token<AMRMTokenIdentifier>) token;
+ }
+ }
+ return null;
+ }
}
@Test
public void testAuthorizedAccess() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- final MockRM rm =
- new MockRMWithAMS(confWithSecurityEnabled, containerManager);
+ rm =
+ new MockRMWithAMS(conf, containerManager);
rm.start();
MockNM nm1 = rm.registerNode("localhost:1234", 5120);
@@ -131,11 +178,11 @@ public class TestAMAuthorization {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amTokens == null && waitCount++ < 20) {
+ while (containerManager.containerTokens == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amTokens);
+ Assert.assertNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -147,13 +194,13 @@ public class TestAMAuthorization {
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- Credentials credentials = new Credentials();
- DataInputByteBuffer buf = new DataInputByteBuffer();
- containerManager.amTokens.rewind();
- buf.reset(containerManager.amTokens);
- credentials.readTokenStorageStream(buf);
- currentUser.addCredentials(credentials);
-
+ Credentials credentials = containerManager.getContainerCredentials();
+ final InetSocketAddress rmBindAddress =
+ rm.getApplicationMasterService().getBindAddress();
+ Token<? extends TokenIdentifier> amRMToken =
+ MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
+ credentials.getAllTokens());
+ currentUser.addToken(amRMToken);
ApplicationMasterProtocol client = currentUser
.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
@Override
@@ -165,22 +212,21 @@ public class TestAMAuthorization {
RegisterApplicationMasterRequest request = Records
.newRecord(RegisterApplicationMasterRequest.class);
- request.setApplicationAttemptId(applicationAttemptId);
RegisterApplicationMasterResponse response =
client.registerApplicationMaster(request);
Assert.assertNotNull(response.getClientToAMTokenMasterKey());
- Assert
+ if (UserGroupInformation.isSecurityEnabled()) {
+ Assert
.assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
+ }
Assert.assertEquals("Register response has bad ACLs", "*",
response.getApplicationACLs().get(ApplicationAccessType.VIEW_APP));
-
- rm.stop();
}
@Test
public void testUnauthorizedAccess() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- MockRM rm = new MockRMWithAMS(confWithSecurityEnabled, containerManager);
+ rm = new MockRMWithAMS(conf, containerManager);
rm.start();
MockNM nm1 = rm.registerNode("localhost:1234", 5120);
@@ -190,11 +236,11 @@ public class TestAMAuthorization {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amTokens == null && waitCount++ < 40) {
+ while (containerManager.containerTokens == null && waitCount++ < 40) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amTokens);
+ Assert.assertNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -222,52 +268,24 @@ public class TestAMAuthorization {
RegisterApplicationMasterRequest request = Records
.newRecord(RegisterApplicationMasterRequest.class);
- request.setApplicationAttemptId(applicationAttemptId);
try {
client.registerApplicationMaster(request);
Assert.fail("Should fail with authorization error");
} catch (Exception e) {
// Because there are no tokens, the request should be rejected as the
// server side will assume we are trying simple auth.
- Assert.assertTrue(e.getCause().getMessage().contains(
- "SIMPLE authentication is not enabled. "
- + "Available:[TOKEN, KERBEROS]"));
+ String expectedMessage = "";
+ if (UserGroupInformation.isSecurityEnabled()) {
+ expectedMessage = "Client cannot authenticate via:[TOKEN]";
+ } else {
+ expectedMessage =
+ "SIMPLE authentication is not enabled. Available:[TOKEN]";
+ }
+ Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage));
}
- // Now try to validate invalid authorization.
- Credentials credentials = new Credentials();
- DataInputByteBuffer buf = new DataInputByteBuffer();
- containerManager.amTokens.rewind();
- buf.reset(containerManager.amTokens);
- credentials.readTokenStorageStream(buf);
- currentUser.addCredentials(credentials);
-
- // Create a client to the RM.
- client = currentUser
- .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
- @Override
- public ApplicationMasterProtocol run() {
- return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class,
- serviceAddr, conf);
- }
- });
-
- request = Records.newRecord(RegisterApplicationMasterRequest.class);
- ApplicationAttemptId otherAppAttemptId = BuilderUtils
- .newApplicationAttemptId(applicationAttemptId.getApplicationId(), 42);
- request.setApplicationAttemptId(otherAppAttemptId);
- try {
- client.registerApplicationMaster(request);
- Assert.fail("Should fail with authorization error");
- } catch (YarnException e) {
- Assert.assertTrue(e.getMessage().contains(
- "Unauthorized request from ApplicationMaster. "
- + "Expected ApplicationAttemptID: "
- + applicationAttemptId.toString() + " Found: "
- + otherAppAttemptId.toString()));
- } finally {
- rm.stop();
- }
+ // TODO: Add validation of invalid authorization when there's more data in
+ // the AMRMToken
}
private void waitForLaunchedState(RMAppAttempt attempt)
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Mon Aug 12 21:25:49 2013
@@ -19,7 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -27,16 +29,18 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -69,9 +73,10 @@ public class TestApplicationMasterLaunch
int maxAppAttempts;
@Override
- public StartContainerResponse
- startContainer(StartContainerRequest request)
+ public StartContainersResponse
+ startContainers(StartContainersRequest requests)
throws YarnException {
+ StartContainerRequest request = requests.getStartContainerRequests().get(0);
LOG.info("Container started by MyContainerManager: " + request);
launched = true;
Map<String, String> env =
@@ -95,11 +100,13 @@ public class TestApplicationMasterLaunch
Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
maxAppAttempts =
Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
- return null;
+ return StartContainersResponse.newInstance(
+ new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
+ new HashMap<ContainerId, SerializedException>());
}
@Override
- public StopContainerResponse stopContainer(StopContainerRequest request)
+ public StopContainersResponse stopContainers(StopContainersRequest request)
throws YarnException {
LOG.info("Container cleaned up by MyContainerManager");
cleanedup = true;
@@ -107,11 +114,10 @@ public class TestApplicationMasterLaunch
}
@Override
- public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException {
+ public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException {
return null;
}
-
}
@Test
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Mon Aug 12 21:25:49 2013
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -186,10 +187,14 @@ public class TestClientRMService {
GetApplicationReportRequest request = recordFactory
.newRecordInstance(GetApplicationReportRequest.class);
request.setApplicationId(ApplicationId.newInstance(0, 0));
- GetApplicationReportResponse applicationReport = rmService
- .getApplicationReport(request);
- Assert.assertNull("It should return null as application report for absent application.",
- applicationReport.getApplicationReport());
+ try {
+ rmService.getApplicationReport(request);
+ Assert.fail();
+ } catch (ApplicationNotFoundException ex) {
+ Assert.assertEquals(ex.getMessage(),
+ "Application with id '" + request.getApplicationId()
+ + "' doesn't exist in RM.");
+ }
}
@Test
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java Mon Aug 12 21:25:49 2013
@@ -222,8 +222,9 @@ public class TestRMAuditLogger {
public void testRMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
// start the IPC server
- Server server = RPC.getServer(TestProtocol.class,
- new MyTestRPCServer(), "0.0.0.0", 0, 5, true, conf, null);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
+ .setPort(0).setNumHandlers(5).setVerbose(true).build();
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java Mon Aug 12 21:25:49 2013
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
@@ -269,6 +270,15 @@ public class TestRMNodeTransitions {
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
Assert.assertEquals(NodeState.LOST, node.getState());
}
+
+ @Test
+ public void testUnhealthyExpireForSchedulerRemove() {
+ RMNodeImpl node = getUnhealthyNode();
+ verify(scheduler,times(2)).handle(any(NodeRemovedSchedulerEvent.class));
+ node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
+ verify(scheduler,times(2)).handle(any(NodeRemovedSchedulerEvent.class));
+ Assert.assertEquals(NodeState.LOST, node.getState());
+ }
@Test
public void testRunningDecommission() {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationmasterservice/TestApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationmasterservice/TestApplicationMasterService.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationmasterservice/TestApplicationMasterService.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationmasterservice/TestApplicationMasterService.java Mon Aug 12 21:25:49 2013
@@ -24,7 +24,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -88,4 +91,63 @@ public class TestApplicationMasterServic
Assert.assertEquals(MockRM.clusterTimeStamp, tokenId.getRMIdentifer());
rm.stop();
}
+
+ @Test(timeout=600000)
+ public void testInvalidContainerReleaseRequest() throws Exception {
+ MockRM rm = new MockRM(conf);
+
+ try {
+ rm.start();
+
+ // Register node1
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+
+ // Submit an application
+ RMApp app1 = rm.submitApp(1024);
+
+ // kick the scheduling
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ am1.registerAppAttempt();
+
+ am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
+ AllocateResponse alloc1Response = am1.schedule(); // send the request
+
+ // kick the scheduler
+ nm1.nodeHeartbeat(true);
+ while (alloc1Response.getAllocatedContainers().size() < 1) {
+ LOG.info("Waiting for containers to be created for app 1...");
+ Thread.sleep(1000);
+ alloc1Response = am1.schedule();
+ }
+
+ Assert.assertTrue(alloc1Response.getAllocatedContainers().size() > 0);
+
+ RMApp app2 = rm.submitApp(1024);
+
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
+ MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId());
+ am2.registerAppAttempt();
+
+ // Now trying to release container allocated for app1 -> appAttempt1.
+ ContainerId cId = alloc1Response.getAllocatedContainers().get(0).getId();
+ am2.addContainerToBeReleased(cId);
+ try {
+ am2.schedule();
+ Assert.fail("Exception was expected!!");
+ } catch (InvalidContainerReleaseException e) {
+ StringBuilder sb = new StringBuilder("Cannot release container : ");
+ sb.append(cId.toString());
+ sb.append(" not belonging to this application attempt : ");
+ sb.append(attempt2.getAppAttemptId().toString());
+ Assert.assertTrue(e.getMessage().contains(sb.toString()));
+ }
+ } finally {
+ if (rm != null) {
+ rm.stop();
+ }
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java Mon Aug 12 21:25:49 2013
@@ -18,17 +18,22 @@
package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
+import java.security.PrivilegedExceptionAction;
import java.util.List;
import junit.framework.Assert;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@@ -87,6 +92,22 @@ public class TestAMRMRPCNodeUpdates {
dispatcher.await();
}
+ private AllocateResponse allocate(final ApplicationAttemptId attemptId,
+ final AllocateRequest req) throws Exception {
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(attemptId.toString());
+ Token<AMRMTokenIdentifier> token =
+ rm.getRMContext().getRMApps().get(attemptId.getApplicationId())
+ .getRMAppAttempt(attemptId).getAMRMToken();
+ ugi.addTokenIdentifier(token.decodeIdentifier());
+ return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() {
+ @Override
+ public AllocateResponse run() throws Exception {
+ return amService.allocate(req);
+ }
+ });
+ }
+
@Test
public void testAMRMUnusableNodes() throws Exception {
@@ -107,18 +128,20 @@ public class TestAMRMRPCNodeUpdates {
am1.registerAppAttempt();
// allocate request returns no updated node
- AllocateRequest allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), 0, 0F, null, null, null);
- AllocateResponse response1 = amService.allocate(allocateRequest1);
+ AllocateRequest allocateRequest1 =
+ AllocateRequest.newInstance(0, 0F, null, null, null);
+ AllocateResponse response1 =
+ allocate(attempt1.getAppAttemptId(), allocateRequest1);
List<NodeReport> updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
syncNodeHeartbeat(nm4, false);
// allocate request returns updated node
- allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null);
- response1 = amService.allocate(allocateRequest1);
+ allocateRequest1 =
+ AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null,
+ null);
+ response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
NodeReport nr = updatedNodes.iterator().next();
@@ -126,7 +149,7 @@ public class TestAMRMRPCNodeUpdates {
Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());
// resending the allocate request returns the same result
- response1 = amService.allocate(allocateRequest1);
+ response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
@@ -136,9 +159,10 @@ public class TestAMRMRPCNodeUpdates {
syncNodeLost(nm3);
// subsequent allocate request returns delta
- allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null);
- response1 = amService.allocate(allocateRequest1);
+ allocateRequest1 =
+ AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null,
+ null);
+ response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
@@ -156,27 +180,30 @@ public class TestAMRMRPCNodeUpdates {
am2.registerAppAttempt();
// allocate request returns no updated node
- AllocateRequest allocateRequest2 = AllocateRequest.newInstance(attempt2
- .getAppAttemptId(), 0, 0F, null, null, null);
- AllocateResponse response2 = amService.allocate(allocateRequest2);
+ AllocateRequest allocateRequest2 =
+ AllocateRequest.newInstance(0, 0F, null, null, null);
+ AllocateResponse response2 =
+ allocate(attempt2.getAppAttemptId(), allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
syncNodeHeartbeat(nm4, true);
// both AM's should get delta updated nodes
- allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null);
- response1 = amService.allocate(allocateRequest1);
+ allocateRequest1 =
+ AllocateRequest.newInstance(response1.getResponseId(), 0F, null, null,
+ null);
+ response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
- allocateRequest2 = AllocateRequest.newInstance(attempt2
- .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null);
- response2 = amService.allocate(allocateRequest2);
+ allocateRequest2 =
+ AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null,
+ null);
+ response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
nr = updatedNodes.iterator().next();
@@ -184,9 +211,10 @@ public class TestAMRMRPCNodeUpdates {
Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
// subsequent allocate calls should return no updated nodes
- allocateRequest2 = AllocateRequest.newInstance(attempt2
- .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null);
- response2 = amService.allocate(allocateRequest2);
+ allocateRequest2 =
+ AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null,
+ null);
+ response2 = allocate(attempt2.getAppAttemptId(), allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java Mon Aug 12 21:25:49 2013
@@ -18,19 +18,20 @@
package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
+import java.security.PrivilegedExceptionAction;
+
import junit.framework.Assert;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.AMCommand;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
-import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.junit.After;
@@ -39,20 +40,13 @@ import org.junit.Test;
public class TestAMRMRPCResponseId {
- private static final RecordFactory recordFactory = RecordFactoryProvider
- .getRecordFactory(null);
-
private MockRM rm;
ApplicationMasterService amService = null;
- private ClientRMService clientService;
-
- private RMContext context;
@Before
public void setUp() {
this.rm = new MockRM();
rm.start();
- this.clientService = rm.getClientRMService();
amService = rm.getApplicationMasterService();
}
@@ -63,6 +57,22 @@ public class TestAMRMRPCResponseId {
}
}
+ private AllocateResponse allocate(ApplicationAttemptId attemptId,
+ final AllocateRequest req) throws Exception {
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(attemptId.toString());
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token =
+ rm.getRMContext().getRMApps().get(attemptId.getApplicationId())
+ .getRMAppAttempt(attemptId).getAMRMToken();
+ ugi.addTokenIdentifier(token.decodeIdentifier());
+ return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() {
+ @Override
+ public AllocateResponse run() throws Exception {
+ return amService.allocate(req);
+ }
+ });
+ }
+
@Test
public void testARRMResponseId() throws Exception {
@@ -78,25 +88,26 @@ public class TestAMRMRPCResponseId {
am.registerAppAttempt();
- AllocateRequest allocateRequest = AllocateRequest.newInstance(attempt
- .getAppAttemptId(), 0, 0F, null, null, null);
+ AllocateRequest allocateRequest =
+ AllocateRequest.newInstance(0, 0F, null, null, null);
- AllocateResponse response = amService.allocate(allocateRequest);
+ AllocateResponse response =
+ allocate(attempt.getAppAttemptId(), allocateRequest);
Assert.assertEquals(1, response.getResponseId());
Assert.assertTrue(response.getAMCommand() == null);
- allocateRequest = AllocateRequest.newInstance(attempt
- .getAppAttemptId(), response.getResponseId(), 0F, null, null, null);
+ allocateRequest =
+ AllocateRequest.newInstance(response.getResponseId(), 0F, null, null,
+ null);
- response = amService.allocate(allocateRequest);
+ response = allocate(attempt.getAppAttemptId(), allocateRequest);
Assert.assertEquals(2, response.getResponseId());
/* try resending */
- response = amService.allocate(allocateRequest);
+ response = allocate(attempt.getAppAttemptId(), allocateRequest);
Assert.assertEquals(2, response.getResponseId());
/** try sending old request again **/
- allocateRequest = AllocateRequest.newInstance(attempt
- .getAppAttemptId(), 0, 0F, null, null, null);
- response = amService.allocate(allocateRequest);
+ allocateRequest = AllocateRequest.newInstance(0, 0F, null, null, null);
+ response = allocate(attempt.getAppAttemptId(), allocateRequest);
Assert.assertTrue(response.getAMCommand() == AMCommand.AM_RESYNC);
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Mon Aug 12 21:25:49 2013
@@ -105,8 +105,6 @@ public class TestRMStateStore {
interface RMStateStoreHelper {
RMStateStore getRMStateStore() throws Exception;
- void addOrphanAttemptIfNeeded(RMStateStore testStore,
- TestDispatcher dispatcher) throws Exception;
boolean isFinalStateValid() throws Exception;
}
@@ -154,15 +152,6 @@ public class TestRMStateStore {
}
@Override
- public void addOrphanAttemptIfNeeded(RMStateStore testStore,
- TestDispatcher dispatcher) throws Exception {
- ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(
- "appattempt_1352994193343_0003_000001");
- storeAttempt(testStore, attemptId,
- "container_1352994193343_0003_01_000001", null, null, dispatcher);
- }
-
- @Override
public boolean isFinalStateValid() throws Exception {
FileSystem fs = cluster.getFileSystem();
FileStatus[] files = fs.listStatus(workingDirPathURI);
@@ -289,9 +278,6 @@ public class TestRMStateStore {
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
- // add orphan attempt file to simulate incomplete removal of app state
- stateStoreHelper.addOrphanAttemptIfNeeded(store, dispatcher);
-
// let things settle down
Thread.sleep(1000);
store.close();
@@ -301,9 +287,6 @@ public class TestRMStateStore {
RMState state = store.loadState();
Map<ApplicationId, ApplicationState> rmAppState = state.getApplicationState();
- // removed app or orphan attempt is not loaded
- assertEquals(1, rmAppState.size());
-
ApplicationState appState = rmAppState.get(appId1);
// app is loaded
assertNotNull(appState);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Mon Aug 12 21:25:49 2013
@@ -68,7 +68,7 @@ public class TestQueueMetrics {
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
@@ -77,7 +77,7 @@ public class TestQueueMetrics {
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
@@ -86,7 +86,7 @@ public class TestQueueMetrics {
checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
assertNull(userSource);
}
@@ -102,37 +102,37 @@ public class TestQueueMetrics {
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.finishApp(app, RMAppAttemptState.FAILED);
- checkApps(queueSource, 1, 0, 0, 0, 1, 0);
+ checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.submitApp(user, 2);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has failed this time as well.
metrics.finishApp(app, RMAppAttemptState.FAILED);
- checkApps(queueSource, 1, 0, 0, 0, 1, 0);
+ checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.submitApp(user, 3);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has finished.
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
assertNull(userSource);
}
@@ -149,8 +149,8 @@ public class TestQueueMetrics {
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
- checkApps(userSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
@@ -161,8 +161,8 @@ public class TestQueueMetrics {
checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
- checkApps(userSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
@@ -173,8 +173,8 @@ public class TestQueueMetrics {
checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
- checkApps(userSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
}
@Test public void testTwoLevelWithUserMetrics() {
@@ -196,10 +196,10 @@ public class TestQueueMetrics {
MetricsSource userSource = userSource(ms, leafQueueName, user);
MetricsSource parentUserSource = userSource(ms, parentQueueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
- checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0);
- checkApps(userSource, 1, 1, 0, 0, 0, 0);
- checkApps(parentUserSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
parentMetrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
@@ -212,8 +212,8 @@ public class TestQueueMetrics {
checkResources(parentUserSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
- checkApps(userSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
metrics.reserveResource(user, Resources.createResource(3*GB, 3));
@@ -232,10 +232,10 @@ public class TestQueueMetrics {
checkResources(parentUserSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
- checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0);
- checkApps(userSource, 1, 0, 0, 1, 0, 0);
- checkApps(parentUserSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
}
@Test
@@ -275,13 +275,35 @@ public class TestQueueMetrics {
FifoScheduler.class, ResourceScheduler.class);
MockRM rm = new MockRM(conf);
QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
- checkApps(metrics, 0, 0, 0, 0, 0, 0);
+ checkApps(metrics, 0, 0, 0, 0, 0, 0, true);
MetricsAsserts.assertGauge("ReservedContainers", 0, metrics);
}
+ // This is to test all metrics can consistently show up if specified true to
+ // collect all metrics, even though they are not modified from last time they
+ // are collected. If not collecting all metrics, only modified metrics will show up.
+ @Test
+ public void testCollectAllMetrics() {
+ String queueName = "single";
+ QueueMetrics.forQueue(ms, queueName, null, false, conf);
+ MetricsSource queueSource = queueSource(ms, queueName);
+
+ checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
+ try {
+ // do not collect all metrics
+ checkApps(queueSource, 0, 0, 0, 0, 0, 0, false);
+ Assert.fail();
+ } catch (AssertionError e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Expected exactly one metric for name "));
+ }
+ // collect all metrics
+ checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
+ }
+
public static void checkApps(MetricsSource source, int submitted, int pending,
- int running, int completed, int failed, int killed) {
- MetricsRecordBuilder rb = getMetrics(source);
+ int running, int completed, int failed, int killed, boolean all) {
+ MetricsRecordBuilder rb = getMetrics(source, all);
assertCounter("AppsSubmitted", submitted, rb);
assertGauge("AppsPending", pending, rb);
assertGauge("AppsRunning", running, rb);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java Mon Aug 12 21:25:49 2013
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
+import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.util.Collections;
import java.util.HashMap;
@@ -30,7 +31,10 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
@@ -42,9 +46,10 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -271,7 +276,7 @@ public class TestSchedulerUtils {
public void testValidateResourceBlacklistRequest() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- final MockRM rm =
+ final MockRMWithAMS rm =
new MockRMWithAMS(new YarnConfiguration(), containerManager);
rm.start();
@@ -294,19 +299,24 @@ public class TestSchedulerUtils {
UserGroupInformation currentUser =
UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
-
- ApplicationMasterProtocol client = currentUser
- .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
+ Credentials credentials = containerManager.getContainerCredentials();
+ final InetSocketAddress rmBindAddress =
+ rm.getApplicationMasterService().getBindAddress();
+ Token<? extends TokenIdentifier> amRMToken =
+ MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
+ credentials.getAllTokens());
+ currentUser.addToken(amRMToken);
+ ApplicationMasterProtocol client =
+ currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
@Override
public ApplicationMasterProtocol run() {
- return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm
- .getApplicationMasterService().getBindAddress(), conf);
+ return (ApplicationMasterProtocol) rpc.getProxy(
+ ApplicationMasterProtocol.class, rmBindAddress, conf);
}
});
RegisterApplicationMasterRequest request = Records
.newRecord(RegisterApplicationMasterRequest.class);
- request.setApplicationAttemptId(applicationAttemptId);
client.registerApplicationMaster(request);
ResourceBlacklistRequest blacklistRequest =
@@ -314,8 +324,7 @@ public class TestSchedulerUtils {
Collections.singletonList(ResourceRequest.ANY), null);
AllocateRequest allocateRequest =
- AllocateRequest.newInstance(applicationAttemptId, 0, 0.0f, null, null,
- blacklistRequest);
+ AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
boolean error = false;
try {
client.allocate(allocateRequest);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Mon Aug 12 21:25:49 2013
@@ -43,7 +43,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -227,7 +226,7 @@ public class TestLeafQueue {
doNothing().when(parent).completedContainer(
any(Resource.class), any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class),
any(RMContainer.class), any(ContainerStatus.class),
- any(RMContainerEventType.class));
+ any(RMContainerEventType.class), any(CSQueue.class));
return queue;
}
@@ -480,7 +479,7 @@ public class TestLeafQueue {
// Release each container from app_0
for (RMContainer rmContainer : app_0.getLiveContainers()) {
a.completedContainer(clusterResource, app_0, node_0, rmContainer,
- null, RMContainerEventType.KILL);
+ null, RMContainerEventType.KILL, null);
}
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -491,7 +490,7 @@ public class TestLeafQueue {
// Release each container from app_1
for (RMContainer rmContainer : app_1.getLiveContainers()) {
a.completedContainer(clusterResource, app_1, node_0, rmContainer,
- null, RMContainerEventType.KILL);
+ null, RMContainerEventType.KILL, null);
}
assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -850,7 +849,7 @@ public class TestLeafQueue {
// 8. Release each container from app_0
for (RMContainer rmContainer : app_0.getLiveContainers()) {
a.completedContainer(clusterResource, app_0, node_0, rmContainer,
- null, RMContainerEventType.KILL);
+ null, RMContainerEventType.KILL, null);
}
assertEquals(5*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -861,7 +860,7 @@ public class TestLeafQueue {
// 9. Release each container from app_2
for (RMContainer rmContainer : app_2.getLiveContainers()) {
a.completedContainer(clusterResource, app_2, node_0, rmContainer,
- null, RMContainerEventType.KILL);
+ null, RMContainerEventType.KILL, null);
}
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -872,7 +871,7 @@ public class TestLeafQueue {
// 10. Release each container from app_3
for (RMContainer rmContainer : app_3.getLiveContainers()) {
a.completedContainer(clusterResource, app_3, node_0, rmContainer,
- null, RMContainerEventType.KILL);
+ null, RMContainerEventType.KILL, null);
}
assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -959,7 +958,8 @@ public class TestLeafQueue {
// Now free 1 container from app_0 i.e. 1G
a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(), null, RMContainerEventType.KILL);
+ app_0.getLiveContainers().iterator().next(),
+ null, RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(5*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
@@ -971,7 +971,8 @@ public class TestLeafQueue {
// Now finish another container from app_0 and fulfill the reservation
a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(), null, RMContainerEventType.KILL);
+ app_0.getLiveContainers().iterator().next(),
+ null, RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -1069,7 +1070,8 @@ public class TestLeafQueue {
// Now free 1 container from app_0 and try to assign to node_0
a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(), null, RMContainerEventType.KILL);
+ app_0.getLiveContainers().iterator().next(),
+ null, RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(8*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -1160,7 +1162,8 @@ public class TestLeafQueue {
// Now free 1 container from app_0 i.e. 1G, and re-reserve it
a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(), null, RMContainerEventType.KILL);
+ app_0.getLiveContainers().iterator().next(),
+ null, RMContainerEventType.KILL, null);
a.assignContainers(clusterResource, node_0);
assertEquals(5*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
@@ -1191,7 +1194,8 @@ public class TestLeafQueue {
// Now finish another container from app_0 and see the reservation cancelled
a.completedContainer(clusterResource, app_0, node_0,
- app_0.getLiveContainers().iterator().next(), null, RMContainerEventType.KILL);
+ app_0.getLiveContainers().iterator().next(),
+ null, RMContainerEventType.KILL, null);
CSAssignment assignment = a.assignContainers(clusterResource, node_0);
assertEquals(8*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
@@ -1956,6 +1960,50 @@ public class TestLeafQueue {
assertEquals(0, app_0.getTotalRequiredResources(priority));
}
+
+ @Test
+ public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
+ throws Exception {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ Resource clusterResource = Resources
+ .createResource(100 * 16 * GB, 100 * 32);
+ CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
+ csConf.setFloat(CapacitySchedulerConfiguration.
+ MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
+ ParentQueue root = new ParentQueue(csContext,
+ CapacitySchedulerConfiguration.ROOT, null, null);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
+ LeafQueue a = new LeafQueue(csContext, A, root, null);
+ assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
+ assertEquals(160, a.getMaximumActiveApplications());
+
+ csConf.setFloat(CapacitySchedulerConfiguration.
+ MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
+ LeafQueue newA = new LeafQueue(csContext, A, root, null);
+ a.reinitialize(newA, clusterResource);
+ assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
+ assertEquals(320, a.getMaximumActiveApplications());
+
+ Resource newClusterResource = Resources.createResource(100 * 20 * GB,
+ 100 * 32);
+ a.updateClusterResource(newClusterResource);
+ // 100 * 20 * 0.2 = 400
+ assertEquals(400, a.getMaximumActiveApplications());
+ }
+
+ private CapacitySchedulerContext mockCSContext(
+ CapacitySchedulerConfiguration csConf, Resource clusterResource) {
+ CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
+ when(csContext.getConfiguration()).thenReturn(csConf);
+ when(csContext.getConf()).thenReturn(new YarnConfiguration());
+ when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
+ when(csContext.getClusterResources()).thenReturn(clusterResource);
+ when(csContext.getMinimumResourceCapability()).thenReturn(
+ Resources.createResource(GB, 1));
+ when(csContext.getMaximumResourceCapability()).thenReturn(
+ Resources.createResource(2 * GB, 2));
+ return csContext;
+ }
@After
public void tearDown() throws Exception {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Mon Aug 12 21:25:49 2013
@@ -558,7 +558,26 @@ public class TestFairScheduler {
assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2")
.getAppSchedulables().size());
}
-
+
+ @Test
+ public void testEmptyQueueName() throws Exception {
+ Configuration conf = createConfiguration();
+
+ // only default queue
+ assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+
+ // submit app with empty queue
+ ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
+ AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
+ appAttemptId, "", "user1");
+ scheduler.handle(appAddedEvent);
+
+ // submission rejected
+ assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
+ assertNull(scheduler.getSchedulerApp(appAttemptId));
+ assertEquals(0, resourceManager.getRMContext().getRMApps().size());
+ }
+
@Test
public void testAssignToQueue() throws Exception {
Configuration conf = createConfiguration();
@@ -1929,7 +1948,7 @@ public class TestFairScheduler {
scheduler.handle(node2UpdateEvent);
assertEquals(1, app.getLiveContainers().size());
}
-
+
/**
* If we update our ask to strictly request a node, it doesn't make sense to keep
* a reservation on another.
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java Mon Aug 12 21:25:49 2013
@@ -18,7 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
+import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collection;
import javax.crypto.SecretKey;
@@ -26,9 +29,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -42,21 +46,32 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+@RunWith(Parameterized.class)
public class TestAMRMTokens {
private static final Log LOG = LogFactory.getLog(TestAMRMTokens.class);
- private static final Configuration confWithSecurityEnabled =
- new Configuration();
- static {
- confWithSecurityEnabled.set(
+ private final Configuration conf;
+
+ @Parameters
+ public static Collection<Object[]> configs() {
+ Configuration conf = new Configuration();
+ Configuration confWithSecurity = new Configuration();
+ confWithSecurity.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- UserGroupInformation.setConfiguration(confWithSecurityEnabled);
+ return Arrays.asList(new Object[][] {{ conf }, { confWithSecurity } });
+ }
+
+ public TestAMRMTokens(Configuration conf) {
+ this.conf = conf;
+ UserGroupInformation.setConfiguration(conf);
}
/**
@@ -69,8 +84,8 @@ public class TestAMRMTokens {
public void testTokenExpiry() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- final MockRM rm =
- new MockRMWithAMS(confWithSecurityEnabled, containerManager);
+ final MockRMWithAMS rm =
+ new MockRMWithAMS(conf, containerManager);
rm.start();
final Configuration conf = rm.getConfig();
@@ -85,11 +100,11 @@ public class TestAMRMTokens {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amTokens == null && waitCount++ < 20) {
+ while (containerManager.containerTokens == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amTokens);
+ Assert.assertNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -98,23 +113,21 @@ public class TestAMRMTokens {
UserGroupInformation currentUser =
UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- Credentials credentials = new Credentials();
- DataInputByteBuffer buf = new DataInputByteBuffer();
- containerManager.amTokens.rewind();
- buf.reset(containerManager.amTokens);
- credentials.readTokenStorageStream(buf);
- currentUser.addCredentials(credentials);
-
+ Credentials credentials = containerManager.getContainerCredentials();
+ final InetSocketAddress rmBindAddress =
+ rm.getApplicationMasterService().getBindAddress();
+ Token<? extends TokenIdentifier> amRMToken =
+ MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
+ credentials.getAllTokens());
+ currentUser.addToken(amRMToken);
rmClient = createRMClient(rm, conf, rpc, currentUser);
RegisterApplicationMasterRequest request =
Records.newRecord(RegisterApplicationMasterRequest.class);
- request.setApplicationAttemptId(applicationAttemptId);
rmClient.registerApplicationMaster(request);
FinishApplicationMasterRequest finishAMRequest =
Records.newRecord(FinishApplicationMasterRequest.class);
- finishAMRequest.setAppAttemptId(applicationAttemptId);
finishAMRequest
.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
finishAMRequest.setDiagnostics("diagnostics");
@@ -125,11 +138,8 @@ public class TestAMRMTokens {
// exception.
rpc.stopProxy(rmClient, conf); // To avoid using cached client
rmClient = createRMClient(rm, conf, rpc, currentUser);
- request.setApplicationAttemptId(BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(12345, 78), 987));
AllocateRequest allocateRequest =
Records.newRecord(AllocateRequest.class);
- allocateRequest.setApplicationAttemptId(applicationAttemptId);
try {
rmClient.allocate(allocateRequest);
Assert.fail("You got to be kidding me! "
@@ -161,8 +171,8 @@ public class TestAMRMTokens {
public void testMasterKeyRollOver() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- final MockRM rm =
- new MockRMWithAMS(confWithSecurityEnabled, containerManager);
+ final MockRMWithAMS rm =
+ new MockRMWithAMS(conf, containerManager);
rm.start();
final Configuration conf = rm.getConfig();
@@ -177,11 +187,11 @@ public class TestAMRMTokens {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.amTokens == null && waitCount++ < 20) {
+ while (containerManager.containerTokens == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.amTokens);
+ Assert.assertNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -190,24 +200,22 @@ public class TestAMRMTokens {
UserGroupInformation currentUser =
UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- Credentials credentials = new Credentials();
- DataInputByteBuffer buf = new DataInputByteBuffer();
- containerManager.amTokens.rewind();
- buf.reset(containerManager.amTokens);
- credentials.readTokenStorageStream(buf);
- currentUser.addCredentials(credentials);
-
+ Credentials credentials = containerManager.getContainerCredentials();
+ final InetSocketAddress rmBindAddress =
+ rm.getApplicationMasterService().getBindAddress();
+ Token<? extends TokenIdentifier> amRMToken =
+ MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
+ credentials.getAllTokens());
+ currentUser.addToken(amRMToken);
rmClient = createRMClient(rm, conf, rpc, currentUser);
RegisterApplicationMasterRequest request =
Records.newRecord(RegisterApplicationMasterRequest.class);
- request.setApplicationAttemptId(applicationAttemptId);
rmClient.registerApplicationMaster(request);
// One allocate call.
AllocateRequest allocateRequest =
Records.newRecord(AllocateRequest.class);
- allocateRequest.setApplicationAttemptId(applicationAttemptId);
Assert.assertTrue(
rmClient.allocate(allocateRequest).getAMCommand() == null);
@@ -224,7 +232,6 @@ public class TestAMRMTokens {
rpc.stopProxy(rmClient, conf); // To avoid using cached client
rmClient = createRMClient(rm, conf, rpc, currentUser);
allocateRequest = Records.newRecord(AllocateRequest.class);
- allocateRequest.setApplicationAttemptId(applicationAttemptId);
Assert.assertTrue(
rmClient.allocate(allocateRequest).getAMCommand() == null);
} finally {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java Mon Aug 12 21:25:49 2013
@@ -19,7 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.lang.annotation.Annotation;
@@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.Contai
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -158,6 +162,9 @@ public class TestClientToAMTokens {
ContainerManagementProtocol containerManager =
mock(ContainerManagementProtocol.class);
+ StartContainersResponse mockResponse = mock(StartContainersResponse.class);
+ when(containerManager.startContainers((StartContainersRequest) any()))
+ .thenReturn(mockResponse);
final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Mon Aug 12 21:25:49 2013
@@ -408,8 +408,7 @@ public class TestRMWebServices extends J
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
long totalMBExpect =
- metrics.getReservedMB()+ metrics.getAvailableMB()
- + metrics.getAllocatedMB();
+ metrics.getAvailableMB() + metrics.getAllocatedMB();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);